Lock/Mutex fixes, documentation, new resource wait state, dtor cleanups

Went through all the code and tried to document lock and thread usage
in the module.  There's a huge comment block introducing all of this
at the beginning and I believe it's correct (though not quite complete).
Keep it updated, people.  Added a new state, WAIT_HTTP_RESOURCE, that's
sort of a side-state of SEND_HTTP_REQ.  If we hit a high-water mark
for HTTP requests, the extra are shunted to the new state once.  Once
levels fall to a low-water mark, we run through a wait list of UUIDs,
sort the valid ones by priority and release them for service.  This
keeps the HTTP layer busy while leaving the active queue shallow enough
that requests can still be re-prioritzed cheaply.  Priority model
changed.  The new state uses the PRIORITY_LOW mask, the old users
of _LOW are now at PRIORITY_NORMAL and sleepers woken up after an
external event are kicked off at PRIORITY_HIGH.  This combination
along with the new state should avoid priority inversion and keep
things running without resorting to an infinite pipeline.  New
state displays as "HTW" with green text in the texture console.
Request cancelation and worker run-down should now be more
correct but this edge case may need more attention.

1 files changed, 135 insertions, 18 deletions

diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h
index cfea3aad9d..53b0f7885f 100644
--- a/indra/newview/lltexturefetch.h
+++ b/indra/newview/lltexturefetch.h
@@ -45,6 +45,7 @@ class LLHost;
 class LLViewerAssetStats;
 
 // Interface class
+
 class LLTextureFetch : public LLWorkerThread
 {
 	friend class LLTextureFetchWorker;
@@ -55,67 +56,174 @@ public:
 
 	class TFRequest;
 	
-	/*virtual*/ S32 update(F32 max_time_ms);	
-	void shutDownTextureCacheThread() ; //called in the main thread after the TextureCacheThread shuts down.
-	void shutDownImageDecodeThread() ;  //called in the main thread after the ImageDecodeThread shuts down.
+    // Threads:  Tmain
+	/*virtual*/ S32 update(F32 max_time_ms);
+	
+	// called in the main thread after the TextureCacheThread shuts down.
+    // Threads:  Tmain
+	void shutDownTextureCacheThread();
+
+	//called in the main thread after the ImageDecodeThread shuts down.
+    // Threads:  Tmain
+	void shutDownImageDecodeThread();
 
+	// Threads:  T* (but Tmain mostly)
 	bool createRequest(const std::string& url, const LLUUID& id, const LLHost& host, F32 priority,
 					   S32 w, S32 h, S32 c, S32 discard, bool needs_aux, bool can_use_http);
+
+	// Requests that a fetch operation be deleted from the queue.
+	// If @cancel is true, also stops any I/O operations pending.
+	// Actual delete will be scheduled and performed later.
+	//
+	// Note:  This *looks* like an override/variant of the
+	// base class's deleteRequest() but is functionally quite
+	// different.
+	//
+	// Threads:  T*
 	void deleteRequest(const LLUUID& id, bool cancel);
+
+	// Threads:  T*
 	bool getRequestFinished(const LLUUID& id, S32& discard_level,
 							LLPointer<LLImageRaw>& raw, LLPointer<LLImageRaw>& aux);
+
+	// Threads:  T*
 	bool updateRequestPriority(const LLUUID& id, F32 priority);
 
+    // Threads:  T*
 	bool receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data);
+
+    // Threads:  T*
 	bool receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data);
 
+    // Threads:  T* (but not safe)
 	void setTextureBandwidth(F32 bandwidth) { mTextureBandwidth = bandwidth; }
+	
+    // Threads:  T* (but not safe)
 	F32 getTextureBandwidth() { return mTextureBandwidth; }
 	
-	// Debug
+    // Threads:  T*
 	BOOL isFromLocalCache(const LLUUID& id);
+
+	// @return	Magic number giving the internal state of the
+	//			request.  We should make these codes public if we're
+	//			going to return them as a status value.
+	//
+    // Threads:  T*
 	S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p,
 					  U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http);
+
+	// Debug utility - generally not safe
 	void dump();
-	S32 getNumRequests() ;
-	S32 getNumHTTPRequests() ;
-	U32 getTotalNumHTTPRequests() ;
+
+	// Threads:  T*
+	S32 getNumRequests();
+
+	// Threads:  T*
+	S32 getNumHTTPRequests();
+
+	// Threads:  T*
+	U32 getTotalNumHTTPRequests();
 	
-	// Public for access by callbacks
+    // Threads:  T*
     S32 getPending();
+
+    // Threads:  T*
 	void lockQueue() { mQueueMutex.lock(); }
+
+    // Threads:  T*
 	void unlockQueue() { mQueueMutex.unlock(); }
+
+	// Threads:  T*
 	LLTextureFetchWorker* getWorker(const LLUUID& id);
+	
+	// Threads:  T*
+	// Locks:  Mfq
 	LLTextureFetchWorker* getWorkerAfterLock(const LLUUID& id);
 
 	// Commands available to other threads to control metrics gathering operations.
+
+	// Threads:  T*
 	void commandSetRegion(U64 region_handle);
+
+	// Threads:  T*
 	void commandSendMetrics(const std::string & caps_url,
 							const LLUUID & session_id,
 							const LLUUID & agent_id,
 							LLViewerAssetStats * main_stats);
+
+	// Threads:  T*
 	void commandDataBreak();
 
+	// Threads:  T*
 	LLCore::HttpRequest & getHttpRequest()	{ return *mHttpRequest; }
 
 	bool isQAMode() const				{ return mQAMode; }
 
+	// ----------------------------------
+	// HTTP resource waiting methods
+
+    // Threads:  T*
+	void addHttpWaiter(const LLUUID & tid);
+
+    // Threads:  T*
+	void removeHttpWaiter(const LLUUID & tid);
+
+	// If there are slots, release one or more LLTextureFetchWorker
+	// requests from resource wait state (WAIT_HTTP_RESOURCE) to
+	// active (SEND_HTTP_REQ).
+	//
+	// Because this will modify state of many workers, you may not
+	// hold any Mw lock while calling.  This makes it a little
+	// inconvenient to use but that's the rule.
+	//
+    // Threads:  T*
+	// Locks:  -Mw (must not hold any worker when called)
+	void releaseHttpWaiters();
+
+    // Threads:  T*
+	void cancelHttpWaiters();
+
+    // Threads:  T*
+	int getHttpWaitersCount();
+	// ----------------------------------
+	
 protected:
+	// Threads:  T* (but Ttf in practice)
 	void addToNetworkQueue(LLTextureFetchWorker* worker);
+
+	// Threads:  T*
 	void removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel);
+
+    // Threads:  T*
 	void addToHTTPQueue(const LLUUID& id);
+
+    // Threads:  T*
 	void removeFromHTTPQueue(const LLUUID& id, S32 received_size = 0);
+
+	// Identical to @deleteRequest but with different arguments
+	// (caller already has the worker pointer).
+	//
+	// Threads:  T*
 	void removeRequest(LLTextureFetchWorker* worker, bool cancel);
-	void cancelHttpRequests();
 	
 	// Overrides from the LLThread tree
+	// Locks:  Ct
 	bool runCondition();
 
 private:
+    // Threads:  Tmain
 	void sendRequestListToSimulators();
+	
+	// Threads:  Ttf
 	/*virtual*/ void startThread(void);
+	
+	// Threads:  Ttf
 	/*virtual*/ void endThread(void);
+	
+	// Threads:  Ttf
 	/*virtual*/ void threadedUpdate(void);
+
+	// Threads:  Ttf
 	void commonUpdate();
 
 	// Metrics command helpers
@@ -126,6 +234,8 @@ private:
 	 * Takes ownership of the TFRequest object.
 	 *
 	 * Method locks the command queue.
+	 *
+	 * Threads:  T*
 	 */
 	void cmdEnqueue(TFRequest *);
 
@@ -136,6 +246,8 @@ private:
 	 * Caller acquires ownership of the object and must dispose of it.
 	 *
 	 * Method locks the command queue.
+	 *
+	 * Threads:  T*
 	 */
 	TFRequest * cmdDequeue();
 
@@ -145,6 +257,8 @@ private:
 	 * additional commands.
 	 *
 	 * Method locks the command queue.
+	 *
+	 * Threads:  Ttf
 	 */
 	void cmdDoWork();
 	
@@ -164,29 +278,29 @@ private:
 	
 	// Map of all requests by UUID
 	typedef std::map<LLUUID,LLTextureFetchWorker*> map_t;
-	map_t mRequestMap;
+	map_t mRequestMap;													// Mfq
 
 	// Set of requests that require network data
 	typedef std::set<LLUUID> queue_t;
-	queue_t mNetworkQueue;
-	queue_t mHTTPTextureQueue;
+	queue_t mNetworkQueue;												// Mfnq
+	queue_t mHTTPTextureQueue;											// Mfnq
 	typedef std::map<LLHost,std::set<LLUUID> > cancel_queue_t;
-	cancel_queue_t mCancelQueue;
-	F32 mTextureBandwidth;
-	F32 mMaxBandwidth;
+	cancel_queue_t mCancelQueue;										// Mfnq
+	F32 mTextureBandwidth;												// <none>
+	F32 mMaxBandwidth;													// Mfnq
 	LLTextureInfo mTextureInfo;
 
-	U32 mHTTPTextureBits;
+	U32 mHTTPTextureBits;												// Mfnq
 
 	//debug use
-	U32 mTotalHTTPRequests ;
+	U32 mTotalHTTPRequests;
 
 	// Out-of-band cross-thread command queue.  This command queue
 	// is logically tied to LLQueuedThread's list of
 	// QueuedRequest instances and so must be covered by the
 	// same locks.
 	typedef std::vector<TFRequest *> command_queue_t;
-	command_queue_t mCommands;
+	command_queue_t mCommands;											// Mfq
 
 	// If true, modifies some behaviors that help with QA tasks.
 	const bool mQAMode;
@@ -197,6 +311,9 @@ private:
 	LLCore::HttpRequest *		mHttpRequest;
 	LLCore::HttpOptions *		mHttpOptions;
 	LLCore::HttpHeaders *		mHttpHeaders;
+
+	typedef std::set<LLUUID> wait_http_res_queue_t;
+	wait_http_res_queue_t		mHttpWaitResource;						// Mfnq
 	
 public:
 	// A probabilistically-correct indicator that the current