From 4eef1c8a2e2a8abcc463b9df38b66f025da57589 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 15 Apr 2013 16:55:35 +0000 Subject: SH-4106 Significantly upgrade the HttpHeaders interface for SSB. Header container moves from a vector of raw lines to a vector of string pairs representing name/value pairs in headers. For incoming headers, we normalize the name to lowercase and trim it. Values are only left-trimmed. Outgoing headers are left as-is. Simple find() method for the common case, forward and reverse iterators for those few who need to do it themselves. The HTTP status line (e.g. 'HTTP/1.1 200 Ok') is no longer treated as a header to be returned to caller. Unit tests, as usual, were a bear but they absolutely ensured outgoing HTTP header conformance after the change. Grunt work paid off. LLTextureFetch was also given a second options structure for texture fetches. Same as the original but with header return to caller requested. Baked textures should use this, the other 20,000 texture fetch requests should continue to use the original. --- indra/llcorehttp/examples/http_texture_load.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcorehttp/examples') diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 40ad4f047d..909dc5b0cb 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -328,7 +328,7 @@ WorkingSet::WorkingSet() mTextures.reserve(30000); mHeaders = new LLCore::HttpHeaders; - mHeaders->mHeaders.push_back("Accept: image/x-j2c"); + mHeaders->append("Accept", "image/x-j2c"); } -- cgit v1.2.3 From d8e32c58ce020f4d26af417d927a266209a40483 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 3 Jul 2013 15:17:26 -0400 Subject: Update the example program so it handles meshes as well. Fix the request feed logic to use high/low-water level logic as is done in viewer code. --- indra/llcorehttp/examples/http_texture_load.cpp | 72 +++++++++++++++---------- 1 file changed, 44 insertions(+), 28 deletions(-) (limited to 'indra/llcorehttp/examples') diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 909dc5b0cb..119e34f7ec 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -39,6 +39,7 @@ #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" +#include "httpoptions.h" #include "httpheaders.h" #include "bufferarray.h" #include "_mutex.h" @@ -79,11 +80,11 @@ public: WorkingSet(); ~WorkingSet(); - bool reload(LLCore::HttpRequest *); + bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *); virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - void loadTextureUuids(FILE * in); + void loadAssetUuids(FILE * in); public: struct Spec @@ -93,18 +94,19 @@ public: int mLength; }; typedef std::set handle_set_t; - typedef std::vector texture_list_t; + typedef std::vector asset_list_t; public: bool mVerbose; bool mRandomRange; - int mMaxConcurrency; + int mRequestLowWater; + int mRequestHighWater; handle_set_t mHandles; int mRemaining; int mLimit; int mAt; std::string mUrl; - texture_list_t mTextures; + asset_list_t mAssets; int mErrorsApi; int mErrorsHttp; int mErrorsHttp404; @@ -224,17 +226,23 @@ int main(int argc, char** argv) // Get service point LLCore::HttpRequest * hr = new LLCore::HttpRequest(); + // Get request options + LLCore::HttpOptions * opt = new LLCore::HttpOptions(); + opt->setRetries(12); + opt->setUseRetryAfter(true); + // Get a handler/working set WorkingSet ws; // Fill the working set with work ws.mUrl = url_format; - ws.loadTextureUuids(uuids); + ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; ws.mVerbose = do_verbose; - ws.mMaxConcurrency = 100; + ws.mRequestHighWater = 100; + ws.mRequestLowWater = ws.mRequestHighWater / 2; - if (! ws.mTextures.size()) + if (! ws.mAssets.size()) { std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; return 1; @@ -246,9 +254,9 @@ int main(int argc, char** argv) // Run it int passes(0); - while (! ws.reload(hr)) + while (! ws.reload(hr, opt)) { - hr->update(5000000); + hr->update(0); ms_sleep(2); if (0 == (++passes % 200)) { @@ -275,6 +283,8 @@ int main(int argc, char** argv) // Clean up hr->requestStopThread(NULL); ms_sleep(1000); + opt->release(); + opt = NULL; delete hr; LLCore::HttpRequest::destroyService(); term_curl(); @@ -325,7 +335,7 @@ WorkingSet::WorkingSet() mSuccesses(0), mByteCount(0L) { - mTextures.reserve(30000); + mAssets.reserve(30000); mHeaders = new LLCore::HttpHeaders; mHeaders->append("Accept", "image/x-j2c"); @@ -342,29 +352,35 @@ WorkingSet::~WorkingSet() } -bool WorkingSet::reload(LLCore::HttpRequest * hr) +bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) { - int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); + if (mRequestLowWater <= mHandles.size()) + { + // Haven't fallen below low-water level yet. + return false; + } + + int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size()))); for (int i(0); i < to_do; ++i) { char buffer[1024]; #if defined(WIN32) - _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #else - snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #endif - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); + int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); + int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); LLCore::HttpHandle handle; if (offset || length) { - handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this); + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this); } else { - handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this); + handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this); } if (! handle) { @@ -459,21 +475,21 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r } -void WorkingSet::loadTextureUuids(FILE * in) +void WorkingSet::loadAssetUuids(FILE * in) { char buffer[1024]; while (fgets(buffer, sizeof(buffer), in)) { - WorkingSet::Spec texture; + WorkingSet::Spec asset; char * state(NULL); char * token = strtok_r(buffer, " \t\n,", &state); if (token && 36 == strlen(token)) { // Close enough for this function - texture.mUuid = token; - texture.mOffset = 0; - texture.mLength = 0; + asset.mUuid = token; + asset.mOffset = 0; + asset.mLength = 0; token = strtok_r(buffer, " \t\n,", &state); if (token) { @@ -482,14 +498,14 @@ void WorkingSet::loadTextureUuids(FILE * in) if (token) { int length(atoi(token)); - texture.mOffset = offset; - texture.mLength = length; + asset.mOffset = offset; + asset.mLength = length; } } - mTextures.push_back(texture); + mAssets.push_back(asset); } } - mRemaining = mLimit = mTextures.size(); + mRemaining = mLimit = mAssets.size(); } -- cgit v1.2.3 From fb734d621e6fa2004d191849783e81da75992d06 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 8 Jul 2013 20:31:09 +0000 Subject: Found the memory corruptor. String trimmer didn't have a valid termination test. Sheesh. Also get some more numbers out of the example/load test program which drives traffic. This should give some useful insights into how the current http throttle works (or doesn't) with varying client demands. --- indra/llcorehttp/examples/http_texture_load.cpp | 36 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) (limited to 'indra/llcorehttp/examples') diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 119e34f7ec..88c8102b27 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -58,6 +58,7 @@ void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); +static int highwater(100); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) @@ -113,6 +114,8 @@ public: int mErrorsHttp416; int mErrorsHttp500; int mErrorsHttp503; + int mRetries; + int mRetriesHttp503; int mSuccesses; long mByteCount; LLCore::HttpHeaders * mHeaders; @@ -160,7 +163,7 @@ int main(int argc, char** argv) bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) { switch (option) { @@ -184,6 +187,21 @@ int main(int argc, char** argv) } break; + case 'H': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + highwater = value; + } + break; + case 'R': do_random = true; break; @@ -239,7 +257,7 @@ int main(int argc, char** argv) ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; ws.mVerbose = do_verbose; - ws.mRequestHighWater = 100; + ws.mRequestHighWater = highwater; ws.mRequestLowWater = ws.mRequestHighWater / 2; if (! ws.mAssets.size()) @@ -273,6 +291,8 @@ int main(int argc, char** argv) std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416 << " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503 << std::endl; + std::cout << "Retries: " << ws.mRetries << " Retries on 503: " << ws.mRetriesHttp503 + << std::endl; std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime) << " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime) << " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime) @@ -310,8 +330,10 @@ void usage(std::ostream & out) " -u printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" - " -c Maximum request concurrency. Range: [1..100]\n" + " -c Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" + " -H HTTP request highwater (requests fed to llcorehttp).\n" + " Range: [1..100] Default: " << highwater << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" @@ -332,6 +354,8 @@ WorkingSet::WorkingSet() mErrorsHttp416(0), mErrorsHttp500(0), mErrorsHttp503(0), + mRetries(0), + mRetriesHttp503(0), mSuccesses(0), mByteCount(0L) { @@ -426,7 +450,7 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r { // More success LLCore::BufferArray * data(response->getBody()); - mByteCount += data->size(); + mByteCount += data ? data->size() : 0; ++mSuccesses; } else @@ -462,6 +486,10 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r ++mErrorsApi; } } + unsigned int retry(0U), retry_503(0U); + response->getRetries(&retry, &retry_503); + mRetries += int(retry); + mRetriesHttp503 += int(retry_503); mHandles.erase(it); } -- cgit v1.2.3 From eff651cffca60f2b69f6c596a8e9aa9e1ab44d3c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 12 Jul 2013 15:00:24 -0400 Subject: SH-4312 Configuration data between viewer and llcorehttp is clumsy. Much improved. Unified the global and class options into a single option list. Implemented static and dynamic setting paths as much as possible. Dynamic path does require packet/RPC but otherwise there's near unification. Dynamic modes can't get values back yet due to the response/notifier scheme but this doesn't bother me. Flatten global and class options into simpler struct-like entities. Setter/getter available on these when needed (external APIs) but code can otherwise fiddle directly when it knows what to do. Much duplicated options/state removed from HttpPolicy. Comments cleaned up. Threads better described and consistently mentioned in API docs. Integration test extended for 503 responses with Reply-After headers. --- indra/llcorehttp/examples/http_texture_load.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'indra/llcorehttp/examples') diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 88c8102b27..73c49687d7 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -236,9 +236,10 @@ int main(int argc, char** argv) // Initialization init_curl(); LLCore::HttpRequest::createService(); - LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - concurrency_limit); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); LLCore::HttpRequest::startThread(); // Get service point -- cgit v1.2.3