From 888e2587e57bf296c0b9ee227a4baa3efb3f4892 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 15 Jun 2012 13:05:24 -0400 Subject: Ported example (freestanding) program to drive API & generate performance numbers. This is a command-line utility to pull content down from a service through the llcorehttp library to produce timings and resource footprints. --- indra/llcorehttp/CMakeLists.txt | 28 ++ indra/llcorehttp/examples/http_texture_load.cpp | 494 ++++++++++++++++++++++++ 2 files changed, 522 insertions(+) create mode 100644 indra/llcorehttp/examples/http_texture_load.cpp (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 4273b32fe3..9d8bae973e 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -134,5 +134,33 @@ if (LL_TESTS) "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_llcorehttp_peer.py" ) + # + # Example Programs + # + SET(llcorehttp_EXAMPLE_SOURCE_FILES + examples/http_texture_load.cpp + ) + + set(example_libs + ${LLCOREHTTP_LIBRARIES} + ${WINDOWS_LIBRARIES} + ${LLMESSAGE_LIBRARIES} + ${LLCOMMON_LIBRARIES} + ${GOOGLEMOCK_LIBRARIES} + ${CURL_LIBRARIES} + ${CARES_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${CRYPTO_LIBRARIES} + ${BOOST_THREAD_LIBRARY} + ) + + add_executable(http_texture_load + ${llcorehttp_EXAMPLE_SOURCE_FILES} + ) + set_target_properties(http_texture_load + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${EXE_STAGING_DIR}") + + target_link_libraries(http_texture_load ${example_libs}) + endif (LL_TESTS) diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp new file mode 100644 index 0000000000..83139912c9 --- /dev/null +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -0,0 +1,494 @@ +/** + * @file http_texture_load.cpp + * @brief Texture download example for core-http library + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include +#include +#include +#include +#include +#if !defined(WIN32) +#include +#endif + +#include "httpcommon.h" +#include "httprequest.h" +#include "httphandler.h" +#include "httpresponse.h" +#include "bufferarray.h" +#include "_mutex.h" + +#include +#include + + +void init_curl(); +void term_curl(); +unsigned long ssl_thread_id_callback(void); +void ssl_locking_callback(int mode, int type, const char * file, int line); +void usage(std::ostream & out); + +// Default command line settings +static int concurrency_limit(40); +static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; + +#if defined(WIN32) + +#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) +#define strtok_r(_a, _b, _c) strtok_s(_a, _b, _c) + +int getopt(int argc, char * const argv[], const char *optstring); +char *optarg(NULL); +int optind(1); + +#endif + +class WorkingSet : public LLCore::HttpHandler +{ +public: + WorkingSet(); + + bool reload(LLCore::HttpRequest *); + + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + void loadTextureUuids(FILE * in); + +public: + struct Spec + { + std::string mUuid; + int mOffset; + int mLength; + }; + typedef std::set handle_set_t; + typedef std::vector texture_list_t; + +public: + bool mVerbose; + bool mRandomRange; + int mMaxConcurrency; + handle_set_t mHandles; + int mRemaining; + int mLimit; + int mAt; + std::string mUrl; + texture_list_t mTextures; + int mErrorsApi; + int mErrorsHttp; + int mSuccesses; + long mByteCount; +}; + + +// +// +// +int main(int argc, char** argv) +{ + bool do_random(false); + bool do_verbose(false); + + int option(-1); + while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) + { + switch (option) + { + case 'u': + strncpy(url_format, optarg, sizeof(url_format)); + url_format[sizeof(url_format) - 1] = '\0'; + break; + + case 'c': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + concurrency_limit = value; + } + break; + + case 'R': + do_random = true; + break; + + case 'v': + do_verbose = true; + break; + + case 'h': + case '?': + usage(std::cout); + return 0; + } + } + + if ((optind + 1) != argc) + { + usage(std::cerr); + return 1; + } + + FILE * uuids(fopen(argv[optind], "r")); + if (! uuids) + { + const char * errstr(strerror(errno)); + + std::cerr << "Couldn't open UUID file '" << argv[optind] << "'. Reason: " + << errstr << std::endl; + return 1; + } + + // Initialization + init_curl(); + LLCore::HttpRequest::createService(); + LLCore::HttpRequest::startThread(); + + // Get service point + LLCore::HttpRequest * hr = new LLCore::HttpRequest(); + + // Get a handler/working set + WorkingSet ws; + + // Fill the working set with work + ws.mUrl = url_format; + ws.loadTextureUuids(uuids); + ws.mRandomRange = do_random; + ws.mVerbose = do_verbose; + ws.mMaxConcurrency = concurrency_limit; + + if (! ws.mTextures.size()) + { + std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; + return 1; + } + + // Run it + while (! ws.reload(hr)) + { + hr->update(1000); +#if defined(WIN32) + Sleep(5); +#else + usleep(5000); +#endif + } + + // Report + std::cout << "HTTP errors: " << ws.mErrorsHttp << " API errors: " << ws.mErrorsApi + << " Successes: " << ws.mSuccesses << " Byte count: " << ws.mByteCount + << std::endl; + + // Clean up + delete hr; + term_curl(); + + return 0; +} + + +void usage(std::ostream & out) +{ + out << "\n" + "usage:\thttp_texture_load [options] uuid_file\n" + "\n" + "This is a standalone program to drive the New Platform HTTP Library.\n" + "The program is supplied with a file of texture UUIDs, one per line\n" + "These are fetched sequentially using a pool of concurrent connection\n" + "until all are fetched. The default URL format is only useful from\n" + "within Linden Lab but this can be overriden with a printf-style\n" + "URL formatting string on the command line.\n" + "\n" + "Options:\n" + "\n" + " -u printf-style format string for URL generation\n" + " Default: " << url_format << "\n" + " -R Issue GETs with random Range: headers\n" + " -c Maximum request concurrency. Range: [1..100]\n" + " Default: " << concurrency_limit << "\n" + " -v Verbose mode. Issue some chatter while running\n" + " -h print this help\n" + "\n" + << std::endl; +} + + +WorkingSet::WorkingSet() + : LLCore::HttpHandler(), + mVerbose(false), + mRandomRange(false), + mRemaining(200), + mLimit(200), + mAt(0), + mErrorsApi(0), + mErrorsHttp(0), + mSuccesses(0), + mByteCount(0L) +{ + mTextures.reserve(30000); +} + + +bool WorkingSet::reload(LLCore::HttpRequest * hr) +{ + int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); + + for (int i(0); i < to_do; ++i) + { + char buffer[1024]; +#if defined(WIN32) + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); +#else + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mUuids[mAt].c_str()); +#endif + int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); + int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); + + LLCore::HttpHandle handle; + if (offset || length) + { + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, NULL, this); + } + else + { + handle = hr->requestGet(0, 0, buffer, NULL, NULL, this); + } + if (! handle) + { + // Fatal. Couldn't queue up something. + std::cerr << "Failed to queue work to HTTP Service. Reason: " + << hr->getStatus().toString() << std::endl; + exit(1); + } + else + { + mHandles.insert(handle); + } + mAt++; + mRemaining--; + + if (mVerbose) + { + static int count(0); + ++count; + if (0 == (count %5)) + std::cout << "Queued " << count << std::endl; + } + } + + // Are we done? + return (! mRemaining) && mHandles.empty(); +} + + +void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + handle_set_t::iterator it(mHandles.find(handle)); + if (mHandles.end() == it) + { + // Wha? + std::cerr << "Failed to find handle in request list. Fatal." << std::endl; + exit(1); + } + else + { + LLCore::HttpStatus status(response->getStatus()); + if (status) + { + // More success + LLCore::BufferArray * data(response->getBody()); + mByteCount += data->size(); + ++mSuccesses; + } + else + { + // Something in this library or libcurl + if (status.isHttpStatus()) + { + ++mErrorsHttp; + } + else + { + ++mErrorsApi; + } + } + mHandles.erase(it); + } + + if (mVerbose) + { + static int count(0); + ++count; + if (0 == (count %5)) + std::cout << "Handled " << count << std::endl; + } +} + + +void WorkingSet::loadTextureUuids(FILE * in) +{ + char buffer[1024]; + + while (fgets(buffer, sizeof(buffer), in)) + { + WorkingSet::Spec texture; + char * state(NULL); + char * token = strtok_r(buffer, " \t\n,", &state); + if (token && 36 == strlen(token)) + { + // Close enough for this function + texture.mUuid = token; + texture.mOffset = 0; + texture.mLength = 0; + token = strtok_r(buffer, " \t\n,", &state); + if (token) + { + int offset(atoi(token)); + token = strtok_r(buffer, " \t\n,", &state); + if (token) + { + int length(atoi(token)); + texture.mOffset = offset; + texture.mLength = length; + } + } + mTextures.push_back(texture); + } + } + mRemaining = mLimit = mTextures.size(); +} + + +int ssl_mutex_count(0); +LLCoreInt::HttpMutex ** ssl_mutex_list = NULL; + +void init_curl() +{ + curl_global_init(CURL_GLOBAL_ALL); + + ssl_mutex_count = CRYPTO_num_locks(); + if (ssl_mutex_count > 0) + { + ssl_mutex_list = new LLCoreInt::HttpMutex * [ssl_mutex_count]; + + for (int i(0); i < ssl_mutex_count; ++i) + { + ssl_mutex_list[i] = new LLCoreInt::HttpMutex; + } + + CRYPTO_set_locking_callback(ssl_locking_callback); + CRYPTO_set_id_callback(ssl_thread_id_callback); + } +} + + +void term_curl() +{ + CRYPTO_set_locking_callback(NULL); + for (int i(0); i < ssl_mutex_count; ++i) + { + delete ssl_mutex_list[i]; + } + delete [] ssl_mutex_list; +} + + +unsigned long ssl_thread_id_callback(void) +{ +#if defined(WIN32) + return (unsigned long) GetCurrentThread(); +#else + return (unsigned long) pthread_self(); +#endif +} + + +void ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) +{ + if (type >= 0 && type < ssl_mutex_count) + { + if (mode & CRYPTO_LOCK) + { + ssl_mutex_list[type]->lock(); + } + else + { + ssl_mutex_list[type]->unlock(); + } + } +} + + +#if defined(WIN32) + +// Very much a subset of posix functionality. Don't push +// it too hard... +int getopt(int argc, char * const argv[], const char *optstring) +{ + static int pos(0); + while (optind < argc) + { + if (pos == 0) + { + if (argv[optind][0] != '-') + return -1; + pos = 1; + } + if (! argv[optind][pos]) + { + ++optind; + pos = 0; + continue; + } + const char * thing(strchr(optstring, argv[optind][pos])); + if (! thing) + { + ++optind; + return -1; + } + if (thing[1] == ':') + { + optarg = argv[++optind]; + ++optind; + pos = 0; + } + else + { + optarg = NULL; + ++pos; + } + return *thing; + } + return -1; +} + +#endif + -- cgit v1.2.3