/** * @file http_texture_load.cpp * @brief Texture download example for core-http library * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2012, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ #include #include #include #include #include #if !defined(WIN32) #include #endif #include "httpcommon.h" #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" #include "bufferarray.h" #include "_mutex.h" #include #include void init_curl(); void term_curl(); unsigned long ssl_thread_id_callback(void); void ssl_locking_callback(int mode, int type, const char * file, int line); void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) #define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) #define strtok_r(_a, _b, _c) strtok_s(_a, _b, _c) int getopt(int argc, char * const argv[], const char *optstring); char *optarg(NULL); int optind(1); #endif class WorkingSet : public LLCore::HttpHandler { public: WorkingSet(); bool reload(LLCore::HttpRequest *); virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); void loadTextureUuids(FILE * in); public: struct Spec { std::string mUuid; int mOffset; int mLength; }; typedef std::set handle_set_t; typedef std::vector texture_list_t; public: bool mVerbose; bool mRandomRange; int mMaxConcurrency; handle_set_t mHandles; int mRemaining; int mLimit; int mAt; std::string mUrl; texture_list_t mTextures; int mErrorsApi; int mErrorsHttp; int mSuccesses; long mByteCount; }; // // // int main(int argc, char** argv) { bool do_random(false); bool do_verbose(false); int option(-1); while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) { switch (option) { case 'u': strncpy(url_format, optarg, sizeof(url_format)); url_format[sizeof(url_format) - 1] = '\0'; break; case 'c': { unsigned long value; char * end; value = strtoul(optarg, &end, 10); if (value < 1 || value > 100 || *end != '\0') { usage(std::cerr); return 1; } concurrency_limit = value; } break; case 'R': do_random = true; break; case 'v': do_verbose = true; break; case 'h': case '?': usage(std::cout); return 0; } } if ((optind + 1) != argc) { usage(std::cerr); return 1; } FILE * uuids(fopen(argv[optind], "r")); if (! uuids) { const char * errstr(strerror(errno)); std::cerr << "Couldn't open UUID file '" << argv[optind] << "'. Reason: " << errstr << std::endl; return 1; } // Initialization init_curl(); LLCore::HttpRequest::createService(); LLCore::HttpRequest::startThread(); // Get service point LLCore::HttpRequest * hr = new LLCore::HttpRequest(); // Get a handler/working set WorkingSet ws; // Fill the working set with work ws.mUrl = url_format; ws.loadTextureUuids(uuids); ws.mRandomRange = do_random; ws.mVerbose = do_verbose; ws.mMaxConcurrency = concurrency_limit; if (! ws.mTextures.size()) { std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; return 1; } // Run it while (! ws.reload(hr)) { hr->update(1000); #if defined(WIN32) Sleep(5); #else usleep(5000); #endif } // Report std::cout << "HTTP errors: " << ws.mErrorsHttp << " API errors: " << ws.mErrorsApi << " Successes: " << ws.mSuccesses << " Byte count: " << ws.mByteCount << std::endl; // Clean up delete hr; term_curl(); return 0; } void usage(std::ostream & out) { out << "\n" "usage:\thttp_texture_load [options] uuid_file\n" "\n" "This is a standalone program to drive the New Platform HTTP Library.\n" "The program is supplied with a file of texture UUIDs, one per line\n" "These are fetched sequentially using a pool of concurrent connection\n" "until all are fetched. The default URL format is only useful from\n" "within Linden Lab but this can be overriden with a printf-style\n" "URL formatting string on the command line.\n" "\n" "Options:\n" "\n" " -u printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" " -c Maximum request concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" << std::endl; } WorkingSet::WorkingSet() : LLCore::HttpHandler(), mVerbose(false), mRandomRange(false), mRemaining(200), mLimit(200), mAt(0), mErrorsApi(0), mErrorsHttp(0), mSuccesses(0), mByteCount(0L) { mTextures.reserve(30000); } bool WorkingSet::reload(LLCore::HttpRequest * hr) { int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); for (int i(0); i < to_do; ++i) { char buffer[1024]; #if defined(WIN32) _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); #else snprintf(buffer, sizeof(buffer), mUrl.c_str(), mUuids[mAt].c_str()); #endif int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); LLCore::HttpHandle handle; if (offset || length) { handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, NULL, this); } else { handle = hr->requestGet(0, 0, buffer, NULL, NULL, this); } if (! handle) { // Fatal. Couldn't queue up something. std::cerr << "Failed to queue work to HTTP Service. Reason: " << hr->getStatus().toString() << std::endl; exit(1); } else { mHandles.insert(handle); } mAt++; mRemaining--; if (mVerbose) { static int count(0); ++count; if (0 == (count %5)) std::cout << "Queued " << count << std::endl; } } // Are we done? return (! mRemaining) && mHandles.empty(); } void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { handle_set_t::iterator it(mHandles.find(handle)); if (mHandles.end() == it) { // Wha? std::cerr << "Failed to find handle in request list. Fatal." << std::endl; exit(1); } else { LLCore::HttpStatus status(response->getStatus()); if (status) { // More success LLCore::BufferArray * data(response->getBody()); mByteCount += data->size(); ++mSuccesses; } else { // Something in this library or libcurl if (status.isHttpStatus()) { ++mErrorsHttp; } else { ++mErrorsApi; } } mHandles.erase(it); } if (mVerbose) { static int count(0); ++count; if (0 == (count %5)) std::cout << "Handled " << count << std::endl; } } void WorkingSet::loadTextureUuids(FILE * in) { char buffer[1024]; while (fgets(buffer, sizeof(buffer), in)) { WorkingSet::Spec texture; char * state(NULL); char * token = strtok_r(buffer, " \t\n,", &state); if (token && 36 == strlen(token)) { // Close enough for this function texture.mUuid = token; texture.mOffset = 0; texture.mLength = 0; token = strtok_r(buffer, " \t\n,", &state); if (token) { int offset(atoi(token)); token = strtok_r(buffer, " \t\n,", &state); if (token) { int length(atoi(token)); texture.mOffset = offset; texture.mLength = length; } } mTextures.push_back(texture); } } mRemaining = mLimit = mTextures.size(); } int ssl_mutex_count(0); LLCoreInt::HttpMutex ** ssl_mutex_list = NULL; void init_curl() { curl_global_init(CURL_GLOBAL_ALL); ssl_mutex_count = CRYPTO_num_locks(); if (ssl_mutex_count > 0) { ssl_mutex_list = new LLCoreInt::HttpMutex * [ssl_mutex_count]; for (int i(0); i < ssl_mutex_count; ++i) { ssl_mutex_list[i] = new LLCoreInt::HttpMutex; } CRYPTO_set_locking_callback(ssl_locking_callback); CRYPTO_set_id_callback(ssl_thread_id_callback); } } void term_curl() { CRYPTO_set_locking_callback(NULL); for (int i(0); i < ssl_mutex_count; ++i) { delete ssl_mutex_list[i]; } delete [] ssl_mutex_list; } unsigned long ssl_thread_id_callback(void) { #if defined(WIN32) return (unsigned long) GetCurrentThread(); #else return (unsigned long) pthread_self(); #endif } void ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) { if (type >= 0 && type < ssl_mutex_count) { if (mode & CRYPTO_LOCK) { ssl_mutex_list[type]->lock(); } else { ssl_mutex_list[type]->unlock(); } } } #if defined(WIN32) // Very much a subset of posix functionality. Don't push // it too hard... int getopt(int argc, char * const argv[], const char *optstring) { static int pos(0); while (optind < argc) { if (pos == 0) { if (argv[optind][0] != '-') return -1; pos = 1; } if (! argv[optind][pos]) { ++optind; pos = 0; continue; } const char * thing(strchr(optstring, argv[optind][pos])); if (! thing) { ++optind; return -1; } if (thing[1] == ':') { optarg = argv[++optind]; ++optind; pos = 0; } else { optarg = NULL; ++pos; } return *thing; } return -1; } #endif