diff options
Diffstat (limited to 'indra/llcorehttp')
62 files changed, 14802 insertions, 0 deletions
diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt new file mode 100644 index 0000000000..9adfd0ed62 --- /dev/null +++ b/indra/llcorehttp/CMakeLists.txt @@ -0,0 +1,186 @@ +# -*- cmake -*- + +project(llcorehttp) + +include(00-Common) +include(GoogleMock) +include(CURL) +include(CARes) +include(OpenSSL) +include(ZLIB) +include(LLCoreHttp) +include(LLAddBuildTest) +include(LLMessage) +include(LLCommon) +include(Tut) + +include_directories (${CMAKE_CURRENT_SOURCE_DIR}) + +include_directories( + ${LLMESSAGE_INCLUDE_DIRS} + ${LLCOMMON_INCLUDE_DIRS} + ${LLCOREHTTP_INCLUDE_DIRS} + ) + +set(llcorehttp_SOURCE_FILES + bufferarray.cpp + bufferstream.cpp + httpcommon.cpp + httpheaders.cpp + httpoptions.cpp + httprequest.cpp + httpresponse.cpp + _httplibcurl.cpp + _httpopcancel.cpp + _httpoperation.cpp + _httpoprequest.cpp + _httpopsetget.cpp + _httpopsetpriority.cpp + _httppolicy.cpp + _httppolicyclass.cpp + _httppolicyglobal.cpp + _httpreplyqueue.cpp + _httprequestqueue.cpp + _httpservice.cpp + _refcounted.cpp + ) + +set(llcorehttp_HEADER_FILES + CMakeLists.txt + + bufferarray.h + bufferstream.h + httpcommon.h + httphandler.h + httpheaders.h + httpoptions.h + httprequest.h + httpresponse.h + _httpinternal.h + _httplibcurl.h + _httpopcancel.h + _httpoperation.h + _httpoprequest.h + _httpopsetget.h + _httpopsetpriority.h + _httppolicy.h + _httppolicyclass.h + _httppolicyglobal.h + _httpreadyqueue.h + _httpreplyqueue.h + _httprequestqueue.h + _httpservice.h + _mutex.h + _refcounted.h + _thread.h + ) + +set_source_files_properties(${llcorehttp_HEADER_FILES} + PROPERTIES HEADER_FILE_ONLY TRUE) +if (DARWIN OR LINUX) + # Boost headers define unused members in condition_variable so... + set_source_files_properties(${llcorehttp_SOURCE_FILES} + PROPERTIES COMPILE_FLAGS -Wno-unused-variable) +endif (DARWIN OR LINUX) + +list(APPEND llcorehttp_SOURCE_FILES ${llcorehttp_HEADER_FILES}) + +add_library (llcorehttp ${llcorehttp_SOURCE_FILES}) +target_link_libraries( + llcorehttp + ${CURL_LIBRARIES} + ${CARES_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${CRYPTO_LIBRARIES} + ${BOOST_THREAD_LIBRARY} + ) + +# tests +if (LL_TESTS) + SET(llcorehttp_TEST_SOURCE_FILES + tests/test_allocator.cpp + ) + + set(llcorehttp_TEST_HEADER_FILS + tests/test_httpstatus.hpp + tests/test_refcounted.hpp + tests/test_httpoperation.hpp + tests/test_httprequest.hpp + tests/test_httprequestqueue.hpp + tests/test_httpheaders.hpp + tests/test_bufferarray.hpp + tests/test_bufferstream.hpp + ) + + set_source_files_properties(${llcorehttp_TEST_HEADER_FILES} + PROPERTIES HEADER_FILE_ONLY TRUE) + + list(APPEND llcorehttp_TEST_SOURCE_FILES ${llcorehttp_TEST_HEADER_FILES}) + + # LL_ADD_PROJECT_UNIT_TESTS(llcorehttp "${llcorehttp_TEST_SOURCE_FILES}") + + # set(TEST_DEBUG on) + set(test_libs + ${LLCOREHTTP_LIBRARIES} + ${WINDOWS_LIBRARIES} + ${LLMESSAGE_LIBRARIES} + ${LLCOMMON_LIBRARIES} + ${GOOGLEMOCK_LIBRARIES} + ${CURL_LIBRARIES} + ${CARES_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${CRYPTO_LIBRARIES} + ${BOOST_SYSTEM_LIBRARY} + ${BOOST_THREAD_LIBRARY} + ) + + LL_ADD_INTEGRATION_TEST(llcorehttp + "${llcorehttp_TEST_SOURCE_FILES}" + "${test_libs}" + ${PYTHON_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_llcorehttp_peer.py" + ) + + # + # Example Programs + # + SET(llcorehttp_EXAMPLE_SOURCE_FILES + examples/http_texture_load.cpp + ) + + set(example_libs + ${LLCOREHTTP_LIBRARIES} + ${WINDOWS_LIBRARIES} + ${LLMESSAGE_LIBRARIES} + ${LLCOMMON_LIBRARIES} + ${GOOGLEMOCK_LIBRARIES} + ${CURL_LIBRARIES} + ${CARES_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${CRYPTO_LIBRARIES} + ${BOOST_SYSTEM_LIBRARY} + ${BOOST_THREAD_LIBRARY} + ) + + add_executable(http_texture_load + ${llcorehttp_EXAMPLE_SOURCE_FILES} + ) + set_target_properties(http_texture_load + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${EXE_STAGING_DIR}" + ) + + if (WINDOWS) + # The following come from LLAddBuildTest.cmake's INTEGRATION_TEST_xxxx target. + set_target_properties(http_texture_load + PROPERTIES + LINK_FLAGS "/debug /NODEFAULTLIB:LIBCMT /SUBSYSTEM:WINDOWS ${TCMALLOC_LINK_FLAGS}" + LINK_FLAGS_DEBUG "/NODEFAULTLIB:\"LIBCMT;LIBCMTD;MSVCRT\" /INCREMENTAL:NO" + LINK_FLAGS_RELEASE "" + ) + endif (WINDOWS) + + target_link_libraries(http_texture_load ${example_libs}) + +endif (LL_TESTS) + diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h new file mode 100644 index 0000000000..14f744a9f1 --- /dev/null +++ b/indra/llcorehttp/_httpinternal.h @@ -0,0 +1,154 @@ +/** + * @file _httpinternal.h + * @brief Implementation constants and magic numbers + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_INTERNAL_H_ +#define _LLCORE_HTTP_INTERNAL_H_ + + +// If you find this included in a public interface header, +// something wrong is probably happening. + + +// -------------------------------------------------------------------- +// General library to-do list +// +// - Implement policy classes. Structure is mostly there just didn't +// need it for the first consumer. +// - Consider Removing 'priority' from the request interface. Its use +// in an always active class can lead to starvation of low-priority +// requests. Requires coodination of priority values across all +// components that share a class. Changing priority across threads +// is slightly expensive (relative to gain) and hasn't been completely +// implemented. And the major user of priority, texture fetches, +// may not really need it. +// - Set/get for global policy and policy classes is clumsy. Rework +// it heading in a direction that allows for more dynamic behavior. +// - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the +// pedantic. +// - Update downloader and other long-duration services are going to +// need a progress notification. Initial idea is to introduce a +// 'repeating request' which can piggyback on another request and +// persist until canceled or carrier completes. Current queue +// structures allow an HttpOperation object to be enqueued +// repeatedly, so... +// - Investigate making c-ares' re-implementation of a resolver library +// more resilient or more intelligent on Mac. Part of the DNS failure +// lies in here. The mechanism also looks a little less dynamic +// than needed in an environments where networking is changing. +// - Global optimizations: 'borrowing' connections from other classes, +// HTTP pipelining. +// - Dynamic/control system stuff: detect problems and self-adjust. +// This won't help in the face of the router problems we've looked +// at, however. Detect starvation due to UDP activity and provide +// feedback to it. +// +// Integration to-do list +// - LLTextureFetch still needs a major refactor. The use of +// LLQueuedThread makes it hard to inspect workers and do the +// resource waiting we're now doing. Rebuild along simpler lines +// some of which are suggested in new commentary at the top of +// the main source file. +// - Expand areas of usage eventually leading to the removal of LLCurl. +// Rough order of expansion: +// . Mesh fetch +// . Avatar names +// . Group membership lists +// . Caps access in general +// . 'The rest' +// - Adapt texture cache, image decode and other image consumers to +// the BufferArray model to reduce data copying. Alternatively, +// adapt this library to something else. +// +// -------------------------------------------------------------------- + + +// If '1', internal ready queues will not order ready +// requests by priority, instead it's first-come-first-served. +// Reprioritization requests have the side-effect of then +// putting the modified request at the back of the ready queue. + +#define LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY 1 + + +namespace LLCore +{ + +// Maxium number of policy classes that can be defined. +// *TODO: Currently limited to the default class, extend. +const int HTTP_POLICY_CLASS_LIMIT = 1; + +// Debug/informational tracing. Used both +// as a global option and in per-request traces. +const int HTTP_TRACE_OFF = 0; +const int HTTP_TRACE_LOW = 1; +const int HTTP_TRACE_CURL_HEADERS = 2; +const int HTTP_TRACE_CURL_BODIES = 3; + +const int HTTP_TRACE_MIN = HTTP_TRACE_OFF; +const int HTTP_TRACE_MAX = HTTP_TRACE_CURL_BODIES; + +// Request retry limits +// +// At a minimum, retries need to extend past any throttling +// window we're expecting from central services. In the case +// of Linden services running through the caps routers, there's +// a five-second or so window for throttling with some spillover. +// We want to span a few windows to allow transport to slow +// after onset of the throttles and then recover without a final +// failure. Other systems may need other constants. +const int HTTP_RETRY_COUNT_DEFAULT = 8; +const int HTTP_RETRY_COUNT_MIN = 0; +const int HTTP_RETRY_COUNT_MAX = 100; + +const int HTTP_REDIRECTS_DEFAULT = 10; + +// Timeout value used for both connect and protocol exchange. +// Retries and time-on-queue are not included and aren't +// accounted for. +const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_TIMEOUT_MIN = 0L; +const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; + +// Limits on connection counts +const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; +const int HTTP_CONNECTION_LIMIT_MIN = 1; +const int HTTP_CONNECTION_LIMIT_MAX = 256; + +// Tuning parameters + +// Time worker thread sleeps after a pass through the +// request, ready and active queues. +const int HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS = 2; + +// Block allocation size (a tuning parameter) is found +// in bufferarray.h. + +// Compatibility controls +const bool HTTP_ENABLE_LINKSYS_WRT54G_V5_DNS_FIX = true; + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_INTERNAL_H_ diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp new file mode 100644 index 0000000000..6fe0bfc7d1 --- /dev/null +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -0,0 +1,373 @@ +/** + * @file _httplibcurl.cpp + * @brief Internal definitions of the Http libcurl thread + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httplibcurl.h" + +#include "httpheaders.h" +#include "bufferarray.h" +#include "_httpoprequest.h" +#include "_httppolicy.h" + +#include "llhttpstatuscodes.h" + + +namespace LLCore +{ + + +HttpLibcurl::HttpLibcurl(HttpService * service) + : mService(service), + mPolicyCount(0), + mMultiHandles(NULL) +{} + + +HttpLibcurl::~HttpLibcurl() +{ + shutdown(); + + mService = NULL; +} + + +void HttpLibcurl::shutdown() +{ + while (! mActiveOps.empty()) + { + HttpOpRequest * op(* mActiveOps.begin()); + mActiveOps.erase(mActiveOps.begin()); + + cancelRequest(op); + op->release(); + } + + if (mMultiHandles) + { + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) + { + if (mMultiHandles[policy_class]) + { + curl_multi_cleanup(mMultiHandles[policy_class]); + mMultiHandles[policy_class] = 0; + } + } + + delete [] mMultiHandles; + mMultiHandles = NULL; + } + + mPolicyCount = 0; +} + + +void HttpLibcurl::start(int policy_count) +{ + llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT); + llassert_always(! mMultiHandles); // One-time call only + + mPolicyCount = policy_count; + mMultiHandles = new CURLM * [mPolicyCount]; + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) + { + mMultiHandles[policy_class] = curl_multi_init(); + } +} + + +// Give libcurl some cycles, invoke it's callbacks, process +// completed requests finalizing or issuing retries as needed. +// +// If active list goes empty *and* we didn't queue any +// requests for retry, we return a request for a hard +// sleep otherwise ask for a normal polling interval. +HttpService::ELoopSpeed HttpLibcurl::processTransport() +{ + HttpService::ELoopSpeed ret(HttpService::REQUEST_SLEEP); + + // Give libcurl some cycles to do I/O & callbacks + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) + { + if (! mMultiHandles[policy_class]) + continue; + + int running(0); + CURLMcode status(CURLM_CALL_MULTI_PERFORM); + do + { + running = 0; + status = curl_multi_perform(mMultiHandles[policy_class], &running); + } + while (0 != running && CURLM_CALL_MULTI_PERFORM == status); + + // Run completion on anything done + CURLMsg * msg(NULL); + int msgs_in_queue(0); + while ((msg = curl_multi_info_read(mMultiHandles[policy_class], &msgs_in_queue))) + { + if (CURLMSG_DONE == msg->msg) + { + CURL * handle(msg->easy_handle); + CURLcode result(msg->data.result); + + if (completeRequest(mMultiHandles[policy_class], handle, result)) + { + // Request is still active, don't get too sleepy + ret = HttpService::NORMAL; + } + handle = NULL; // No longer valid on return + } + else if (CURLMSG_NONE == msg->msg) + { + // Ignore this... it shouldn't mean anything. + ; + } + else + { + LL_WARNS_ONCE("CoreHttp") << "Unexpected message from libcurl. Msg code: " + << msg->msg + << LL_ENDL; + } + msgs_in_queue = 0; + } + } + + if (! mActiveOps.empty()) + { + ret = HttpService::NORMAL; + } + return ret; +} + + +// Caller has provided us with a ref count on op. +void HttpLibcurl::addOp(HttpOpRequest * op) +{ + llassert_always(op->mReqPolicy < mPolicyCount); + llassert_always(mMultiHandles[op->mReqPolicy] != NULL); + + // Create standard handle + if (! op->prepareRequest(mService)) + { + // Couldn't issue request, fail with notification + // *TODO: Need failure path + return; + } + + // Make the request live + curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + op->mCurlActive = true; + + if (op->mTracing > HTTP_TRACE_OFF) + { + HttpPolicy & policy(mService->getPolicy()); + + LL_INFOS("CoreHttp") << "TRACE, ToActiveQueue, Handle: " + << static_cast<HttpHandle>(op) + << ", Actives: " << mActiveOps.size() + << ", Readies: " << policy.getReadyCount(op->mReqPolicy) + << LL_ENDL; + } + + // On success, make operation active + mActiveOps.insert(op); +} + + +// Implements the transport part of any cancel operation. +// See if the handle is an active operation and if so, +// use the more complicated transport-based cancelation +// method to kill the request. +bool HttpLibcurl::cancel(HttpHandle handle) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(handle)); + active_set_t::iterator it(mActiveOps.find(op)); + if (mActiveOps.end() == it) + { + return false; + } + + // Cancel request + cancelRequest(op); + + // Drop references + mActiveOps.erase(it); + op->release(); + + return true; +} + + +// *NOTE: cancelRequest logic parallels completeRequest logic. +// Keep them synchronized as necessary. Caller is expected to +// remove the op from the active list and release the op *after* +// calling this method. It must be called first to deliver the +// op to the reply queue with refcount intact. +void HttpLibcurl::cancelRequest(HttpOpRequest * op) +{ + // Deactivate request + op->mCurlActive = false; + + // Detach from multi and recycle handle + curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + curl_easy_cleanup(op->mCurlHandle); + op->mCurlHandle = NULL; + + // Tracing + if (op->mTracing > HTTP_TRACE_OFF) + { + LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " + << static_cast<HttpHandle>(op) + << ", Status: " << op->mStatus.toHex() + << LL_ENDL; + } + + // Cancel op and deliver for notification + op->cancel(); +} + + +// *NOTE: cancelRequest logic parallels completeRequest logic. +// Keep them synchronized as necessary. +bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status) +{ + HttpOpRequest * op(NULL); + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); + + if (handle != op->mCurlHandle || ! op->mCurlActive) + { + LL_WARNS("CoreHttp") << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." + << " Handle: " << static_cast<HttpHandle>(handle) + << LL_ENDL; + return false; + } + + active_set_t::iterator it(mActiveOps.find(op)); + if (mActiveOps.end() == it) + { + LL_WARNS("CoreHttp") << "libcurl completion for request not on active list. Continuing." + << " Handle: " << static_cast<HttpHandle>(handle) + << LL_ENDL; + return false; + } + + // Deactivate request + mActiveOps.erase(it); + op->mCurlActive = false; + + // Set final status of request if it hasn't failed by other mechanisms yet + if (op->mStatus) + { + op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status); + } + if (op->mStatus) + { + int http_status(HTTP_OK); + + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); + if (http_status >= 100 && http_status <= 999) + { + char * cont_type(NULL); + curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &cont_type); + if (cont_type) + { + op->mReplyConType = cont_type; + } + op->mStatus = HttpStatus(http_status); + } + else + { + LL_WARNS("CoreHttp") << "Invalid HTTP response code (" + << http_status << ") received from server." + << LL_ENDL; + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + } + } + + // Detach from multi and recycle handle + curl_multi_remove_handle(multi_handle, handle); + curl_easy_cleanup(handle); + op->mCurlHandle = NULL; + + // Tracing + if (op->mTracing > HTTP_TRACE_OFF) + { + LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " + << static_cast<HttpHandle>(op) + << ", Status: " << op->mStatus.toHex() + << LL_ENDL; + } + + // Dispatch to next stage + HttpPolicy & policy(mService->getPolicy()); + bool still_active(policy.stageAfterCompletion(op)); + + return still_active; +} + + +int HttpLibcurl::getActiveCount() const +{ + return mActiveOps.size(); +} + + +int HttpLibcurl::getActiveCountInClass(int policy_class) const +{ + int count(0); + + for (active_set_t::const_iterator iter(mActiveOps.begin()); + mActiveOps.end() != iter; + ++iter) + { + if ((*iter)->mReqPolicy == policy_class) + { + ++count; + } + } + + return count; +} + + +// --------------------------------------- +// Free functions +// --------------------------------------- + + +struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist) +{ + for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin()); + + headers->mHeaders.end() != it; + ++it) + { + slist = curl_slist_append(slist, (*it).c_str()); + } + return slist; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h new file mode 100644 index 0000000000..611f029ef5 --- /dev/null +++ b/indra/llcorehttp/_httplibcurl.h @@ -0,0 +1,129 @@ +/** + * @file _httplibcurl.h + * @brief Declarations for internal class providing libcurl transport. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_LIBCURL_H_ +#define _LLCORE_HTTP_LIBCURL_H_ + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include <curl/curl.h> +#include <curl/multi.h> + +#include <set> + +#include "httprequest.h" +#include "_httpservice.h" +#include "_httpinternal.h" + + +namespace LLCore +{ + + +class HttpPolicy; +class HttpOpRequest; +class HttpHeaders; + + +/// Implements libcurl-based transport for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. + +class HttpLibcurl +{ +public: + HttpLibcurl(HttpService * service); + virtual ~HttpLibcurl(); + +private: + HttpLibcurl(const HttpLibcurl &); // Not defined + void operator=(const HttpLibcurl &); // Not defined + +public: + /// Give cycles to libcurl to run active requests. Completed + /// operations (successful or failed) will be retried or handed + /// over to the reply queue as final responses. + /// + /// @return Indication of how long this method is + /// willing to wait for next service call. + HttpService::ELoopSpeed processTransport(); + + /// Add request to the active list. Caller is expected to have + /// provided us with a reference count on the op to hold the + /// request. (No additional references will be added.) + void addOp(HttpOpRequest * op); + + /// One-time call to set the number of policy classes to be + /// serviced and to create the resources for each. Value + /// must agree with HttpPolicy::setPolicies() call. + void start(int policy_count); + + /// Synchronously stop libcurl operations. All active requests + /// are canceled and removed from libcurl's handling. Easy + /// handles are detached from their multi handles and released. + /// Multi handles are also released. Canceled requests are + /// completed with canceled status and made available on their + /// respective reply queues. + /// + /// Can be restarted with a start() call. + void shutdown(); + + /// Return global and per-class counts of active requests. + int getActiveCount() const; + int getActiveCountInClass(int policy_class) const; + + /// Attempt to cancel a request identified by handle. + /// + /// Interface shadows HttpService's method. + /// + /// @return True if handle was found and operation canceled. + /// + bool cancel(HttpHandle handle); + +protected: + /// Invoked when libcurl has indicated a request has been processed + /// to completion and we need to move the request to a new state. + bool completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); + + /// Invoked to cancel an active request, mainly during shutdown + /// and destroy. + void cancelRequest(HttpOpRequest * op); + +protected: + typedef std::set<HttpOpRequest *> active_set_t; + +protected: + HttpService * mService; // Simple reference, not owner + active_set_t mActiveOps; + int mPolicyCount; + CURLM ** mMultiHandles; +}; // end class HttpLibcurl + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_LIBCURL_H_ diff --git a/indra/llcorehttp/_httpopcancel.cpp b/indra/llcorehttp/_httpopcancel.cpp new file mode 100644 index 0000000000..c1912eb3db --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.cpp @@ -0,0 +1,73 @@ +/** + * @file _httpopcancel.cpp + * @brief Definitions for internal class HttpOpCancel + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopcancel.h" + +#include "httpcommon.h" +#include "httphandler.h" +#include "httpresponse.h" + +#include "_httpservice.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOpCancel +// ================================== + + +HttpOpCancel::HttpOpCancel(HttpHandle handle) + : HttpOperation(), + mHandle(handle) +{} + + +HttpOpCancel::~HttpOpCancel() +{} + + +// Immediately search for the request on various queues +// and cancel operations if found. Return the status of +// the search and cancel as the status of this request. +// The canceled request will return a canceled status to +// its handler. +void HttpOpCancel::stageFromRequest(HttpService * service) +{ + if (! service->cancel(mHandle)) + { + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + + addAsReply(); +} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h new file mode 100644 index 0000000000..336dfdc573 --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.h @@ -0,0 +1,78 @@ +/** + * @file _httpopcancel.h + * @brief Internal declarations for the HttpOpCancel subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPCANCEL_H_ +#define _LLCORE_HTTP_OPCANCEL_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include "httpcommon.h" + +#include <curl/curl.h> + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpCancel requests that a previously issued request +/// be canceled, if possible. This includes active requests +/// that may be in the middle of an HTTP transaction. Any +/// completed request will not be canceled and will return +/// its final status unchanged and *this* request will complete +/// with an HE_HANDLE_NOT_FOUND error status. + +class HttpOpCancel : public HttpOperation +{ +public: + /// @param handle Handle of previously-issued request to + /// be canceled. + HttpOpCancel(HttpHandle handle); + +protected: + virtual ~HttpOpCancel(); // Use release() + +private: + HttpOpCancel(const HttpOpCancel &); // Not defined + void operator=(const HttpOpCancel &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +public: + // Request data + HttpHandle mHandle; +}; // end class HttpOpCancel + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPCANCEL_H_ + diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp new file mode 100644 index 0000000000..5cf5bc5930 --- /dev/null +++ b/indra/llcorehttp/_httpoperation.cpp @@ -0,0 +1,248 @@ +/** + * @file _httpoperation.cpp + * @brief Definitions for internal classes based on HttpOperation + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpoperation.h" + +#include "httphandler.h" +#include "httpresponse.h" +#include "httprequest.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httpinternal.h" + +#include "lltimer.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOperation +// ================================== + + +HttpOperation::HttpOperation() + : LLCoreInt::RefCounted(true), + mReplyQueue(NULL), + mUserHandler(NULL), + mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), + mReqPriority(0U), + mTracing(0) +{ + mMetricCreated = totalTime(); +} + + +HttpOperation::~HttpOperation() +{ + setReplyPath(NULL, NULL); +} + + +void HttpOperation::setReplyPath(HttpReplyQueue * reply_queue, + HttpHandler * user_handler) +{ + if (reply_queue != mReplyQueue) + { + if (mReplyQueue) + { + mReplyQueue->release(); + } + + if (reply_queue) + { + reply_queue->addRef(); + } + + mReplyQueue = reply_queue; + } + + // Not refcounted + mUserHandler = user_handler; +} + + + +void HttpOperation::stageFromRequest(HttpService *) +{ + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called." + << LL_ENDL; +} + + +void HttpOperation::stageFromReady(HttpService *) +{ + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LL_ERRS("HttpCore") << "Default stageFromReady method may not be called." + << LL_ENDL; +} + + +void HttpOperation::stageFromActive(HttpService *) +{ + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LL_ERRS("HttpCore") << "Default stageFromActive method may not be called." + << LL_ENDL; +} + + +void HttpOperation::visitNotifier(HttpRequest *) +{ + if (mUserHandler) + { + HttpResponse * response = new HttpResponse(); + + response->setStatus(mStatus); + mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); + + response->release(); + } +} + + +HttpStatus HttpOperation::cancel() +{ + HttpStatus status; + + return status; +} + + +void HttpOperation::addAsReply() +{ + if (mTracing > HTTP_TRACE_OFF) + { + LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " + << static_cast<HttpHandle>(this) + << LL_ENDL; + } + + if (mReplyQueue) + { + addRef(); + mReplyQueue->addOp(this); + } +} + + +// ================================== +// HttpOpStop +// ================================== + + +HttpOpStop::HttpOpStop() + : HttpOperation() +{} + + +HttpOpStop::~HttpOpStop() +{} + + +void HttpOpStop::stageFromRequest(HttpService * service) +{ + // Do operations + service->stopRequested(); + + // Prepare response if needed + addAsReply(); +} + + +// ================================== +// HttpOpNull +// ================================== + + +HttpOpNull::HttpOpNull() + : HttpOperation() +{} + + +HttpOpNull::~HttpOpNull() +{} + + +void HttpOpNull::stageFromRequest(HttpService * service) +{ + // Perform op + // Nothing to perform. This doesn't fall into the libcurl + // ready/active queues, it just bounces over to the reply + // queue directly. + + // Prepare response if needed + addAsReply(); +} + + +// ================================== +// HttpOpSpin +// ================================== + + +HttpOpSpin::HttpOpSpin(int mode) + : HttpOperation(), + mMode(mode) +{} + + +HttpOpSpin::~HttpOpSpin() +{} + + +void HttpOpSpin::stageFromRequest(HttpService * service) +{ + if (0 == mMode) + { + // Spin forever + while (true) + { + ms_sleep(100); + } + } + else + { + ms_sleep(1); // backoff interlock plumbing a bit + this->addRef(); + if (! service->getRequestQueue().addOp(this)) + { + this->release(); + } + } +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h new file mode 100644 index 0000000000..914627fad0 --- /dev/null +++ b/indra/llcorehttp/_httpoperation.h @@ -0,0 +1,262 @@ +/** + * @file _httpoperation.h + * @brief Internal declarations for HttpOperation and sub-classes + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPERATION_H_ +#define _LLCORE_HTTP_OPERATION_H_ + + +#include "httpcommon.h" +#include "httprequest.h" +#include "_refcounted.h" + + +namespace LLCore +{ + +class HttpReplyQueue; +class HttpHandler; +class HttpService; + +/// HttpOperation is the base class for all request/reply +/// pairs. +/// +/// Operations are expected to be of two types: immediate +/// and queued. Immediate requests go to the singleton +/// request queue and when picked up by the worker thread +/// are executed immediately and there results placed on +/// the supplied reply queue. Queued requests (namely for +/// HTTP operations), go to the request queue, are picked +/// up and moved to a ready queue where they're ordered by +/// priority and managed by the policy component, are +/// then activated issuing HTTP requests and moved to an +/// active list managed by the transport (libcurl) component +/// and eventually finalized when a response is available +/// and status and data return via reply queue. +/// +/// To manage these transitions, derived classes implement +/// three methods: stageFromRequest, stageFromReady and +/// stageFromActive. Immediate requests will only override +/// stageFromRequest which will perform the operation and +/// return the result by invoking addAsReply() to put the +/// request on a reply queue. Queued requests will involve +/// all three stage methods. +/// +/// Threading: not thread-safe. Base and derived classes +/// provide no locking. Instances move across threads +/// via queue-like interfaces that are thread compatible +/// and those interfaces establish the access rules. + +class HttpOperation : public LLCoreInt::RefCounted +{ +public: + /// Threading: called by a consumer/application thread. + HttpOperation(); + +protected: + /// Threading: called by any thread. + virtual ~HttpOperation(); // Use release() + +private: + HttpOperation(const HttpOperation &); // Not defined + void operator=(const HttpOperation &); // Not defined + +public: + /// Register a reply queue and a handler for completion notifications. + /// + /// Invokers of operations that want to receive notification that an + /// operation has been completed do so by binding a reply queue and + /// a handler object to the request. + /// + /// @param reply_queue Pointer to the reply queue where completion + /// notifications are to be queued (typically + /// by addAsReply()). This will typically be + /// the reply queue referenced by the request + /// object. This method will increment the + /// refcount on the queue holding the queue + /// until delivery is complete. Using a reply_queue + /// even if the handler is NULL has some benefits + /// for memory deallocation by keeping it in the + /// originating thread. + /// + /// @param handler Possibly NULL pointer to a non-refcounted + //// handler object to be invoked (onCompleted) + /// when the operation is finished. Note that + /// the handler object is never dereferenced + /// by the worker thread. This is passible data + /// until notification is performed. + /// + /// Threading: called by application thread. + /// + void setReplyPath(HttpReplyQueue * reply_queue, + HttpHandler * handler); + + /// The three possible staging steps in an operation's lifecycle. + /// Asynchronous requests like HTTP operations move from the + /// request queue to the ready queue via stageFromRequest. Then + /// from the ready queue to the active queue by stageFromReady. And + /// when complete, to the reply queue via stageFromActive and the + /// addAsReply utility. + /// + /// Immediate mode operations (everything else) move from the + /// request queue to the reply queue directly via stageFromRequest + /// and addAsReply with no existence on the ready or active queues. + /// + /// These methods will take out a reference count on the request, + /// caller only needs to dispose of its reference when done with + /// the request. + /// + /// Threading: called by worker thread. + /// + virtual void stageFromRequest(HttpService *); + virtual void stageFromReady(HttpService *); + virtual void stageFromActive(HttpService *); + + /// Delivers a notification to a handler object on completion. + /// + /// Once a request is complete and it has been removed from its + /// reply queue, a handler notification may be delivered by a + /// call to HttpRequest::update(). This method does the necessary + /// dispatching. + /// + /// Threading: called by application thread. + /// + virtual void visitNotifier(HttpRequest *); + + /// Cancels the operation whether queued or active. + /// Final status of the request becomes canceled (an error) and + /// that will be delivered to caller via notification scheme. + /// + /// Threading: called by worker thread. + /// + virtual HttpStatus cancel(); + +protected: + /// Delivers request to reply queue on completion. After this + /// call, worker thread no longer accesses the object and it + /// is owned by the reply queue. + /// + /// Threading: called by worker thread. + /// + void addAsReply(); + +protected: + HttpReplyQueue * mReplyQueue; // Have refcount + HttpHandler * mUserHandler; // Naked pointer + +public: + // Request Data + HttpRequest::policy_t mReqPolicy; + HttpRequest::priority_t mReqPriority; + + // Reply Data + HttpStatus mStatus; + + // Tracing, debug and metrics + HttpTime mMetricCreated; + int mTracing; +}; // end class HttpOperation + + +/// HttpOpStop requests the servicing thread to shutdown +/// operations, cease pulling requests from the request +/// queue and release shared resources (particularly +/// those shared via reference count). The servicing +/// thread will then exit. The underlying thread object +/// remains so that another thread can join on the +/// servicing thread prior to final cleanup. The +/// request *does* generate a reply on the response +/// queue, if requested. + +class HttpOpStop : public HttpOperation +{ +public: + HttpOpStop(); + +protected: + virtual ~HttpOpStop(); + +private: + HttpOpStop(const HttpOpStop &); // Not defined + void operator=(const HttpOpStop &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +}; // end class HttpOpStop + + +/// HttpOpNull is a do-nothing operation used for testing via +/// a basic loopback pattern. It's executed immediately by +/// the servicing thread which bounces a reply back to the +/// caller without any further delay. + +class HttpOpNull : public HttpOperation +{ +public: + HttpOpNull(); + +protected: + virtual ~HttpOpNull(); + +private: + HttpOpNull(const HttpOpNull &); // Not defined + void operator=(const HttpOpNull &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +}; // end class HttpOpNull + + +/// HttpOpSpin is a test-only request that puts the worker +/// thread into a cpu spin. Used for unit tests and cleanup +/// evaluation. You do not want to use this in production. +class HttpOpSpin : public HttpOperation +{ +public: + // 0 does a hard spin in the operation + // 1 does a soft spin continuously requeuing itself + HttpOpSpin(int mode); + +protected: + virtual ~HttpOpSpin(); + +private: + HttpOpSpin(const HttpOpSpin &); // Not defined + void operator=(const HttpOpSpin &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +protected: + int mMode; +}; // end class HttpOpSpin + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPERATION_H_ + diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp new file mode 100644 index 0000000000..51a8eaf998 --- /dev/null +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -0,0 +1,911 @@ +/** + * @file _httpoprequest.cpp + * @brief Definitions for internal class HttpOpRequest + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpoprequest.h" + +#include <cstdio> +#include <algorithm> + +#include "httpcommon.h" +#include "httphandler.h" +#include "httpresponse.h" +#include "bufferarray.h" +#include "httpheaders.h" +#include "httpoptions.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httppolicy.h" +#include "_httppolicyglobal.h" +#include "_httplibcurl.h" +#include "_httpinternal.h" + +#include "llhttpstatuscodes.h" +#include "llproxy.h" + +namespace +{ + +// Attempts to parse a 'Content-Range:' header. Caller must already +// have verified that the header tag is present. The 'buffer' argument +// will be processed by strtok_r calls which will modify the buffer. +// +// @return -1 if invalid and response should be dropped, 0 if valid an +// correct, 1 if couldn't be parsed. If 0, the first, last, +// and length arguments are also written. 'length' may be +// 0 if the length wasn't available to the server. +// +int parse_content_range_header(char * buffer, + unsigned int * first, + unsigned int * last, + unsigned int * length); + + +// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and +// escape and format it for a tracing line in logging. Absolutely +// anything including NULs can be in the data. If @scrub is true, +// non-printing or non-ascii characters are replaced with spaces +// otherwise a %XX form of escaping is used. +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, + std::string & safe_line); + + +// OS-neutral string comparisons of various types +int os_strncasecmp(const char *s1, const char *s2, size_t n); +int os_strcasecmp(const char *s1, const char *s2); +char * os_strtok_r(char *str, const char *delim, char **saveptr); + + +static const char * const hdr_whitespace(" \t"); +static const char * const hdr_separator(": \t"); + +} // end anonymous namespace + + +namespace LLCore +{ + + +HttpOpRequest::HttpOpRequest() + : HttpOperation(), + mProcFlags(0U), + mReqMethod(HOR_GET), + mReqBody(NULL), + mReqOffset(0), + mReqLength(0), + mReqHeaders(NULL), + mReqOptions(NULL), + mCurlActive(false), + mCurlHandle(NULL), + mCurlService(NULL), + mCurlHeaders(NULL), + mCurlBodyPos(0), + mReplyBody(NULL), + mReplyOffset(0), + mReplyLength(0), + mReplyFullLength(0), + mReplyHeaders(NULL), + mPolicyRetries(0), + mPolicyRetryAt(HttpTime(0)), + mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) +{ + // *NOTE: As members are added, retry initialization/cleanup + // may need to be extended in @see prepareRequest(). +} + + + +HttpOpRequest::~HttpOpRequest() +{ + if (mReqBody) + { + mReqBody->release(); + mReqBody = NULL; + } + + if (mReqOptions) + { + mReqOptions->release(); + mReqOptions = NULL; + } + + if (mReqHeaders) + { + mReqHeaders->release(); + mReqHeaders = NULL; + } + + if (mCurlHandle) + { + curl_easy_cleanup(mCurlHandle); + mCurlHandle = NULL; + } + + mCurlService = NULL; + + if (mCurlHeaders) + { + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; + } + + if (mReplyBody) + { + mReplyBody->release(); + mReplyBody = NULL; + } + + if (mReplyHeaders) + { + mReplyHeaders->release(); + mReplyHeaders = NULL; + } +} + + +void HttpOpRequest::stageFromRequest(HttpService * service) +{ + addRef(); + service->getPolicy().addOp(this); // transfers refcount +} + + +void HttpOpRequest::stageFromReady(HttpService * service) +{ + addRef(); + service->getTransport().addOp(this); // transfers refcount +} + + +void HttpOpRequest::stageFromActive(HttpService * service) +{ + if (mReplyLength) + { + // If non-zero, we received and processed a Content-Range + // header with the response. Verify that what it says + // is consistent with the received data. + if (mReplyLength != mReplyBody->size()) + { + // Not as expected, fail the request + mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + } + + if (mCurlHeaders) + { + // We take these headers out of the request now as they were + // allocated originally in this thread and the notifier doesn't + // need them. This eliminates one source of heap moving across + // threads. + + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; + } + + addAsReply(); +} + + +void HttpOpRequest::visitNotifier(HttpRequest * request) +{ + if (mUserHandler) + { + HttpResponse * response = new HttpResponse(); + response->setStatus(mStatus); + response->setBody(mReplyBody); + response->setHeaders(mReplyHeaders); + if (mReplyOffset || mReplyLength) + { + // Got an explicit offset/length in response + response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); + } + response->setContentType(mReplyConType); + + mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); + + response->release(); + } +} + + +HttpStatus HttpOpRequest::cancel() +{ + mStatus = HttpStatus(HttpStatus::LLCORE, HE_OP_CANCELED); + + addAsReply(); + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, NULL, options, headers); + mReqMethod = HOR_GET; + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, NULL, options, headers); + mReqMethod = HOR_GET; + mReqOffset = offset; + mReqLength = len; + if (offset || len) + { + mProcFlags |= PF_SCAN_RANGE_HEADER; + } + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, body, options, headers); + mReqMethod = HOR_POST; + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, body, options, headers); + mReqMethod = HOR_PUT; + + return HttpStatus(); +} + + +void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + mProcFlags = 0U; + mReqPolicy = policy_id; + mReqPriority = priority; + mReqURL = url; + if (body) + { + body->addRef(); + mReqBody = body; + } + if (headers && ! mReqHeaders) + { + headers->addRef(); + mReqHeaders = headers; + } + if (options && ! mReqOptions) + { + options->addRef(); + mReqOptions = options; + if (options->getWantHeaders()) + { + mProcFlags |= PF_SAVE_HEADERS; + } + mPolicyRetryLimit = options->getRetries(); + mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); + mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); + } +} + + +// Sets all libcurl options and data for a request. +// +// Used both for initial requests and to 'reload' for +// a retry, generally with a different CURL handle. +// Junk may be left around from a failed request and that +// needs to be cleaned out. +// +HttpStatus HttpOpRequest::prepareRequest(HttpService * service) +{ + // Scrub transport and result data for retried op case + mCurlActive = false; + mCurlHandle = NULL; + mCurlService = NULL; + if (mCurlHeaders) + { + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; + } + mCurlBodyPos = 0; + + if (mReplyBody) + { + mReplyBody->release(); + mReplyBody = NULL; + } + mReplyOffset = 0; + mReplyLength = 0; + mReplyFullLength = 0; + if (mReplyHeaders) + { + mReplyHeaders->release(); + mReplyHeaders = NULL; + } + mReplyConType.clear(); + + // *FIXME: better error handling later + HttpStatus status; + + // Get policy options + HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); + + mCurlHandle = curl_easy_init(); + curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); + curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + + if (HTTP_ENABLE_LINKSYS_WRT54G_V5_DNS_FIX) + { + // The Linksys WRT54G V5 router has an issue with frequent + // DNS lookups from LAN machines. If they happen too often, + // like for every HTTP request, the router gets annoyed after + // about 700 or so requests and starts issuing TCP RSTs to + // new connections. Reuse the DNS lookups for even a few + // seconds and no RSTs. + curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); + } + else + { + // *TODO: Revisit this old DNS timeout setting - may no longer be valid + // I don't think this is valid anymore, the Multi shared DNS + // cache is working well. For the case of naked easy handles, + // consider using a shared DNS object. + curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + } + curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); + curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); + curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); + curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); + curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); + curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); + + const std::string * opt_value(NULL); + long opt_long(0L); + policy.get(HttpRequest::GP_LLPROXY, &opt_long); + if (opt_long) + { + // Use the viewer-based thread-safe API which has a + // fast/safe check for proxy enable. Would like to + // encapsulate this someway... + LLProxy::getInstance()->applyProxySettings(mCurlHandle); + } + else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) + { + // *TODO: This is fine for now but get fuller socks5/ + // authentication thing going later.... + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + } + if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + } + if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); + } + + switch (mReqMethod) + { + case HOR_GET: + curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); + break; + + case HOR_POST: + { + curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + long data_size(0); + if (mReqBody) + { + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); + } + break; + + case HOR_PUT: + { + curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + long data_size(0); + if (mReqBody) + { + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); + } + break; + + default: + LL_ERRS("CoreHttp") << "Invalid HTTP method in request: " + << int(mReqMethod) << ". Can't recover." + << LL_ENDL; + break; + } + + // Tracing + if (mTracing >= HTTP_TRACE_CURL_HEADERS) + { + curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); + curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + } + + // There's a CURLOPT for this now... + if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) + { + static const char * const fmt1("Range: bytes=%lu-%lu"); + static const char * const fmt2("Range: bytes=%lu-"); + + char range_line[64]; + +#if LL_WINDOWS + _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, + (mReqLength ? fmt1 : fmt2), + (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); +#else + snprintf(range_line, sizeof(range_line), + (mReqLength ? fmt1 : fmt2), + (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); +#endif // LL_WINDOWS + range_line[sizeof(range_line) - 1] = '\0'; + mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); + } + + mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); + + // Request options + long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); + if (mReqOptions) + { + timeout = mReqOptions->getTimeout(); + timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + } + curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); + curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); + + // Request headers + if (mReqHeaders) + { + // Caller's headers last to override + mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); + } + curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); + + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); + curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); + } + + if (status) + { + mCurlService = service; + } + return status; +} + + +size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + if (! op->mReplyBody) + { + op->mReplyBody = new BufferArray(); + } + const size_t req_size(size * nmemb); + const size_t write_size(op->mReplyBody->append(static_cast<char *>(data), req_size)); + return write_size; +} + + +size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + if (! op->mReqBody) + { + return 0; + } + const size_t req_size(size * nmemb); + const size_t body_size(op->mReqBody->size()); + if (body_size <= op->mCurlBodyPos) + { + if (body_size < op->mCurlBodyPos) + { + // Warn but continue if the read position moves beyond end-of-body + // for some reason. + LL_WARNS("HttpCore") << "Request body position beyond body size. Truncating request body." + << LL_ENDL; + } + return 0; + } + + const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos)); + const size_t read_size(op->mReqBody->read(op->mCurlBodyPos, static_cast<char *>(data), do_size)); + op->mCurlBodyPos += read_size; + return read_size; +} + + +size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + static const char status_line[] = "HTTP/"; + static const size_t status_line_len = sizeof(status_line) - 1; + + static const char con_ran_line[] = "content-range:"; + static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; + + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + const size_t hdr_size(size * nmemb); + const char * hdr_data(static_cast<const char *>(data)); // Not null terminated + + if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) + { + // One of possibly several status lines. Reset what we know and start over + // taking results from the last header stanza we receive. + op->mReplyOffset = 0; + op->mReplyLength = 0; + op->mReplyFullLength = 0; + op->mStatus = HttpStatus(); + if (op->mReplyHeaders) + { + op->mReplyHeaders->mHeaders.clear(); + } + } + + // Nothing in here wants a final CR/LF combination. Remove + // it as much as possible. + size_t wanted_hdr_size(hdr_size); + if (wanted_hdr_size && '\n' == hdr_data[wanted_hdr_size - 1]) + { + if (--wanted_hdr_size && '\r' == hdr_data[wanted_hdr_size - 1]) + { + --wanted_hdr_size; + } + } + + // Save header if caller wants them in the response + if (op->mProcFlags & PF_SAVE_HEADERS) + { + // Save headers in response + if (! op->mReplyHeaders) + { + op->mReplyHeaders = new HttpHeaders; + } + op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size)); + } + + // Detect and parse 'Content-Range' headers + if (op->mProcFlags & PF_SCAN_RANGE_HEADER) + { + char hdr_buffer[128]; // Enough for a reasonable header + size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); + + memcpy(hdr_buffer, hdr_data, frag_size); + hdr_buffer[frag_size] = '\0'; + if (frag_size > con_ran_line_len && + ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) + { + unsigned int first(0), last(0), length(0); + int status; + + if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) + { + // Success, record the fragment position + op->mReplyOffset = first; + op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; + } + else if (-1 == status) + { + // Response is badly formed and shouldn't be accepted + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + else + { + // Ignore the unparsable. + LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, frag_size) + << "'. Ignoring." + << LL_ENDL; + } + } + } + + return hdr_size; +} + + +int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + std::string safe_line; + std::string tag; + bool logit(false); + len = (std::min)(len, size_t(256)); // Keep things reasonable in all cases + + switch (info) + { + case CURLINFO_TEXT: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "TEXT"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_IN: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "HEADERIN"; + escape_libcurl_debug_data(buffer, len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_OUT: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "HEADEROUT"; + escape_libcurl_debug_data(buffer, 2 * len, true, safe_line); // Goes out as one line + logit = true; + } + break; + + case CURLINFO_DATA_IN: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "DATAIN"; + logit = true; + if (op->mTracing >= HTTP_TRACE_CURL_BODIES) + { + escape_libcurl_debug_data(buffer, len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + case CURLINFO_DATA_OUT: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "DATAOUT"; + logit = true; + if (op->mTracing >= HTTP_TRACE_CURL_BODIES) + { + escape_libcurl_debug_data(buffer, len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + default: + logit = false; + break; + } + + if (logit) + { + LL_INFOS("CoreHttp") << "TRACE, LibcurlDebug, Handle: " + << static_cast<HttpHandle>(op) + << ", Type: " << tag + << ", Data: " << safe_line + << LL_ENDL; + } + + return 0; +} + + +} // end namespace LLCore + + +// ======================================= +// Anonymous Namespace +// ======================================= + +namespace +{ + +int parse_content_range_header(char * buffer, + unsigned int * first, + unsigned int * last, + unsigned int * length) +{ + char * tok_state(NULL), * tok(NULL); + bool match(true); + + if (! os_strtok_r(buffer, hdr_separator, &tok_state)) + match = false; + if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) + match = 0 == os_strcasecmp("bytes", tok); + if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) + match = false; + if (match) + { + unsigned int lcl_first(0), lcl_last(0), lcl_len(0); + +#if LL_WINDOWS + if (3 == sscanf_s(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) +#else + if (3 == sscanf(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) +#endif // LL_WINDOWS + { + if (lcl_first > lcl_last || lcl_last >= lcl_len) + return -1; + *first = lcl_first; + *last = lcl_last; + *length = lcl_len; + return 0; + } +#if LL_WINDOWS + if (2 == sscanf_s(tok, "%u-%u/*", &lcl_first, &lcl_last)) +#else + if (2 == sscanf(tok, "%u-%u/*", &lcl_first, &lcl_last)) +#endif // LL_WINDOWS + { + if (lcl_first > lcl_last) + return -1; + *first = lcl_first; + *last = lcl_last; + *length = 0; + return 0; + } + } + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) +{ + std::string out; + len = (std::min)(len, size_t(200)); + out.reserve(3 * len); + for (int i(0); i < len; ++i) + { + unsigned char uc(static_cast<unsigned char>(buffer[i])); + + if (uc < 32 || uc > 126) + { + if (scrub) + { + out.append(1, ' '); + } + else + { + static const char hex[] = "0123456789ABCDEF"; + char convert[4]; + + convert[0] = '%'; + convert[1] = hex[(uc >> 4) % 16]; + convert[2] = hex[uc % 16]; + convert[3] = '\0'; + out.append(convert); + } + } + else + { + out.append(1, buffer[i]); + } + } + safe_line.swap(out); +} + + +int os_strncasecmp(const char *s1, const char *s2, size_t n) +{ +#if LL_WINDOWS + return _strnicmp(s1, s2, n); +#else + return strncasecmp(s1, s2, n); +#endif // LL_WINDOWS +} + + +int os_strcasecmp(const char *s1, const char *s2) +{ +#if LL_WINDOWS + return _stricmp(s1, s2); +#else + return strcasecmp(s1, s2); +#endif // LL_WINDOWS +} + + +char * os_strtok_r(char *str, const char *delim, char ** savestate) +{ +#if LL_WINDOWS + return strtok_s(str, delim, savestate); +#else + return strtok_r(str, delim, savestate); +#endif +} + + +} // end anonymous namespace + + diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h new file mode 100644 index 0000000000..7b65d17783 --- /dev/null +++ b/indra/llcorehttp/_httpoprequest.h @@ -0,0 +1,219 @@ +/** + * @file _httpoprequest.h + * @brief Internal declarations for the HttpOpRequest subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPREQUEST_H_ +#define _LLCORE_HTTP_OPREQUEST_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include <string> +#include <curl/curl.h> + +#include "httpcommon.h" +#include "httprequest.h" +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +class BufferArray; +class HttpHeaders; +class HttpOptions; + + +/// HttpOpRequest requests a supported HTTP method invocation with +/// option and header overrides. +/// +/// Essentially an RPC to get an HTTP GET, POST or PUT executed +/// asynchronously with options to override behaviors and HTTP +/// headers. +/// +/// Constructor creates a raw object incapable of useful work. +/// A subsequent call to one of the setupXXX() methods provides +/// the information needed to make a working request which can +/// then be enqueued to a request queue. +/// + +class HttpOpRequest : public HttpOperation +{ +public: + HttpOpRequest(); + +protected: + virtual ~HttpOpRequest(); // Use release() + +private: + HttpOpRequest(const HttpOpRequest &); // Not defined + void operator=(const HttpOpRequest &); // Not defined + +public: + enum EMethod + { + HOR_GET, + HOR_POST, + HOR_PUT + }; + + virtual void stageFromRequest(HttpService *); + virtual void stageFromReady(HttpService *); + virtual void stageFromActive(HttpService *); + + virtual void visitNotifier(HttpRequest * request); + +public: + /// Setup Methods + /// + /// Basically an RPC setup for each type of HTTP method + /// invocation with one per method type. These are + /// generally invoked right after construction. + /// + /// Threading: called by application thread + /// + HttpStatus setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus setupGetByteRange(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus setupPost(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus setupPut(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + // Internal method used to setup the libcurl options for a request. + // Does all the libcurl handle setup in one place. + // + // Threading: called by worker thread + // + HttpStatus prepareRequest(HttpService * service); + + virtual HttpStatus cancel(); + +protected: + // Common setup for all the request methods. + // + // Threading: called by application thread + // + void setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + // libcurl operational callbacks + // + // Threading: called by worker thread + // + static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); + static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); + static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); + static int debugCallback(CURL *, curl_infotype info, char * buffer, size_t len, void * userdata); + +protected: + unsigned int mProcFlags; + static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; + static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + +public: + // Request data + EMethod mReqMethod; + std::string mReqURL; + BufferArray * mReqBody; + off_t mReqOffset; + size_t mReqLength; + HttpHeaders * mReqHeaders; + HttpOptions * mReqOptions; + + // Transport data + bool mCurlActive; + CURL * mCurlHandle; + HttpService * mCurlService; + curl_slist * mCurlHeaders; + size_t mCurlBodyPos; + + // Result data + HttpStatus mStatus; + BufferArray * mReplyBody; + off_t mReplyOffset; + size_t mReplyLength; + size_t mReplyFullLength; + HttpHeaders * mReplyHeaders; + std::string mReplyConType; + + // Policy data + int mPolicyRetries; + HttpTime mPolicyRetryAt; + int mPolicyRetryLimit; +}; // end class HttpOpRequest + + +/// HttpOpRequestCompare isn't an operation but a uniform comparison +/// functor for STL containers that order by priority. Mainly +/// used for the ready queue container but defined here. +class HttpOpRequestCompare +{ +public: + bool operator()(const HttpOpRequest * lhs, const HttpOpRequest * rhs) + { + return lhs->mReqPriority > rhs->mReqPriority; + } +}; // end class HttpOpRequestCompare + + +// --------------------------------------- +// Free functions +// --------------------------------------- + +// Internal function to append the contents of an HttpHeaders +// instance to a curl_slist object. +curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPREQUEST_H_ + diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp new file mode 100644 index 0000000000..8198528a9b --- /dev/null +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -0,0 +1,97 @@ +/** + * @file _httpopsetget.cpp + * @brief Definitions for internal class HttpOpSetGet + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetget.h" + +#include "httpcommon.h" + +#include "_httpservice.h" +#include "_httppolicy.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOpSetget +// ================================== + + +HttpOpSetGet::HttpOpSetGet() + : HttpOperation(), + mIsGlobal(false), + mDoSet(false), + mSetting(-1), // Nothing requested + mLongValue(0L) +{} + + +HttpOpSetGet::~HttpOpSetGet() +{} + + +void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) +{ + mIsGlobal = true; + mSetting = setting; +} + + +void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) +{ + mIsGlobal = true; + mDoSet = true; + mSetting = setting; + mStrValue = value; +} + + +void HttpOpSetGet::stageFromRequest(HttpService * service) +{ + HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); + HttpRequest::EGlobalPolicy setting(static_cast<HttpRequest::EGlobalPolicy>(mSetting)); + + if (mDoSet) + { + mStatus = pol_opt.set(setting, mStrValue); + } + if (mStatus) + { + const std::string * value(NULL); + if ((mStatus = pol_opt.get(setting, &value))) + { + mStrValue = *value; + } + } + + addAsReply(); +} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h new file mode 100644 index 0000000000..6966b9d94e --- /dev/null +++ b/indra/llcorehttp/_httpopsetget.h @@ -0,0 +1,83 @@ +/** + * @file _httpopsetget.h + * @brief Internal declarations for the HttpOpSetGet subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPSETGET_H_ +#define _LLCORE_HTTP_OPSETGET_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include "httpcommon.h" + +#include <curl/curl.h> + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetGet requests dynamic changes to policy and +/// configuration settings. +/// +/// *NOTE: Expect this to change. Don't really like it yet. + +class HttpOpSetGet : public HttpOperation +{ +public: + HttpOpSetGet(); + +protected: + virtual ~HttpOpSetGet(); // Use release() + +private: + HttpOpSetGet(const HttpOpSetGet &); // Not defined + void operator=(const HttpOpSetGet &); // Not defined + +public: + /// Threading: called by application thread + void setupGet(HttpRequest::EGlobalPolicy setting); + void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); + + virtual void stageFromRequest(HttpService *); + +public: + // Request data + bool mIsGlobal; + bool mDoSet; + int mSetting; + long mLongValue; + std::string mStrValue; + +}; // end class HttpOpSetGet + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPSETGET_H_ + diff --git a/indra/llcorehttp/_httpopsetpriority.cpp b/indra/llcorehttp/_httpopsetpriority.cpp new file mode 100644 index 0000000000..d48c7a0b7d --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.cpp @@ -0,0 +1,63 @@ +/** + * @file _httpopsetpriority.cpp + * @brief Definitions for internal classes based on HttpOpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetpriority.h" + +#include "httpresponse.h" +#include "httphandler.h" +#include "_httpservice.h" + + +namespace LLCore +{ + + +HttpOpSetPriority::HttpOpSetPriority(HttpHandle handle, HttpRequest::priority_t priority) + : HttpOperation(), + mHandle(handle), + mPriority(priority) +{} + + +HttpOpSetPriority::~HttpOpSetPriority() +{} + + +void HttpOpSetPriority::stageFromRequest(HttpService * service) +{ + // Do operations + if (! service->changePriority(mHandle, mPriority)) + { + // Request not found, fail the final status + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + + // Move directly to response queue + addAsReply(); +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpopsetpriority.h b/indra/llcorehttp/_httpopsetpriority.h new file mode 100644 index 0000000000..31706b737c --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.h @@ -0,0 +1,73 @@ +/** + * @file _httpsetpriority.h + * @brief Internal declarations for HttpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_SETPRIORITY_H_ +#define _LLCORE_HTTP_SETPRIORITY_H_ + + +#include "httpcommon.h" +#include "httprequest.h" +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetPriority is an immediate request that +/// searches the various queues looking for a given +/// request handle and changing it's priority if +/// found. +/// +/// *NOTE: This will very likely be removed in the near future +/// when priority is removed from the library. + +class HttpOpSetPriority : public HttpOperation +{ +public: + HttpOpSetPriority(HttpHandle handle, HttpRequest::priority_t priority); + +protected: + virtual ~HttpOpSetPriority(); + +private: + HttpOpSetPriority(const HttpOpSetPriority &); // Not defined + void operator=(const HttpOpSetPriority &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +protected: + // Request Data + HttpHandle mHandle; + HttpRequest::priority_t mPriority; +}; // end class HttpOpSetPriority + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_SETPRIORITY_H_ + diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp new file mode 100644 index 0000000000..76c1e22431 --- /dev/null +++ b/indra/llcorehttp/_httppolicy.cpp @@ -0,0 +1,387 @@ +/** + * @file _httppolicy.cpp + * @brief Internal definitions of the Http policy thread + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "_httppolicy.h" + +#include "_httpoprequest.h" +#include "_httpservice.h" +#include "_httplibcurl.h" +#include "_httppolicyclass.h" + +#include "lltimer.h" + + +namespace LLCore +{ + + +// Per-policy-class data for a running system. +// Collection of queues, parameters, history, metrics, etc. +// for a single policy class. +// +// Threading: accessed only by worker thread +struct HttpPolicy::State +{ +public: + State() + : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), + mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), + mConnMin(1), + mNextSample(0), + mErrorCount(0), + mErrorFactor(0) + {} + + HttpReadyQueue mReadyQueue; + HttpRetryQueue mRetryQueue; + + HttpPolicyClass mOptions; + + long mConnMax; + long mConnAt; + long mConnMin; + + HttpTime mNextSample; + unsigned long mErrorCount; + unsigned long mErrorFactor; +}; + + +HttpPolicy::HttpPolicy(HttpService * service) + : mActiveClasses(0), + mState(NULL), + mService(service) +{} + + +HttpPolicy::~HttpPolicy() +{ + shutdown(); + + mService = NULL; +} + + +void HttpPolicy::shutdown() +{ + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + { + HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + while (! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + retryq.pop(); + + op->cancel(); + op->release(); + } + + HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + while (! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->cancel(); + op->release(); + } + } + delete [] mState; + mState = NULL; + mActiveClasses = 0; +} + + +void HttpPolicy::start(const HttpPolicyGlobal & global, + const std::vector<HttpPolicyClass> & classes) +{ + llassert_always(! mState); + + mGlobalOptions = global; + mActiveClasses = classes.size(); + mState = new State [mActiveClasses]; + for (int i(0); i < mActiveClasses; ++i) + { + mState[i].mOptions = classes[i]; + mState[i].mConnMax = classes[i].mConnectionLimit; + mState[i].mConnAt = mState[i].mConnMax; + mState[i].mConnMin = 2; + } +} + + +void HttpPolicy::addOp(HttpOpRequest * op) +{ + const int policy_class(op->mReqPolicy); + + op->mPolicyRetries = 0; + mState[policy_class].mReadyQueue.push(op); +} + + +void HttpPolicy::retryOp(HttpOpRequest * op) +{ + static const HttpTime retry_deltas[] = + { + 250000, // 1st retry in 0.25 S, etc... + 500000, + 1000000, + 2000000, + 5000000 // ... to every 5.0 S. + }; + static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); + + const HttpTime now(totalTime()); + const int policy_class(op->mReqPolicy); + + const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + op->mPolicyRetryAt = now + delta; + ++op->mPolicyRetries; + LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) + << " retry " << op->mPolicyRetries + << " scheduled for +" << (delta / HttpTime(1000)) + << " mS. Status: " << op->mStatus.toHex() + << LL_ENDL; + if (op->mTracing > 0) + { + LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " + << static_cast<HttpHandle>(op) + << LL_ENDL; + } + mState[policy_class].mRetryQueue.push(op); +} + + +// Attempt to deliver requests to the transport layer. +// +// Tries to find HTTP requests for each policy class with +// available capacity. Starts with the retry queue first +// looking for requests that have waited long enough then +// moves on to the ready queue. +// +// If all queues are empty, will return an indication that +// the worker thread may sleep hard otherwise will ask for +// normal polling frequency. +// +HttpService::ELoopSpeed HttpPolicy::processReadyQueue() +{ + const HttpTime now(totalTime()); + HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); + HttpLibcurl & transport(mService->getTransport()); + + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + { + State & state(mState[policy_class]); + int active(transport.getActiveCountInClass(policy_class)); + int needed(state.mConnAt - active); // Expect negatives here + + HttpRetryQueue & retryq(state.mRetryQueue); + HttpReadyQueue & readyq(state.mReadyQueue); + + if (needed > 0) + { + // First see if we have any retries... + while (needed > 0 && ! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + if (op->mPolicyRetryAt > now) + break; + + retryq.pop(); + + op->stageFromReady(mService); + op->release(); + + --needed; + } + + // Now go on to the new requests... + while (needed > 0 && ! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->stageFromReady(mService); + op->release(); + + --needed; + } + } + + if (! readyq.empty() || ! retryq.empty()) + { + // If anything is ready, continue looping... + result = HttpService::NORMAL; + } + } // end foreach policy_class + + return result; +} + + +bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) +{ + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + { + State & state(mState[policy_class]); + // We don't scan retry queue because a priority change there + // is meaningless. The request will be issued based on retry + // intervals not priority value, which is now moot. + + // Scan ready queue for requests that match policy + HttpReadyQueue::container_type & c(state.mReadyQueue.get_container()); + for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast<HttpHandle>(*cur) == handle) + { + HttpOpRequest * op(*cur); + c.erase(cur); // All iterators are now invalidated + op->mReqPriority = priority; + state.mReadyQueue.push(op); // Re-insert using adapter class + return true; + } + } + } + + return false; +} + + +bool HttpPolicy::cancel(HttpHandle handle) +{ + for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + { + State & state(mState[policy_class]); + + // Scan retry queue + HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); + for (HttpRetryQueue::container_type::iterator iter(c1.begin()); c1.end() != iter;) + { + HttpRetryQueue::container_type::iterator cur(iter++); + + if (static_cast<HttpHandle>(*cur) == handle) + { + HttpOpRequest * op(*cur); + c1.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + + // Scan ready queue + HttpReadyQueue::container_type & c2(state.mReadyQueue.get_container()); + for (HttpReadyQueue::container_type::iterator iter(c2.begin()); c2.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast<HttpHandle>(*cur) == handle) + { + HttpOpRequest * op(*cur); + c2.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + } + + return false; +} + + +bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) +{ + static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + static const HttpStatus cant_res_proxy(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); + static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); + static const HttpStatus send_error(HttpStatus::EXT_CURL_EASY, CURLE_SEND_ERROR); + static const HttpStatus recv_error(HttpStatus::EXT_CURL_EASY, CURLE_RECV_ERROR); + static const HttpStatus upload_failed(HttpStatus::EXT_CURL_EASY, CURLE_UPLOAD_FAILED); + static const HttpStatus op_timedout(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); + static const HttpStatus post_error(HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR); + + // Retry or finalize + if (! op->mStatus) + { + // If this failed, we might want to retry. Have to inspect + // the status a little more deeply for those reasons worth retrying... + if (op->mPolicyRetries < op->mPolicyRetryLimit && + ((op->mStatus.isHttpStatus() && op->mStatus.mType >= 499 && op->mStatus.mType <= 599) || + cant_connect == op->mStatus || + cant_res_proxy == op->mStatus || + cant_res_host == op->mStatus || + send_error == op->mStatus || + recv_error == op->mStatus || + upload_failed == op->mStatus || + op_timedout == op->mStatus || + post_error == op->mStatus)) + { + // Okay, worth a retry. We include 499 in this test as + // it's the old 'who knows?' error from many grid services... + retryOp(op); + return true; // still active/ready + } + } + + // This op is done, finalize it delivering it to the reply queue... + if (! op->mStatus) + { + LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) + << " failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << " (" << op->mStatus.toHex() << ")" + << LL_ENDL; + } + else if (op->mPolicyRetries) + { + LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; + } + + op->stageFromActive(mService); + op->release(); + return false; // not active +} + + +int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const +{ + if (policy_class < mActiveClasses) + { + return (mState[policy_class].mReadyQueue.size() + + mState[policy_class].mRetryQueue.size()); + } + return 0; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h new file mode 100644 index 0000000000..03d92c0b8e --- /dev/null +++ b/indra/llcorehttp/_httppolicy.h @@ -0,0 +1,161 @@ +/** + * @file _httppolicy.h + * @brief Declarations for internal class enforcing policy decisions. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_POLICY_H_ +#define _LLCORE_HTTP_POLICY_H_ + + +#include "httprequest.h" +#include "_httpservice.h" +#include "_httpreadyqueue.h" +#include "_httpretryqueue.h" +#include "_httppolicyglobal.h" +#include "_httppolicyclass.h" +#include "_httpinternal.h" + + +namespace LLCore +{ + +class HttpReadyQueue; +class HttpOpRequest; + + +/// Implements class-based queuing policies for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. +class HttpPolicy +{ +public: + HttpPolicy(HttpService *); + virtual ~HttpPolicy(); + +private: + HttpPolicy(const HttpPolicy &); // Not defined + void operator=(const HttpPolicy &); // Not defined + +public: + /// Cancel all ready and retry requests sending them to + /// their notification queues. Release state resources + /// making further request handling impossible. + /// + /// Threading: called by worker thread + void shutdown(); + + /// Deliver policy definitions and enable handling of + /// requests. One-time call invoked before starting + /// the worker thread. + /// + /// Threading: called by application thread + void start(const HttpPolicyGlobal & global, + const std::vector<HttpPolicyClass> & classes); + + /// Give the policy layer some cycles to scan the ready + /// queue promoting higher-priority requests to active + /// as permited. + /// + /// @return Indication of how soon this method + /// should be called again. + /// + /// Threading: called by worker thread + HttpService::ELoopSpeed processReadyQueue(); + + /// Add request to a ready queue. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) + /// + /// OpRequest is owned by the request queue after this call + /// and should not be modified by anyone until retrieved + /// from queue. + /// + /// Threading: called by any thread + void addOp(HttpOpRequest *); + + /// Similar to addOp, used when a caller wants to retry a + /// request that has failed. It's placed on a special retry + /// queue but ordered by retry time not priority. Otherwise, + /// handling is the same and retried operations are considered + /// before new ones but that doesn't guarantee completion + /// order. + /// + /// Threading: called by worker thread + void retryOp(HttpOpRequest *); + + /// Attempt to change the priority of an earlier request. + /// Request that Shadows HttpService's method + /// + /// Threading: called by worker thread + bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + + /// Attempt to cancel a previous request. + /// Shadows HttpService's method as well + /// + /// Threading: called by worker thread + bool cancel(HttpHandle handle); + + /// When transport is finished with an op and takes it off the + /// active queue, it is delivered here for dispatch. Policy + /// may send it back to the ready/retry queues if it needs another + /// go or we may finalize it and send it on to the reply queue. + /// + /// @return Returns true of the request is still active + /// or ready after staging, false if has been + /// sent on to the reply queue. + /// + /// Threading: called by worker thread + bool stageAfterCompletion(HttpOpRequest * op); + + // Get pointer to global policy options. Caller is expected + // to do context checks like no setting once running. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running. + /// + HttpPolicyGlobal & getGlobalOptions() + { + return mGlobalOptions; + } + + /// Get ready counts for a particular policy class + /// + /// Threading: called by worker thread + int getReadyCount(HttpRequest::policy_t policy_class) const; + +protected: + struct State; + + int mActiveClasses; + State * mState; + HttpService * mService; // Naked pointer, not refcounted, not owner + HttpPolicyGlobal mGlobalOptions; + +}; // end class HttpPolicy + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_POLICY_H_ diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp new file mode 100644 index 0000000000..a23b81322c --- /dev/null +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -0,0 +1,125 @@ +/** + * @file _httppolicyclass.cpp + * @brief Definitions for internal class defining class policy option. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httppolicyclass.h" + +#include "_httpinternal.h" + + +namespace LLCore +{ + + +HttpPolicyClass::HttpPolicyClass() + : mSetMask(0UL), + mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mPipelining(0) +{} + + +HttpPolicyClass::~HttpPolicyClass() +{} + + +HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) +{ + if (this != &other) + { + mSetMask = other.mSetMask; + mConnectionLimit = other.mConnectionLimit; + mPerHostConnectionLimit = other.mPerHostConnectionLimit; + mPipelining = other.mPipelining; + } + return *this; +} + + +HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) + : mSetMask(other.mSetMask), + mConnectionLimit(other.mConnectionLimit), + mPerHostConnectionLimit(other.mPerHostConnectionLimit), + mPipelining(other.mPipelining) +{} + + +HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) +{ + switch (opt) + { + case HttpRequest::CP_CONNECTION_LIMIT: + mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); + break; + + case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: + mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); + break; + + case HttpRequest::CP_ENABLE_PIPELINING: + mPipelining = llclamp(value, 0L, 1L); + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + mSetMask |= 1UL << int(opt); + return HttpStatus(); +} + + +HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value) +{ + static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); + long * src(NULL); + + switch (opt) + { + case HttpRequest::CP_CONNECTION_LIMIT: + src = &mConnectionLimit; + break; + + case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: + src = &mPerHostConnectionLimit; + break; + + case HttpRequest::CP_ENABLE_PIPELINING: + src = &mPipelining; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + if (! (mSetMask & (1UL << int(opt)))) + return not_set; + + *value = *src; + return HttpStatus(); +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h new file mode 100644 index 0000000000..d175413cbd --- /dev/null +++ b/indra/llcorehttp/_httppolicyclass.h @@ -0,0 +1,59 @@ +/** + * @file _httppolicyclass.h + * @brief Declarations for internal class defining policy class options. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_POLICY_CLASS_H_ +#define _LLCORE_HTTP_POLICY_CLASS_H_ + + +#include "httprequest.h" + + +namespace LLCore +{ + +class HttpPolicyClass +{ +public: + HttpPolicyClass(); + ~HttpPolicyClass(); + + HttpPolicyClass & operator=(const HttpPolicyClass &); + HttpPolicyClass(const HttpPolicyClass &); // Not defined + +public: + HttpStatus set(HttpRequest::EClassPolicy opt, long value); + HttpStatus get(HttpRequest::EClassPolicy opt, long * value); + +public: + unsigned long mSetMask; + long mConnectionLimit; + long mPerHostConnectionLimit; + long mPipelining; +}; // end class HttpPolicyClass + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_POLICY_CLASS_H_ diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp new file mode 100644 index 0000000000..72f409d3b1 --- /dev/null +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -0,0 +1,175 @@ +/** + * @file _httppolicyglobal.cpp + * @brief Definitions for internal class defining global policy option. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httppolicyglobal.h" + +#include "_httpinternal.h" + + +namespace LLCore +{ + + +HttpPolicyGlobal::HttpPolicyGlobal() + : mSetMask(0UL), + mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mTrace(HTTP_TRACE_OFF), + mUseLLProxy(0) +{} + + +HttpPolicyGlobal::~HttpPolicyGlobal() +{} + + +HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) +{ + if (this != &other) + { + mSetMask = other.mSetMask; + mConnectionLimit = other.mConnectionLimit; + mCAPath = other.mCAPath; + mCAFile = other.mCAFile; + mHttpProxy = other.mHttpProxy; + mTrace = other.mTrace; + mUseLLProxy = other.mUseLLProxy; + } + return *this; +} + + +HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) +{ + switch (opt) + { + case HttpRequest::GP_CONNECTION_LIMIT: + mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); + break; + + case HttpRequest::GP_TRACE: + mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); + break; + + case HttpRequest::GP_LLPROXY: + mUseLLProxy = llclamp(value, 0L, 1L); + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + mSetMask |= 1UL << int(opt); + return HttpStatus(); +} + + +HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value) +{ + switch (opt) + { + case HttpRequest::GP_CA_PATH: + mCAPath = value; + break; + + case HttpRequest::GP_CA_FILE: + mCAFile = value; + break; + + case HttpRequest::GP_HTTP_PROXY: + mCAFile = value; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + mSetMask |= 1UL << int(opt); + return HttpStatus(); +} + + +HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) +{ + static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); + long * src(NULL); + + switch (opt) + { + case HttpRequest::GP_CONNECTION_LIMIT: + src = &mConnectionLimit; + break; + + case HttpRequest::GP_TRACE: + src = &mTrace; + break; + + case HttpRequest::GP_LLPROXY: + src = &mUseLLProxy; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + if (! (mSetMask & (1UL << int(opt)))) + return not_set; + + *value = *src; + return HttpStatus(); +} + + +HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) +{ + static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); + const std::string * src(NULL); + + switch (opt) + { + case HttpRequest::GP_CA_PATH: + src = &mCAPath; + break; + + case HttpRequest::GP_CA_FILE: + src = &mCAFile; + break; + + case HttpRequest::GP_HTTP_PROXY: + src = &mHttpProxy; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + if (! (mSetMask & (1UL << int(opt)))) + return not_set; + + *value = src; + return HttpStatus(); +} + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h new file mode 100644 index 0000000000..a50d0e4188 --- /dev/null +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -0,0 +1,66 @@ +/** + * @file _httppolicyglobal.h + * @brief Declarations for internal class defining global policy option. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_POLICY_GLOBAL_H_ +#define _LLCORE_HTTP_POLICY_GLOBAL_H_ + + +#include "httprequest.h" + + +namespace LLCore +{ + +class HttpPolicyGlobal +{ +public: + HttpPolicyGlobal(); + ~HttpPolicyGlobal(); + + HttpPolicyGlobal & operator=(const HttpPolicyGlobal &); + +private: + HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined + +public: + HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); + HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); + HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); + HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); + +public: + unsigned long mSetMask; + long mConnectionLimit; + std::string mCAPath; + std::string mCAFile; + std::string mHttpProxy; + long mTrace; + long mUseLLProxy; +}; // end class HttpPolicyGlobal + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_POLICY_GLOBAL_H_ diff --git a/indra/llcorehttp/_httpreadyqueue.h b/indra/llcorehttp/_httpreadyqueue.h new file mode 100644 index 0000000000..5f19a9c5f9 --- /dev/null +++ b/indra/llcorehttp/_httpreadyqueue.h @@ -0,0 +1,124 @@ +/** + * @file _httpreadyqueue.h + * @brief Internal declaration for the operation ready queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_READY_QUEUE_H_ +#define _LLCORE_HTTP_READY_QUEUE_H_ + + +#include <queue> + +#include "_httpinternal.h" +#include "_httpoprequest.h" + + +namespace LLCore +{ + +/// HttpReadyQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// If LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY tests true, the class +/// implements a std::priority_queue interface but on std::deque +/// behavior to eliminate sensitivity to priority. In the future, +/// this will likely become the only behavior or it may become +/// a run-time election. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +#if LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + +typedef std::deque<HttpOpRequest *> HttpReadyQueueBase; + +#else + +typedef std::priority_queue<HttpOpRequest *, + std::deque<HttpOpRequest *>, + LLCore::HttpOpRequestCompare> HttpReadyQueueBase; + +#endif // LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + +class HttpReadyQueue : public HttpReadyQueueBase +{ +public: + HttpReadyQueue() + : HttpReadyQueueBase() + {} + + ~HttpReadyQueue() + {} + +protected: + HttpReadyQueue(const HttpReadyQueue &); // Not defined + void operator=(const HttpReadyQueue &); // Not defined + +public: + +#if LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + // Types and methods needed to make a std::deque look + // more like a std::priority_queue, at least for our + // purposes. + typedef HttpReadyQueueBase container_type; + + const_reference top() const + { + return front(); + } + + void pop() + { + pop_front(); + } + + void push(const value_type & v) + { + push_back(v); + } + +#endif // LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + + const container_type & get_container() const + { + return *this; + } + + container_type & get_container() + { + return *this; + } + +}; // end class HttpReadyQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_READY_QUEUE_H_ diff --git a/indra/llcorehttp/_httpreplyqueue.cpp b/indra/llcorehttp/_httpreplyqueue.cpp new file mode 100644 index 0000000000..558b7bdee9 --- /dev/null +++ b/indra/llcorehttp/_httpreplyqueue.cpp @@ -0,0 +1,107 @@ +/** + * @file _httpreplyqueue.cpp + * @brief Internal definitions for the operation reply queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpreplyqueue.h" + + +#include "_mutex.h" +#include "_thread.h" +#include "_httpoperation.h" + +using namespace LLCoreInt; + + +namespace LLCore +{ + + +HttpReplyQueue::HttpReplyQueue() + : RefCounted(true) +{ +} + + +HttpReplyQueue::~HttpReplyQueue() +{ + while (! mQueue.empty()) + { + HttpOperation * op = mQueue.back(); + mQueue.pop_back(); + op->release(); + } +} + + +void HttpReplyQueue::addOp(HttpOperation * op) +{ + { + HttpScopedLock lock(mQueueMutex); + + mQueue.push_back(op); + } + mQueueCV.notify_all(); +} + + +HttpOperation * HttpReplyQueue::fetchOp() +{ + HttpOperation * result(NULL); + + { + HttpScopedLock lock(mQueueMutex); + + if (mQueue.empty()) + return NULL; + + result = mQueue.front(); + mQueue.erase(mQueue.begin()); + } + + // Caller also acquires the reference count + return result; +} + + +void HttpReplyQueue::fetchAll(OpContainer & ops) +{ + // Not valid putting something back on the queue... + llassert_always(ops.empty()); + + { + HttpScopedLock lock(mQueueMutex); + + if (! mQueue.empty()) + { + mQueue.swap(ops); + } + } + + // Caller also acquires the reference counts on each op. + return; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpreplyqueue.h b/indra/llcorehttp/_httpreplyqueue.h new file mode 100644 index 0000000000..4220a09a3b --- /dev/null +++ b/indra/llcorehttp/_httpreplyqueue.h @@ -0,0 +1,108 @@ +/** + * @file _httpreplyqueue.h + * @brief Internal declarations for the operation reply queue. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_REPLY_QUEUE_H_ +#define _LLCORE_HTTP_REPLY_QUEUE_H_ + + +#include "_refcounted.h" +#include "_mutex.h" + + +namespace LLCore +{ + + +class HttpOperation; + + +/// Almost identical to the HttpRequestQueue class but +/// whereas that class is a singleton and is known to the +/// HttpService object, this queue is 1:1 with HttpRequest +/// instances and isn't explicitly referenced by the +/// service object. Instead, HttpOperation objects that +/// want to generate replies back to their creators also +/// keep references to the corresponding HttpReplyQueue. +/// The HttpService plumbing then simply delivers replies +/// to the requested reply queue. +/// +/// One result of that is that the fetch operations do +/// not have a wait forever option. The service object +/// doesn't keep handles on everything it would need to +/// notify so it can't wake up sleepers should it need to +/// shutdown. So only non-blocking or timed-blocking modes +/// are anticipated. These are how most application consumers +/// will be coded anyway so it shouldn't be too much of a +/// burden. + +class HttpReplyQueue : public LLCoreInt::RefCounted +{ +public: + /// Caller acquires a Refcount on construction + HttpReplyQueue(); + +protected: + virtual ~HttpReplyQueue(); // Use release() + +private: + HttpReplyQueue(const HttpReplyQueue &); // Not defined + void operator=(const HttpReplyQueue &); // Not defined + +public: + typedef std::vector<HttpOperation *> OpContainer; + + /// Insert an object at the back of the reply queue. + /// + /// Library also takes possession of one reference count to pass + /// through the queue. + /// + /// Threading: callable by any thread. + void addOp(HttpOperation * op); + + /// Fetch an operation from the head of the queue. Returns + /// NULL if none exists. + /// + /// Caller acquires reference count on returned operation. + /// + /// Threading: callable by any thread. + HttpOperation * fetchOp(); + + /// Caller acquires reference count on each returned operation + /// + /// Threading: callable by any thread. + void fetchAll(OpContainer & ops); + +protected: + OpContainer mQueue; + LLCoreInt::HttpMutex mQueueMutex; + LLCoreInt::HttpConditionVariable mQueueCV; + +}; // end class HttpReplyQueue + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_REPLY_QUEUE_H_ diff --git a/indra/llcorehttp/_httprequestqueue.cpp b/indra/llcorehttp/_httprequestqueue.cpp new file mode 100644 index 0000000000..c16966d078 --- /dev/null +++ b/indra/llcorehttp/_httprequestqueue.cpp @@ -0,0 +1,161 @@ +/** + * @file _httprequestqueue.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httprequestqueue.h" + +#include "_httpoperation.h" +#include "_mutex.h" + + +using namespace LLCoreInt; + +namespace LLCore +{ + +HttpRequestQueue * HttpRequestQueue::sInstance(NULL); + + +HttpRequestQueue::HttpRequestQueue() + : RefCounted(true), + mQueueStopped(false) +{ +} + + +HttpRequestQueue::~HttpRequestQueue() +{ + while (! mQueue.empty()) + { + HttpOperation * op = mQueue.back(); + mQueue.pop_back(); + op->release(); + } +} + + +void HttpRequestQueue::init() +{ + llassert_always(! sInstance); + sInstance = new HttpRequestQueue(); +} + + +void HttpRequestQueue::term() +{ + if (sInstance) + { + sInstance->release(); + sInstance = NULL; + } +} + + +HttpStatus HttpRequestQueue::addOp(HttpOperation * op) +{ + bool wake(false); + { + HttpScopedLock lock(mQueueMutex); + + if (mQueueStopped) + { + // Return op and error to caller + return HttpStatus(HttpStatus::LLCORE, HE_SHUTTING_DOWN); + } + wake = mQueue.empty(); + mQueue.push_back(op); + } + if (wake) + { + mQueueCV.notify_all(); + } + return HttpStatus(); +} + + +HttpOperation * HttpRequestQueue::fetchOp(bool wait) +{ + HttpOperation * result(NULL); + + { + HttpScopedLock lock(mQueueMutex); + + while (mQueue.empty()) + { + if (! wait || mQueueStopped) + return NULL; + mQueueCV.wait(lock); + } + + result = mQueue.front(); + mQueue.erase(mQueue.begin()); + } + + // Caller also acquires the reference count + return result; +} + + +void HttpRequestQueue::fetchAll(bool wait, OpContainer & ops) +{ + // Not valid putting something back on the queue... + llassert_always(ops.empty()); + + { + HttpScopedLock lock(mQueueMutex); + + while (mQueue.empty()) + { + if (! wait || mQueueStopped) + return; + mQueueCV.wait(lock); + } + + mQueue.swap(ops); + } + + // Caller also acquires the reference counts on each op. + return; +} + + +void HttpRequestQueue::wakeAll() +{ + mQueueCV.notify_all(); +} + + +void HttpRequestQueue::stopQueue() +{ + { + HttpScopedLock lock(mQueueMutex); + + mQueueStopped = true; + wakeAll(); + } +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httprequestqueue.h b/indra/llcorehttp/_httprequestqueue.h new file mode 100644 index 0000000000..c9c52b7233 --- /dev/null +++ b/indra/llcorehttp/_httprequestqueue.h @@ -0,0 +1,141 @@ +/** + * @file _httprequestqueue.h + * @brief Internal declaration for the operation request queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_REQUEST_QUEUE_H_ +#define _LLCORE_HTTP_REQUEST_QUEUE_H_ + + +#include <vector> + +#include "httpcommon.h" +#include "_refcounted.h" +#include "_mutex.h" + + +namespace LLCore +{ + + +class HttpOperation; + + +/// Thread-safe queue of HttpOperation objects. Just +/// a simple queue that handles the transfer of operation +/// requests from all HttpRequest instances into the +/// singleton HttpService instance. + +class HttpRequestQueue : public LLCoreInt::RefCounted +{ +protected: + /// Caller acquires a Refcount on construction + HttpRequestQueue(); + +protected: + virtual ~HttpRequestQueue(); // Use release() + +private: + HttpRequestQueue(const HttpRequestQueue &); // Not defined + void operator=(const HttpRequestQueue &); // Not defined + +public: + static void init(); + static void term(); + + /// Threading: callable by any thread once inited. + inline static HttpRequestQueue * instanceOf() + { + return sInstance; + } + +public: + typedef std::vector<HttpOperation *> OpContainer; + + /// Insert an object at the back of the request queue. + /// + /// Caller must provide one refcount to the queue which takes + /// possession of the count on success. + /// + /// @return Standard status. On failure, caller + /// must dispose of the operation with + /// an explicit release() call. + /// + /// Threading: callable by any thread. + HttpStatus addOp(HttpOperation * op); + + /// Return the operation on the front of the queue. If + /// the queue is empty and @wait is false, call returns + /// immediately and a NULL pointer is returned. If true, + /// caller will sleep until explicitly woken. Wakeups + /// can be spurious and callers must expect NULL pointers + /// even if waiting is indicated. + /// + /// Caller acquires reference count any returned operation + /// + /// Threading: callable by any thread. + HttpOperation * fetchOp(bool wait); + + /// Return all queued requests to caller. The @ops argument + /// should be empty when called and will be swap()'d with + /// current contents. Handling of the @wait argument is + /// identical to @fetchOp. + /// + /// Caller acquires reference count on each returned operation + /// + /// Threading: callable by any thread. + void fetchAll(bool wait, OpContainer & ops); + + /// Wake any sleeping threads. Normal queuing operations + /// won't require this but it may be necessary for termination + /// requests. + /// + /// Threading: callable by any thread. + void wakeAll(); + + /// Disallow further request queuing. Callers to @addOp will + /// get a failure status (LLCORE, HE_SHUTTING_DOWN). Callers + /// to @fetchAll or @fetchOp will get requests that are on the + /// queue but the calls will no longer wait. Instead they'll + /// return immediately. Also wakes up all sleepers to send + /// them on their way. + /// + /// Threading: callable by any thread. + void stopQueue(); + +protected: + static HttpRequestQueue * sInstance; + +protected: + OpContainer mQueue; + LLCoreInt::HttpMutex mQueueMutex; + LLCoreInt::HttpConditionVariable mQueueCV; + bool mQueueStopped; + +}; // end class HttpRequestQueue + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_REQUEST_QUEUE_H_ diff --git a/indra/llcorehttp/_httpretryqueue.h b/indra/llcorehttp/_httpretryqueue.h new file mode 100644 index 0000000000..745adec09d --- /dev/null +++ b/indra/llcorehttp/_httpretryqueue.h @@ -0,0 +1,94 @@ +/** + * @file _httpretryqueue.h + * @brief Internal declaration for the operation retry queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_RETRY_QUEUE_H_ +#define _LLCORE_HTTP_RETRY_QUEUE_H_ + + +#include <queue> + +#include "_httpoprequest.h" + + +namespace LLCore +{ + +/// HttpRetryQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +struct HttpOpRetryCompare +{ + bool operator()(const HttpOpRequest * lhs, const HttpOpRequest * rhs) + { + return lhs->mPolicyRetryAt < rhs->mPolicyRetryAt; + } +}; + + +typedef std::priority_queue<HttpOpRequest *, + std::deque<HttpOpRequest *>, + LLCore::HttpOpRetryCompare> HttpRetryQueueBase; + +class HttpRetryQueue : public HttpRetryQueueBase +{ +public: + HttpRetryQueue() + : HttpRetryQueueBase() + {} + + ~HttpRetryQueue() + {} + +protected: + HttpRetryQueue(const HttpRetryQueue &); // Not defined + void operator=(const HttpRetryQueue &); // Not defined + +public: + const container_type & get_container() const + { + return c; + } + + container_type & get_container() + { + return c; + } + +}; // end class HttpRetryQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_RETRY_QUEUE_H_ diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp new file mode 100644 index 0000000000..0825888d0f --- /dev/null +++ b/indra/llcorehttp/_httpservice.cpp @@ -0,0 +1,348 @@ +/** + * @file _httpservice.cpp + * @brief Internal definitions of the Http service thread + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpservice.h" + +#include <boost/bind.hpp> +#include <boost/function.hpp> + +#include "_httpoperation.h" +#include "_httprequestqueue.h" +#include "_httppolicy.h" +#include "_httplibcurl.h" +#include "_thread.h" +#include "_httpinternal.h" + +#include "lltimer.h" +#include "llthread.h" + + +namespace LLCore +{ + +HttpService * HttpService::sInstance(NULL); +volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); + +HttpService::HttpService() + : mRequestQueue(NULL), + mExitRequested(0U), + mThread(NULL), + mPolicy(NULL), + mTransport(NULL) +{ + // Create the default policy class + HttpPolicyClass pol_class; + pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); + pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); + pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); + mPolicyClasses.push_back(pol_class); +} + + +HttpService::~HttpService() +{ + mExitRequested = 1U; + if (RUNNING == sState) + { + // Trying to kill the service object with a running thread + // is a bit tricky. + if (mRequestQueue) + { + mRequestQueue->stopQueue(); + } + + if (mThread) + { + if (! mThread->timedJoin(250)) + { + // Failed to join, expect problems ahead so do a hard termination. + mThread->cancel(); + + LL_WARNS("CoreHttp") << "Destroying HttpService with running thread. Expect problems." + << LL_ENDL; + } + } + } + + if (mRequestQueue) + { + mRequestQueue->release(); + mRequestQueue = NULL; + } + + delete mTransport; + mTransport = NULL; + + delete mPolicy; + mPolicy = NULL; + + if (mThread) + { + mThread->release(); + mThread = NULL; + } +} + + +void HttpService::init(HttpRequestQueue * queue) +{ + llassert_always(! sInstance); + llassert_always(NOT_INITIALIZED == sState); + sInstance = new HttpService(); + + queue->addRef(); + sInstance->mRequestQueue = queue; + sInstance->mPolicy = new HttpPolicy(sInstance); + sInstance->mTransport = new HttpLibcurl(sInstance); + sState = INITIALIZED; +} + + +void HttpService::term() +{ + if (sInstance) + { + if (RUNNING == sState && sInstance->mThread) + { + // Unclean termination. Thread appears to be running. We'll + // try to give the worker thread a chance to cancel using the + // exit flag... + sInstance->mExitRequested = 1U; + sInstance->mRequestQueue->stopQueue(); + + // And a little sleep + for (int i(0); i < 10 && RUNNING == sState; ++i) + { + ms_sleep(100); + } + } + + delete sInstance; + sInstance = NULL; + } + sState = NOT_INITIALIZED; +} + + +HttpRequest::policy_t HttpService::createPolicyClass() +{ + const HttpRequest::policy_t policy_class(mPolicyClasses.size()); + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) + { + return 0; + } + mPolicyClasses.push_back(HttpPolicyClass()); + return policy_class; +} + + +bool HttpService::isStopped() +{ + // What is really wanted here is something like: + // + // HttpService * service = instanceOf(); + // return STOPPED == sState && (! service || ! service->mThread || ! service->mThread->joinable()); + // + // But boost::thread is not giving me a consistent story on joinability + // of a thread after it returns. Debug and non-debug builds are showing + // different behavior on Linux/Etch so we do a weaker test that may + // not be globally correct (i.e. thread *is* stopping, may not have + // stopped but will very soon): + + return STOPPED == sState; +} + + +/// Threading: callable by consumer thread *once*. +void HttpService::startThread() +{ + llassert_always(! mThread || STOPPED == sState); + llassert_always(INITIALIZED == sState || STOPPED == sState); + + if (mThread) + { + mThread->release(); + } + + // Push current policy definitions, enable policy & transport components + mPolicy->start(mPolicyGlobal, mPolicyClasses); + mTransport->start(mPolicyClasses.size()); + + mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); + sState = RUNNING; +} + + +/// Threading: callable by worker thread. +void HttpService::stopRequested() +{ + mExitRequested = 1U; +} + + +/// Threading: callable by worker thread. +bool HttpService::changePriority(HttpHandle handle, HttpRequest::priority_t priority) +{ + bool found(false); + + // Skip the request queue as we currently don't leave earlier + // requests sitting there. Start with the ready queue... + found = mPolicy->changePriority(handle, priority); + + // If not there, we could try the transport/active queue but priority + // doesn't really have much effect there so we don't waste cycles. + + return found; +} + + +/// Try to find the given request handle on any of the request +/// queues and cancel the operation. +/// +/// @return True if the request was canceled. +/// +/// Threading: callable by worker thread. +bool HttpService::cancel(HttpHandle handle) +{ + bool canceled(false); + + // Request can't be on request queue so skip that. + + // Check the policy component's queues first + canceled = mPolicy->cancel(handle); + + if (! canceled) + { + // If that didn't work, check transport's. + canceled = mTransport->cancel(handle); + } + + return canceled; +} + + +/// Threading: callable by worker thread. +void HttpService::shutdown() +{ + // Disallow future enqueue of requests + mRequestQueue->stopQueue(); + + // Cancel requests already on the request queue + HttpRequestQueue::OpContainer ops; + mRequestQueue->fetchAll(false, ops); + while (! ops.empty()) + { + HttpOperation * op(ops.front()); + ops.erase(ops.begin()); + + op->cancel(); + op->release(); + } + + // Shutdown transport canceling requests, freeing resources + mTransport->shutdown(); + + // And now policy + mPolicy->shutdown(); +} + + +// Working thread loop-forever method. Gives time to +// each of the request queue, policy layer and transport +// layer pieces and then either sleeps for a small time +// or waits for a request to come in. Repeats until +// requested to stop. +void HttpService::threadRun(LLCoreInt::HttpThread * thread) +{ + boost::this_thread::disable_interruption di; + + LLThread::registerThreadID(); + + ELoopSpeed loop(REQUEST_SLEEP); + while (! mExitRequested) + { + loop = processRequestQueue(loop); + + // Process ready queue issuing new requests as needed + ELoopSpeed new_loop = mPolicy->processReadyQueue(); + loop = (std::min)(loop, new_loop); + + // Give libcurl some cycles + new_loop = mTransport->processTransport(); + loop = (std::min)(loop, new_loop); + + // Determine whether to spin, sleep briefly or sleep for next request + if (REQUEST_SLEEP != loop) + { + ms_sleep(HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS); + } + } + + shutdown(); + sState = STOPPED; +} + + +HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) +{ + HttpRequestQueue::OpContainer ops; + const bool wait_for_req(REQUEST_SLEEP == loop); + + mRequestQueue->fetchAll(wait_for_req, ops); + while (! ops.empty()) + { + HttpOperation * op(ops.front()); + ops.erase(ops.begin()); + + // Process operation + if (! mExitRequested) + { + // Setup for subsequent tracing + long tracing(HTTP_TRACE_OFF); + mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); + op->mTracing = (std::max)(op->mTracing, int(tracing)); + + if (op->mTracing > HTTP_TRACE_OFF) + { + LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " + << static_cast<HttpHandle>(op) + << LL_ENDL; + } + + // Stage + op->stageFromRequest(this); + } + + // Done with operation + op->release(); + } + + // Queue emptied, allow polling loop to sleep + return REQUEST_SLEEP; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h new file mode 100644 index 0000000000..ffe0349d4d --- /dev/null +++ b/indra/llcorehttp/_httpservice.h @@ -0,0 +1,224 @@ +/** + * @file _httpservice.h + * @brief Declarations for internal class providing HTTP service. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_SERVICE_H_ +#define _LLCORE_HTTP_SERVICE_H_ + + +#include <vector> + +#include "linden_common.h" +#include "llapr.h" +#include "httpcommon.h" +#include "httprequest.h" +#include "_httppolicyglobal.h" +#include "_httppolicyclass.h" + + +namespace LLCoreInt +{ + +class HttpThread; + +} + + +namespace LLCore +{ + + +class HttpRequestQueue; +class HttpPolicy; +class HttpLibcurl; + + +/// The HttpService class does the work behind the request queue. It +/// oversees the HTTP workflow carrying out a number of tasks: +/// - Pulling requests from the global request queue +/// - Executing 'immediate' requests directly +/// - Prioritizing and re-queuing on internal queues the slower requests +/// - Providing cpu cycles to the libcurl plumbing +/// - Overseeing retry operations +/// +/// Note that the service object doesn't have a pointer to any +/// reply queue. These are kept by HttpRequest and HttpOperation +/// only. +/// +/// Service, Policy and Transport +/// +/// HttpService could have been a monolithic class combining a request +/// queue servicer, request policy manager and network transport. +/// Instead, to prevent monolithic growth and allow for easier +/// replacement, it was developed as three separate classes: HttpService, +/// HttpPolicy and HttpLibcurl (transport). These always exist in a +/// 1:1:1 relationship with HttpService managing instances of the other +/// two. So, these classes do not use reference counting to refer +/// to one another, their lifecycles are always managed together. + +class HttpService +{ +protected: + HttpService(); + virtual ~HttpService(); + +private: + HttpService(const HttpService &); // Not defined + void operator=(const HttpService &); // Not defined + +public: + enum EState + { + NOT_INITIALIZED = -1, + INITIALIZED, ///< init() has been called + RUNNING, ///< thread created and running + STOPPED ///< thread has committed to exiting + }; + + // Ordered enumeration of idling strategies available to + // threadRun's loop. Ordered so that std::min on values + // produces the most conservative result of multiple + // requests. + enum ELoopSpeed + { + NORMAL, ///< continuous polling of request, ready, active queues + REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write + }; + + static void init(HttpRequestQueue *); + static void term(); + + /// Threading: callable by any thread once inited. + inline static HttpService * instanceOf() + { + return sInstance; + } + + /// Return the state of the worker thread. Note that the + /// transition from RUNNING to STOPPED is performed by the + /// worker thread itself. This has two weaknesses: + /// - race where the thread hasn't really stopped but will + /// - data ordering between threads where a non-worker thread + /// may see a stale RUNNING status. + /// + /// This transition is generally of interest only to unit tests + /// and these weaknesses shouldn't be any real burden. + /// + /// Threading: callable by any thread with above exceptions. + static EState getState() + { + return sState; + } + + /// Threading: callable by any thread but uses @see getState() and + /// acquires its weaknesses. + static bool isStopped(); + + /// Threading: callable by consumer thread *once*. + void startThread(); + + /// Threading: callable by worker thread. + void stopRequested(); + + /// Threading: callable by worker thread. + void shutdown(); + + /// Try to find the given request handle on any of the request + /// queues and reset the priority (and queue position) of the + /// request if found. + /// + /// @return True if the request was found somewhere. + /// + /// Threading: callable by worker thread. + bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + + /// Try to find the given request handle on any of the request + /// queues and cancel the operation. + /// + /// @return True if the request was found and canceled. + /// + /// Threading: callable by worker thread. + bool cancel(HttpHandle handle); + + /// Threading: callable by worker thread. + HttpPolicy & getPolicy() + { + return *mPolicy; + } + + /// Threading: callable by worker thread. + HttpLibcurl & getTransport() + { + return *mTransport; + } + + /// Threading: callable by worker thread. + HttpRequestQueue & getRequestQueue() + { + return *mRequestQueue; + } + + /// Threading: callable by consumer thread. + HttpPolicyGlobal & getGlobalOptions() + { + return mPolicyGlobal; + } + + /// Threading: callable by consumer thread. + HttpRequest::policy_t createPolicyClass(); + + /// Threading: callable by consumer thread. + HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class) + { + llassert(policy_class >= 0 && policy_class < mPolicyClasses.size()); + return mPolicyClasses[policy_class]; + } + +protected: + void threadRun(LLCoreInt::HttpThread * thread); + + ELoopSpeed processRequestQueue(ELoopSpeed loop); + +protected: + static HttpService * sInstance; + + // === shared data === + static volatile EState sState; + HttpRequestQueue * mRequestQueue; // Refcounted + LLAtomicU32 mExitRequested; + LLCoreInt::HttpThread * mThread; + + // === consumer-thread-only data === + HttpPolicyGlobal mPolicyGlobal; + std::vector<HttpPolicyClass> mPolicyClasses; + + // === working-thread-only data === + HttpPolicy * mPolicy; // Simple pointer, has ownership + HttpLibcurl * mTransport; // Simple pointer, has ownership +}; // end class HttpService + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_SERVICE_H_ diff --git a/indra/llcorehttp/_mutex.h b/indra/llcorehttp/_mutex.h new file mode 100644 index 0000000000..4be4d016d4 --- /dev/null +++ b/indra/llcorehttp/_mutex.h @@ -0,0 +1,55 @@ +/** + * @file _mutex.hpp + * @brief mutex type abstraction + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLCOREINT_MUTEX_H_ +#define LLCOREINT_MUTEX_H_ + + +#include <boost/thread.hpp> + + +namespace LLCoreInt +{ + +// MUTEX TYPES + +// unique mutex type +typedef boost::mutex HttpMutex; + +// CONDITION VARIABLES + +// standard condition variable +typedef boost::condition_variable HttpConditionVariable; + +// LOCKS AND FENCES + +// scoped unique lock +typedef boost::unique_lock<HttpMutex> HttpScopedLock; + +} + +#endif // LLCOREINT_MUTEX_H + diff --git a/indra/llcorehttp/_refcounted.cpp b/indra/llcorehttp/_refcounted.cpp new file mode 100644 index 0000000000..e7d0b72741 --- /dev/null +++ b/indra/llcorehttp/_refcounted.cpp @@ -0,0 +1,45 @@ +/** + * @file _refcounted.cpp + * @brief Atomic, thread-safe ref counting and destruction mixin class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_refcounted.h" + + +namespace LLCoreInt +{ + +#if ! LL_WINDOWS + +const S32 RefCounted::NOT_REF_COUNTED; + +#endif // ! LL_WINDOWS + +RefCounted::~RefCounted() +{} + + +} // end namespace LLCoreInt + + diff --git a/indra/llcorehttp/_refcounted.h b/indra/llcorehttp/_refcounted.h new file mode 100644 index 0000000000..21a916b13b --- /dev/null +++ b/indra/llcorehttp/_refcounted.h @@ -0,0 +1,126 @@ +/** + * @file _refcounted.h + * @brief Atomic, thread-safe ref counting and destruction mixin class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLCOREINT__REFCOUNTED_H_ +#define LLCOREINT__REFCOUNTED_H_ + + +#include "linden_common.h" + +#include "fix_macros.h" +#include <boost/thread.hpp> + +#include "llapr.h" + + +namespace LLCoreInt +{ + + +class RefCounted +{ +private: + RefCounted(); // Not defined - may not be default constructed + void operator=(const RefCounted &); // Not defined + +public: + explicit RefCounted(bool const implicit) + : mRefCount(implicit) + {} + + // ref-count interface + void addRef() const; + void release() const; + bool isLastRef() const; + S32 getRefCount() const; + void noRef() const; + + static const S32 NOT_REF_COUNTED = -1; + +protected: + virtual ~RefCounted(); + virtual void destroySelf(); + +private: + mutable LLAtomicS32 mRefCount; + +}; // end class RefCounted + + +inline void RefCounted::addRef() const +{ + S32 count(mRefCount++); + llassert_always(count >= 0); +} + + +inline void RefCounted::release() const +{ + S32 count(mRefCount); + llassert_always(count != NOT_REF_COUNTED); + llassert_always(count > 0); + count = mRefCount--; + + // clean ourselves up if that was the last reference + if (0 == count) + { + const_cast<RefCounted *>(this)->destroySelf(); + } +} + + +inline bool RefCounted::isLastRef() const +{ + const S32 count(mRefCount); + llassert_always(count != NOT_REF_COUNTED); + llassert_always(count >= 1); + return (1 == count); +} + + +inline S32 RefCounted::getRefCount() const +{ + const S32 result(mRefCount); + return result; +} + + +inline void RefCounted::noRef() const +{ + llassert_always(mRefCount <= 1); + mRefCount = NOT_REF_COUNTED; +} + + +inline void RefCounted::destroySelf() +{ + delete this; +} + +} // end namespace LLCoreInt + +#endif // LLCOREINT__REFCOUNTED_H_ + diff --git a/indra/llcorehttp/_thread.h b/indra/llcorehttp/_thread.h new file mode 100644 index 0000000000..e058d660e5 --- /dev/null +++ b/indra/llcorehttp/_thread.h @@ -0,0 +1,123 @@ +/** + * @file _thread.h + * @brief thread type abstraction + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLCOREINT_THREAD_H_ +#define LLCOREINT_THREAD_H_ + +#include "linden_common.h" + +#include <boost/thread.hpp> +#include <boost/function.hpp> +#include <boost/date_time/posix_time/posix_time_types.hpp> + +#include "_refcounted.h" + +namespace LLCoreInt +{ + +class HttpThread : public RefCounted +{ +private: + HttpThread(); // Not defined + void operator=(const HttpThread &); // Not defined + + void at_exit() + { + // the thread function has exited so we need to release our reference + // to ourself so that we will be automagically cleaned up. + release(); + } + + void run() + { // THREAD CONTEXT + + // Take out additional reference for the at_exit handler + addRef(); + boost::this_thread::at_thread_exit(boost::bind(&HttpThread::at_exit, this)); + + // run the thread function + mThreadFunc(this); + + } // THREAD CONTEXT + +protected: + virtual ~HttpThread() + { + delete mThread; + } + +public: + /// Constructs a thread object for concurrent execution but does + /// not start running. Caller receives on refcount on the thread + /// instance. If the thread is started, another will be taken + /// out for the exit handler. + explicit HttpThread(boost::function<void (HttpThread *)> threadFunc) + : RefCounted(true), // implicit reference + mThreadFunc(threadFunc) + { + // this creates a boost thread that will call HttpThread::run on this instance + // and pass it the threadfunc callable... + boost::function<void()> f = boost::bind(&HttpThread::run, this); + + mThread = new boost::thread(f); + } + + inline void join() + { + mThread->join(); + } + + inline bool timedJoin(S32 millis) + { + return mThread->timed_join(boost::posix_time::milliseconds(millis)); + } + + inline bool joinable() const + { + return mThread->joinable(); + } + + // A very hostile method to force a thread to quit + inline void cancel() + { + boost::thread::native_handle_type thread(mThread->native_handle()); +#if LL_WINDOWS + TerminateThread(thread, 0); +#else + pthread_cancel(thread); +#endif + } + +private: + boost::function<void(HttpThread *)> mThreadFunc; + boost::thread * mThread; +}; // end class HttpThread + +} // end namespace LLCoreInt + +#endif // LLCOREINT_THREAD_H_ + + diff --git a/indra/llcorehttp/bufferarray.cpp b/indra/llcorehttp/bufferarray.cpp new file mode 100644 index 0000000000..8eaaeed710 --- /dev/null +++ b/indra/llcorehttp/bufferarray.cpp @@ -0,0 +1,352 @@ +/** + * @file bufferarray.cpp + * @brief Implements the BufferArray scatter/gather buffer + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "bufferarray.h" + + +// BufferArray is a list of chunks, each a BufferArray::Block, of contiguous +// data presented as a single array. Chunks are at least BufferArray::BLOCK_ALLOC_SIZE +// in length and can be larger. Any chunk may be partially filled or even +// empty. +// +// The BufferArray itself is sharable as a RefCounted entity. As shared +// reads don't work with the concept of a current position/seek value, +// none is kept with the object. Instead, the read and write operations +// all take position arguments. Single write/shared read isn't supported +// directly and any such attempts have to be serialized outside of this +// implementation. + +namespace LLCore +{ + + +// ================================== +// BufferArray::Block Declaration +// ================================== + +class BufferArray::Block +{ +public: + ~Block(); + + void operator delete(void *); + void operator delete(void *, size_t len); + +protected: + Block(size_t len); + + Block(const Block &); // Not defined + void operator=(const Block &); // Not defined + + // Allocate the block with the additional space for the + // buffered data at the end of the object. + void * operator new(size_t len, size_t addl_len); + +public: + // Only public entry to get a block. + static Block * alloc(size_t len); + +public: + size_t mUsed; + size_t mAlloced; + + // *NOTE: Must be last member of the object. We'll + // overallocate as requested via operator new and index + // into the array at will. + char mData[1]; +}; + + +// ================================== +// BufferArray Definitions +// ================================== + + +#if ! LL_WINDOWS +const size_t BufferArray::BLOCK_ALLOC_SIZE; +#endif // ! LL_WINDOWS + +BufferArray::BufferArray() + : LLCoreInt::RefCounted(true), + mLen(0) +{} + + +BufferArray::~BufferArray() +{ + for (container_t::iterator it(mBlocks.begin()); + it != mBlocks.end(); + ++it) + { + delete *it; + *it = NULL; + } + mBlocks.clear(); +} + + +size_t BufferArray::append(const void * src, size_t len) +{ + const size_t ret(len); + const char * c_src(static_cast<const char *>(src)); + + // First, try to copy into the last block + if (len && ! mBlocks.empty()) + { + Block & last(*mBlocks.back()); + if (last.mUsed < last.mAlloced) + { + // Some will fit... + const size_t copy_len((std::min)(len, (last.mAlloced - last.mUsed))); + + memcpy(&last.mData[last.mUsed], c_src, copy_len); + last.mUsed += copy_len; + llassert_always(last.mUsed <= last.mAlloced); + mLen += copy_len; + c_src += copy_len; + len -= copy_len; + } + } + + // Then get new blocks as needed + while (len) + { + const size_t copy_len((std::min)(len, BLOCK_ALLOC_SIZE)); + + if (mBlocks.size() >= mBlocks.capacity()) + { + mBlocks.reserve(mBlocks.size() + 5); + } + Block * block = Block::alloc(BLOCK_ALLOC_SIZE); + memcpy(block->mData, c_src, copy_len); + block->mUsed = copy_len; + llassert_always(block->mUsed <= block->mAlloced); + mBlocks.push_back(block); + mLen += copy_len; + c_src += copy_len; + len -= copy_len; + } + return ret; +} + + +void * BufferArray::appendBufferAlloc(size_t len) +{ + // If someone asks for zero-length, we give them a valid pointer. + if (mBlocks.size() >= mBlocks.capacity()) + { + mBlocks.reserve(mBlocks.size() + 5); + } + Block * block = Block::alloc((std::max)(BLOCK_ALLOC_SIZE, len)); + block->mUsed = len; + mBlocks.push_back(block); + mLen += len; + return block->mData; +} + + +size_t BufferArray::read(size_t pos, void * dst, size_t len) +{ + char * c_dst(static_cast<char *>(dst)); + + if (pos >= mLen) + return 0; + size_t len_limit(mLen - pos); + len = (std::min)(len, len_limit); + if (0 == len) + return 0; + + size_t result(0), offset(0); + const int block_limit(mBlocks.size()); + int block_start(findBlock(pos, &offset)); + if (block_start < 0) + return 0; + + do + { + Block & block(*mBlocks[block_start]); + size_t block_limit(block.mUsed - offset); + size_t block_len((std::min)(block_limit, len)); + + memcpy(c_dst, &block.mData[offset], block_len); + result += block_len; + len -= block_len; + c_dst += block_len; + offset = 0; + ++block_start; + } + while (len && block_start < block_limit); + + return result; +} + + +size_t BufferArray::write(size_t pos, const void * src, size_t len) +{ + const char * c_src(static_cast<const char *>(src)); + + if (pos > mLen || 0 == len) + return 0; + + size_t result(0), offset(0); + const int block_limit(mBlocks.size()); + int block_start(findBlock(pos, &offset)); + + if (block_start >= 0) + { + // Some or all of the write will be on top of + // existing data. + do + { + Block & block(*mBlocks[block_start]); + size_t block_limit(block.mUsed - offset); + size_t block_len((std::min)(block_limit, len)); + + memcpy(&block.mData[offset], c_src, block_len); + result += block_len; + c_src += block_len; + len -= block_len; + offset = 0; + ++block_start; + } + while (len && block_start < block_limit); + } + + // Something left, see if it will fit in the free + // space of the last block. + if (len && ! mBlocks.empty()) + { + Block & last(*mBlocks.back()); + if (last.mUsed < last.mAlloced) + { + // Some will fit... + const size_t copy_len((std::min)(len, (last.mAlloced - last.mUsed))); + + memcpy(&last.mData[last.mUsed], c_src, copy_len); + last.mUsed += copy_len; + result += copy_len; + llassert_always(last.mUsed <= last.mAlloced); + mLen += copy_len; + c_src += copy_len; + len -= copy_len; + } + } + + if (len) + { + // Some or all of the remaining write data will + // be an append. + result += append(c_src, len); + } + + return result; +} + + +int BufferArray::findBlock(size_t pos, size_t * ret_offset) +{ + *ret_offset = 0; + if (pos >= mLen) + return -1; // Doesn't exist + + const int block_limit(mBlocks.size()); + for (int i(0); i < block_limit; ++i) + { + if (pos < mBlocks[i]->mUsed) + { + *ret_offset = pos; + return i; + } + pos -= mBlocks[i]->mUsed; + } + + // Shouldn't get here but... + return -1; +} + + +bool BufferArray::getBlockStartEnd(int block, const char ** start, const char ** end) +{ + if (block < 0 || block >= mBlocks.size()) + { + return false; + } + + const Block & b(*mBlocks[block]); + *start = &b.mData[0]; + *end = &b.mData[b.mUsed]; + return true; +} + + +// ================================== +// BufferArray::Block Definitions +// ================================== + + +BufferArray::Block::Block(size_t len) + : mUsed(0), + mAlloced(len) +{ + memset(mData, 0, len); +} + + +BufferArray::Block::~Block() +{ + mUsed = 0; + mAlloced = 0; +} + + +void * BufferArray::Block::operator new(size_t len, size_t addl_len) +{ + void * mem = new char[len + addl_len + sizeof(void *)]; + return mem; +} + + +void BufferArray::Block::operator delete(void * mem) +{ + char * cmem = static_cast<char *>(mem); + delete [] cmem; +} + + +void BufferArray::Block::operator delete(void * mem, size_t) +{ + operator delete(mem); +} + + +BufferArray::Block * BufferArray::Block::alloc(size_t len) +{ + Block * block = new (len) Block(len); + return block; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h new file mode 100644 index 0000000000..1094a435b4 --- /dev/null +++ b/indra/llcorehttp/bufferarray.h @@ -0,0 +1,137 @@ +/** + * @file bufferarray.h + * @brief Public-facing declaration for the BufferArray scatter/gather class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_BUFFER_ARRAY_H_ +#define _LLCORE_BUFFER_ARRAY_H_ + + +#include <cstdlib> +#include <vector> + +#include "_refcounted.h" + + +namespace LLCore +{ + +class BufferArrayStreamBuf; + +/// A very simple scatter/gather type map for bulk data. The motivation +/// for this class is the writedata callback used by libcurl. Response +/// bodies are delivered to the caller in a sequence of sequential write +/// operations and this class captures them without having to reallocate +/// and move data. +/// +/// The interface looks a little like a unix file descriptor but only +/// just. There is a notion of a current position, starting from 0, +/// which is used as the position in the data when performing read and +/// write operations. The position also moves after various operations: +/// - seek(...) +/// - read(...) +/// - write(...) +/// - append(...) +/// - appendBufferAlloc(...) +/// The object also keeps a total length value which is updated after +/// write and append operations and beyond which the current position +/// cannot be set. +/// +/// Threading: not thread-safe +/// +/// Allocation: Refcounted, heap only. Caller of the constructor +/// is given a single refcount. +/// +class BufferArray : public LLCoreInt::RefCounted +{ +public: + // BufferArrayStreamBuf has intimate knowledge of this + // implementation to implement a buffer-free adapter. + // Changes here will likely need to be reflected there. + friend class BufferArrayStreamBuf; + + BufferArray(); + +protected: + virtual ~BufferArray(); // Use release() + +private: + BufferArray(const BufferArray &); // Not defined + void operator=(const BufferArray &); // Not defined + +public: + // Internal magic number, may be used by unit tests. + static const size_t BLOCK_ALLOC_SIZE = 65540; + + /// Appends the indicated data to the BufferArray + /// modifying current position and total size. New + /// position is one beyond the final byte of the buffer. + /// + /// @return Count of bytes copied to BufferArray + size_t append(const void * src, size_t len); + + /// Similar to @see append(), this call guarantees a + /// contiguous block of memory of requested size placed + /// at the current end of the BufferArray. On return, + /// the data in the memory is considered valid whether + /// the caller writes to it or not. + /// + /// @return Pointer to contiguous region at end + /// of BufferArray of 'len' size. + void * appendBufferAlloc(size_t len); + + /// Current count of bytes in BufferArray instance. + size_t size() const + { + return mLen; + } + + /// Copies data from the given position in the instance + /// to the caller's buffer. Will return a short count of + /// bytes copied if the 'len' extends beyond the data. + size_t read(size_t pos, void * dst, size_t len); + + /// Copies data from the caller's buffer to the instance + /// at the current position. May overwrite existing data, + /// append data when current position is equal to the + /// size of the instance or do a mix of both. + size_t write(size_t pos, const void * src, size_t len); + +protected: + int findBlock(size_t pos, size_t * ret_offset); + + bool getBlockStartEnd(int block, const char ** start, const char ** end); + +protected: + class Block; + typedef std::vector<Block *> container_t; + + container_t mBlocks; + size_t mLen; +}; // end class BufferArray + + +} // end namespace LLCore + +#endif // _LLCORE_BUFFER_ARRAY_H_ diff --git a/indra/llcorehttp/bufferstream.cpp b/indra/llcorehttp/bufferstream.cpp new file mode 100644 index 0000000000..6553900eef --- /dev/null +++ b/indra/llcorehttp/bufferstream.cpp @@ -0,0 +1,285 @@ +/** + * @file bufferstream.cpp + * @brief Implements the BufferStream adapter class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "bufferstream.h" + +#include "bufferarray.h" + + +namespace LLCore +{ + +BufferArrayStreamBuf::BufferArrayStreamBuf(BufferArray * array) + : mBufferArray(array), + mReadCurPos(0), + mReadCurBlock(-1), + mReadBegin(NULL), + mReadCur(NULL), + mReadEnd(NULL), + mWriteCurPos(0) +{ + if (array) + { + array->addRef(); + mWriteCurPos = array->mLen; + } +} + + +BufferArrayStreamBuf::~BufferArrayStreamBuf() +{ + if (mBufferArray) + { + mBufferArray->release(); + mBufferArray = NULL; + } +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::underflow() +{ + if (! mBufferArray) + { + return traits_type::eof(); + } + + if (mReadCur == mReadEnd) + { + // Find the next block with actual data or leave + // mCurBlock/mCur/mEnd unchanged if we're at the end + // of any block chain. + const char * new_begin(NULL), * new_end(NULL); + int new_cur_block(mReadCurBlock + 1); + + while (mBufferArray->getBlockStartEnd(new_cur_block, &new_begin, &new_end)) + { + if (new_begin != new_end) + { + break; + } + ++new_cur_block; + } + if (new_begin == new_end) + { + return traits_type::eof(); + } + + mReadCurBlock = new_cur_block; + mReadBegin = mReadCur = new_begin; + mReadEnd = new_end; + } + + return traits_type::to_int_type(*mReadCur); +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::uflow() +{ + const int_type ret(underflow()); + + if (traits_type::eof() != ret) + { + ++mReadCur; + ++mReadCurPos; + } + return ret; +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::pbackfail(int_type ch) +{ + if (! mBufferArray) + { + return traits_type::eof(); + } + + if (mReadCur == mReadBegin) + { + // Find the previous block with actual data or leave + // mCurBlock/mBegin/mCur/mEnd unchanged if we're at the + // beginning of any block chain. + const char * new_begin(NULL), * new_end(NULL); + int new_cur_block(mReadCurBlock - 1); + + while (mBufferArray->getBlockStartEnd(new_cur_block, &new_begin, &new_end)) + { + if (new_begin != new_end) + { + break; + } + --new_cur_block; + } + if (new_begin == new_end) + { + return traits_type::eof(); + } + + mReadCurBlock = new_cur_block; + mReadBegin = new_begin; + mReadEnd = mReadCur = new_end; + } + + if (traits_type::eof() != ch && mReadCur[-1] != ch) + { + return traits_type::eof(); + } + --mReadCurPos; + return traits_type::to_int_type(*--mReadCur); +} + + +std::streamsize BufferArrayStreamBuf::showmanyc() +{ + if (! mBufferArray) + { + return -1; + } + return mBufferArray->mLen - mReadCurPos; +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::overflow(int c) +{ + if (! mBufferArray || mWriteCurPos > mBufferArray->mLen) + { + return traits_type::eof(); + } + const size_t wrote(mBufferArray->write(mWriteCurPos, &c, 1)); + mWriteCurPos += wrote; + return wrote ? c : traits_type::eof(); +} + + +std::streamsize BufferArrayStreamBuf::xsputn(const char * src, std::streamsize count) +{ + if (! mBufferArray || mWriteCurPos > mBufferArray->mLen) + { + return 0; + } + const size_t wrote(mBufferArray->write(mWriteCurPos, src, count)); + mWriteCurPos += wrote; + return wrote; +} + + +std::streampos BufferArrayStreamBuf::seekoff(std::streamoff off, + std::ios_base::seekdir way, + std::ios_base::openmode which) +{ + std::streampos ret(-1); + + if (! mBufferArray) + { + return ret; + } + + if (std::ios_base::in == which) + { + size_t pos(0); + + switch (way) + { + case std::ios_base::beg: + pos = off; + break; + + case std::ios_base::cur: + pos = mReadCurPos += off; + break; + + case std::ios_base::end: + pos = mBufferArray->mLen - off; + break; + + default: + return ret; + } + + if (pos >= mBufferArray->size()) + { + pos = (std::max)(size_t(0), mBufferArray->size() - 1); + } + size_t ba_offset(0); + int block(mBufferArray->findBlock(pos, &ba_offset)); + if (block < 0) + return ret; + const char * start(NULL), * end(NULL); + if (! mBufferArray->getBlockStartEnd(block, &start, &end)) + return ret; + mReadCurBlock = block; + mReadBegin = start; + mReadCur = start + ba_offset; + mReadEnd = end; + ret = mReadCurPos = pos; + } + else if (std::ios_base::out == which) + { + size_t pos(0); + + switch (way) + { + case std::ios_base::beg: + pos = off; + break; + + case std::ios_base::cur: + pos = mWriteCurPos += off; + break; + + case std::ios_base::end: + pos = mBufferArray->mLen - off; + break; + + default: + return ret; + } + + if (pos < 0) + return ret; + if (pos > mBufferArray->size()) + { + pos = mBufferArray->size(); + } + ret = mWriteCurPos = pos; + } + + return ret; +} + + +BufferArrayStream::BufferArrayStream(BufferArray * ba) + : std::iostream(&mStreamBuf), + mStreamBuf(ba) +{} + + +BufferArrayStream::~BufferArrayStream() +{} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/bufferstream.h b/indra/llcorehttp/bufferstream.h new file mode 100644 index 0000000000..9327a798aa --- /dev/null +++ b/indra/llcorehttp/bufferstream.h @@ -0,0 +1,153 @@ +/** + * @file bufferstream.h + * @brief Public-facing declaration for the BufferStream adapter class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_BUFFER_STREAM_H_ +#define _LLCORE_BUFFER_STREAM_H_ + + +#include <sstream> +#include <cstdlib> + +#include "bufferarray.h" + + +/// @file bufferstream.h +/// +/// std::streambuf and std::iostream adapters for BufferArray +/// objects. +/// +/// BufferArrayStreamBuf inherits std::streambuf and implements +/// an unbuffered interface for streambuf. This may or may not +/// be the most time efficient implementation and it is a little +/// challenging. +/// +/// BufferArrayStream inherits std::iostream and will be the +/// adapter object most callers will be interested in (though +/// it uses BufferArrayStreamBuf internally). Instances allow +/// for the usual streaming operators ('<<', '>>') and serialization +/// methods. +/// +/// Example of LLSD serialization to a BufferArray: +/// +/// BufferArray * ba = new BufferArray; +/// BufferArrayStream bas(ba); +/// LLSDSerialize::toXML(llsd, bas); +/// operationOnBufferArray(ba); +/// ba->release(); +/// ba = NULL; +/// // operationOnBufferArray and bas are each holding +/// // references to the ba instance at this point. +/// + +namespace LLCore +{ + + +// ===================================================== +// BufferArrayStreamBuf +// ===================================================== + +/// Adapter class to put a std::streambuf interface on a BufferArray +/// +/// Application developers will rarely be interested in anything +/// other than the constructor and even that will rarely be used +/// except indirectly via the @BufferArrayStream class. The +/// choice of interfaces implemented yields a bufferless adapter +/// that doesn't used either the input or output pointer triplets +/// of the more common buffered implementations. This may or may +/// not be faster and that question could stand to be looked at +/// sometime. +/// + +class BufferArrayStreamBuf : public std::streambuf +{ +public: + /// Constructor increments the reference count on the + /// BufferArray argument and calls release() on destruction. + BufferArrayStreamBuf(BufferArray * array); + virtual ~BufferArrayStreamBuf(); + +private: + BufferArrayStreamBuf(const BufferArrayStreamBuf &); // Not defined + void operator=(const BufferArrayStreamBuf &); // Not defined + +public: + // Input interfaces from std::streambuf + int_type underflow(); + int_type uflow(); + int_type pbackfail(int_type ch); + std::streamsize showmanyc(); + + // Output interfaces from std::streambuf + int_type overflow(int c); + std::streamsize xsputn(const char * src, std::streamsize count); + + // Common/misc interfaces from std::streambuf + std::streampos seekoff(std::streamoff off, std::ios_base::seekdir way, std::ios_base::openmode which); + +protected: + BufferArray * mBufferArray; // Ref counted + size_t mReadCurPos; + int mReadCurBlock; + const char * mReadBegin; + const char * mReadCur; + const char * mReadEnd; + size_t mWriteCurPos; + +}; // end class BufferArrayStreamBuf + + +// ===================================================== +// BufferArrayStream +// ===================================================== + +/// Adapter class that supplies streaming operators to BufferArray +/// +/// Provides a streaming adapter to an existing BufferArray +/// instance so that the convenient '<<' and '>>' conversions +/// can be applied to a BufferArray. Very convenient for LLSD +/// serialization and parsing as well. + +class BufferArrayStream : public std::iostream +{ +public: + /// Constructor increments the reference count on the + /// BufferArray argument and calls release() on destruction. + BufferArrayStream(BufferArray * ba); + ~BufferArrayStream(); + +protected: + BufferArrayStream(const BufferArrayStream &); + void operator=(const BufferArrayStream &); + +protected: + BufferArrayStreamBuf mStreamBuf; +}; // end class BufferArrayStream + + +} // end namespace LLCore + +#endif // _LLCORE_BUFFER_STREAM_H_ diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp new file mode 100644 index 0000000000..40ad4f047d --- /dev/null +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -0,0 +1,947 @@ +/** + * @file http_texture_load.cpp + * @brief Texture download example for core-http library + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include <iostream> +#include <cstdio> +#include <cstdlib> +#include <set> +#include <map> +#if !defined(WIN32) +#include <pthread.h> +#endif + +#include "linden_common.h" + +#include "httpcommon.h" +#include "httprequest.h" +#include "httphandler.h" +#include "httpresponse.h" +#include "httpheaders.h" +#include "bufferarray.h" +#include "_mutex.h" + +#include <curl/curl.h> +#include <openssl/crypto.h> + +#include "lltimer.h" + + +void init_curl(); +void term_curl(); +unsigned long ssl_thread_id_callback(void); +void ssl_locking_callback(int mode, int type, const char * file, int line); +void usage(std::ostream & out); + +// Default command line settings +static int concurrency_limit(40); +static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; + +#if defined(WIN32) + +#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) +#define strtok_r(_a, _b, _c) strtok_s(_a, _b, _c) + +int getopt(int argc, char * const argv[], const char *optstring); +char *optarg(NULL); +int optind(1); + +#endif + + +// Mostly just a container for the texture IDs and fetch +// parameters.... +class WorkingSet : public LLCore::HttpHandler +{ +public: + WorkingSet(); + ~WorkingSet(); + + bool reload(LLCore::HttpRequest *); + + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + void loadTextureUuids(FILE * in); + +public: + struct Spec + { + std::string mUuid; + int mOffset; + int mLength; + }; + typedef std::set<LLCore::HttpHandle> handle_set_t; + typedef std::vector<Spec> texture_list_t; + +public: + bool mVerbose; + bool mRandomRange; + int mMaxConcurrency; + handle_set_t mHandles; + int mRemaining; + int mLimit; + int mAt; + std::string mUrl; + texture_list_t mTextures; + int mErrorsApi; + int mErrorsHttp; + int mErrorsHttp404; + int mErrorsHttp416; + int mErrorsHttp500; + int mErrorsHttp503; + int mSuccesses; + long mByteCount; + LLCore::HttpHeaders * mHeaders; +}; + + +// Gather process information while we run. Process +// size, cpu consumed, wallclock time. + +class Metrics +{ +public: + class MetricsImpl; + +public: + Metrics(); + ~Metrics(); + + void init(); + void sample(); + void term(); + +protected: + MetricsImpl * mImpl; + +public: + U64 mMaxVSZ; + U64 mMinVSZ; + U64 mStartWallTime; + U64 mEndWallTime; + U64 mStartUTime; + U64 mEndUTime; + U64 mStartSTime; + U64 mEndSTime; +}; + + +// +// +// +int main(int argc, char** argv) +{ + LLCore::HttpStatus status; + bool do_random(false); + bool do_verbose(false); + + int option(-1); + while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) + { + switch (option) + { + case 'u': + strncpy(url_format, optarg, sizeof(url_format)); + url_format[sizeof(url_format) - 1] = '\0'; + break; + + case 'c': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + concurrency_limit = value; + } + break; + + case 'R': + do_random = true; + break; + + case 'v': + do_verbose = true; + break; + + case 'h': + case '?': + usage(std::cout); + return 0; + } + } + + if ((optind + 1) != argc) + { + usage(std::cerr); + return 1; + } + + FILE * uuids(fopen(argv[optind], "r")); + if (! uuids) + { + const char * errstr(strerror(errno)); + + std::cerr << "Couldn't open UUID file '" << argv[optind] << "'. Reason: " + << errstr << std::endl; + return 1; + } + + // Initialization + init_curl(); + LLCore::HttpRequest::createService(); + LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, + LLCore::HttpRequest::CP_CONNECTION_LIMIT, + concurrency_limit); + LLCore::HttpRequest::startThread(); + + // Get service point + LLCore::HttpRequest * hr = new LLCore::HttpRequest(); + + // Get a handler/working set + WorkingSet ws; + + // Fill the working set with work + ws.mUrl = url_format; + ws.loadTextureUuids(uuids); + ws.mRandomRange = do_random; + ws.mVerbose = do_verbose; + ws.mMaxConcurrency = 100; + + if (! ws.mTextures.size()) + { + std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; + return 1; + } + + // Setup metrics + Metrics metrics; + metrics.init(); + + // Run it + int passes(0); + while (! ws.reload(hr)) + { + hr->update(5000000); + ms_sleep(2); + if (0 == (++passes % 200)) + { + metrics.sample(); + } + } + metrics.sample(); + metrics.term(); + + // Report + std::cout << "HTTP errors: " << ws.mErrorsHttp << " API errors: " << ws.mErrorsApi + << " Successes: " << ws.mSuccesses << " Byte count: " << ws.mByteCount + << std::endl; + std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416 + << " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503 + << std::endl; + std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime) + << " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime) + << " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime) + << " uS Maximum VSZ: " << metrics.mMaxVSZ + << " Bytes Minimum VSZ: " << metrics.mMinVSZ << " Bytes" + << std::endl; + + // Clean up + hr->requestStopThread(NULL); + ms_sleep(1000); + delete hr; + LLCore::HttpRequest::destroyService(); + term_curl(); + + return 0; +} + + +void usage(std::ostream & out) +{ + out << "\n" + "usage:\thttp_texture_load [options] uuid_file\n" + "\n" + "This is a standalone program to drive the New Platform HTTP Library.\n" + "The program is supplied with a file of texture UUIDs, one per line\n" + "These are fetched sequentially using a pool of concurrent connection\n" + "until all are fetched. The default URL format is only useful from\n" + "within Linden Lab but this can be overriden with a printf-style\n" + "URL formatting string on the command line.\n" + "\n" + "Options:\n" + "\n" + " -u <url_format> printf-style format string for URL generation\n" + " Default: " << url_format << "\n" + " -R Issue GETs with random Range: headers\n" + " -c <limit> Maximum request concurrency. Range: [1..100]\n" + " Default: " << concurrency_limit << "\n" + " -v Verbose mode. Issue some chatter while running\n" + " -h print this help\n" + "\n" + << std::endl; +} + + +WorkingSet::WorkingSet() + : LLCore::HttpHandler(), + mVerbose(false), + mRandomRange(false), + mRemaining(200), + mLimit(200), + mAt(0), + mErrorsApi(0), + mErrorsHttp(0), + mErrorsHttp404(0), + mErrorsHttp416(0), + mErrorsHttp500(0), + mErrorsHttp503(0), + mSuccesses(0), + mByteCount(0L) +{ + mTextures.reserve(30000); + + mHeaders = new LLCore::HttpHeaders; + mHeaders->mHeaders.push_back("Accept: image/x-j2c"); +} + + +WorkingSet::~WorkingSet() +{ + if (mHeaders) + { + mHeaders->release(); + mHeaders = NULL; + } +} + + +bool WorkingSet::reload(LLCore::HttpRequest * hr) +{ + int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); + + for (int i(0); i < to_do; ++i) + { + char buffer[1024]; +#if defined(WIN32) + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); +#else + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str()); +#endif + int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); + int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); + + LLCore::HttpHandle handle; + if (offset || length) + { + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this); + } + else + { + handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this); + } + if (! handle) + { + // Fatal. Couldn't queue up something. + std::cerr << "Failed to queue work to HTTP Service. Reason: " + << hr->getStatus().toString() << std::endl; + exit(1); + } + else + { + mHandles.insert(handle); + } + mAt++; + mRemaining--; + + if (mVerbose) + { + static int count(0); + ++count; + if (0 == (count %5)) + std::cout << "Queued " << count << std::endl; + } + } + + // Are we done? + return (! mRemaining) && mHandles.empty(); +} + + +void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + handle_set_t::iterator it(mHandles.find(handle)); + if (mHandles.end() == it) + { + // Wha? + std::cerr << "Failed to find handle in request list. Fatal." << std::endl; + exit(1); + } + else + { + LLCore::HttpStatus status(response->getStatus()); + if (status) + { + // More success + LLCore::BufferArray * data(response->getBody()); + mByteCount += data->size(); + ++mSuccesses; + } + else + { + // Something in this library or libcurl + if (status.isHttpStatus()) + { + static const LLCore::HttpStatus hs404(404); + static const LLCore::HttpStatus hs416(416); + static const LLCore::HttpStatus hs500(500); + static const LLCore::HttpStatus hs503(503); + + ++mErrorsHttp; + if (hs404 == status) + { + ++mErrorsHttp404; + } + else if (hs416 == status) + { + ++mErrorsHttp416; + } + else if (hs500 == status) + { + ++mErrorsHttp500; + } + else if (hs503 == status) + { + ++mErrorsHttp503; + } + } + else + { + ++mErrorsApi; + } + } + mHandles.erase(it); + } + + if (mVerbose) + { + static int count(0); + ++count; + if (0 == (count %5)) + std::cout << "Handled " << count << std::endl; + } +} + + +void WorkingSet::loadTextureUuids(FILE * in) +{ + char buffer[1024]; + + while (fgets(buffer, sizeof(buffer), in)) + { + WorkingSet::Spec texture; + char * state(NULL); + char * token = strtok_r(buffer, " \t\n,", &state); + if (token && 36 == strlen(token)) + { + // Close enough for this function + texture.mUuid = token; + texture.mOffset = 0; + texture.mLength = 0; + token = strtok_r(buffer, " \t\n,", &state); + if (token) + { + int offset(atoi(token)); + token = strtok_r(buffer, " \t\n,", &state); + if (token) + { + int length(atoi(token)); + texture.mOffset = offset; + texture.mLength = length; + } + } + mTextures.push_back(texture); + } + } + mRemaining = mLimit = mTextures.size(); +} + + +int ssl_mutex_count(0); +LLCoreInt::HttpMutex ** ssl_mutex_list = NULL; + +void init_curl() +{ + curl_global_init(CURL_GLOBAL_ALL); + + ssl_mutex_count = CRYPTO_num_locks(); + if (ssl_mutex_count > 0) + { + ssl_mutex_list = new LLCoreInt::HttpMutex * [ssl_mutex_count]; + + for (int i(0); i < ssl_mutex_count; ++i) + { + ssl_mutex_list[i] = new LLCoreInt::HttpMutex; + } + + CRYPTO_set_locking_callback(ssl_locking_callback); + CRYPTO_set_id_callback(ssl_thread_id_callback); + } +} + + +void term_curl() +{ + CRYPTO_set_locking_callback(NULL); + for (int i(0); i < ssl_mutex_count; ++i) + { + delete ssl_mutex_list[i]; + } + delete [] ssl_mutex_list; +} + + +unsigned long ssl_thread_id_callback(void) +{ +#if defined(WIN32) + return (unsigned long) GetCurrentThread(); +#else + return (unsigned long) pthread_self(); +#endif +} + + +void ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) +{ + if (type >= 0 && type < ssl_mutex_count) + { + if (mode & CRYPTO_LOCK) + { + ssl_mutex_list[type]->lock(); + } + else + { + ssl_mutex_list[type]->unlock(); + } + } +} + + +#if defined(WIN32) + +// Very much a subset of posix functionality. Don't push +// it too hard... +int getopt(int argc, char * const argv[], const char *optstring) +{ + static int pos(0); + while (optind < argc) + { + if (pos == 0) + { + if (argv[optind][0] != '-') + return -1; + pos = 1; + } + if (! argv[optind][pos]) + { + ++optind; + pos = 0; + continue; + } + const char * thing(strchr(optstring, argv[optind][pos])); + if (! thing) + { + ++optind; + return -1; + } + if (thing[1] == ':') + { + optarg = argv[++optind]; + ++optind; + pos = 0; + } + else + { + optarg = NULL; + ++pos; + } + return *thing; + } + return -1; +} + +#endif + + + +#if LL_WINDOWS + +#define PSAPI_VERSION 1 +#include "windows.h" +#include "psapi.h" + +class Metrics::MetricsImpl +{ +public: + MetricsImpl() + {} + + ~MetricsImpl() + {} + + void init(Metrics * metrics) + { + HANDLE self(GetCurrentProcess()); // Does not have to be closed + FILETIME ft_dummy, ft_system, ft_user; + GetProcessTimes(self, &ft_dummy, &ft_dummy, &ft_system, &ft_user); + ULARGE_INTEGER uli; + uli.u.LowPart = ft_system.dwLowDateTime; + uli.u.HighPart = ft_system.dwHighDateTime; + metrics->mStartSTime = uli.QuadPart / U64L(10); // Convert to uS + uli.u.LowPart = ft_user.dwLowDateTime; + uli.u.HighPart = ft_user.dwHighDateTime; + metrics->mStartUTime = uli.QuadPart / U64L(10); + metrics->mStartWallTime = totalTime(); + } + + void sample(Metrics * metrics) + { + PROCESS_MEMORY_COUNTERS_EX counters; + + GetProcessMemoryInfo(GetCurrentProcess(), + (PROCESS_MEMORY_COUNTERS *) &counters, + sizeof(counters)); + // Okay, PrivateUsage isn't truly VSZ but it will be + // a good tracker for leaks and fragmentation. Work on + // a better estimator later... + SIZE_T vsz(counters.PrivateUsage); + metrics->mMaxVSZ = (std::max)(metrics->mMaxVSZ, U64(vsz)); + metrics->mMinVSZ = (std::min)(metrics->mMinVSZ, U64(vsz)); + } + + void term(Metrics * metrics) + { + HANDLE self(GetCurrentProcess()); // Does not have to be closed + FILETIME ft_dummy, ft_system, ft_user; + GetProcessTimes(self, &ft_dummy, &ft_dummy, &ft_system, &ft_user); + ULARGE_INTEGER uli; + uli.u.LowPart = ft_system.dwLowDateTime; + uli.u.HighPart = ft_system.dwHighDateTime; + metrics->mEndSTime = uli.QuadPart / U64L(10); + uli.u.LowPart = ft_user.dwLowDateTime; + uli.u.HighPart = ft_user.dwHighDateTime; + metrics->mEndUTime = uli.QuadPart / U64L(10); + metrics->mEndWallTime = totalTime(); + } + +protected: +}; + +#elif LL_DARWIN + +#include <sys/resource.h> +#include <mach/mach.h> + +class Metrics::MetricsImpl +{ +public: + MetricsImpl() + {} + + ~MetricsImpl() + {} + + void init(Metrics * metrics) + { + U64 utime, stime; + + if (getTimes(&utime, &stime)) + { + metrics->mStartSTime = stime; + metrics->mStartUTime = utime; + } + metrics->mStartWallTime = totalTime(); + sample(metrics); + } + + void sample(Metrics * metrics) + { + U64 vsz; + + if (getVM(&vsz)) + { + metrics->mMaxVSZ = (std::max)(metrics->mMaxVSZ, vsz); + metrics->mMinVSZ = (std::min)(metrics->mMinVSZ, vsz); + } + } + + void term(Metrics * metrics) + { + U64 utime, stime; + + if (getTimes(&utime, &stime)) + { + metrics->mEndSTime = stime; + metrics->mEndUTime = utime; + } + metrics->mEndWallTime = totalTime(); + } + +protected: + bool getVM(U64 * vsz) + { + task_basic_info task_info_block; + mach_msg_type_number_t task_info_count(TASK_BASIC_INFO_COUNT); + + if (KERN_SUCCESS != task_info(mach_task_self(), + TASK_BASIC_INFO, + (task_info_t) &task_info_block, + &task_info_count)) + { + return false; + } + * vsz = task_info_block.virtual_size; + return true; + } + + bool getTimes(U64 * utime, U64 * stime) + { + struct rusage usage; + + if (getrusage(RUSAGE_SELF, &usage)) + { + return false; + } + * utime = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; + * stime = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; + return true; + } + +}; + +#else + +class Metrics::MetricsImpl +{ +public: + MetricsImpl() + : mProcFS(NULL), + mUsecsPerTick(U64L(0)) + {} + + + ~MetricsImpl() + { + if (mProcFS) + { + fclose(mProcFS); + mProcFS = NULL; + } + } + + void init(Metrics * metrics) + { + if (! mProcFS) + { + mProcFS = fopen("/proc/self/stat", "r"); + if (! mProcFS) + { + const int errnum(errno); + LL_ERRS("Main") << "Error opening proc fs: " << strerror(errnum) << LL_ENDL; + } + } + + long ticks_per_sec(sysconf(_SC_CLK_TCK)); + mUsecsPerTick = U64L(1000000) / ticks_per_sec; + U64 usecs_per_sec(mUsecsPerTick * ticks_per_sec); + if (900000 > usecs_per_sec || 1100000 < usecs_per_sec) + { + LL_ERRS("Main") << "Resolution problems using uSecs for ticks" << LL_ENDL; + } + + U64 utime, stime; + if (scanProcFS(&utime, &stime, NULL)) + { + metrics->mStartSTime = stime; + metrics->mStartUTime = utime; + } + metrics->mStartWallTime = totalTime(); + + sample(metrics); + } + + + void sample(Metrics * metrics) + { + U64 vsz; + if (scanProcFS(NULL, NULL, &vsz)) + { + metrics->mMaxVSZ = (std::max)(metrics->mMaxVSZ, vsz); + metrics->mMinVSZ = (std::min)(metrics->mMinVSZ, vsz); + } + } + + + void term(Metrics * metrics) + { + U64 utime, stime; + if (scanProcFS(&utime, &stime, NULL)) + { + metrics->mEndSTime = stime; + metrics->mEndUTime = utime; + } + metrics->mEndWallTime = totalTime(); + + sample(metrics); + + if (mProcFS) + { + fclose(mProcFS); + mProcFS = NULL; + } + } + +protected: + bool scanProcFS(U64 * utime, U64 * stime, U64 * vsz) + { + if (mProcFS) + { + int i_dummy; + unsigned int ui_dummy; + unsigned long ul_dummy, user_ticks, sys_ticks, vsize; + long l_dummy, rss; + unsigned long long ull_dummy; + char c_dummy; + + char buffer[256]; + + static const char * format("%d %*s %c %d %d %d %d %d %u %lu %lu %lu %lu %lu %lu %ld %ld %ld %ld %ld %ld %llu %lu %ld"); + + fseek(mProcFS, 0L, SEEK_SET); + size_t len = fread(buffer, 1, sizeof(buffer) - 1, mProcFS); + if (! len) + { + return false; + } + buffer[len] = '\0'; + if (23 == sscanf(buffer, format, + &i_dummy, // pid + // &s_dummy, // command name + &c_dummy, // state + &i_dummy, // ppid + &i_dummy, // pgrp + &i_dummy, // session + &i_dummy, // terminal + &i_dummy, // terminal group id + &ui_dummy, // flags + &ul_dummy, // minor faults + &ul_dummy, // minor faults in children + &ul_dummy, // major faults + &ul_dummy, // major faults in children + &user_ticks, + &sys_ticks, + &l_dummy, // cutime + &l_dummy, // cstime + &l_dummy, // process priority + &l_dummy, // nice value + &l_dummy, // thread count + &l_dummy, // time to SIGALRM + &ull_dummy, // start time + &vsize, + &rss)) + { + // Looks like we understand the line + if (utime) + { + *utime = user_ticks * mUsecsPerTick; + } + + if (stime) + { + *stime = sys_ticks * mUsecsPerTick; + } + + if (vsz) + { + *vsz = vsize; + } + return true; + } + } + return false; + } + +protected: + FILE * mProcFS; + U64 mUsecsPerTick; + +}; + + +#endif // LL_WINDOWS + +Metrics::Metrics() + : mMaxVSZ(U64(0)), + mMinVSZ(U64L(0xffffffffffffffff)), + mStartWallTime(U64(0)), + mEndWallTime(U64(0)), + mStartUTime(U64(0)), + mEndUTime(U64(0)), + mStartSTime(U64(0)), + mEndSTime(U64(0)) +{ + mImpl = new MetricsImpl(); +} + + +Metrics::~Metrics() +{ + delete mImpl; + mImpl = NULL; +} + + +void Metrics::init() +{ + mImpl->init(this); +} + + +void Metrics::sample() +{ + mImpl->sample(this); +} + + +void Metrics::term() +{ + mImpl->term(this); +} + + diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp new file mode 100644 index 0000000000..f2fcbf77a3 --- /dev/null +++ b/indra/llcorehttp/httpcommon.cpp @@ -0,0 +1,179 @@ +/** + * @file httpcommon.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpcommon.h" + +#include <curl/curl.h> +#include <string> +#include <sstream> + + +namespace LLCore +{ + +HttpStatus::type_enum_t EXT_CURL_EASY; +HttpStatus::type_enum_t EXT_CURL_MULTI; +HttpStatus::type_enum_t LLCORE; + +HttpStatus::operator unsigned long() const +{ + static const int shift(sizeof(unsigned long) * 4); + + unsigned long result(((unsigned long) mType) << shift | (unsigned long) (int) mStatus); + return result; +} + + +std::string HttpStatus::toHex() const +{ + std::ostringstream result; + result.width(8); + result.fill('0'); + result << std::hex << operator unsigned long(); + return result.str(); +} + + +std::string HttpStatus::toString() const +{ + static const char * llcore_errors[] = + { + "", + "HTTP error reply status", + "Services shutting down", + "Operation canceled", + "Invalid Content-Range header encountered", + "Request handle not found", + "Invalid datatype for argument or option", + "Option has not been explicitly set", + "Option is not dynamic and must be set early", + "Invalid HTTP status code received from server" + }; + static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); + + static const struct + { + type_enum_t mCode; + const char * mText; + } + http_errors[] = + { + // Keep sorted by mCode, we binary search this list. + { 100, "Continue" }, + { 101, "Switching Protocols" }, + { 200, "OK" }, + { 201, "Created" }, + { 202, "Accepted" }, + { 203, "Non-Authoritative Information" }, + { 204, "No Content" }, + { 205, "Reset Content" }, + { 206, "Partial Content" }, + { 300, "Multiple Choices" }, + { 301, "Moved Permanently" }, + { 302, "Found" }, + { 303, "See Other" }, + { 304, "Not Modified" }, + { 305, "Use Proxy" }, + { 307, "Temporary Redirect" }, + { 400, "Bad Request" }, + { 401, "Unauthorized" }, + { 402, "Payment Required" }, + { 403, "Forbidden" }, + { 404, "Not Found" }, + { 405, "Method Not Allowed" }, + { 406, "Not Acceptable" }, + { 407, "Proxy Authentication Required" }, + { 408, "Request Time-out" }, + { 409, "Conflict" }, + { 410, "Gone" }, + { 411, "Length Required" }, + { 412, "Precondition Failed" }, + { 413, "Request Entity Too Large" }, + { 414, "Request-URI Too Large" }, + { 415, "Unsupported Media Type" }, + { 416, "Requested range not satisfiable" }, + { 417, "Expectation Failed" }, + { 500, "Internal Server Error" }, + { 501, "Not Implemented" }, + { 502, "Bad Gateway" }, + { 503, "Service Unavailable" }, + { 504, "Gateway Time-out" }, + { 505, "HTTP Version not supported" } + }; + static const int http_errors_count(sizeof(http_errors) / sizeof(http_errors[0])); + + if (*this) + { + return std::string(""); + } + switch (mType) + { + case EXT_CURL_EASY: + return std::string(curl_easy_strerror(CURLcode(mStatus))); + + case EXT_CURL_MULTI: + return std::string(curl_multi_strerror(CURLMcode(mStatus))); + + case LLCORE: + if (mStatus >= 0 && mStatus < llcore_errors_count) + { + return std::string(llcore_errors[mStatus]); + } + break; + + default: + if (isHttpStatus()) + { + // Binary search for the error code and string + int bottom(0), top(http_errors_count); + while (true) + { + int at((bottom + top) / 2); + if (mType == http_errors[at].mCode) + { + return std::string(http_errors[at].mText); + } + if (at == bottom) + { + break; + } + else if (mType < http_errors[at].mCode) + { + top = at; + } + else + { + bottom = at; + } + } + } + break; + } + return std::string("Unknown error"); +} + +} // end namespace LLCore + diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h new file mode 100644 index 0000000000..c0d4ec5aad --- /dev/null +++ b/indra/llcorehttp/httpcommon.h @@ -0,0 +1,311 @@ +/** + * @file httpcommon.h + * @brief Public-facing declarations and definitions of common types + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_COMMON_H_ +#define _LLCORE_HTTP_COMMON_H_ + +/// @package LLCore::HTTP +/// +/// This library implements a high-level, Indra-code-free client interface to +/// HTTP services based on actual patterns found in the viewer and simulator. +/// Interfaces are similar to those supplied by the legacy classes +/// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that +/// allows an application to specify connection behaviors: limits on +/// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. +/// +/// Features of the library include: +/// - Single, private working thread where all transport and processing occurs. +/// - Support for multiple consumers running in multiple threads. +/// - Scatter/gather (a.k.a. buffer array) model for bulk data movement. +/// - Reference counting used for many object instance lifetimes. +/// - Minimal data sharing across threads for correctness and low latency. +/// +/// The public interface is declared in a few key header files: +/// - "llcorehttp/bufferarray.h" +/// - "llcorehttp/httpcommon.h" +/// - "llcorehttp/httphandler.h" +/// - "llcorehttp/httpheaders.h" +/// - "llcorehttp/httpoptions.h" +/// - "llcorehttp/httprequest.h" +/// - "llcorehttp/httpresponse.h" +/// +/// The library is still under early development and particular users +/// may need access to internal implementation details that are found +/// in the _*.h header files. But this is a crutch to be avoided if at +/// all possible and probably indicates some interface work is neeeded. +/// +/// Using the library is fairly easy. Global setup needs a few +/// steps: +/// +/// - libcurl initialization including thread-safely callbacks for SSL: +/// . curl_global_init(...) +/// . CRYPTO_set_locking_callback(...) +/// . CRYPTO_set_id_callback(...) +/// - HttpRequest::createService() called to instantiate singletons +/// and support objects. +/// +/// An HTTP consumer in an application, and an application may have many +/// consumers, does a few things: +/// +/// - Instantiate and retain an object based on HttpRequest. This +/// object becomes the portal into runtime services for the consumer. +/// - Derive or mixin the HttpHandler class if you want notification +/// when requests succeed or fail. This object's onCompleted() +/// method is invoked and an instance can be shared across +/// requests. +/// +/// Issuing a request is straightforward: +/// - Construct a suitable URL. +/// - Configure HTTP options for the request. (optional) +/// - Build a list of additional headers. (optional) +/// - Invoke one of the requestXXXX() methods (requestGetByteRange, +/// requestPost, etc.) on the HttpRequest instance supplying the +/// above along with a policy class, a priority and an optional +/// pointer to an HttpHandler instance. Work is then queued to +/// the worker thread and occurs asynchronously. +/// - Periodically invoke the update() method on the HttpRequest +/// instance which performs completion notification to HttpHandler +/// objects. +/// - Do completion processing in your onCompletion() method. +/// +/// Code fragments: +/// Rather than a poorly-maintained example in comments, look in the +/// example subdirectory which is a minimal yet functional tool to do +/// GET request performance testing. With four calls: +/// +/// init_curl(); +/// LLCore::HttpRequest::createService(); +/// LLCore::HttpRequest::startThread(); +/// LLCore::HttpRequest * hr = new LLCore::HttpRequest(); +/// +/// the program is basically ready to issue requests. +/// + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include <string> + + +namespace LLCore +{ + + +/// All queued requests are represented by an HttpHandle value. +/// The invalid value is returned when a request failed to queue. +/// The actual status for these failures is then fetched with +/// HttpRequest::getStatus(). +/// +/// The handle is valid only for the life of a request. On +/// return from any HttpHandler notification, the handle immediately +/// becomes invalid and may be recycled for other queued requests. + +typedef void * HttpHandle; +#define LLCORE_HTTP_HANDLE_INVALID (NULL) + +/// For internal scheduling and metrics, we use a microsecond +/// timebase compatible with the environment. +typedef U64 HttpTime; + +/// Error codes defined by the library itself as distinct from +/// libcurl (or any other transport provider). +enum HttpError +{ + // Successful value compatible with the libcurl codes. + HE_SUCCESS = 0, + + // Intended for HTTP reply codes 100-999, indicates that + // the reply should be considered an error by the application. + HE_REPLY_ERROR = 1, + + // Service is shutting down and requested operation will + // not be queued or performed. + HE_SHUTTING_DOWN = 2, + + // Operation was canceled by request. + HE_OP_CANCELED = 3, + + // Invalid content range header received. + HE_INV_CONTENT_RANGE_HDR = 4, + + // Request handle not found + HE_HANDLE_NOT_FOUND = 5, + + // Invalid datatype for option/setting + HE_INVALID_ARG = 6, + + // Option hasn't been explicitly set + HE_OPT_NOT_SET = 7, + + // Option not dynamic, must be set during init phase + HE_OPT_NOT_DYNAMIC = 8, + + // Invalid HTTP status code returned by server + HE_INVALID_HTTP_STATUS = 9 + +}; // end enum HttpError + + +/// HttpStatus encapsulates errors from libcurl (easy, multi), HTTP +/// reply status codes and internal errors as well. The encapsulation +/// isn't expected to completely isolate the caller from libcurl but +/// basic operational tests (success or failure) are provided. +/// +/// Non-HTTP status are encoded as (type, status) with type being +/// one of: EXT_CURL_EASY, EXT_CURL_MULTI or LLCORE and status +/// being the success/error code from that domain. HTTP status +/// is encoded as (status, error_flag). Status should be in the +/// range [100, 999] and error_flag is either HE_SUCCESS or +/// HE_REPLY_ERROR to indicate whether this should be treated as +/// a successful status or an error. The application is responsible +/// for making that determination and a range like [200, 299] isn't +/// automatically assumed to be definitive. +/// +/// Examples: +/// +/// 1. Construct a default, successful status code: +/// HttpStatus(); +/// +/// 2. Construct a successful, HTTP 200 status code: +/// HttpStatus(200); +/// +/// 3. Construct a failed, HTTP 404 not-found status code: +/// HttpStatus(404); +/// +/// 4. Construct a failed libcurl couldn't connect status code: +/// HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); +/// +/// 5. Construct an HTTP 301 status code to be treated as success: +/// HttpStatus(301, HE_SUCCESS); +/// + +struct HttpStatus +{ + typedef unsigned short type_enum_t; + + HttpStatus() + : mType(LLCORE), + mStatus(HE_SUCCESS) + {} + + HttpStatus(type_enum_t type, short status) + : mType(type), + mStatus(status) + {} + + HttpStatus(int http_status) + : mType(http_status), + mStatus(http_status >= 200 && http_status <= 299 + ? HE_SUCCESS + : HE_REPLY_ERROR) + { + llassert(http_status >= 100 && http_status <= 999); + } + + HttpStatus(const HttpStatus & rhs) + : mType(rhs.mType), + mStatus(rhs.mStatus) + {} + + HttpStatus & operator=(const HttpStatus & rhs) + { + // Don't care if lhs & rhs are the same object + + mType = rhs.mType; + mStatus = rhs.mStatus; + return *this; + } + + static const type_enum_t EXT_CURL_EASY = 0; + static const type_enum_t EXT_CURL_MULTI = 1; + static const type_enum_t LLCORE = 2; + + type_enum_t mType; + short mStatus; + + /// Test for successful status in the code regardless + /// of error source (internal, libcurl). + /// + /// @return 'true' when status is successful. + /// + operator bool() const + { + return 0 == mStatus; + } + + /// Inverse of previous operator. + /// + /// @return 'true' on any error condition + bool operator !() const + { + return 0 != mStatus; + } + + /// Equality and inequality tests to bypass bool conversion + /// which will do the wrong thing in conditional expressions. + bool operator==(const HttpStatus & rhs) const + { + return mType == rhs.mType && mStatus == rhs.mStatus; + } + + bool operator!=(const HttpStatus & rhs) const + { + return ! operator==(rhs); + } + + /// Convert to single numeric representation. Mainly + /// for logging or other informal purposes. Also + /// creates an ambiguous second path to integer conversion + /// which tends to find programming errors such as formatting + /// the status to a stream (operator<<). + operator unsigned long() const; + unsigned long toULong() const + { + return operator unsigned long(); + } + + /// And to convert to a hex string. + std::string toHex() const; + + /// Convert status to a string representation. For + /// success, returns an empty string. For failure + /// statuses, a string as appropriate for the source of + /// the error code (libcurl easy, libcurl multi, or + /// LLCore itself). + std::string toString() const; + + /// Returns true if the status value represents an + /// HTTP response status (100 - 999). + bool isHttpStatus() const + { + return mType >= type_enum_t(100) && mType <= type_enum_t(999); + } + +}; // end struct HttpStatus + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_COMMON_H_ diff --git a/indra/llcorehttp/httphandler.h b/indra/llcorehttp/httphandler.h new file mode 100644 index 0000000000..9171e4e7b9 --- /dev/null +++ b/indra/llcorehttp/httphandler.h @@ -0,0 +1,88 @@ +/** + * @file httphandler.h + * @brief Public-facing declarations for the HttpHandler class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_HANDLER_H_ +#define _LLCORE_HTTP_HANDLER_H_ + + +#include "httpcommon.h" + + +namespace LLCore +{ + +class HttpResponse; + + +/// HttpHandler defines an interface used by the library to +/// notify library callers of significant events, currently +/// request completion. Callers must derive or mixin this class +/// then provide an implementation of the @see onCompleted +/// method to receive such notifications. An instance may +/// be shared by any number of requests and across instances +/// of HttpRequest running in the same thread. +/// +/// Threading: HttpHandler itself is pure interface and is +/// tread-compatible. Most derivations, however, will have +/// different constraints. +/// +/// Allocation: Not refcounted, may be stack allocated though +/// that is rarely a good idea. Queued requests and replies keep +/// a naked pointer to the handler and this can result in a +/// dangling pointer if lifetimes aren't managed correctly. + +class HttpHandler +{ +public: + virtual ~HttpHandler() + {} + + /// Method invoked during calls to @see update(). Each invocation + /// represents the completion of some requested operation. Caller + /// can identify the request from the handle and interrogate the + /// response argument for success/failure, data and other information. + /// + /// @param handle Identifier of the request generating + /// the notification. + /// @param response Supplies detailed information about + /// the request including status codes + /// (both programming and HTTP), HTTP body + /// data and encodings, headers, etc. + /// The response object is refcounted and + /// the called code may retain the object + /// by invoking @see addRef() on it. The + /// library itself drops all references to + /// to object on return and never touches + /// it again. + /// + virtual void onCompleted(HttpHandle handle, HttpResponse * response) = 0; + +}; // end class HttpHandler + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_HANDLER_H_ diff --git a/indra/llcorehttp/httpheaders.cpp b/indra/llcorehttp/httpheaders.cpp new file mode 100644 index 0000000000..2832696271 --- /dev/null +++ b/indra/llcorehttp/httpheaders.cpp @@ -0,0 +1,44 @@ +/** + * @file httpheaders.cpp + * @brief Implementation of the HTTPHeaders class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpheaders.h" + + +namespace LLCore +{ + + +HttpHeaders::HttpHeaders() + : RefCounted(true) +{} + + +HttpHeaders::~HttpHeaders() +{} + + +} // end namespace LLCore + diff --git a/indra/llcorehttp/httpheaders.h b/indra/llcorehttp/httpheaders.h new file mode 100644 index 0000000000..3449daa3a1 --- /dev/null +++ b/indra/llcorehttp/httpheaders.h @@ -0,0 +1,87 @@ +/** + * @file httpheaders.h + * @brief Public-facing declarations for the HttpHeaders class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_HEADERS_H_ +#define _LLCORE_HTTP_HEADERS_H_ + + +#include <string> + +#include "_refcounted.h" + + +namespace LLCore +{ + +/// +/// Maintains an ordered list of name/value pairs representing +/// HTTP header lines. This is used both to provide additional +/// headers when making HTTP requests and in responses when the +/// caller has asked that headers be returned (not the default +/// option). +/// +/// @note +/// This is a minimally-functional placeholder at the moment +/// to fill out the class hierarchy. The final class will be +/// something else, probably more pair-oriented. It's also +/// an area where shared values are desirable so refcounting is +/// already specced and a copy-on-write scheme imagined. +/// Expect changes here. +/// +/// Threading: Not intrinsically thread-safe. It *is* expected +/// that callers will build these objects and then share them +/// via reference counting with the worker thread. The implication +/// is that once an HttpHeader instance is handed to a request, +/// the object must be treated as read-only. +/// +/// Allocation: Refcounted, heap only. Caller of the +/// constructor is given a refcount. +/// + +class HttpHeaders : public LLCoreInt::RefCounted +{ +public: + /// @post In addition to the instance, caller has a refcount + /// to the instance. A call to @see release() will destroy + /// the instance. + HttpHeaders(); + +protected: + virtual ~HttpHeaders(); // Use release() + + HttpHeaders(const HttpHeaders &); // Not defined + void operator=(const HttpHeaders &); // Not defined + +public: + typedef std::vector<std::string> container_t; + container_t mHeaders; + +}; // end class HttpHeaders + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_HEADERS_H_ diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp new file mode 100644 index 0000000000..1699d19f8d --- /dev/null +++ b/indra/llcorehttp/httpoptions.cpp @@ -0,0 +1,73 @@ +/** + * @file httpoptions.cpp + * @brief Implementation of the HTTPOptions class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpoptions.h" + +#include "_httpinternal.h" + + +namespace LLCore +{ + + +HttpOptions::HttpOptions() + : RefCounted(true), + mWantHeaders(false), + mTracing(HTTP_TRACE_OFF), + mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), + mRetries(HTTP_RETRY_COUNT_DEFAULT) +{} + + +HttpOptions::~HttpOptions() +{} + + +void HttpOptions::setWantHeaders(bool wanted) +{ + mWantHeaders = wanted; +} + + +void HttpOptions::setTrace(long level) +{ + mTracing = int(level); +} + + +void HttpOptions::setTimeout(unsigned int timeout) +{ + mTimeout = timeout; +} + + +void HttpOptions::setRetries(unsigned int retries) +{ + mRetries = retries; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h new file mode 100644 index 0000000000..97e46a8cd3 --- /dev/null +++ b/indra/llcorehttp/httpoptions.h @@ -0,0 +1,106 @@ +/** + * @file httpoptions.h + * @brief Public-facing declarations for the HTTPOptions class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPTIONS_H_ +#define _LLCORE_HTTP_OPTIONS_H_ + + +#include "httpcommon.h" + +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// Really a struct in spirit, it provides options that +/// modify HTTP requests. +/// +/// Sharing instances across requests. It's intended that +/// these be shared across requests: caller can create one +/// of these, set it up as needed and then reference it +/// repeatedly in HTTP operations. But see the Threading +/// note about references. +/// +/// Threading: While this class does nothing to ensure thread +/// safety, it *is* intended to be shared between the application +/// thread and the worker thread. This means that once an instance +/// is delivered to the library in request operations, the +/// option data must not be written until all such requests +/// complete and relinquish their references. +/// +/// Allocation: Refcounted, heap only. Caller of the constructor +/// is given a refcount. +/// +class HttpOptions : public LLCoreInt::RefCounted +{ +public: + HttpOptions(); + +protected: + virtual ~HttpOptions(); // Use release() + + HttpOptions(const HttpOptions &); // Not defined + void operator=(const HttpOptions &); // Not defined + +public: + void setWantHeaders(bool wanted); + bool getWantHeaders() const + { + return mWantHeaders; + } + + void setTrace(int long); + int getTrace() const + { + return mTracing; + } + + void setTimeout(unsigned int timeout); + unsigned int getTimeout() const + { + return mTimeout; + } + + void setRetries(unsigned int retries); + unsigned int getRetries() const + { + return mRetries; + } + +protected: + bool mWantHeaders; + int mTracing; + unsigned int mTimeout; + unsigned int mRetries; + +}; // end class HttpOptions + + +} // end namespace HttpOptions + +#endif // _LLCORE_HTTP_OPTIONS_H_ diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp new file mode 100644 index 0000000000..9b739a8825 --- /dev/null +++ b/indra/llcorehttp/httprequest.cpp @@ -0,0 +1,504 @@ +/** + * @file httprequest.cpp + * @brief Implementation of the HTTPRequest class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httprequest.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httppolicy.h" +#include "_httpoperation.h" +#include "_httpoprequest.h" +#include "_httpopsetpriority.h" +#include "_httpopcancel.h" +#include "_httpopsetget.h" + +#include "lltimer.h" + + +namespace +{ + +bool has_inited(false); + +} + +namespace LLCore +{ + +// ==================================== +// HttpRequest Implementation +// ==================================== + + +HttpRequest::policy_t HttpRequest::sNextPolicyID(1); + + +HttpRequest::HttpRequest() + : //HttpHandler(), + mReplyQueue(NULL), + mRequestQueue(NULL) +{ + mRequestQueue = HttpRequestQueue::instanceOf(); + mRequestQueue->addRef(); + + mReplyQueue = new HttpReplyQueue(); +} + + +HttpRequest::~HttpRequest() +{ + if (mRequestQueue) + { + mRequestQueue->release(); + mRequestQueue = NULL; + } + + if (mReplyQueue) + { + mReplyQueue->release(); + mReplyQueue = NULL; + } +} + + +// ==================================== +// Policy Methods +// ==================================== + + +HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + return HttpService::instanceOf()->getGlobalOptions().set(opt, value); +} + + +HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + return HttpService::instanceOf()->getGlobalOptions().set(opt, value); +} + + +HttpRequest::policy_t HttpRequest::createPolicyClass() +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return 0; + } + return HttpService::instanceOf()->createPolicyClass(); +} + + +HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id, + EClassPolicy opt, + long value) +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value); +} + + +// ==================================== +// Request Methods +// ==================================== + + +HttpStatus HttpRequest::getStatus() const +{ + return mLastReqStatus; +} + + +HttpHandle HttpRequest::requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupGet(policy_id, priority, url, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id, + priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupGetByteRange(policy_id, priority, url, offset, len, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestPost(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPost(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestPut(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPut(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpNull * op = new HttpOpNull(); + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpStatus HttpRequest::update(long usecs) +{ + HttpOperation * op(NULL); + + if (usecs) + { + const HttpTime limit(totalTime() + HttpTime(usecs)); + while (limit >= totalTime() && (op = mReplyQueue->fetchOp())) + { + // Process operation + op->visitNotifier(this); + + // We're done with the operation + op->release(); + } + } + else + { + // Same as above, just no time limit + HttpReplyQueue::OpContainer replies; + mReplyQueue->fetchAll(replies); + if (! replies.empty()) + { + for (HttpReplyQueue::OpContainer::iterator iter(replies.begin()); + replies.end() != iter; + ++iter) + { + // Swap op pointer for NULL; + op = *iter; *iter = NULL; + + // Process operation + op->visitNotifier(this); + + // We're done with the operation + op->release(); + } + } + } + + return HttpStatus(); +} + + + + +// ==================================== +// Request Management Methods +// ==================================== + +HttpHandle HttpRequest::requestCancel(HttpHandle request, HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpCancel * op = new HttpOpCancel(request); + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return ret_handle; + } + + mLastReqStatus = status; + ret_handle = static_cast<HttpHandle>(op); + + return ret_handle; +} + + +HttpHandle HttpRequest::requestSetPriority(HttpHandle request, priority_t priority, + HttpHandler * handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetPriority * op = new HttpOpSetPriority(request, priority); + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return ret_handle; + } + + mLastReqStatus = status; + ret_handle = static_cast<HttpHandle>(op); + + return ret_handle; +} + + +// ==================================== +// Utility Methods +// ==================================== + +HttpStatus HttpRequest::createService() +{ + HttpStatus status; + + if (! has_inited) + { + HttpRequestQueue::init(); + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + HttpService::init(rq); + has_inited = true; + } + + return status; +} + + +HttpStatus HttpRequest::destroyService() +{ + HttpStatus status; + + if (has_inited) + { + HttpService::term(); + HttpRequestQueue::term(); + has_inited = false; + } + + return status; +} + + +HttpStatus HttpRequest::startThread() +{ + HttpStatus status; + + HttpService::instanceOf()->startThread(); + + return status; +} + + +HttpHandle HttpRequest::requestStopThread(HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpStop * op = new HttpOpStop(); + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestSpin(int mode) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSpin * op = new HttpOpSpin(mode); + op->setReplyPath(mReplyQueue, NULL); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + +// ==================================== +// Dynamic Policy Methods +// ==================================== + +HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + op->setupSet(GP_HTTP_PROXY, proxy); + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +} // end namespace LLCore + diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h new file mode 100644 index 0000000000..ab2f302d34 --- /dev/null +++ b/indra/llcorehttp/httprequest.h @@ -0,0 +1,535 @@ +/** + * @file httprequest.h + * @brief Public-facing declarations for HttpRequest class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_REQUEST_H_ +#define _LLCORE_HTTP_REQUEST_H_ + + +#include "httpcommon.h" +#include "httphandler.h" + + +namespace LLCore +{ + +class HttpRequestQueue; +class HttpReplyQueue; +class HttpService; +class HttpOptions; +class HttpHeaders; +class HttpOperation; +class BufferArray; + +/// HttpRequest supplies the entry into the HTTP transport +/// services in the LLCore libraries. Services provided include: +/// +/// - Some, but not all, global initialization of libcurl. +/// - Starting asynchronous, threaded HTTP requests. +/// - Definition of policy classes affect request handling. +/// - Utilities to control request options and headers +/// +/// Requests +/// +/// The class supports the current HTTP request operations: +/// +/// - requestGetByteRange: GET with Range header for a single range of bytes +/// +/// Policy Classes +/// +/// <TBD> +/// +/// Usage +/// +/// <TBD> +/// +/// Threading: An instance may only be used by one application/ +/// consumer thread. But a thread may have as many instances of +/// this as it likes. +/// +/// Allocation: Not refcounted, may be stack allocated though that +/// hasn't been tested. Queued requests can still run and any +/// queued replies will keep refcounts to the reply queue leading +/// to memory leaks. +/// +/// @pre Before using this class (static or instances), some global +/// initialization is required. See @see httpcommon.h for more information. +/// +/// @nosubgrouping +/// + +class HttpRequest +{ +public: + HttpRequest(); + virtual ~HttpRequest(); + +private: + HttpRequest(const HttpRequest &); // Disallowed + void operator=(const HttpRequest &); // Disallowed + +public: + typedef unsigned int policy_t; + typedef unsigned int priority_t; + +public: + /// @name PolicyMethods + /// @{ + + /// Represents a default, catch-all policy class that guarantees + /// eventual service for any HTTP request. + static const int DEFAULT_POLICY_ID = 0; + + enum EGlobalPolicy + { + /// Maximum number of connections the library will use to + /// perform operations. This is somewhat soft as the underlying + /// transport will cache some connections (up to 5). + + /// A long value setting the maximum number of connections + /// allowed over all policy classes. Note that this will be + /// a somewhat soft value. There may be an additional five + /// connections per policy class depending upon runtime + /// behavior. + GP_CONNECTION_LIMIT, + + /// String containing a system-appropriate directory name + /// where SSL certs are stored. + GP_CA_PATH, + + /// String giving a full path to a file containing SSL certs. + GP_CA_FILE, + + /// String of host/port to use as simple HTTP proxy. This is + /// going to change in the future into something more elaborate + /// that may support richer schemes. + GP_HTTP_PROXY, + + /// Long value that if non-zero enables the use of the + /// traditional LLProxy code for http/socks5 support. If + /// enabled, has priority over GP_HTTP_PROXY. + GP_LLPROXY, + + /// Long value setting the logging trace level for the + /// library. Possible values are: + /// 0 - No tracing (default) + /// 1 - Basic tracing of request start, stop and major events. + /// 2 - Connection, header and payload size information from + /// HTTP transactions. + /// 3 - Partial logging of payload itself. + /// + /// These values are also used in the trace modes for + /// individual requests in HttpOptions. Also be aware that + /// tracing tends to impact performance of the viewer. + GP_TRACE + }; + + /// Set a parameter on a global policy option. Calls + /// made after the start of the servicing thread are + /// not honored and return an error status. + /// + /// @param opt Enum of option to be set. + /// @param value Desired value of option. + /// @return Standard status code. + static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); + static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); + + /// Create a new policy class into which requests can be made. + /// + /// @return If positive, the policy_id used to reference + /// the class in other methods. If 0, an error + /// occurred and @see getStatus() may provide more + /// detail on the reason. + static policy_t createPolicyClass(); + + enum EClassPolicy + { + /// Limits the number of connections used for the class. + CP_CONNECTION_LIMIT, + + /// Limits the number of connections used for a single + /// literal address/port pair within the class. + CP_PER_HOST_CONNECTION_LIMIT, + + /// Suitable requests are allowed to pipeline on their + /// connections when they ask for it. + CP_ENABLE_PIPELINING + }; + + /// Set a parameter on a class-based policy option. Calls + /// made after the start of the servicing thread are + /// not honored and return an error status. + /// + /// @param policy_id ID of class as returned by @see createPolicyClass(). + /// @param opt Enum of option to be set. + /// @param value Desired value of option. + /// @return Standard status code. + static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); + + /// @} + + /// @name RequestMethods + /// + /// @{ + + /// Some calls expect to succeed as the normal part of operation and so + /// return a useful value rather than a status. When they do fail, the + /// status is saved and can be fetched with this method. + /// + /// @return Status of the failing method invocation. If the + /// preceding call succeeded or other HttpStatus + /// returning calls immediately preceded this method, + /// the returned value may not be reliable. + /// + HttpStatus getStatus() const; + + /// Queue a full HTTP GET request to be issued for entire entity. + /// The request is queued and serviced by the working thread and + /// notification of completion delivered to the optional HttpHandler + /// argument during @see update() calls. + /// + /// With a valid handle returned, it can be used to reference the + /// request in other requests (like cancellation) and will be an + /// argument when any HttpHandler object is invoked. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-alive: 300 + /// - Host: <stuff> + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-control: + /// - Range: + /// - Transfer-Encoding: + /// - Referer: + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base (U32-type scheme). + /// @param url URL with any encoded query parameters to + /// be accessed. + /// @param options Optional instance of an HttpOptions object + /// to provide additional controls over the request + /// function for this request only. Any such + /// object then becomes shared-read across threads + /// and no code should modify the HttpOptions + /// instance. + /// @param headers Optional instance of an HttpHeaders object + /// to provide additional and/or overridden + /// headers for the request. As with options, + /// the instance becomes shared-read across threads + /// and no code should modify the HttpHeaders + /// instance. + /// @param handler Optional pointer to an HttpHandler instance + /// whose onCompleted() method will be invoked + /// during calls to update(). This is a non- + /// reference-counted object which would be a + /// problem for shutdown and other edge cases but + /// the pointer is only dereferenced during + /// calls to update(). + /// + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a full HTTP GET request to be issued with a 'Range' header. + /// The request is queued and serviced by the working thread and + /// notification of completion delivered to the optional HttpHandler + /// argument during @see update() calls. + /// + /// With a valid handle returned, it can be used to reference the + /// request in other requests (like cancellation) and will be an + /// argument when any HttpHandler object is invoked. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-alive: 300 + /// - Host: <stuff> + /// - Range: <stuff> (will be omitted if offset == 0 and len == 0) + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-control: + /// - Transfer-Encoding: + /// - Referer: + /// + /// @param policy_id @see requestGet() + /// @param priority " + /// @param url " + /// @param offset Offset of first byte into resource to be returned. + /// @param len Count of bytes to be returned + /// @param options @see requestGet() + /// @param headers " + /// @param handler " + /// @return " + /// + HttpHandle requestGetByteRange(policy_t policy_id, + priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a full HTTP POST. Query arguments and body may + /// be provided. Caller is responsible for escaping and + /// encoding and communicating the content types. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-Alive: 300 + /// - Host: <stuff> + /// - Content-Length: <digits> + /// - Content-Type: application/x-www-form-urlencoded + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-Control: + /// - Transfer-Encoding: ... chunked ... + /// - Referer: + /// - Content-Encoding: + /// - Expect: + /// + /// @param policy_id @see requestGet() + /// @param priority " + /// @param url " + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options @see requestGet()K(optional) + /// @param headers " + /// @param handler " + /// @return " + /// + HttpHandle requestPost(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a full HTTP PUT. Query arguments and body may + /// be provided. Caller is responsible for escaping and + /// encoding and communicating the content types. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-Alive: 300 + /// - Host: <stuff> + /// - Content-Length: <digits> + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-Control: + /// - Transfer-Encoding: ... chunked ... + /// - Referer: + /// - Content-Encoding: + /// - Expect: + /// - Content-Type: + /// + /// @param policy_id @see requestGet() + /// @param priority " + /// @param url " + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options @see requestGet()K(optional) + /// @param headers " + /// @param handler " + /// @return " + /// + HttpHandle requestPut(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a NoOp request. + /// The request is queued and serviced by the working thread which + /// immediately processes it and returns the request to the reply + /// queue. + /// + /// @param handler @see requestGet() + /// @return " + /// + HttpHandle requestNoOp(HttpHandler * handler); + + /// While all the heavy work is done by the worker thread, notifications + /// must be performed in the context of the application thread. These + /// are done synchronously during calls to this method which gives the + /// library control so notification can be performed. Application handlers + /// are expected to return 'quickly' and do any significant processing + /// outside of the notification callback to onCompleted(). + /// + /// @param usecs Maximum number of wallclock microseconds to + /// spend in the call. As hinted at above, this + /// is partly a function of application code so it's + /// a soft limit. A '0' value will run without + /// time limit until everything queued has been + /// delivered. + /// + /// @return Standard status code. + HttpStatus update(long usecs); + + /// @} + + /// @name RequestMgmtMethods + /// + /// @{ + + HttpHandle requestCancel(HttpHandle request, HttpHandler *); + + /// Request that a previously-issued request be reprioritized. + /// The status of whether the change itself succeeded arrives + /// via notification. + /// + /// @param request Handle of previously-issued request to + /// be changed. + /// @param priority New priority value. + /// @param handler @see requestGet() + /// @return " + /// + HttpHandle requestSetPriority(HttpHandle request, priority_t priority, HttpHandler * handler); + + /// @} + + /// @name UtilityMethods + /// + /// @{ + + /// Initialization method that needs to be called before queueing any + /// requests. Doesn't start the worker thread and may be called befoer + /// or after policy setup. + static HttpStatus createService(); + + /// Mostly clean shutdown of services prior to exit. Caller is expected + /// to have stopped a running worker thread before calling this. + static HttpStatus destroyService(); + + /// Called once after @see createService() to start the worker thread. + /// Stopping the thread is achieved by requesting it via @see requestStopThread(). + /// May be called before or after requests are issued. + static HttpStatus startThread(); + + /// Queues a request to the worker thread to have it stop processing + /// and exit (without exiting the program). When the operation is + /// picked up by the worker thread, it immediately processes it and + /// begins detaching from refcounted resources like request and + /// reply queues and then returns to the host OS. It *does* queue a + /// reply to give the calling application thread a notification that + /// the operation has been performed. + /// + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// As the request cannot be cancelled, the handle + /// is generally not useful. + /// + HttpHandle requestStopThread(HttpHandler * handler); + + /// Queue a Spin request. + /// DEBUG/TESTING ONLY. This puts the worker into a CPU spin for + /// test purposes. + /// + /// @param mode 0 for hard spin, 1 for soft spin + /// @return Standard handle return cases. + /// + HttpHandle requestSpin(int mode); + + /// @} + + /// @name DynamicPolicyMethods + /// + /// @{ + + /// Request that a running transport pick up a new proxy setting. + /// An empty string will indicate no proxy is to be used. + HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); + + /// @} + +protected: + void generateNotification(HttpOperation * op); + +private: + /// @name InstanceData + /// + /// @{ + HttpStatus mLastReqStatus; + HttpReplyQueue * mReplyQueue; + HttpRequestQueue * mRequestQueue; + + /// @} + + // ==================================== + /// @name GlobalState + /// + /// @{ + /// + /// Must be established before any threading is allowed to + /// start. + /// + static policy_t sNextPolicyID; + + /// @} + // End Global State + // ==================================== + +}; // end class HttpRequest + + +} // end namespace LLCore + + + +#endif // _LLCORE_HTTP_REQUEST_H_ diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp new file mode 100644 index 0000000000..a552e48a1b --- /dev/null +++ b/indra/llcorehttp/httpresponse.cpp @@ -0,0 +1,91 @@ +/** + * @file httpresponse.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpresponse.h" +#include "bufferarray.h" +#include "httpheaders.h" + + +namespace LLCore +{ + + +HttpResponse::HttpResponse() + : LLCoreInt::RefCounted(true), + mReplyOffset(0U), + mReplyLength(0U), + mReplyFullLength(0U), + mBufferArray(NULL), + mHeaders(NULL) +{} + + +HttpResponse::~HttpResponse() +{ + setBody(NULL); + setHeaders(NULL); +} + + +void HttpResponse::setBody(BufferArray * ba) +{ + if (mBufferArray == ba) + return; + + if (mBufferArray) + { + mBufferArray->release(); + } + + if (ba) + { + ba->addRef(); + } + + mBufferArray = ba; +} + + +void HttpResponse::setHeaders(HttpHeaders * headers) +{ + if (mHeaders == headers) + return; + + if (mHeaders) + { + mHeaders->release(); + } + + if (headers) + { + headers->addRef(); + } + + mHeaders = headers; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h new file mode 100644 index 0000000000..4a481db6ac --- /dev/null +++ b/indra/llcorehttp/httpresponse.h @@ -0,0 +1,161 @@ +/** + * @file httpresponse.h + * @brief Public-facing declarations for the HttpResponse class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_RESPONSE_H_ +#define _LLCORE_HTTP_RESPONSE_H_ + + +#include <string> + +#include "httpcommon.h" + +#include "_refcounted.h" + + +namespace LLCore +{ + +class BufferArray; +class HttpHeaders; + +/// HttpResponse is instantiated by the library and handed to +/// the caller during callbacks to the handler. It supplies +/// all the status, header and HTTP body data the caller is +/// interested in. Methods provide simple getters to return +/// individual pieces of the response. +/// +/// Typical usage will have the caller interrogate the object +/// and return from the handler callback. Instances are refcounted +/// and callers can bump the count and retain the object as needed. +/// +/// Threading: Not intrinsically thread-safe. +/// +/// Allocation: Refcounted, heap only. Caller of the constructor +/// is given a refcount. +/// +class HttpResponse : public LLCoreInt::RefCounted +{ +public: + HttpResponse(); + +protected: + virtual ~HttpResponse(); // Use release() + + HttpResponse(const HttpResponse &); // Not defined + void operator=(const HttpResponse &); // Not defined + +public: + /// Returns the final status of the requested operation. + /// + HttpStatus getStatus() const + { + return mStatus; + } + + void setStatus(const HttpStatus & status) + { + mStatus = status; + } + + /// Simple getter for the response body returned as a scatter/gather + /// buffer. If the operation doesn't produce data (such as the Null + /// or StopThread operations), this may be NULL. + /// + /// Caller can hold onto the response by incrementing the reference + /// count of the returned object. + BufferArray * getBody() const + { + return mBufferArray; + } + + /// Set the response data in the instance. Will drop the reference + /// count to any existing data and increment the count of that passed + /// in. It is legal to set the data to NULL. + void setBody(BufferArray * ba); + + /// And a getter for the headers. And as with @see getResponse(), + /// if headers aren't available because the operation doesn't produce + /// any or delivery of headers wasn't requested in the options, this + /// will be NULL. + /// + /// Caller can hold onto the headers by incrementing the reference + /// count of the returned object. + HttpHeaders * getHeaders() const + { + return mHeaders; + } + + /// Behaves like @see setResponse() but for header data. + void setHeaders(HttpHeaders * headers); + + /// If a 'Range:' header was used, these methods are involved + /// in setting and returning data about the actual response. + /// If both @offset and @length are returned as 0, we probably + /// didn't get a Content-Range header in the response. This + /// occurs with various Capabilities-based services and the + /// caller is going to have to make assumptions on receipt of + /// a 206 status. The @full value may also be zero in cases of + /// parsing problems or a wild-carded length response. + void getRange(unsigned int * offset, unsigned int * length, unsigned int * full) const + { + *offset = mReplyOffset; + *length = mReplyLength; + *full = mReplyFullLength; + } + + void setRange(unsigned int offset, unsigned int length, unsigned int full_length) + { + mReplyOffset = offset; + mReplyLength = length; + mReplyFullLength = full_length; + } + + /// + const std::string & getContentType() const + { + return mContentType; + } + + void setContentType(const std::string & con_type) + { + mContentType = con_type; + } + +protected: + // Response data here + HttpStatus mStatus; + unsigned int mReplyOffset; + unsigned int mReplyLength; + unsigned int mReplyFullLength; + BufferArray * mBufferArray; + HttpHeaders * mHeaders; + std::string mContentType; +}; + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_RESPONSE_H_ diff --git a/indra/llcorehttp/tests/llcorehttp_test.cpp b/indra/llcorehttp/tests/llcorehttp_test.cpp new file mode 100644 index 0000000000..e863ddd13f --- /dev/null +++ b/indra/llcorehttp/tests/llcorehttp_test.cpp @@ -0,0 +1,175 @@ +/** + * @file llcorehttp_test + * @brief Main test runner + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "llcorehttp_test.h" + +#include <iostream> +#include <sstream> + +// These are not the right way in viewer for some reason: +// #include <tut/tut.hpp> +// #include <tut/tut_reporter.hpp> +// This works: +#include "../test/lltut.h" + +// Pull in each of the test sets +#include "test_bufferarray.hpp" +#include "test_bufferstream.hpp" +#include "test_httpstatus.hpp" +#include "test_refcounted.hpp" +#include "test_httpoperation.hpp" +#include "test_httprequest.hpp" +#include "test_httpheaders.hpp" +#include "test_httprequestqueue.hpp" + +#include "llproxy.h" + +unsigned long ssl_thread_id_callback(void); +void ssl_locking_callback(int mode, int type, const char * file, int line); + +#if 0 // lltut provides main and runner + +namespace tut +{ + test_runner_singleton runner; +} + +int main() +{ + curl_global_init(CURL_GLOBAL_ALL); + + // *FIXME: Need threaded/SSL curl setup here. + + tut::reporter reporter; + + tut::runner.get().set_callback(&reporter); + tut::runner.get().run_tests(); + return !reporter.all_ok(); + + curl_global_cleanup(); +} + +#endif // 0 + +int ssl_mutex_count(0); +LLCoreInt::HttpMutex ** ssl_mutex_list = NULL; + +void init_curl() +{ + curl_global_init(CURL_GLOBAL_ALL); + + ssl_mutex_count = CRYPTO_num_locks(); + if (ssl_mutex_count > 0) + { + ssl_mutex_list = new LLCoreInt::HttpMutex * [ssl_mutex_count]; + + for (int i(0); i < ssl_mutex_count; ++i) + { + ssl_mutex_list[i] = new LLCoreInt::HttpMutex; + } + + CRYPTO_set_locking_callback(ssl_locking_callback); + CRYPTO_set_id_callback(ssl_thread_id_callback); + } + + LLProxy::getInstance(); +} + + +void term_curl() +{ + LLProxy::cleanupClass(); + + CRYPTO_set_locking_callback(NULL); + for (int i(0); i < ssl_mutex_count; ++i) + { + delete ssl_mutex_list[i]; + } + delete [] ssl_mutex_list; +} + + +unsigned long ssl_thread_id_callback(void) +{ +#if defined(WIN32) + return (unsigned long) GetCurrentThread(); +#else + return (unsigned long) pthread_self(); +#endif +} + + +void ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) +{ + if (type >= 0 && type < ssl_mutex_count) + { + if (mode & CRYPTO_LOCK) + { + ssl_mutex_list[type]->lock(); + } + else + { + ssl_mutex_list[type]->unlock(); + } + } +} + + +std::string get_base_url() +{ + const char * env(getenv("LL_TEST_PORT")); + + if (! env) + { + std::cerr << "LL_TEST_PORT environment variable missing." << std::endl; + std::cerr << "Test expects to run in test_llcorehttp_peer.py script." << std::endl; + tut::ensure("LL_TEST_PORT set in environment", NULL != env); + } + + int port(atoi(env)); + std::ostringstream out; + out << "http://localhost:" << port << "/"; + return out.str(); +} + + +void stop_thread(LLCore::HttpRequest * req) +{ + if (req) + { + req->requestStopThread(NULL); + + int count = 0; + int limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + req->update(1000); + usleep(100000); + } + } +} + + diff --git a/indra/llcorehttp/tests/llcorehttp_test.h b/indra/llcorehttp/tests/llcorehttp_test.h new file mode 100644 index 0000000000..a9567435ce --- /dev/null +++ b/indra/llcorehttp/tests/llcorehttp_test.h @@ -0,0 +1,64 @@ +/** + * @file llcorehttp_test.h + * @brief Main test runner + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + + +#ifndef _LLCOREHTTP_TEST_H_ +#define _LLCOREHTTP_TEST_H_ + +#include "linden_common.h" // Modifies curl interfaces + +#include <curl/curl.h> +#include <openssl/crypto.h> +#include <string> + +#include "httprequest.h" + +// Initialization and cleanup for libcurl. Mainly provides +// a mutex callback for SSL and a thread ID hash for libcurl. +// If you don't use these (or equivalent) and do use libcurl, +// you'll see stalls and other anomalies when performing curl +// operations. +extern void init_curl(); +extern void term_curl(); +extern std::string get_base_url(); +extern void stop_thread(LLCore::HttpRequest * req); + +class ScopedCurlInit +{ +public: + ScopedCurlInit() + { + init_curl(); + } + + ~ScopedCurlInit() + { + term_curl(); + } +}; + + +#endif // _LLCOREHTTP_TEST_H_ diff --git a/indra/llcorehttp/tests/test_allocator.cpp b/indra/llcorehttp/tests/test_allocator.cpp new file mode 100644 index 0000000000..ea12dc58eb --- /dev/null +++ b/indra/llcorehttp/tests/test_allocator.cpp @@ -0,0 +1,184 @@ +/** + * @file test_allocator.cpp + * @brief quick and dirty allocator for tracking memory allocations + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "test_allocator.h" + +#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 +#include <libkern/OSAtomic.h> +#elif defined(_MSC_VER) +#include <Windows.h> +#elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ ) > 40100 +// atomic extensions are built into GCC on posix platforms +#endif + +#include <cassert> +#include <cstdlib> +#include <cstring> +#include <vector> +#include <iostream> +#include <new> + +#include <boost/thread.hpp> + + +#if defined(WIN32) +#define THROW_BAD_ALLOC() _THROW1(std::bad_alloc) +#define THROW_NOTHING() _THROW0() +#else +#define THROW_BAD_ALLOC() throw(std::bad_alloc) +#define THROW_NOTHING() throw() +#endif + + +struct BlockHeader +{ + struct Block * next; + std::size_t size; + bool in_use; +}; + +struct Block +{ + BlockHeader hdr; + unsigned char data[1]; +}; + +#define TRACE_MSG(val) std::cout << __FUNCTION__ << "(" << val << ") [" << __FILE__ << ":" << __LINE__ << "]" << std::endl; + +static unsigned char MemBuf[ 4096 * 1024 ]; +Block * pNext = static_cast<Block *>(static_cast<void *>(MemBuf)); +volatile std::size_t MemTotal = 0; + +// cross-platform compare and swap operation +static bool CAS(void * volatile * ptr, void * expected, void * new_value) +{ +#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwapPtr( expected, new_value, ptr ); +#elif defined(_MSC_VER) + return expected == InterlockedCompareExchangePointer( ptr, new_value, expected ); +#elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ ) > 40100 + return __sync_bool_compare_and_swap( ptr, expected, new_value ); +#endif +} + +static void * GetMem(std::size_t size) +{ + // TRACE_MSG(size); + volatile Block * pBlock = NULL; + volatile Block * pNewNext = NULL; + + // do a lock-free update of the global next pointer + do + { + pBlock = pNext; + pNewNext = (volatile Block *)(pBlock->data + size); + + } while(! CAS((void * volatile *) &pNext, (void *) pBlock, (void *) pNewNext)); + + // if we get here, we safely carved out a block of memory in the + // memory pool... + + // initialize our block + pBlock->hdr.next = (Block *)(pBlock->data + size); + pBlock->hdr.size = size; + pBlock->hdr.in_use = true; + memset((void *) pBlock->data, 0, pBlock->hdr.size); + + // do a lock-free update of the global memory total + volatile size_t total = 0; + volatile size_t new_total = 0; + do + { + total = MemTotal; + new_total = total + size; + + } while (! CAS((void * volatile *) &MemTotal, (void *) total, (void *) new_total)); + + return (void *) pBlock->data; +} + + +static void FreeMem(void * p) +{ + // get the pointer to the block record + Block * pBlock = (Block *)((unsigned char *) p - sizeof(BlockHeader)); + + // TRACE_MSG(pBlock->hdr.size); + bool * cur_in_use = &(pBlock->hdr.in_use); + volatile bool in_use = false; + bool new_in_use = false; + do + { + in_use = pBlock->hdr.in_use; + } while (! CAS((void * volatile *) cur_in_use, (void *) in_use, (void *) new_in_use)); + + // do a lock-free update of the global memory total + volatile size_t total = 0; + volatile size_t new_total = 0; + do + { + total = MemTotal; + new_total = total - pBlock->hdr.size; + } while (! CAS((void * volatile *)&MemTotal, (void *) total, (void *) new_total)); +} + + +std::size_t GetMemTotal() +{ + return MemTotal; +} + + +void * operator new(std::size_t size) THROW_BAD_ALLOC() +{ + return GetMem( size ); +} + + +void * operator new[](std::size_t size) THROW_BAD_ALLOC() +{ + return GetMem( size ); +} + + +void operator delete(void * p) THROW_NOTHING() +{ + if (p) + { + FreeMem( p ); + } +} + + +void operator delete[](void * p) THROW_NOTHING() +{ + if (p) + { + FreeMem( p ); + } +} + + diff --git a/indra/llcorehttp/tests/test_allocator.h b/indra/llcorehttp/tests/test_allocator.h new file mode 100644 index 0000000000..3572bbc5c5 --- /dev/null +++ b/indra/llcorehttp/tests/test_allocator.h @@ -0,0 +1,47 @@ +/** + * @file test_allocator.h + * @brief quick and dirty allocator for tracking memory allocations + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef TEST_ALLOCATOR_H +#define TEST_ALLOCATOR_H + +#include <cstdlib> +#include <new> + +size_t GetMemTotal(); +#if defined(WIN32) +void * operator new(std::size_t size) _THROW1(std::bad_alloc); +void * operator new[](std::size_t size) _THROW1(std::bad_alloc); +void operator delete(void * p) _THROW0(); +void operator delete[](void * p) _THROW0(); +#else +void * operator new(std::size_t size) throw (std::bad_alloc); +void * operator new[](std::size_t size) throw (std::bad_alloc); +void operator delete(void * p) throw (); +void operator delete[](void * p) throw (); +#endif + +#endif // TEST_ALLOCATOR_H + diff --git a/indra/llcorehttp/tests/test_bufferarray.hpp b/indra/llcorehttp/tests/test_bufferarray.hpp new file mode 100644 index 0000000000..8a2a64d970 --- /dev/null +++ b/indra/llcorehttp/tests/test_bufferarray.hpp @@ -0,0 +1,432 @@ +/** + * @file test_bufferarray.hpp + * @brief unit tests for the LLCore::BufferArray class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_BUFFER_ARRAY_H_ +#define TEST_LLCORE_BUFFER_ARRAY_H_ + +#include "bufferarray.h" + +#include <iostream> + +#include "test_allocator.h" + + +using namespace LLCore; + + + +namespace tut +{ + +struct BufferArrayTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<BufferArrayTestData> BufferArrayTestGroupType; +typedef BufferArrayTestGroupType::object BufferArrayTestObjectType; +BufferArrayTestGroupType BufferArrayTestGroup("BufferArray Tests"); + +template <> template <> +void BufferArrayTestObjectType::test<1>() +{ + set_test_name("BufferArray construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + ensure("One ref on construction of BufferArray", ba->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + ensure("Nothing in BA", 0 == ba->size()); + + // Try to read + char buffer[20]; + size_t read_len(ba->read(0, buffer, sizeof(buffer))); + ensure("Read returns empty", 0 == read_len); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<2>() +{ + set_test_name("BufferArray single write"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + char buffer[256]; + + size_t len = ba->write(0, str1, strlen(str1)); + ensure("Wrote length correct", strlen(str1) == len); + ensure("Recorded size correct", strlen(str1) == ba->size()); + + // read some data back + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(2, buffer, 2); + ensure("Read length correct", 2 == len); + ensure("Read content correct", 'c' == buffer[0] && 'd' == buffer[1]); + ensure("Read didn't overwrite", 'X' == buffer[2]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferArrayTestObjectType::test<3>() +{ + set_test_name("BufferArray multiple writes"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, strlen(str1)); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // read some data back + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(8, buffer, 4); + ensure("Read length correct", 4 == len); + ensure("Read content correct", 'i' == buffer[0] && 'j' == buffer[1]); + ensure("Read content correct", 'a' == buffer[2] && 'b' == buffer[3]); + ensure("Read didn't overwrite", 'X' == buffer[4]); + + // Read whole thing + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Read length correct", (2 * str1_len) == len); + ensure("Read content correct (3)", 0 == strncmp(buffer, str1, str1_len)); + ensure("Read content correct (4)", 0 == strncmp(&buffer[str1_len], str1, str1_len)); + ensure("Read didn't overwrite (5)", 'X' == buffer[2 * str1_len]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<4>() +{ + set_test_name("BufferArray overwriting"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJ"; + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, strlen(str1)); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // reposition and overwrite + len = ba->write(8, str2, 4); + ensure("Overwrite length correct", 4 == len); + + // Leave position and read verifying content (stale really from seek() days) + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(12, buffer, 4); + ensure("Read length correct", 4 == len); + ensure("Read content correct", 'c' == buffer[0] && 'd' == buffer[1]); + ensure("Read content correct.2", 'e' == buffer[2] && 'f' == buffer[3]); + ensure("Read didn't overwrite", 'X' == buffer[4]); + + // reposition and check + len = ba->read(6, buffer, 8); + ensure("Read length correct.2", 8 == len); + ensure("Read content correct.3", 'g' == buffer[0] && 'h' == buffer[1]); + ensure("Read content correct.4", 'A' == buffer[2] && 'B' == buffer[3]); + ensure("Read content correct.5", 'C' == buffer[4] && 'D' == buffer[5]); + ensure("Read content correct.6", 'c' == buffer[6] && 'd' == buffer[7]); + ensure("Read didn't overwrite.7", 'X' == buffer[8]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<5>() +{ + set_test_name("BufferArray multiple writes - sequential reads"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // read some data back + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(8, buffer, 4); + ensure("Read length correct", 4 == len); + ensure("Read content correct", 'i' == buffer[0] && 'j' == buffer[1]); + ensure("Read content correct.2", 'a' == buffer[2] && 'b' == buffer[3]); + ensure("Read didn't overwrite", 'X' == buffer[4]); + + // Read some more without repositioning + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(12, buffer, sizeof(buffer)); + ensure("Read length correct", (str1_len - 2) == len); + ensure("Read content correct.3", 0 == strncmp(buffer, str1+2, str1_len-2)); + ensure("Read didn't overwrite.2", 'X' == buffer[str1_len-1]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<6>() +{ + set_test_name("BufferArray overwrite spanning blocks and appending"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJKLMNOPQRST"; + size_t str2_len(strlen(str2)); + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, strlen(str1)); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // reposition and overwrite + len = ba->write(8, str2, str2_len); + ensure("Overwrite length correct", str2_len == len); + + // Leave position and read verifying content + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(8 + str2_len, buffer, 0); + ensure("Read length correct", 0 == len); + ensure("Read didn't overwrite", 'X' == buffer[0]); + + // reposition and check + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Read length correct.2", (str1_len + str2_len - 2) == len); + ensure("Read content correct", 0 == strncmp(buffer, str1, str1_len-2)); + ensure("Read content correct.2", 0 == strncmp(buffer+str1_len-2, str2, str2_len)); + ensure("Read didn't overwrite.2", 'X' == buffer[str1_len + str2_len - 2]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure("All memory released", mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<7>() +{ + set_test_name("BufferArray overwrite spanning blocks and sequential writes"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJKLMNOPQRST"; + size_t str2_len(strlen(str2)); + char buffer[256]; + + // 2x str1 + size_t len = ba->write(0, str1, str1_len); + len = ba->write(str1_len, str1, str1_len); + + // reposition and overwrite + len = ba->write(6, str2, 2); + ensure("Overwrite length correct", 2 == len); + + len = ba->write(8, str2, 2); + ensure("Overwrite length correct.2", 2 == len); + + len = ba->write(10, str2, 2); + ensure("Overwrite length correct.3", 2 == len); + + // append some data + len = ba->append(str2, str2_len); + ensure("Append length correct", str2_len == len); + + // append some more + void * out_buf(ba->appendBufferAlloc(str1_len)); + memcpy(out_buf, str1, str1_len); + + // And some final writes + len = ba->write(3 * str1_len + str2_len, str2, 2); + ensure("Write length correct.2", 2 == len); + + // Check contents + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Final buffer length correct", (3 * str1_len + str2_len + 2) == len); + ensure("Read content correct", 0 == strncmp(buffer, str1, 6)); + ensure("Read content correct.2", 0 == strncmp(buffer + 6, str2, 2)); + ensure("Read content correct.3", 0 == strncmp(buffer + 8, str2, 2)); + ensure("Read content correct.4", 0 == strncmp(buffer + 10, str2, 2)); + ensure("Read content correct.5", 0 == strncmp(buffer + str1_len + 2, str1 + 2, str1_len - 2)); + ensure("Read content correct.6", 0 == strncmp(buffer + str1_len + str1_len, str2, str2_len)); + ensure("Read content correct.7", 0 == strncmp(buffer + str1_len + str1_len + str2_len, str1, str1_len)); + ensure("Read content correct.8", 0 == strncmp(buffer + str1_len + str1_len + str2_len + str1_len, str2, 2)); + ensure("Read didn't overwrite", 'X' == buffer[str1_len + str1_len + str2_len + str1_len + 2]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure("All memory released", mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<8>() +{ + set_test_name("BufferArray zero-length appendBufferAlloc"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJKLMNOPQRST"; + size_t str2_len(strlen(str2)); + char buffer[256]; + + // 2x str1 + size_t len = ba->write(0, str1, str1_len); + len = ba->write(str1_len, str1, str1_len); + + // zero-length allocate (we allow this with a valid pointer returned) + void * out_buf(ba->appendBufferAlloc(0)); + ensure("Buffer from zero-length appendBufferAlloc non-NULL", NULL != out_buf); + + // Do it again + void * out_buf2(ba->appendBufferAlloc(0)); + ensure("Buffer from zero-length appendBufferAlloc non-NULL.2", NULL != out_buf2); + ensure("Two zero-length appendBufferAlloc buffers distinct", out_buf != out_buf2); + + // And some final writes + len = ba->write(2 * str1_len, str2, str2_len); + + // Check contents + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Final buffer length correct", (2 * str1_len + str2_len) == len); + ensure("Read content correct.1", 0 == strncmp(buffer, str1, str1_len)); + ensure("Read content correct.2", 0 == strncmp(buffer + str1_len, str1, str1_len)); + ensure("Read content correct.3", 0 == strncmp(buffer + str1_len + str1_len, str2, str2_len)); + ensure("Read didn't overwrite", 'X' == buffer[str1_len + str1_len + str2_len]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure("All memory released", mMemTotal == GetMemTotal()); +} + +} // end namespace tut + + +#endif // TEST_LLCORE_BUFFER_ARRAY_H_ diff --git a/indra/llcorehttp/tests/test_bufferstream.hpp b/indra/llcorehttp/tests/test_bufferstream.hpp new file mode 100644 index 0000000000..831c901b9d --- /dev/null +++ b/indra/llcorehttp/tests/test_bufferstream.hpp @@ -0,0 +1,304 @@ +/** + * @file test_bufferstream.hpp + * @brief unit tests for the LLCore::BufferArrayStreamBuf/BufferArrayStream classes + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_BUFFER_STREAM_H_ +#define TEST_LLCORE_BUFFER_STREAM_H_ + +#include "bufferstream.h" + +#include <iostream> + +#include "test_allocator.h" +#include "llsd.h" +#include "llsdserialize.h" + + +using namespace LLCore; + + +namespace tut +{ + +struct BufferStreamTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<BufferStreamTestData> BufferStreamTestGroupType; +typedef BufferStreamTestGroupType::object BufferStreamTestObjectType; +BufferStreamTestGroupType BufferStreamTestGroup("BufferStream Tests"); +typedef BufferArrayStreamBuf::traits_type tst_traits_t; + + +template <> template <> +void BufferStreamTestObjectType::test<1>() +{ + set_test_name("BufferArrayStreamBuf construction with NULL BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArrayStreamBuf * bsb = new BufferArrayStreamBuf(NULL); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Not much will work with a NULL + ensure("underflow() on NULL fails", tst_traits_t::eof() == bsb->underflow()); + ensure("uflow() on NULL fails", tst_traits_t::eof() == bsb->uflow()); + ensure("pbackfail() on NULL fails", tst_traits_t::eof() == bsb->pbackfail('c')); + ensure("showmanyc() on NULL fails", bsb->showmanyc() == -1); + ensure("overflow() on NULL fails", tst_traits_t::eof() == bsb->overflow('c')); + ensure("xsputn() on NULL fails", bsb->xsputn("blah", 4) == 0); + ensure("seekoff() on NULL fails", bsb->seekoff(0, std::ios_base::beg, std::ios_base::in) == std::streampos(-1)); + + // release the implicit reference, causing the object to be released + delete bsb; + bsb = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<2>() +{ + set_test_name("BufferArrayStream construction with NULL BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArrayStream * bas = new BufferArrayStream(NULL); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Not much will work with a NULL here + ensure("eof() is false on NULL", ! bas->eof()); + ensure("fail() is false on NULL", ! bas->fail()); + ensure("good() on NULL", bas->good()); + + // release the implicit reference, causing the object to be released + delete bas; + bas = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<3>() +{ + set_test_name("BufferArrayStreamBuf construction with empty BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + BufferArrayStreamBuf * bsb = new BufferArrayStreamBuf(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // I can release my ref on the BA + ba->release(); + ba = NULL; + + // release the implicit reference, causing the object to be released + delete bsb; + bsb = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<4>() +{ + set_test_name("BufferArrayStream construction with empty BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + + { + // create a new ref counted object with an implicit reference + BufferArrayStream bas(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + } + + // release the implicit reference, causing the object to be released + ba->release(); + ba = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<5>() +{ + set_test_name("BufferArrayStreamBuf construction with real BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + const char * content("This is a string. A fragment."); + const size_t c_len(strlen(content)); + ba->append(content, c_len); + + // Creat an adapter for the BufferArray + BufferArrayStreamBuf * bsb = new BufferArrayStreamBuf(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // I can release my ref on the BA + ba->release(); + ba = NULL; + + // Various static state + ensure("underflow() returns 'T'", bsb->underflow() == 'T'); + ensure("underflow() returns 'T' again", bsb->underflow() == 'T'); + ensure("uflow() returns 'T'", bsb->uflow() == 'T'); + ensure("uflow() returns 'h'", bsb->uflow() == 'h'); + ensure("pbackfail('i') fails", tst_traits_t::eof() == bsb->pbackfail('i')); + ensure("pbackfail('T') fails", tst_traits_t::eof() == bsb->pbackfail('T')); + ensure("pbackfail('h') succeeds", bsb->pbackfail('h') == 'h'); + ensure("showmanyc() is everything but the 'T'", bsb->showmanyc() == (c_len - 1)); + ensure("overflow() appends", bsb->overflow('c') == 'c'); + ensure("showmanyc() reflects append", bsb->showmanyc() == (c_len - 1 + 1)); + ensure("xsputn() appends some more", bsb->xsputn("bla!", 4) == 4); + ensure("showmanyc() reflects 2nd append", bsb->showmanyc() == (c_len - 1 + 5)); + ensure("seekoff() succeeds", bsb->seekoff(0, std::ios_base::beg, std::ios_base::in) == std::streampos(0)); + ensure("seekoff() succeeds 2", bsb->seekoff(4, std::ios_base::cur, std::ios_base::in) == std::streampos(4)); + ensure("showmanyc() picks up seekoff", bsb->showmanyc() == (c_len + 5 - 4)); + ensure("seekoff() succeeds 3", bsb->seekoff(0, std::ios_base::end, std::ios_base::in) == std::streampos(c_len + 4)); + ensure("pbackfail('!') succeeds", tst_traits_t::eof() == bsb->pbackfail('!')); + + // release the implicit reference, causing the object to be released + delete bsb; + bsb = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<6>() +{ + set_test_name("BufferArrayStream construction with real BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + //const char * content("This is a string. A fragment."); + //const size_t c_len(strlen(content)); + //ba->append(content, strlen(content)); + + { + // Creat an adapter for the BufferArray + BufferArrayStream bas(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Basic operations + bas << "Hello" << 27 << "."; + ensure("BA length 8", ba->size() == 8); + + std::string str; + bas >> str; + ensure("reads correctly", str == "Hello27."); + } + + // release the implicit reference, causing the object to be released + ba->release(); + ba = NULL; + + // make sure we didn't leak any memory + // ensure("Allocated memory returned", mMemTotal == GetMemTotal()); + // static U64 mem = GetMemTotal(); +} + + +template <> template <> +void BufferStreamTestObjectType::test<7>() +{ + set_test_name("BufferArrayStream with LLSD serialization"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + + { + // Creat an adapter for the BufferArray + BufferArrayStream bas(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // LLSD + LLSD llsd = LLSD::emptyMap(); + + llsd["int"] = LLSD::Integer(3); + llsd["float"] = LLSD::Real(923289.28992); + llsd["string"] = LLSD::String("aksjdl;ajsdgfjgfal;sdgjakl;sdfjkl;ajsdfkl;ajsdfkl;jaskl;dfj"); + + LLSD llsd_map = LLSD::emptyMap(); + llsd_map["int"] = LLSD::Integer(-2889); + llsd_map["float"] = LLSD::Real(2.37829e32); + llsd_map["string"] = LLSD::String("OHIGODHSPDGHOSDHGOPSHDGP"); + + llsd["map"] = llsd_map; + + // Serialize it + LLSDSerialize::toXML(llsd, bas); + + std::string str; + bas >> str; + // std::cout << "SERIALIZED LLSD: " << str << std::endl; + ensure("Extracted string has reasonable length", str.size() > 60); + } + + // release the implicit reference, causing the object to be released + ba->release(); + ba = NULL; + + // make sure we didn't leak any memory + // ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +} // end namespace tut + + +#endif // TEST_LLCORE_BUFFER_STREAM_H_ diff --git a/indra/llcorehttp/tests/test_httpheaders.hpp b/indra/llcorehttp/tests/test_httpheaders.hpp new file mode 100644 index 0000000000..ce0d19b058 --- /dev/null +++ b/indra/llcorehttp/tests/test_httpheaders.hpp @@ -0,0 +1,108 @@ +/** + * @file test_httpheaders.hpp + * @brief unit tests for the LLCore::HttpHeaders class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_HEADERS_H_ +#define TEST_LLCORE_HTTP_HEADERS_H_ + +#include "httpheaders.h" + +#include <iostream> + +#include "test_allocator.h" + + +using namespace LLCoreInt; + + + +namespace tut +{ + +struct HttpHeadersTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<HttpHeadersTestData> HttpHeadersTestGroupType; +typedef HttpHeadersTestGroupType::object HttpHeadersTestObjectType; +HttpHeadersTestGroupType HttpHeadersTestGroup("HttpHeaders Tests"); + +template <> template <> +void HttpHeadersTestObjectType::test<1>() +{ + set_test_name("HttpHeaders construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + ensure("One ref on construction of HttpHeaders", headers->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + ensure("Nothing in headers", 0 == headers->mHeaders.size()); + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<2>() +{ + set_test_name("HttpHeaders construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + // Append a few strings + std::string str1("Pragma:"); + headers->mHeaders.push_back(str1); + std::string str2("Accept: application/json"); + headers->mHeaders.push_back(str2); + + ensure("Headers retained", 2 == headers->mHeaders.size()); + ensure("First is first", headers->mHeaders[0] == str1); + ensure("Second is second", headers->mHeaders[1] == str2); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +} // end namespace tut + + +#endif // TEST_LLCORE_HTTP_HEADERS_H_ diff --git a/indra/llcorehttp/tests/test_httpoperation.hpp b/indra/llcorehttp/tests/test_httpoperation.hpp new file mode 100644 index 0000000000..17b1a96878 --- /dev/null +++ b/indra/llcorehttp/tests/test_httpoperation.hpp @@ -0,0 +1,125 @@ +/** + * @file test_httpoperation.hpp + * @brief unit tests for the LLCore::HttpOperation-derived classes + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_OPERATION_H_ +#define TEST_LLCORE_HTTP_OPERATION_H_ + +#include "_httpoperation.h" +#include "httphandler.h" + +#include <iostream> + +#include "test_allocator.h" + + +using namespace LLCoreInt; + + +namespace +{ + +class TestHandler : public LLCore::HttpHandler +{ +public: + virtual void onCompleted(HttpHandle, HttpResponse *) + { + std::cout << "TestHandler::onCompleted() invoked" << std::endl; + } + +}; + + +} // end namespace anonymous + + +namespace tut +{ + struct HttpOperationTestData + { + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; + }; + + typedef test_group<HttpOperationTestData> HttpOperationTestGroupType; + typedef HttpOperationTestGroupType::object HttpOperationTestObjectType; + HttpOperationTestGroupType HttpOperationTestGroup("HttpOperation Tests"); + + template <> template <> + void HttpOperationTestObjectType::test<1>() + { + set_test_name("HttpOpNull construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpOpNull * op = new HttpOpNull(); + ensure(op->getRefCount() == 1); + ensure(mMemTotal < GetMemTotal()); + + // release the implicit reference, causing the object to be released + op->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void HttpOperationTestObjectType::test<2>() + { + set_test_name("HttpOpNull construction with handlers"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // Get some handlers + TestHandler * h1 = new TestHandler(); + + // create a new ref counted object with an implicit reference + HttpOpNull * op = new HttpOpNull(); + + // Add the handlers + op->setReplyPath(NULL, h1); + + // Check ref count + ensure(op->getRefCount() == 1); + + // release the reference, releasing the operation but + // not the handlers. + op->release(); + op = NULL; + ensure(mMemTotal != GetMemTotal()); + + // release the handlers + delete h1; + h1 = NULL; + + ensure(mMemTotal == GetMemTotal()); + } + +} + +#endif // TEST_LLCORE_HTTP_OPERATION_H_ diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp new file mode 100644 index 0000000000..e5488cf941 --- /dev/null +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -0,0 +1,2670 @@ +/** + * @file test_httprequest.hpp + * @brief unit tests for the LLCore::HttpRequest class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_REQUEST_H_ +#define TEST_LLCORE_HTTP_REQUEST_H_ + +#include "httprequest.h" +#include "bufferarray.h" +#include "httphandler.h" +#include "httpheaders.h" +#include "httpresponse.h" +#include "httpoptions.h" +#include "_httpservice.h" +#include "_httprequestqueue.h" + +#include <curl/curl.h> +#include <boost/regex.hpp> +#include <sstream> + +#include "test_allocator.h" +#include "llcorehttp_test.h" + + +using namespace LLCoreInt; + + +namespace +{ + +#if defined(WIN32) + +void usleep(unsigned long usec); + +#endif + +} + +namespace tut +{ + +struct HttpRequestTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; + int mHandlerCalls; + HttpStatus mStatus; +}; + +class TestHandler2 : public LLCore::HttpHandler +{ +public: + TestHandler2(HttpRequestTestData * state, + const std::string & name) + : mState(state), + mName(name), + mExpectHandle(LLCORE_HTTP_HANDLE_INVALID) + {} + + virtual void onCompleted(HttpHandle handle, HttpResponse * response) + { + if (LLCORE_HTTP_HANDLE_INVALID != mExpectHandle) + { + ensure("Expected handle received in handler", mExpectHandle == handle); + } + ensure("Handler got a response", NULL != response); + if (response && mState) + { + const HttpStatus actual_status(response->getStatus()); + std::ostringstream test; + test << "Expected HttpStatus received in response. Wanted: " + << mState->mStatus.toHex() << " Received: " << actual_status.toHex(); + ensure(test.str().c_str(), actual_status == mState->mStatus); + } + if (mState) + { + mState->mHandlerCalls++; + } + if (! mHeadersRequired.empty() || ! mHeadersDisallowed.empty()) + { + ensure("Response required with header check", response != NULL); + HttpHeaders * header(response->getHeaders()); // Will not hold onto this + ensure("Some quantity of headers returned", header != NULL); + + if (! mHeadersRequired.empty()) + { + for (int i(0); i < mHeadersRequired.size(); ++i) + { + bool found = false; + for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); + header->mHeaders.end() != iter; + ++iter) + { + if (boost::regex_match(*iter, mHeadersRequired[i])) + { + found = true; + break; + } + } + std::ostringstream str; + str << "Required header # " << i << " found in response"; + ensure(str.str(), found); + } + } + + if (! mHeadersDisallowed.empty()) + { + for (int i(0); i < mHeadersDisallowed.size(); ++i) + { + for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); + header->mHeaders.end() != iter; + ++iter) + { + if (boost::regex_match(*iter, mHeadersDisallowed[i])) + { + std::ostringstream str; + str << "Disallowed header # " << i << " not found in response"; + ensure(str.str(), false); + } + } + } + } + } + + if (! mCheckContentType.empty()) + { + ensure("Response required with content type check", response != NULL); + std::string con_type(response->getContentType()); + ensure("Content-Type as expected (" + mCheckContentType + ")", + mCheckContentType == con_type); + } + + // std::cout << "TestHandler2::onCompleted() invoked" << std::endl; + } + + HttpRequestTestData * mState; + std::string mName; + HttpHandle mExpectHandle; + std::string mCheckContentType; + std::vector<boost::regex> mHeadersRequired; + std::vector<boost::regex> mHeadersDisallowed; +}; + +typedef test_group<HttpRequestTestData> HttpRequestTestGroupType; +typedef HttpRequestTestGroupType::object HttpRequestTestObjectType; +HttpRequestTestGroupType HttpRequestTestGroup("HttpRequest Tests"); + +template <> template <> +void HttpRequestTestObjectType::test<1>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest construction"); + + HttpRequest * req = NULL; + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + try + { + // Get singletons created + HttpRequest::createService(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // release the request object + delete req; + req = NULL; + + HttpRequest::destroyService(); + + // make sure we didn't leak any memory + ensure("Memory returned", mMemTotal == GetMemTotal()); + } + catch (...) + { + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<2>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest and Null Op queued"); + + HttpRequest * req = NULL; + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + try + { + // Get singletons created + HttpRequest::createService(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Issue a NoOp + HttpHandle handle = req->requestNoOp(NULL); + ensure("Request issued", handle != LLCORE_HTTP_HANDLE_INVALID); + + // release the request object + delete req; + req = NULL; + + // We're still holding onto the operation which is + // sitting, unserviced, on the request queue so... + ensure("Memory being used 2", mMemTotal < GetMemTotal()); + + // Request queue should have two references: global singleton & service object + ensure("Two references to request queue", 2 == HttpRequestQueue::instanceOf()->getRefCount()); + + // Okay, tear it down + HttpRequest::destroyService(); + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory returned", mMemTotal == GetMemTotal()); + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<3>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest NoOp + Stop execution"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a NoOp + HttpHandle handle = req->requestNoOp(&handler); + ensure("Valid handle returned for first request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(20); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 100; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<4>() +{ + ScopedCurlInit ready; + + set_test_name("2 HttpRequest instances, one thread"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + TestHandler2 handler1(this, "handler1"); + TestHandler2 handler2(this, "handler2"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req1 = NULL; + HttpRequest * req2 = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req1 = new HttpRequest(); + req2 = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue some NoOps + HttpHandle handle = req1->requestNoOp(&handler1); + ensure("Valid handle returned for first request", handle != LLCORE_HTTP_HANDLE_INVALID); + handler1.mExpectHandle = handle; + + handle = req2->requestNoOp(&handler2); + ensure("Valid handle returned for first request", handle != LLCORE_HTTP_HANDLE_INVALID); + handler2.mExpectHandle = handle; + + // Run the notification pump. + int count(0); + int limit(20); + while (count++ < limit && mHandlerCalls < 2) + { + req1->update(1000000); + req2->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 2); + + // Okay, request a shutdown of the servicing thread + handle = req2->requestStopThread(&handler2); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + handler2.mExpectHandle = handle; + + // Run the notification pump again + count = 0; + limit = 100; + while (count++ < limit && mHandlerCalls < 3) + { + req1->update(1000000); + req2->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 3); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req1; + req1 = NULL; + delete req2; + req2 = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 3 == mHandlerCalls); + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + } + catch (...) + { + stop_thread(req1); + delete req1; + delete req2; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<5>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest Spin (soft) + NoOp + hard termination"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a Spin + HttpHandle handle = req->requestSpin(1); + ensure("Valid handle returned for spin request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Issue a NoOp + handle = req->requestNoOp(&handler); + ensure("Valid handle returned for no-op request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("NoOp notification received", mHandlerCalls == 1); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + // Check memory usage + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + // This memory test should work but could give problems as it + // relies on the worker thread picking up a friendly request + // to shutdown. Doing so, it drops references to things and + // we should go back to where we started. If it gives you + // problems, look into the code before commenting things out. + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<6>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest Spin + NoOp + hard termination"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a Spin + HttpHandle handle = req->requestSpin(0); // Hard spin + ensure("Valid handle returned for spin request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Issue a NoOp + handle = req->requestNoOp(&handler); + ensure("Valid handle returned for no-op request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("No notifications received", mHandlerCalls == 0); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + // Check memory usage + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + // ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + // This memory test won't work because we're killing the thread + // hard with the hard spinner. There's no opportunity to join + // nicely so many things leak or get destroyed unilaterally. + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<7>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET to dead port + Stop execution"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(1); // Don't try for too long - default retries take about 18S + + // Issue a GET that can't connect + mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + "http://127.0.0.1:2/nothing/here", + 0, + 0, + opts, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(50); // With one retry, should fail quickish + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 100; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if 0 // defined(WIN32) + // Can't do this on any platform anymore, the LL logging system holds + // on to memory and produces what looks like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<8>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<9>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with Range: header to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<10>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest PUT to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + BufferArray * body = new BufferArray; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + static const char * body_text("Now is the time for all good men..."); + body->append(body_text, strlen(body_text)); + mStatus = HttpStatus(200); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + body, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // Lose the request body + body->release(); + body = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if 0 // defined(WIN32) + // Can't do this on any platform anymore, the LL logging system holds + // on to memory and produces what looks like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + if (body) + { + body->release(); + } + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<11>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest POST to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + BufferArray * body = new BufferArray; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + static const char * body_text("Now is the time for all good men..."); + body->append(body_text, strlen(body_text)); + mStatus = HttpStatus(200); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + body, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // Lose the request body + body->release(); + body = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + if (body) + { + body->release(); + } + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<12>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with some tracing"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Enable tracing + HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if 0 // defined(WIN32) + // Can't do this on any platform anymore, the LL logging system holds + // on to memory and produces what looks like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<13>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with returned headers"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + handler.mHeadersRequired.reserve(20); // Avoid memory leak test failure + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Enable tracing + HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setWantHeaders(true); + + // Issue a GET that succeeds + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back(boost::regex("\\W*X-LL-Special:.*", boost::regex::icase)); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + opts, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // release options + opts->release(); + opts = NULL; + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<14>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET timeout"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + std::string url_base(get_base_url() + "/sleep/"); // path to a 30-second sleep + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(0); // Don't retry + opts->setTimeout(2); + + // Issue a GET that sleeps + mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + opts, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(50); // With one retry, should fail quickish + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 100; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + +// Test retrieval of Content-Type/Content-Encoding headers +template <> template <> +void HttpRequestTestObjectType::test<15>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with Content-Type"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // Load and clear the string setting to preload std::string object + // for memory return tests. + handler.mCheckContentType = "application/llsd+xml"; + handler.mCheckContentType.clear(); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + handler.mCheckContentType = "application/llsd+xml"; + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mCheckContentType.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on GET requests +template <> template <> +void HttpRequestTestObjectType::test<16>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest GET"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + options, + NULL, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Do a texture-style fetch + headers = new HttpHeaders; + headers->mHeaders.push_back("Accept: image/x-j2c"); + + mStatus = HttpStatus(200); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*image/x-j2c", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("\\W*X-Reflect-range:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + 0, + 47, + options, + headers, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 2); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 3) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 3); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on POST requests +template <> template <> +void HttpRequestTestObjectType::test<17>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest POST"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // And a buffer array + const char * msg("It was the best of times, it was the worst of times."); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default POST + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + NULL, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on PUT requests +template <> template <> +void HttpRequestTestObjectType::test<18>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest PUT"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // And a buffer array + const char * msg("It was the best of times, it was the worst of times."); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default PUT + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:.*", boost::regex::icase)); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + NULL, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on GET requests with overrides +template <> template <> +void HttpRequestTestObjectType::test<19>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest GET with header overrides"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // headers + headers = new HttpHeaders; + headers->mHeaders.push_back("Keep-Alive: 120"); + headers->mHeaders.push_back("Accept-encoding: deflate"); + headers->mHeaders.push_back("Accept: text/plain"); + + // Issue a GET with modified headers + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/plain", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*deflate", boost::regex::icase)); // close enough + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + options, + headers, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on POST requests with overrides +template <> template <> +void HttpRequestTestObjectType::test<20>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest POST with header overrides"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // headers + headers = new HttpHeaders(); + headers->mHeaders.push_back("keep-Alive: 120"); + headers->mHeaders.push_back("Accept: text/html"); + headers->mHeaders.push_back("content-type: application/llsd+xml"); + headers->mHeaders.push_back("cache-control: no-store"); + + // And a buffer array + const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default POST + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/html", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("\\s*X-Reflect-cache-control:\\s*no-store", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + headers, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on PUT requests with overrides +template <> template <> +void HttpRequestTestObjectType::test<21>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest PUT with header overrides"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // headers + headers = new HttpHeaders; + headers->mHeaders.push_back("content-type: text/plain"); + headers->mHeaders.push_back("content-type: text/html"); + headers->mHeaders.push_back("content-type: application/llsd+xml"); + + // And a buffer array + const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default PUT + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); + handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/plain", boost::regex::icase)); + handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/html", boost::regex::icase)); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + headers, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +} // end namespace tut + +namespace +{ + +#if defined(WIN32) + +void usleep(unsigned long usec) +{ + Sleep((DWORD) (usec / 1000UL)); +} + +#endif + +} + +#endif // TEST_LLCORE_HTTP_REQUEST_H_ diff --git a/indra/llcorehttp/tests/test_httprequestqueue.hpp b/indra/llcorehttp/tests/test_httprequestqueue.hpp new file mode 100644 index 0000000000..1de2d8f9ab --- /dev/null +++ b/indra/llcorehttp/tests/test_httprequestqueue.hpp @@ -0,0 +1,186 @@ +/** + * @file test_httprequestqueue.hpp + * @brief unit tests for the LLCore::HttpRequestQueue class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_REQUESTQUEUE_H_ +#define TEST_LLCORE_HTTP_REQUESTQUEUE_H_ + +#include "_httprequestqueue.h" + +#include <iostream> + +#include "test_allocator.h" +#include "_httpoperation.h" + + +using namespace LLCoreInt; + + + +namespace tut +{ + +struct HttpRequestqueueTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<HttpRequestqueueTestData> HttpRequestqueueTestGroupType; +typedef HttpRequestqueueTestGroupType::object HttpRequestqueueTestObjectType; +HttpRequestqueueTestGroupType HttpRequestqueueTestGroup("HttpRequestqueue Tests"); + +template <> template <> +void HttpRequestqueueTestObjectType::test<1>() +{ + set_test_name("HttpRequestQueue construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + ensure("One ref on construction of HttpRequestQueue", HttpRequestQueue::instanceOf()->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // release the implicit reference, causing the object to be released + HttpRequestQueue::term(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpRequestqueueTestObjectType::test<2>() +{ + set_test_name("HttpRequestQueue refcount works"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + rq->addRef(); + + // release the singleton, hold on to the object + HttpRequestQueue::term(); + + ensure("One ref after term() called", rq->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Drop ref + rq->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpRequestqueueTestObjectType::test<3>() +{ + set_test_name("HttpRequestQueue addOp/fetchOp work"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + + HttpOperation * op = new HttpOpNull(); + + rq->addOp(op); // transfer my refcount + + op = rq->fetchOp(true); // Potentially hangs the test on failure + ensure("One goes in, one comes out", NULL != op); + op->release(); + + op = rq->fetchOp(false); + ensure("Better not be two of them", NULL == op); + + // release the singleton, hold on to the object + HttpRequestQueue::term(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpRequestqueueTestObjectType::test<4>() +{ + set_test_name("HttpRequestQueue addOp/fetchAll work"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + + HttpOperation * op = new HttpOpNull(); + rq->addOp(op); // transfer my refcount + + op = new HttpOpNull(); + rq->addOp(op); // transfer my refcount + + op = new HttpOpNull(); + rq->addOp(op); // transfer my refcount + + { + HttpRequestQueue::OpContainer ops; + rq->fetchAll(true, ops); // Potentially hangs the test on failure + ensure("Three go in, three come out", 3 == ops.size()); + + op = rq->fetchOp(false); + ensure("Better not be any more of them", NULL == op); + + // release the singleton, hold on to the object + HttpRequestQueue::term(); + + // We're still holding onto the ops. + ensure(mMemTotal < GetMemTotal()); + + // Release them + while (! ops.empty()) + { + HttpOperation * op = ops.front(); + ops.erase(ops.begin()); + op->release(); + } + } + + // Should be clean + ensure("All memory returned", mMemTotal == GetMemTotal()); +} + +} // end namespace tut + + +#endif // TEST_LLCORE_HTTP_REQUESTQUEUE_H_ diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp new file mode 100644 index 0000000000..f7b542d3b5 --- /dev/null +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -0,0 +1,265 @@ +/** + * @file test_llrefcounted + * @brief unit tests for HttpStatus struct + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef TEST_HTTP_STATUS_H_ +#define TEST_HTTP_STATUS_H_ + +#include "httpcommon.h" + +#include <curl/curl.h> +#include <curl/multi.h> + +using namespace LLCore; + +namespace tut +{ + +struct HttpStatusTestData +{ + HttpStatusTestData() + {} +}; + +typedef test_group<HttpStatusTestData> HttpStatusTestGroupType; +typedef HttpStatusTestGroupType::object HttpStatusTestObjectType; + +HttpStatusTestGroupType HttpStatusTestGroup("HttpStatus Tests"); + +template <> template <> +void HttpStatusTestObjectType::test<1>() +{ + set_test_name("HttpStatus construction"); + + // auto allocation fine for this + HttpStatus status; + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = 0; + + ensure(bool(status)); + ensure(false == !(status)); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = 0; + + ensure(bool(status)); + ensure(false == !(status)); + + status.mType = HttpStatus::LLCORE; + status.mStatus = HE_SUCCESS; + + ensure(bool(status)); + ensure(false == !(status)); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = -1; + + ensure(false == bool(status)); + ensure(!(status)); + + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = CURLE_BAD_DOWNLOAD_RESUME; + + ensure(false == bool(status)); + ensure(!(status)); +} + + +template <> template <> +void HttpStatusTestObjectType::test<2>() +{ + set_test_name("HttpStatus memory structure"); + + // Require that an HttpStatus object can be trivially + // returned as a function return value in registers. + // One should fit in an int on all platforms. + + ensure(sizeof(HttpStatus) <= sizeof(int)); +} + + +template <> template <> +void HttpStatusTestObjectType::test<3>() +{ + set_test_name("HttpStatus valid error string conversion"); + + HttpStatus status; + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = 0; + std::string msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(msg.empty()); + + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = CURLE_BAD_FUNCTION_ARGUMENT; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = CURLM_OUT_OF_MEMORY; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::LLCORE; + status.mStatus = HE_SHUTTING_DOWN; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); +} + + +template <> template <> +void HttpStatusTestObjectType::test<4>() +{ + set_test_name("HttpStatus invalid error string conversion"); + + HttpStatus status; + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = 32726; + std::string msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = -470; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::LLCORE; + status.mStatus = 923; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); +} + +template <> template <> +void HttpStatusTestObjectType::test<5>() +{ + set_test_name("HttpStatus equality/inequality testing"); + + // Make certain equality/inequality tests do not pass + // through the bool conversion. Distinct successful + // and error statuses should compare unequal. + + HttpStatus status1(HttpStatus::LLCORE, HE_SUCCESS); + HttpStatus status2(HttpStatus::EXT_CURL_EASY, HE_SUCCESS); + ensure(status1 != status2); + + status1.mType = HttpStatus::LLCORE; + status1.mStatus = HE_REPLY_ERROR; + status2.mType = HttpStatus::LLCORE; + status2.mStatus= HE_SHUTTING_DOWN; + ensure(status1 != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<6>() +{ + set_test_name("HttpStatus basic HTTP status encoding"); + + HttpStatus status; + status.mType = 200; + status.mStatus = HE_SUCCESS; + std::string msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Normally a success but application says error + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(! msg.empty()); + ensure(! bool(status)); + ensure(status.toULong() > 1UL); // Biggish number, not a bool-to-ulong + + // Same statuses with distinct success/fail are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + HttpStatus status2(200, HE_REPLY_ERROR); + ensure(status != status2); + + // Normally an error but application says okay + status.mType = 406; + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Different statuses but both successful are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + status2.mType = 201; + status2.mStatus = HE_SUCCESS; + ensure(status != status2); + + // Different statuses but both failed are distinct + status.mType = 200; + status.mStatus = HE_REPLY_ERROR; + status2.mType = 201; + status2.mStatus = HE_REPLY_ERROR; + ensure(status != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<7>() +{ + set_test_name("HttpStatus HTTP error text strings"); + + HttpStatus status(100, HE_REPLY_ERROR); + std::string msg(status.toString()); + ensure(! msg.empty()); // Should be something + ensure(msg == "Continue"); + + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); // Success is empty + + status.mType = 199; + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 505; // Last defined string + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "HTTP Version not supported"); + + status.mType = 506; // One beyond + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 999; // Last HTTP status + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); +} + +} // end namespace tut + +#endif // TEST_HTTP_STATUS_H + diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py new file mode 100644 index 0000000000..75a3c39ef2 --- /dev/null +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +"""\ +@file test_llsdmessage_peer.py +@author Nat Goodspeed +@date 2008-10-09 +@brief This script asynchronously runs the executable (with args) specified on + the command line, returning its result code. While that executable is + running, we provide dummy local services for use by C++ tests. + +$LicenseInfo:firstyear=2008&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2012, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +import os +import sys +import time +import select +import getopt +from threading import Thread +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from SocketServer import ThreadingMixIn + +mydir = os.path.dirname(__file__) # expected to be .../indra/llcorehttp/tests/ +sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) +from indra.util.fastest_elementtree import parse as xml_parse +from indra.base import llsd +from testrunner import freeport, run, debug, VERBOSE + +class TestHTTPRequestHandler(BaseHTTPRequestHandler): + """This subclass of BaseHTTPRequestHandler is to receive and echo + LLSD-flavored messages sent by the C++ LLHTTPClient. + """ + def read(self): + # The following logic is adapted from the library module + # SimpleXMLRPCServer.py. + # Get arguments by reading body of request. + # We read this in chunks to avoid straining + # socket.read(); around the 10 or 15Mb mark, some platforms + # begin to have problems (bug #792570). + try: + size_remaining = int(self.headers["content-length"]) + except (KeyError, ValueError): + return "" + max_chunk_size = 10*1024*1024 + L = [] + while size_remaining: + chunk_size = min(size_remaining, max_chunk_size) + chunk = self.rfile.read(chunk_size) + L.append(chunk) + size_remaining -= len(chunk) + return ''.join(L) + # end of swiped read() logic + + def read_xml(self): + # This approach reads the entire POST data into memory first + return llsd.parse(self.read()) +## # This approach attempts to stream in the LLSD XML from self.rfile, +## # assuming that the underlying XML parser reads its input file +## # incrementally. Unfortunately I haven't been able to make it work. +## tree = xml_parse(self.rfile) +## debug("Finished raw parse") +## debug("parsed XML tree %s", tree) +## debug("parsed root node %s", tree.getroot()) +## debug("root node tag %s", tree.getroot().tag) +## return llsd.to_python(tree.getroot()) + + def do_HEAD(self): + self.do_GET(withdata=False) + + def do_GET(self, withdata=True): + # Of course, don't attempt to read data. + self.answer(dict(reply="success", status=200, + reason="Your GET operation worked")) + + def do_POST(self): + # Read the provided POST data. + # self.answer(self.read()) + self.answer(dict(reply="success", status=200, + reason=self.read())) + + def do_PUT(self): + # Read the provided PUT data. + # self.answer(self.read()) + self.answer(dict(reply="success", status=200, + reason=self.read())) + + def answer(self, data, withdata=True): + debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) + if "/sleep/" in self.path: + time.sleep(30) + + if "fail" not in self.path: + data = data.copy() # we're going to modify + # Ensure there's a "reply" key in data, even if there wasn't before + data["reply"] = data.get("reply", llsd.LLSD("success")) + response = llsd.format_xml(data) + debug("success: %s", response) + self.send_response(200) + if "/reflect/" in self.path: + self.reflect_headers() + self.send_header("Content-type", "application/llsd+xml") + self.send_header("Content-Length", str(len(response))) + self.send_header("X-LL-Special", "Mememememe"); + self.end_headers() + if withdata: + self.wfile.write(response) + else: # fail requested + status = data.get("status", 500) + # self.responses maps an int status to a (short, long) pair of + # strings. We want the longer string. That's why we pass a string + # pair to get(): the [1] will select the second string, whether it + # came from self.responses or from our default pair. + reason = data.get("reason", + self.responses.get(status, + ("fail requested", + "Your request specified failure status %s " + "without providing a reason" % status))[1]) + debug("fail requested: %s: %r", status, reason) + self.send_error(status, reason) + if "/reflect/" in self.path: + self.reflect_headers() + self.end_headers() + + def reflect_headers(self): + for name in self.headers.keys(): + # print "Header: %s: %s" % (name, self.headers[name]) + self.send_header("X-Reflect-" + name, self.headers[name]) + + if not VERBOSE: + # When VERBOSE is set, skip both these overrides because they exist to + # suppress output. + + def log_request(self, code, size=None): + # For present purposes, we don't want the request splattered onto + # stderr, as it would upset devs watching the test run + pass + + def log_error(self, format, *args): + # Suppress error output as well + pass + +class Server(ThreadingMixIn, HTTPServer): + # This pernicious flag is on by default in HTTPServer. But proper + # operation of freeport() absolutely depends on it being off. + allow_reuse_address = False + +if __name__ == "__main__": + do_valgrind = False + path_search = False + options, args = getopt.getopt(sys.argv[1:], "V", ["valgrind"]) + for option, value in options: + if option == "-V" or option == "--valgrind": + do_valgrind = True + + # Instantiate a Server(TestHTTPRequestHandler) on the first free port + # in the specified port range. Doing this inline is better than in a + # daemon thread: if it blows up here, we'll get a traceback. If it blew up + # in some other thread, the traceback would get eaten and we'd run the + # subject test program anyway. + httpd, port = freeport(xrange(8000, 8020), + lambda port: Server(('127.0.0.1', port), TestHTTPRequestHandler)) + + # Pass the selected port number to the subject test program via the + # environment. We don't want to impose requirements on the test program's + # command-line parsing -- and anyway, for C++ integration tests, that's + # performed in TUT code rather than our own. + os.environ["LL_TEST_PORT"] = str(port) + debug("$LL_TEST_PORT = %s", port) + if do_valgrind: + args = ["valgrind", "--log-file=./valgrind.log"] + args + path_search = True + sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), use_path=path_search, *args)) diff --git a/indra/llcorehttp/tests/test_refcounted.hpp b/indra/llcorehttp/tests/test_refcounted.hpp new file mode 100644 index 0000000000..cb4b50287a --- /dev/null +++ b/indra/llcorehttp/tests/test_refcounted.hpp @@ -0,0 +1,156 @@ +/** + * @file test_refcounted.hpp + * @brief unit tests for the LLCoreInt::RefCounted class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCOREINT_REF_COUNTED_H_ +#define TEST_LLCOREINT_REF_COUNTED_H_ + +#include "_refcounted.h" + +#include "test_allocator.h" + +using namespace LLCoreInt; + +namespace tut +{ + struct RefCountedTestData + { + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; + }; + + typedef test_group<RefCountedTestData> RefCountedTestGroupType; + typedef RefCountedTestGroupType::object RefCountedTestObjectType; + RefCountedTestGroupType RefCountedTestGroup("RefCounted Tests"); + + template <> template <> + void RefCountedTestObjectType::test<1>() + { + set_test_name("RefCounted construction with implicit count"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + RefCounted * rc = new RefCounted(true); + ensure(rc->getRefCount() == 1); + + // release the implicit reference, causing the object to be released + rc->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<2>() + { + set_test_name("RefCounted construction without implicit count"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + RefCounted * rc = new RefCounted(false); + ensure(rc->getRefCount() == 0); + + // add a reference + rc->addRef(); + ensure(rc->getRefCount() == 1); + + // release the implicit reference, causing the object to be released + rc->release(); + + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<3>() + { + set_test_name("RefCounted addRef and release"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + RefCounted * rc = new RefCounted(false); + + for (int i = 0; i < 1024; ++i) + { + rc->addRef(); + } + + ensure(rc->getRefCount() == 1024); + + for (int i = 0; i < 1024; ++i) + { + rc->release(); + } + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<4>() + { + set_test_name("RefCounted isLastRef check"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + RefCounted * rc = new RefCounted(true); + + // with only one reference, isLastRef should be true + ensure(rc->isLastRef()); + + // release it to clean up memory + rc->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<5>() + { + set_test_name("RefCounted noRef check"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + RefCounted * rc = new RefCounted(false); + + // set the noRef + rc->noRef(); + + // with only one reference, isLastRef should be true + ensure(rc->getRefCount() == RefCounted::NOT_REF_COUNTED); + + // allow this memory leak, but check that we're leaking a known amount + ensure(mMemTotal == (GetMemTotal() - sizeof(RefCounted))); + } +} + +#endif // TEST_LLCOREINT_REF_COUNTED_H_ diff --git a/indra/llcorehttp/tests/testrunner.py b/indra/llcorehttp/tests/testrunner.py new file mode 100644 index 0000000000..9a2de71142 --- /dev/null +++ b/indra/llcorehttp/tests/testrunner.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python +"""\ +@file testrunner.py +@author Nat Goodspeed +@date 2009-03-20 +@brief Utilities for writing wrapper scripts for ADD_COMM_BUILD_TEST unit tests + +$LicenseInfo:firstyear=2009&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2010, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +from __future__ import with_statement + +import os +import sys +import re +import errno +import socket + +VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "0") # default to quiet +# Support usage such as INTEGRATION_TEST_VERBOSE=off -- distressing to user if +# that construct actually turns on verbosity... +VERBOSE = not re.match(r"(0|off|false|quiet)$", VERBOSE, re.IGNORECASE) + +if VERBOSE: + def debug(fmt, *args): + print fmt % args + sys.stdout.flush() +else: + debug = lambda *args: None + +def freeport(portlist, expr): + """ + Find a free server port to use. Specifically, evaluate 'expr' (a + callable(port)) until it stops raising EADDRINUSE exception. + + Pass: + + portlist: an iterable (e.g. xrange()) of ports to try. If you exhaust the + range, freeport() lets the socket.error exception propagate. If you want + unbounded, you could pass itertools.count(baseport), though of course in + practice the ceiling is 2^16-1 anyway. But it seems prudent to constrain + the range much more sharply: if we're iterating an absurd number of times, + probably something else is wrong. + + expr: a callable accepting a port number, specifically one of the items + from portlist. If calling that callable raises socket.error with + EADDRINUSE, freeport() retrieves the next item from portlist and retries. + + Returns: (expr(port), port) + + port: the value from portlist for which expr(port) succeeded + + Raises: + + Any exception raised by expr(port) other than EADDRINUSE. + + socket.error if, for every item from portlist, expr(port) raises + socket.error. The exception you see is the one from the last item in + portlist. + + StopIteration if portlist is completely empty. + + Example: + + class Server(HTTPServer): + # If you use BaseHTTPServer.HTTPServer, turning off this flag is + # essential for proper operation of freeport()! + allow_reuse_address = False + # ... + server, port = freeport(xrange(8000, 8010), + lambda port: Server(("localhost", port), + MyRequestHandler)) + # pass 'port' to client code + # call server.serve_forever() + """ + try: + # If portlist is completely empty, let StopIteration propagate: that's an + # error because we can't return meaningful values. We have no 'port', + # therefore no 'expr(port)'. + portiter = iter(portlist) + port = portiter.next() + + while True: + try: + # If this value of port works, return as promised. + value = expr(port) + + except socket.error, err: + # Anything other than 'Address already in use', propagate + if err.args[0] != errno.EADDRINUSE: + raise + + # Here we want the next port from portiter. But on StopIteration, + # we want to raise the original exception rather than + # StopIteration. So save the original exc_info(). + type, value, tb = sys.exc_info() + try: + try: + port = portiter.next() + except StopIteration: + raise type, value, tb + finally: + # Clean up local traceback, see docs for sys.exc_info() + del tb + + else: + debug("freeport() returning %s on port %s", value, port) + return value, port + + # Recap of the control flow above: + # If expr(port) doesn't raise, return as promised. + # If expr(port) raises anything but EADDRINUSE, propagate that + # exception. + # If portiter.next() raises StopIteration -- that is, if the port + # value we just passed to expr(port) was the last available -- reraise + # the EADDRINUSE exception. + # If we've actually arrived at this point, portiter.next() delivered a + # new port value. Loop back to pass that to expr(port). + + except Exception, err: + debug("*** freeport() raising %s: %s", err.__class__.__name__, err) + raise + +def run(*args, **kwds): + """All positional arguments collectively form a command line, executed as + a synchronous child process. + In addition, pass server=new_thread_instance as an explicit keyword (to + differentiate it from an additional command-line argument). + new_thread_instance should be an instantiated but not yet started Thread + subclass instance, e.g.: + run("python", "-c", 'print "Hello, world!"', server=TestHTTPServer(name="httpd")) + """ + # If there's no server= keyword arg, don't start a server thread: simply + # run a child process. + try: + thread = kwds.pop("server") + except KeyError: + pass + else: + # Start server thread. Note that this and all other comm server + # threads should be daemon threads: we'll let them run "forever," + # confident that the whole process will terminate when the main thread + # terminates, which will be when the child process terminates. + thread.setDaemon(True) + thread.start() + # choice of os.spawnv(): + # - [v vs. l] pass a list of args vs. individual arguments, + # - [no p] don't use the PATH because we specifically want to invoke the + # executable passed as our first arg, + # - [no e] child should inherit this process's environment. + debug("Running %s...", " ".join(args)) + if kwds.get("use_path", False): + rc = os.spawnvp(os.P_WAIT, args[0], args) + else: + rc = os.spawnv(os.P_WAIT, args[0], args) + debug("%s returned %s", args[0], rc) + return rc + +# **************************************************************************** +# test code -- manual at this point, see SWAT-564 +# **************************************************************************** +def test_freeport(): + # ------------------------------- Helpers -------------------------------- + from contextlib import contextmanager + # helper Context Manager for expecting an exception + # with exc(SomeError): + # raise SomeError() + # raises AssertionError otherwise. + @contextmanager + def exc(exception_class, *args): + try: + yield + except exception_class, err: + for i, expected_arg in enumerate(args): + assert expected_arg == err.args[i], \ + "Raised %s, but args[%s] is %r instead of %r" % \ + (err.__class__.__name__, i, err.args[i], expected_arg) + print "Caught expected exception %s(%s)" % \ + (err.__class__.__name__, ', '.join(repr(arg) for arg in err.args)) + else: + assert False, "Failed to raise " + exception_class.__class__.__name__ + + # helper to raise specified exception + def raiser(exception): + raise exception + + # the usual + def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) + + # ------------------------ Sanity check the above ------------------------ + class SomeError(Exception): pass + # Without extra args, accept any err.args value + with exc(SomeError): + raiser(SomeError("abc")) + # With extra args, accept only the specified value + with exc(SomeError, "abc"): + raiser(SomeError("abc")) + with exc(AssertionError): + with exc(SomeError, "abc"): + raiser(SomeError("def")) + with exc(AssertionError): + with exc(socket.error, errno.EADDRINUSE): + raiser(socket.error(errno.ECONNREFUSED, 'Connection refused')) + + # ----------- freeport() without engaging socket functionality ----------- + # If portlist is empty, freeport() raises StopIteration. + with exc(StopIteration): + freeport([], None) + + assert_equals(freeport([17], str), ("17", 17)) + + # This is the magic exception that should prompt us to retry + inuse = socket.error(errno.EADDRINUSE, 'Address already in use') + # Get the iterator to our ports list so we can check later if we've used all + ports = iter(xrange(5)) + with exc(socket.error, errno.EADDRINUSE): + freeport(ports, lambda port: raiser(inuse)) + # did we entirely exhaust 'ports'? + with exc(StopIteration): + ports.next() + + ports = iter(xrange(2)) + # Any exception but EADDRINUSE should quit immediately + with exc(SomeError): + freeport(ports, lambda port: raiser(SomeError())) + assert_equals(ports.next(), 1) + + # ----------- freeport() with platform-dependent socket stuff ------------ + # This is what we should've had unit tests to begin with (see CHOP-661). + def newbind(port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(('127.0.0.1', port)) + return sock + + bound0, port0 = freeport(xrange(7777, 7780), newbind) + assert_equals(port0, 7777) + bound1, port1 = freeport(xrange(7777, 7780), newbind) + assert_equals(port1, 7778) + bound2, port2 = freeport(xrange(7777, 7780), newbind) + assert_equals(port2, 7779) + with exc(socket.error, errno.EADDRINUSE): + bound3, port3 = freeport(xrange(7777, 7780), newbind) + +if __name__ == "__main__": + test_freeport() |