diff options
Diffstat (limited to 'indra/llcorehttp')
63 files changed, 18257 insertions, 0 deletions
diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt new file mode 100755 index 0000000000..a0b1ea13b1 --- /dev/null +++ b/indra/llcorehttp/CMakeLists.txt @@ -0,0 +1,186 @@ +# -*- cmake -*- + +project(llcorehttp) + +include(00-Common) +include(GoogleMock) +include(CURL) +include(CARes) +include(OpenSSL) +include(ZLIB) +include(LLCoreHttp) +include(LLAddBuildTest) +include(LLMessage) +include(LLCommon) +include(Tut) + +include_directories (${CMAKE_CURRENT_SOURCE_DIR}) + +include_directories( + ${LLMESSAGE_INCLUDE_DIRS} + ${LLCOMMON_INCLUDE_DIRS} + ${LLCOREHTTP_INCLUDE_DIRS} + ) + +set(llcorehttp_SOURCE_FILES + bufferarray.cpp + bufferstream.cpp + httpcommon.cpp + httpheaders.cpp + httpoptions.cpp + httprequest.cpp + httpresponse.cpp + _httplibcurl.cpp + _httpopcancel.cpp + _httpoperation.cpp + _httpoprequest.cpp + _httpopsetget.cpp + _httpopsetpriority.cpp + _httppolicy.cpp + _httppolicyclass.cpp + _httppolicyglobal.cpp + _httpreplyqueue.cpp + _httprequestqueue.cpp + _httpservice.cpp + _refcounted.cpp + ) + +set(llcorehttp_HEADER_FILES + CMakeLists.txt + + bufferarray.h + bufferstream.h + httpcommon.h + httphandler.h + httpheaders.h + httpoptions.h + httprequest.h + httpresponse.h + _httpinternal.h + _httplibcurl.h + _httpopcancel.h + _httpoperation.h + _httpoprequest.h + _httpopsetget.h + _httpopsetpriority.h + _httppolicy.h + _httppolicyclass.h + _httppolicyglobal.h + _httpreadyqueue.h + _httpreplyqueue.h + _httprequestqueue.h + _httpservice.h + _mutex.h + _refcounted.h + _thread.h + ) + +set_source_files_properties(${llcorehttp_HEADER_FILES} + PROPERTIES HEADER_FILE_ONLY TRUE) +if (DARWIN OR LINUX) + # Boost headers define unused members in condition_variable so... + set_source_files_properties(${llcorehttp_SOURCE_FILES} + PROPERTIES COMPILE_FLAGS -Wno-unused-variable) +endif (DARWIN OR LINUX) + +list(APPEND llcorehttp_SOURCE_FILES ${llcorehttp_HEADER_FILES}) + +add_library (llcorehttp ${llcorehttp_SOURCE_FILES}) +target_link_libraries( + llcorehttp + ${CURL_LIBRARIES} + ${CARES_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${CRYPTO_LIBRARIES} + ${BOOST_THREAD_LIBRARY} + ) + +# tests +if (LL_TESTS) + SET(llcorehttp_TEST_SOURCE_FILES + tests/test_allocator.cpp + ) + + set(llcorehttp_TEST_HEADER_FILS + tests/test_httpstatus.hpp + tests/test_refcounted.hpp + tests/test_httpoperation.hpp + tests/test_httprequest.hpp + tests/test_httprequestqueue.hpp + tests/test_httpheaders.hpp + tests/test_bufferarray.hpp + tests/test_bufferstream.hpp + ) + + set_source_files_properties(${llcorehttp_TEST_HEADER_FILES} + PROPERTIES HEADER_FILE_ONLY TRUE) + + list(APPEND llcorehttp_TEST_SOURCE_FILES ${llcorehttp_TEST_HEADER_FILES}) + + # LL_ADD_PROJECT_UNIT_TESTS(llcorehttp "${llcorehttp_TEST_SOURCE_FILES}") + + # set(TEST_DEBUG on) + set(test_libs + ${LLCOREHTTP_LIBRARIES} + ${WINDOWS_LIBRARIES} + ${LLMESSAGE_LIBRARIES} + ${LLCOMMON_LIBRARIES} + ${GOOGLEMOCK_LIBRARIES} + ${CURL_LIBRARIES} + ${CARES_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${CRYPTO_LIBRARIES} + ${BOOST_SYSTEM_LIBRARY} + ${BOOST_THREAD_LIBRARY} + ) + + LL_ADD_INTEGRATION_TEST(llcorehttp + "${llcorehttp_TEST_SOURCE_FILES}" + "${test_libs}" + ${PYTHON_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_llcorehttp_peer.py" + ) + + # + # Example Programs + # + SET(llcorehttp_EXAMPLE_SOURCE_FILES + examples/http_texture_load.cpp + ) + + set(example_libs + ${LLCOREHTTP_LIBRARIES} + ${WINDOWS_LIBRARIES} + ${LLMESSAGE_LIBRARIES} + ${LLCOMMON_LIBRARIES} + ${GOOGLEMOCK_LIBRARIES} + ${CURL_LIBRARIES} + ${CARES_LIBRARIES} + ${OPENSSL_LIBRARIES} + ${CRYPTO_LIBRARIES} + ${BOOST_SYSTEM_LIBRARY} + ${BOOST_THREAD_LIBRARY} + ) + + add_executable(http_texture_load + ${llcorehttp_EXAMPLE_SOURCE_FILES} + ) + set_target_properties(http_texture_load + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${EXE_STAGING_DIR}" + ) + + if (WINDOWS) + # The following come from LLAddBuildTest.cmake's INTEGRATION_TEST_xxxx target. + set_target_properties(http_texture_load + PROPERTIES + LINK_FLAGS "/debug /NODEFAULTLIB:LIBCMT /SUBSYSTEM:CONSOLE ${TCMALLOC_LINK_FLAGS}" + LINK_FLAGS_DEBUG "/NODEFAULTLIB:\"LIBCMT;LIBCMTD;MSVCRT\" /INCREMENTAL:NO" + LINK_FLAGS_RELEASE "" + ) + endif (WINDOWS) + + target_link_libraries(http_texture_load ${example_libs}) + +endif (LL_TESTS) + diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden new file mode 100644 index 0000000000..c3aaa9158d --- /dev/null +++ b/indra/llcorehttp/README.Linden @@ -0,0 +1,679 @@ + + + +1. HTTP Fetching in 15 Minutes + + Let's start with a trivial working example. You'll need a throwaway + build of the viewer. And we'll use indra/newview/llappviewer.cpp as + the host module for these hacks. + + First, add some headers: + + + #include "httpcommon.h" + #include "httprequest.h" + #include "httphandler.h" + + + You'll need to derive a class from HttpHandler (not HttpHandle). + This is used to deliver notifications of HTTP completion to your + code. Place it near the top, before LLDeferredTaskList, say: + + + class MyHandler : public LLCore::HttpHandler + { + public: + MyHandler() + : LLCore::HttpHandler() + {} + + virtual void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * /* response */) + { + LL_INFOS("Hack") << "It is happening again." << LL_ENDL; + + delete this; // Last statement + } + }; + + + Add some statics up there as well: + + + // Our request object. Allocate during initialiation. + static LLCore::HttpRequest * my_request(NULL); + + // The policy class for HTTP traffic. + // Use HttpRequest::DEFAULT_POLICY_ID, but DO NOT SHIP WITH THIS VALUE!! + static LLCore::HttpRequest::policy_t my_policy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + + // Priority for HTTP requests. Use 0U. + static LLCore::HttpRequest::priority_t my_priority(0U); + + + In LLAppViewer::init() after mAppCoreHttp.init(), create a request object: + + + my_request = new LLCore::HttpRequest(); + + + In LLAppViewer::mainLoop(), just before entering the while loop, + we'll kick off one HTTP request: + + + // Construct a handler object (we'll use the heap this time): + MyHandler * my_handler = new MyHandler; + + // Issue a GET request to 'http://www.example.com/' kicking off + // all the I/O, retry logic, etc. + LLCore::HttpHandle handle; + handle = my_request->requestGet(my_policy, + my_priority, + "http://www.example.com/", + NULL, + NULL, + my_handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS("Hack") << "Failed to launch HTTP request. Try again." + << LL_ENDL; + } + + + Finally, arrange to periodically call update() on the request object + to find out when the request completes. This will be done by + calling the onCompleted() method with status information and + response data from the HTTP operation. Add this to the + LLAppViewer::idle() method after the ping: + + + my_request->update(0); + + + That's it. Build it, run it and watch the log file. You should get + the "It is happening again." message indicating that the HTTP + operation completed in some manner. + + +2. What Does All That Mean + + MyHandler/HttpHandler. This class replaces the Responder-style in + legacy code. One method is currently defined. It is used for all + request completions, successful or failed: + + + void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * /* response */); + + + The onCompleted() method is invoked as a callback during calls to + HttpRequest::update(). All I/O is completed asynchronously in + another thread. But notifications are polled by calling update() + and invoking a handler for completed requests. + + In this example, the invocation also deletes the handler (which is + never referenced by the llcorehttp code again). But other + allocation models are possible including handlers shared by many + requests, stack-based handlers and handlers mixed in with other, + unrelated classes. + + LLCore::HttpRequest(). Instances of this class are used to request + all major functions of the library. Initialization, starting + requests, delivering final notification of completion and various + utility operations are all done via instances. There is one very + important rule for instances: + + Request objects may NOT be shared between threads. + + my_priority. The APIs support the idea of priority ordering of + requests but it hasn't been implemented and the hope is that this + will become useless and removed from the interface. Use 0U except + as noted. + + my_policy. This is an important one. This library attempts to + manage TCP connection usage more rigorously than in the past. This + is done by issuing requests to a queue that has various settable + properties. These establish connection usage for the queue as well + as how queues compete with one another. (This is patterned after + class-based queueing used in various networking stacks.) Several + classes are pre-defined. Deciding when to use an existing class and + when to create a new one will determine what kind of experience + users have. We'll pick up this question in detail below. + + requestGet(). Issues an ordinary HTTP GET request to a given URL + and associating the request with a policy class, a priority and an + response handler. Two additional arguments, not used here, allow + for additional headers on the request and for per-request options. + If successful, the call returns a handle whose value is other than + LLCORE_HTTP_HANDLE_INVALID. The HTTP operation is then performed + asynchronously by another thread without any additional work by the + caller. If the handle returned is invalid, you can get the status + code by calling my_request->getStatus(). + + update(). To get notification that the request has completed, a + call to update() will invoke onCompleted() methods. + + +3. Refinements, Necessary and Otherwise + + MyHandler::onCompleted(). You'll want to do something useful with + your response. Distinguish errors from successes and getting the + response body back in some form. + + Add a new header: + + + #include "bufferarray.h" + + + Replace the existing MyHandler::onCompleted() definition with: + + + virtual void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * response) + { + LLCore::HttpStatus status = response->getStatus(); + if (status) + { + // Successful request. Try to fetch the data + LLCore::BufferArray * data = response->getBody(); + + if (data && data->size()) + { + // There's some data. A BufferArray is a linked list + // of buckets. We'll create a linear buffer and copy + // the data into it. + size_t data_len = data->size(); + char * data_blob = new char [data_len + 1]; + data->read(0, data_blob, data_len); + data_blob[data_len] = '\0'; + + // Process the data now in NUL-terminated string. + // Needs more scrubbing but this will do. + LL_INFOS("Hack") << "Received: " << data_blob << LL_ENDL; + + // Free the temporary data + delete [] data_blob; + } + } + else + { + // Something went wrong. Translate the status to + // a meaningful message. + LL_WARNS("Hack") << "HTTP GET failed. Status: " + << status.toTerseString() + << ", Reason: " << status.toString() + << LL_ENDL; + } + + delete this; // Last statement + } + + + HttpHeaders. The header file "httprequest.h" documents the expected + important headers that will go out with the request. You can add to + these by including an HttpHeaders object with the requestGet() call. + These are typically setup once as part of init rather than + dynamically created. + + Add another header: + + + #include "httpheaders.h" + + + In LLAppViewer::mainLoop(), add this alongside the allocation of + my_handler: + + + // Additional headers for all requests + LLCore::HttpHeaders * my_headers = new LLCore::HttpHeaders(); + my_headers->append("Accept", "text/html, application/llsd+xml"); + + + HttpOptions. Options are similar and include a mix of value types. + One interesting per-request option is the trace setting. This + enables various debug-type messages in the log file that show the + progress of the request through the library. It takes values from + zero to three with higher values giving more verbose logging. We'll + use '2' and this will also give us a chance to verify that + HttpHeaders works as expected. + + Same as above, a new header: + + + #include "httpoptions.h" + + + And in LLAppView::mainLoop(): + + + // Special options for requests + LLCore::HttpOptions * my_options = new LLCore::HttpOptions(); + my_options->setTrace(2); + + + Now let's put that all together into a more complete requesting + sequence. Replace the existing invocation of requestGet() with this + slightly more elaborate block: + + + LLCore::HttpHandle handle; + handle = my_request->requestGet(my_policy, + my_priority, + "http://www.example.com/", + my_options, + my_headers, + my_handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LLCore::HttpStatus status = my_request->getStatus(); + + LL_WARNS("Hack") << "Failed to request HTTP GET. Status: " + << status.toTerseString() + << ", Reason: " << status.toString() + << LL_ENDL; + + delete my_handler; // No longer needed. + my_handler = NULL; + } + + + Build, run and examine the log file. You'll get some new data with + this run. First, you should get the www.example.com home page + content: + + +---------------------------------------------------------------------------- +2013-09-17T20:26:51Z INFO: MyHandler::onCompleted: Received: <!doctype html> +<html> +<head> + <title>Example Domain</title> + + <meta charset="utf-8" /> + <meta http-equiv="Content-type" content="text/html; charset=utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <style type="text/css"> + body { + background-color: #f0f0f2; + margin: 0; + padding: 0; + font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; + + } + div { + width: 600px; + margin: 5em auto; + padding: 50px; + background-color: #fff; + border-radius: 1em; + } + a:link, a:visited { + color: #38488f; + text-decoration: none; + } + @media (max-width: 700px) { + body { + background-color: #fff; + } + div { + width: auto; + margin: 0 auto; + border-radius: 0; + padding: 1em; + } + } + </style> +</head> + +<body> +<div> + <h1>Example Domain</h1> + <p>This domain is established to be used for illustrative examples in documents. You may use this + domain in examples without prior coordination or asking for permission.</p> + <p><a href="http://www.iana.org/domains/example">More information...</a></p> +</div> +</body> +</html> +---------------------------------------------------------------------------- + + + You'll also get a detailed trace of the HTTP operation itself. Note + the HEADEROUT line which shows the additional header added to the + request. + + +---------------------------------------------------------------------------- +HttpService::processRequestQueue: TRACE, FromRequestQueue, Handle: 086D3148 +HttpLibcurl::addOp: TRACE, ToActiveQueue, Handle: 086D3148, Actives: 0, Readies: 0 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: About to connect() to www.example.com port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Trying 93.184.216.119... +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADEROUT, Data: GET / HTTP/1.1 Host: www.example.com Accept-Encoding: deflate, gzip Connection: keep-alive Keep-alive: 300 Accept: text/html, application/llsd+xml +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: HTTP/1.1 200 OK +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Accept-Ranges: bytes +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Cache-Control: max-age=604800 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Type: text/html +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Date: Tue, 17 Sep 2013 20:26:56 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Etag: "3012602696" +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Expires: Tue, 24 Sep 2013 20:26:56 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Server: ECS (ewr/1590) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: X-Cache: HIT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: x-ec-custom-error: 1 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Length: 1270 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: DATAIN, Data: 256 Bytes +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connection #0 to host www.example.com left intact +HttpLibcurl::completeRequest: TRACE, RequestComplete, Handle: 086D3148, Status: Http_200 +HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 +---------------------------------------------------------------------------- + + +4. What Does All That Mean, Part 2 + + HttpStatus. The HttpStatus object encodes errors from libcurl, the + library itself and HTTP status values. It does this to avoid + collapsing all non-HTTP error into a single '499' HTTP status and to + make errors distinct. + + To aid programming, the usual bool conversions are available so that + you can write 'if (status)' and the expected thing will happen + whether it's an HTTP, libcurl or library error. There's also + provision to override the treatment of HTTP errors (making 404 a + success, say). + + Share data, don't copy it. The library was started with the goal of + avoiding data copies as much as possible. Instead, read-only data + sharing across threads with atomic reference counts is used for a + number of data types. These currently are: + + * BufferArray. Linked list of data blocks/HTTP bodies. + * HttpHeaders. Shared headers for both requests and responses. + * HttpOptions. Request-only data modifying HTTP behavior. + * HttpResponse. HTTP response description given to onCompleted. + + Using objects of these types requires a few rules: + + * Constructor always gives a reference to caller. + * References are dropped with release() not delete. + * Additional references may be taken out with addRef(). + * Unless otherwise stated, once an object is shared with another + thread it should be treated as read-only. There's no + synchronization on the objects themselves. + + HttpResponse. You'll encounter this mainly in onCompleted() methods. + Commonly-used interfaces on this object: + + * getStatus() to return the final status of the request. + * getBody() to retrieve the response body which may be NULL or + zero-length. + * getContentType() to return the value of the 'Content-Type' + header or an empty string if none was sent. + + This is a reference-counted object so you can call addRef() on it + and hold onto the response for an arbitrary time. But you'll + usually just call a few methods and return from onCompleted() whose + caller will release the object. + + BufferArray. The core data representation for request and response + bodies. In HTTP responses, it's fetched with the getBody() method + and may be NULL or non-NULL with zero length. All successful data + handling should check both conditions before attempting to fetch + data from the object. Data access model uses simple read/write + semantics: + + * append() + * size() + * read() + * write() + + (There is a more sophisticated stream adapter that extends these + methods and will be covered below.) So, one way to retrieve data + from a request is as follows: + + + LLCore::BufferArray * data = response->getBody(); + if (data && data->size()) + { + size_t data_len = data->size(); + char * data_blob = new char [data_len + 1]; + data->read(0, data_blob, data_len); + + + HttpOptions and HttpResponse. Really just simple containers of POD + and std::string pairs. But reference counted and the rule about not + modifying after sharing must be followed. You'll have the urge to + change options dynamically at some point. And you'll try to do that + by just writing new values to the shared object. And in tests + everything will appear to work. Then you ship and people in the + real world start hitting read/write races in strings and then crash. + Don't be lazy. + + HttpHandle. Uniquely identifies a request and can be used to + identify it in an onCompleted() method or cancel it if it's still + queued. But as soon as a request's onCompleted() invocation + returns, the handle becomes invalid and may be reused immediately + for new requests. Don't hold on to handles after notification. + + +5. And Still More Refinements + + (Note: The following refinements are just code fragments. They + don't directly fit into the working example above. But they + demonstrate several idioms you'll want to copy.) + + LLSD, std::streambuf, std::iostream. The read(), write() and + append() methods may be adequate for your purposes. But we use a + lot of LLSD. Its interfaces aren't particularly compatible with + BufferArray. And so two adapters are available to give + stream-like behaviors: BufferArrayStreamBuf and BufferArrayStream, + which implement the std::streambuf and std::iostream interfaces, + respectively. + + A std::streambuf interface isn't something you'll want to use + directly. Instead, you'll use the much friendlier std::iostream + interface found in BufferArrayStream. This adapter gives you all + the '>>' and '<<' operators you'll want as well as working + directly with the LLSD conversion operators. + + Some new headers: + + + #include "bufferstream.h" + #include "llsdserialize.h" + + + And an updated fragment based on onCompleted() above: + + + // Successful request. Try to fetch the data + LLCore::BufferArray * data = response->getBody(); + LLSD resp_llsd; + + if (data && data->size()) + { + // There's some data and we expect this to be + // LLSD. Checking of content type and validation + // during parsing would be admirable additions. + // But we'll forgo that now. + LLCore::BufferArrayStream data_stream(data); + LLSDSerialize::fromXML(resp_llsd, data_stream); + } + LL_INFOS("Hack") << "LLSD Received: " << resp_llsd << LL_ENDL; + } + else + { + + + Converting an LLSD object into an XML stream stored in a + BufferArray is just the reverse of the above: + + + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + LLSD src_llsd; + src_llsd["foo"] = "bar"; + + LLSDSerialize::toXML(src_llsd, data_stream); + + // 'data' now contains an XML payload and can be sent + // to a web service using the requestPut() or requestPost() + // methods. + ... requestPost(...); + + // And don't forget to release the BufferArray. + data->release(); + data = NULL; + + + There are now helper functions in llmessage/llcorehttputil.h to + assist with LLSD usage. requestPostWithLLSD(...) provides a + requestPost()-like interface that takes an LLSD object rather than + a BufferArray. And responseToLLSD(...) attempts to convert a + BufferArray received from a server into an LLSD object. You can + find examples in llmeshrepository.cpp, llinventorymodel.cpp, + llinventorymodelbackgroundfetch.cpp and lltexturefetch.cpp. + + LLSD will often go hand-in-hand with BufferArray and data + transport. But you can also do all the streaming I/O you'd expect + of a std::iostream object: + + + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + data_stream << "Hello, World!" << 29.4 << '\n'; + std::string str; + data_stream >> str; + std::cout << str << std::endl; + + data->release(); + // Actual delete will occur when 'data_stream' + // falls out of scope and is destructed. + + + Scoping objects and cleaning up. The examples haven't bothered + with cleanup of objects that are no longer needed. Instead, most + objects have been allocated as if they were global and eternal. + You'll put the objects in more appropriate feature objects and + clean them up as a group. Here's a checklist for actions you may + need to take on cleanup: + + * Call delete on: + o HttpHandlers created on the heap + o HttpRequest objects + * Call release() on: + o BufferArray objects + o HttpHeaders objects + o HttpOptions objects + o HttpResponse objects + + On program exit, as threads wind down, the library continues to + operate safely. Threads don't interact via the library and even + dangling references to HttpHandler objects are safe. If you don't + call HttpRequest::update(), handler references are never + dereferenced. + + You can take a more thorough approach to wind-down. Keep a list + of HttpHandles (not HttpHandlers) of outstanding requests. For + each of these, call HttpRequest::requestCancel() to cancel the + operation. (Don't add the cancel requests' handled to the list.) + This will cancel the outstanding requests that haven't completed. + Canceled or completed, all requests will queue notifications. You + can now cycle calling update() discarding responses. Continue + until all requests notify or a few seconds have passed. + + Global startup and shutdown is handled in the viewer. But you can + learn about it in the code or in the documentation in the headers. + + +6. Choosing a Policy Class + + Now it's time to get rid of the default policy class. Take a look + at the policy class definitions in newview/llappcorehttp.h. + Ideally, you'll find one that's compatible with what you're doing. + Some of the compatibility guidelines are: + + * Destination: Pair of host and port. Mixing requests with + different destinations may cause more connection setup and tear + down. + + * Method: http or https. Usually moot given destination. But + mixing these may also cause connection churn. + + * Transfer size: If you're moving 100MB at a time and you make your + requests to the same policy class as a lot of small, fast event + information that fast traffic is going to get stuck behind you + and someone's experience is going to be miserable. + + * Long poll requests: These are long-lived, must- do operations. + They have a special home called AP_LONG_POLL. + + * Concurrency: High concurrency (5 or more) and large transfer + sizes are incompatible. Another head-of-the-line problem. High + concurrency is tolerated when it's desired to get maximal + throughput. Mesh and texture downloads, for example. + + * Pipelined: If your requests are not idempotent, stay away from + anything marked 'soon' or 'yes'. Hidden retries may be a + problem for you. For now, would also recommend keeping PUT and + POST requests out of classes that may be pipelined. Support for + that is still a bit new. + + If you haven't found a compatible match, you can either create a + new class (llappcorehttp.*) or just use AP_DEFAULT, the catchall + class when all else fails. Inventory query operations might be a + candidate for a new class that supported pipelining on https:. + Same with display name lookups and other bursty-at-login + operations. For other things, AP_DEFAULT will do what it can and + will, in some way or another, tolerate any usage. Whether the + users' experiences are good are for you to determine. + + +7. FAQ + + Q1. What do these policy classes achieve? + + A1. Previously, HTTP-using code in the viewer was written as if + it were some isolated, local operation that didn't have to + consider resources, contention or impact on services and the + larger environment. The result was an application with on the + order of 100 HTTP launch points in its codebase that could create + dozens or even 100's of TCP connections zeroing in on grid + services and disrupting networking equipment, web services and + innocent users. The use of policy classes (modeled on + http://en.wikipedia.org/wiki/Class-based_queueing) is a means to + restrict connection concurrency, good and necessary in itself. In + turn, that reduces demands on an expensive resource (connection + setup and concurrency) which relieves strain on network points. + That enables connection keepalive and opportunites for true + improvements in throughput and user experience. + + Another aspect of the classes is that they give some control over + how competing demands for the network will be apportioned. If + mesh fetches, texture fetches and inventory queries are all being + made at once, the relative weights of their classes' concurrency + limits established that apportioning. We now have an opportunity + to balance the entire viewer system. + + Q2. How's that data sharing with refcounts working for you? + + A2. Meh. It does reduce memory churn and the frequency at which + free blocks must be moved between threads. But it's also a design + for static configuration and dynamic reconfiguration (not + requiring a restart) is favored. Creating new options for every + request isn't too bad, it a sequence of "new, fill, request, + release" for each requested operation. That in contrast to doing + the "new, fill, release" at startup. The bad comes in getting at + the source data. One rule in this work was "no new thread + problems." And one source for those is pulling setting values out + of gSettings in threads. None of that is thread safe though we + tend to get away with it. + + Q3. What needs to be done? + + A3. There's a To-Do list in _httpinternal.h. It has both large + and small projects here if someone would like to try changes. diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h new file mode 100755 index 0000000000..a2a60ca056 --- /dev/null +++ b/indra/llcorehttp/_httpinternal.h @@ -0,0 +1,167 @@ +/** + * @file _httpinternal.h + * @brief Implementation constants and magic numbers + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_INTERNAL_H_ +#define _LLCORE_HTTP_INTERNAL_H_ + + +// If you find this included in a public interface header, +// something wrong is probably happening. + + +// -------------------------------------------------------------------- +// General library to-do list +// +// - Implement policy classes. Structure is mostly there just didn't +// need it for the first consumer. [Classes are there. More +// advanced features, like borrowing, aren't there yet.] +// - Consider Removing 'priority' from the request interface. Its use +// in an always active class can lead to starvation of low-priority +// requests. Requires coodination of priority values across all +// components that share a class. Changing priority across threads +// is slightly expensive (relative to gain) and hasn't been completely +// implemented. And the major user of priority, texture fetches, +// may not really need it. +// - Set/get for global policy and policy classes is clumsy. Rework +// it heading in a direction that allows for more dynamic behavior. +// [Mostly fixed] +// - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the +// pedantic. +// - Update downloader and other long-duration services are going to +// need a progress notification. Initial idea is to introduce a +// 'repeating request' which can piggyback on another request and +// persist until canceled or carrier completes. Current queue +// structures allow an HttpOperation object to be enqueued +// repeatedly, so... +// - Investigate making c-ares' re-implementation of a resolver library +// more resilient or more intelligent on Mac. Part of the DNS failure +// lies in here. The mechanism also looks a little less dynamic +// than needed in an environments where networking is changing. +// - Global optimizations: 'borrowing' connections from other classes, +// HTTP pipelining. +// - Dynamic/control system stuff: detect problems and self-adjust. +// This won't help in the face of the router problems we've looked +// at, however. Detect starvation due to UDP activity and provide +// feedback to it. +// - Change the transfer timeout scheme. We're less interested in +// absolute time, in most cases, than in continuous progress. +// - Many of the policy class settings are currently applied to the +// entire class. Some, like connection limits, would be better +// applied to each destination target making multiple targets +// independent. +// +// Integration to-do list +// - LLTextureFetch still needs a major refactor. The use of +// LLQueuedThread makes it hard to inspect workers and do the +// resource waiting we're now doing. Rebuild along simpler lines +// some of which are suggested in new commentary at the top of +// the main source file. +// - Expand areas of usage eventually leading to the removal of LLCurl. +// Rough order of expansion: +// . Avatar names +// . Group membership lists +// . Caps access in general +// . 'The rest' +// - Adapt texture cache, image decode and other image consumers to +// the BufferArray model to reduce data copying. Alternatively, +// adapt this library to something else. +// +// -------------------------------------------------------------------- + + +// If '1', internal ready queues will not order ready +// requests by priority, instead it's first-come-first-served. +// Reprioritization requests have the side-effect of then +// putting the modified request at the back of the ready queue. + +#define LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY 1 + + +namespace LLCore +{ + +// Maxium number of policy classes that can be defined. +// *TODO: Currently limited to the default class + 1, extend. +const int HTTP_POLICY_CLASS_LIMIT = 8; + +// Debug/informational tracing. Used both +// as a global option and in per-request traces. +const int HTTP_TRACE_OFF = 0; +const int HTTP_TRACE_LOW = 1; +const int HTTP_TRACE_CURL_HEADERS = 2; +const int HTTP_TRACE_CURL_BODIES = 3; + +const int HTTP_TRACE_MIN = HTTP_TRACE_OFF; +const int HTTP_TRACE_MAX = HTTP_TRACE_CURL_BODIES; + +// Request retry limits +// +// At a minimum, retries need to extend past any throttling +// window we're expecting from central services. In the case +// of Linden services running through the caps routers, there's +// a five-second or so window for throttling with some spillover. +// We want to span a few windows to allow transport to slow +// after onset of the throttles and then recover without a final +// failure. Other systems may need other constants. +const int HTTP_RETRY_COUNT_DEFAULT = 8; +const int HTTP_RETRY_COUNT_MIN = 0; +const int HTTP_RETRY_COUNT_MAX = 100; + +const int HTTP_REDIRECTS_DEFAULT = 10; + +// Timeout value used for both connect and protocol exchange. +// Retries and time-on-queue are not included and aren't +// accounted for. +const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; +const long HTTP_REQUEST_TIMEOUT_MIN = 0L; +const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; + +// Limits on connection counts +const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; +const int HTTP_CONNECTION_LIMIT_MIN = 1; +const int HTTP_CONNECTION_LIMIT_MAX = 256; + +// Pipelining limits +const long HTTP_PIPELINING_DEFAULT = 0L; +const long HTTP_PIPELINING_MAX = 20L; + +// Miscellaneous defaults +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; +const long HTTP_THROTTLE_RATE_DEFAULT = 0L; + +// Tuning parameters + +// Time worker thread sleeps after a pass through the +// request, ready and active queues. +const int HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS = 2; + +// Block allocation size (a tuning parameter) is found +// in bufferarray.h. + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_INTERNAL_H_ diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp new file mode 100755 index 0000000000..81b44ab90b --- /dev/null +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -0,0 +1,601 @@ +/** + * @file _httplibcurl.cpp + * @brief Internal definitions of the Http libcurl thread + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httplibcurl.h" + +#include "httpheaders.h" +#include "bufferarray.h" +#include "_httpoprequest.h" +#include "_httppolicy.h" + +#include "llhttpconstants.h" + +namespace +{ + +// Error testing and reporting for libcurl status codes +void check_curl_multi_code(CURLMcode code); +void check_curl_multi_code(CURLMcode code, int curl_setopt_option); + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + +namespace LLCore +{ + + +HttpLibcurl::HttpLibcurl(HttpService * service) + : mService(service), + mHandleCache(), + mPolicyCount(0), + mMultiHandles(NULL), + mActiveHandles(NULL), + mDirtyPolicy(NULL) +{} + + +HttpLibcurl::~HttpLibcurl() +{ + shutdown(); + + mService = NULL; +} + + +void HttpLibcurl::shutdown() +{ + while (! mActiveOps.empty()) + { + HttpOpRequest * op(* mActiveOps.begin()); + mActiveOps.erase(mActiveOps.begin()); + + cancelRequest(op); + op->release(); + } + + if (mMultiHandles) + { + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) + { + if (mMultiHandles[policy_class]) + { + curl_multi_cleanup(mMultiHandles[policy_class]); + mMultiHandles[policy_class] = 0; + } + } + + delete [] mMultiHandles; + mMultiHandles = NULL; + + delete [] mActiveHandles; + mActiveHandles = NULL; + + delete [] mDirtyPolicy; + mDirtyPolicy = NULL; + } + + mPolicyCount = 0; +} + + +void HttpLibcurl::start(int policy_count) +{ + llassert_always(policy_count <= HTTP_POLICY_CLASS_LIMIT); + llassert_always(! mMultiHandles); // One-time call only + + mPolicyCount = policy_count; + mMultiHandles = new CURLM * [mPolicyCount]; + mActiveHandles = new int [mPolicyCount]; + mDirtyPolicy = new bool [mPolicyCount]; + + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) + { + if (NULL == (mMultiHandles[policy_class] = curl_multi_init())) + { + LL_ERRS(LOG_CORE) << "Failed to allocate multi handle in libcurl." + << LL_ENDL; + } + mActiveHandles[policy_class] = 0; + mDirtyPolicy[policy_class] = false; + policyUpdated(policy_class); + } +} + + +// Give libcurl some cycles, invoke it's callbacks, process +// completed requests finalizing or issuing retries as needed. +// +// If active list goes empty *and* we didn't queue any +// requests for retry, we return a request for a hard +// sleep otherwise ask for a normal polling interval. +HttpService::ELoopSpeed HttpLibcurl::processTransport() +{ + HttpService::ELoopSpeed ret(HttpService::REQUEST_SLEEP); + + // Give libcurl some cycles to do I/O & callbacks + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) + { + if (! mMultiHandles[policy_class]) + { + // No handle, nothing to do. + continue; + } + if (! mActiveHandles[policy_class]) + { + // If we've gone quiet and there's a dirty update, apply it, + // otherwise we're done. + if (mDirtyPolicy[policy_class]) + { + policyUpdated(policy_class); + } + continue; + } + + int running(0); + CURLMcode status(CURLM_CALL_MULTI_PERFORM); + do + { + running = 0; + status = curl_multi_perform(mMultiHandles[policy_class], &running); + } + while (0 != running && CURLM_CALL_MULTI_PERFORM == status); + + // Run completion on anything done + CURLMsg * msg(NULL); + int msgs_in_queue(0); + while ((msg = curl_multi_info_read(mMultiHandles[policy_class], &msgs_in_queue))) + { + if (CURLMSG_DONE == msg->msg) + { + CURL * handle(msg->easy_handle); + CURLcode result(msg->data.result); + + completeRequest(mMultiHandles[policy_class], handle, result); + handle = NULL; // No longer valid on return + ret = HttpService::NORMAL; // If anything completes, we may have a free slot. + // Turning around quickly reduces connection gap by 7-10mS. + } + else if (CURLMSG_NONE == msg->msg) + { + // Ignore this... it shouldn't mean anything. + ; + } + else + { + LL_WARNS_ONCE(LOG_CORE) << "Unexpected message from libcurl. Msg code: " + << msg->msg + << LL_ENDL; + } + msgs_in_queue = 0; + } + } + + if (! mActiveOps.empty()) + { + ret = HttpService::NORMAL; + } + return ret; +} + + +// Caller has provided us with a ref count on op. +void HttpLibcurl::addOp(HttpOpRequest * op) +{ + llassert_always(op->mReqPolicy < mPolicyCount); + llassert_always(mMultiHandles[op->mReqPolicy] != NULL); + + // Create standard handle + if (! op->prepareRequest(mService)) + { + // Couldn't issue request, fail with notification + // *TODO: Need failure path + return; + } + + // Make the request live + CURLMcode code; + code = curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + if (CURLM_OK != code) + { + // *TODO: Better cleanup and recovery but not much we can do here. + check_curl_multi_code(code); + return; + } + op->mCurlActive = true; + mActiveOps.insert(op); + ++mActiveHandles[op->mReqPolicy]; + + if (op->mTracing > HTTP_TRACE_OFF) + { + HttpPolicy & policy(mService->getPolicy()); + + LL_INFOS(LOG_CORE) << "TRACE, ToActiveQueue, Handle: " + << static_cast<HttpHandle>(op) + << ", Actives: " << mActiveOps.size() + << ", Readies: " << policy.getReadyCount(op->mReqPolicy) + << LL_ENDL; + } +} + + +// Implements the transport part of any cancel operation. +// See if the handle is an active operation and if so, +// use the more complicated transport-based cancelation +// method to kill the request. +bool HttpLibcurl::cancel(HttpHandle handle) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(handle)); + active_set_t::iterator it(mActiveOps.find(op)); + if (mActiveOps.end() == it) + { + return false; + } + + // Cancel request + cancelRequest(op); + + // Drop references + mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; + op->release(); + + return true; +} + + +// *NOTE: cancelRequest logic parallels completeRequest logic. +// Keep them synchronized as necessary. Caller is expected to +// remove the op from the active list and release the op *after* +// calling this method. It must be called first to deliver the +// op to the reply queue with refcount intact. +void HttpLibcurl::cancelRequest(HttpOpRequest * op) +{ + // Deactivate request + op->mCurlActive = false; + + // Detach from multi and recycle handle + curl_multi_remove_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); + mHandleCache.freeHandle(op->mCurlHandle); + op->mCurlHandle = NULL; + + // Tracing + if (op->mTracing > HTTP_TRACE_OFF) + { + LL_INFOS(LOG_CORE) << "TRACE, RequestCanceled, Handle: " + << static_cast<HttpHandle>(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; + } + + // Cancel op and deliver for notification + op->cancel(); +} + + +// *NOTE: cancelRequest logic parallels completeRequest logic. +// Keep them synchronized as necessary. +bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status) +{ + HttpOpRequest * op(NULL); + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); + + if (handle != op->mCurlHandle || ! op->mCurlActive) + { + LL_WARNS(LOG_CORE) << "libcurl handle and HttpOpRequest handle in disagreement or inactive request." + << " Handle: " << static_cast<HttpHandle>(handle) + << LL_ENDL; + return false; + } + + active_set_t::iterator it(mActiveOps.find(op)); + if (mActiveOps.end() == it) + { + LL_WARNS(LOG_CORE) << "libcurl completion for request not on active list. Continuing." + << " Handle: " << static_cast<HttpHandle>(handle) + << LL_ENDL; + return false; + } + + // Deactivate request + mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; + op->mCurlActive = false; + + // Set final status of request if it hasn't failed by other mechanisms yet + if (op->mStatus) + { + op->mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, status); + } + if (op->mStatus) + { + int http_status(HTTP_OK); + + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); + if (http_status >= 100 && http_status <= 999) + { + char * cont_type(NULL); + curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &cont_type); + if (cont_type) + { + op->mReplyConType = cont_type; + } + op->mStatus = HttpStatus(http_status); + } + else + { + LL_WARNS(LOG_CORE) << "Invalid HTTP response code (" + << http_status << ") received from server." + << LL_ENDL; + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + } + } + + // Detach from multi and recycle handle + curl_multi_remove_handle(multi_handle, handle); + mHandleCache.freeHandle(op->mCurlHandle); + op->mCurlHandle = NULL; + + // Tracing + if (op->mTracing > HTTP_TRACE_OFF) + { + LL_INFOS(LOG_CORE) << "TRACE, RequestComplete, Handle: " + << static_cast<HttpHandle>(op) + << ", Status: " << op->mStatus.toTerseString() + << LL_ENDL; + } + + // Dispatch to next stage + HttpPolicy & policy(mService->getPolicy()); + bool still_active(policy.stageAfterCompletion(op)); + + return still_active; +} + + +int HttpLibcurl::getActiveCount() const +{ + return mActiveOps.size(); +} + + +int HttpLibcurl::getActiveCountInClass(int policy_class) const +{ + llassert_always(policy_class < mPolicyCount); + + return mActiveHandles ? mActiveHandles[policy_class] : 0; +} + +void HttpLibcurl::policyUpdated(int policy_class) +{ + if (policy_class < 0 || policy_class >= mPolicyCount || ! mMultiHandles) + { + return; + } + + HttpPolicy & policy(mService->getPolicy()); + + if (! mActiveHandles[policy_class]) + { + // Clear to set options. As of libcurl 7.37.0, if a pipelining + // multi handle has active requests and you try to set the + // multi handle to non-pipelining, the library gets very angry + // and goes off the rails corrupting memory. A clue that you're + // about to crash is that you'll get a missing server response + // error (curl code 9). So, if options are to be set, we let + // the multi handle run out of requests, then set options, and + // re-enable request processing. + // + // All of this stall mechanism exists for this reason. If + // libcurl becomes more resilient later, it should be possible + // to remove all of this. The connection limit settings are fine, + // it's just that pipelined-to-non-pipelined transition that + // is fatal at the moment. + + HttpPolicyClass & options(policy.getClassOptions(policy_class)); + CURLM * multi_handle(mMultiHandles[policy_class]); + CURLMcode code; + + // Enable policy if stalled + policy.stallPolicy(policy_class, false); + mDirtyPolicy[policy_class] = false; + + if (options.mPipelining > 1) + { + // We'll try to do pipelining on this multihandle + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 1L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_PIPELINE_LENGTH, + long(options.mPipelining)); + check_curl_multi_code(code, CURLMOPT_MAX_PIPELINE_LENGTH); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + long(options.mPerHostConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + else + { + code = curl_multi_setopt(multi_handle, + CURLMOPT_PIPELINING, + 0L); + check_curl_multi_code(code, CURLMOPT_PIPELINING); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_HOST_CONNECTIONS, + 0L); + check_curl_multi_code(code, CURLMOPT_MAX_HOST_CONNECTIONS); + code = curl_multi_setopt(multi_handle, + CURLMOPT_MAX_TOTAL_CONNECTIONS, + long(options.mConnectionLimit)); + check_curl_multi_code(code, CURLMOPT_MAX_TOTAL_CONNECTIONS); + } + } + else if (! mDirtyPolicy[policy_class]) + { + // Mark policy dirty and request a stall in the policy. + // When policy goes idle, we'll re-invoke this method + // and perform the change. Don't allow this thread to + // sleep while we're waiting for quiescence, we'll just + // stop processing. + mDirtyPolicy[policy_class] = true; + policy.stallPolicy(policy_class, true); + } +} + +// --------------------------------------- +// HttpLibcurl::HandleCache +// --------------------------------------- + +HttpLibcurl::HandleCache::HandleCache() + : mHandleTemplate(NULL) +{ + mCache.reserve(50); +} + + +HttpLibcurl::HandleCache::~HandleCache() +{ + if (mHandleTemplate) + { + curl_easy_cleanup(mHandleTemplate); + mHandleTemplate = NULL; + } + + for (handle_cache_t::iterator it(mCache.begin()); mCache.end() != it; ++it) + { + curl_easy_cleanup(*it); + } + mCache.clear(); +} + + +CURL * HttpLibcurl::HandleCache::getHandle() +{ + CURL * ret(NULL); + + if (! mCache.empty()) + { + // Fastest path to handle + ret = mCache.back(); + mCache.pop_back(); + } + else if (mHandleTemplate) + { + // Still fast path + ret = curl_easy_duphandle(mHandleTemplate); + } + else + { + // When all else fails + ret = curl_easy_init(); + } + + return ret; +} + + +void HttpLibcurl::HandleCache::freeHandle(CURL * handle) +{ + if (! handle) + { + return; + } + + curl_easy_reset(handle); + if (! mHandleTemplate) + { + // Save the first freed handle as a template. + mHandleTemplate = handle; + } + else + { + // Otherwise add it to the cache + if (mCache.size() >= mCache.capacity()) + { + mCache.reserve(mCache.capacity() + 50); + } + mCache.push_back(handle); + } +} + + +// --------------------------------------- +// Free functions +// --------------------------------------- + + +struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist) +{ + const HttpHeaders::const_iterator end(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); end != it; ++it) + { + static const char sep[] = ": "; + std::string header; + header.reserve((*it).first.size() + (*it).second.size() + sizeof(sep)); + header.append((*it).first); + header.append(sep); + header.append((*it).second); + + slist = curl_slist_append(slist, header.c_str()); + } + return slist; +} + + +} // end namespace LLCore + + +namespace +{ + +void check_curl_multi_code(CURLMcode code, int curl_setopt_option) +{ + if (CURLM_OK != code) + { + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << ", curl_multi_setopt option: " << curl_setopt_option + << LL_ENDL; + } +} + + +void check_curl_multi_code(CURLMcode code) +{ + if (CURLM_OK != code) + { + LL_WARNS(LOG_CORE) << "libcurl multi error detected: " << curl_multi_strerror(code) + << LL_ENDL; + } +} + +} // end anonymous namespace diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h new file mode 100755 index 0000000000..ffc24c63a8 --- /dev/null +++ b/indra/llcorehttp/_httplibcurl.h @@ -0,0 +1,221 @@ +/** + * @file _httplibcurl.h + * @brief Declarations for internal class providing libcurl transport. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_LIBCURL_H_ +#define _LLCORE_HTTP_LIBCURL_H_ + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include <curl/curl.h> +#include <curl/multi.h> + +#include <set> + +#include "httprequest.h" +#include "_httpservice.h" +#include "_httpinternal.h" + + +namespace LLCore +{ + + +class HttpPolicy; +class HttpOpRequest; +class HttpHeaders; + + +/// Implements libcurl-based transport for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. + +class HttpLibcurl +{ +public: + HttpLibcurl(HttpService * service); + virtual ~HttpLibcurl(); + +private: + HttpLibcurl(const HttpLibcurl &); // Not defined + void operator=(const HttpLibcurl &); // Not defined + +public: + /// Give cycles to libcurl to run active requests. Completed + /// operations (successful or failed) will be retried or handed + /// over to the reply queue as final responses. + /// + /// @return Indication of how long this method is + /// willing to wait for next service call. + /// + /// Threading: called by worker thread. + HttpService::ELoopSpeed processTransport(); + + /// Add request to the active list. Caller is expected to have + /// provided us with a reference count on the op to hold the + /// request. (No additional references will be added.) + /// + /// Threading: called by worker thread. + void addOp(HttpOpRequest * op); + + /// One-time call to set the number of policy classes to be + /// serviced and to create the resources for each. Value + /// must agree with HttpPolicy::setPolicies() call. + /// + /// Threading: called by init thread. + void start(int policy_count); + + /// Synchronously stop libcurl operations. All active requests + /// are canceled and removed from libcurl's handling. Easy + /// handles are detached from their multi handles and released. + /// Multi handles are also released. Canceled requests are + /// completed with canceled status and made available on their + /// respective reply queues. + /// + /// Can be restarted with a start() call. + /// + /// Threading: called by worker thread. + void shutdown(); + + /// Return global and per-class counts of active requests. + /// + /// Threading: called by worker thread. + int getActiveCount() const; + int getActiveCountInClass(int policy_class) const; + + /// Attempt to cancel a request identified by handle. + /// + /// Interface shadows HttpService's method. + /// + /// @return True if handle was found and operation canceled. + /// + /// Threading: called by worker thread. + bool cancel(HttpHandle handle); + + /// Informs transport that a particular policy class has had + /// options changed and so should effect any transport state + /// change necessary to effect those changes. Used mainly for + /// initialization and dynamic option setting. + /// + /// Threading: called by worker thread. + void policyUpdated(int policy_class); + + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. Handle will be in curl_easy_reset() + /// condition. + /// + /// Threading: callable by worker thread. + /// + /// Deprecation: Expect this to go away after _httpoprequest is + /// refactored bringing code into this class. + CURL * getHandle() + { + return mHandleCache.getHandle(); + } + +protected: + /// Invoked when libcurl has indicated a request has been processed + /// to completion and we need to move the request to a new state. + bool completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); + + /// Invoked to cancel an active request, mainly during shutdown + /// and destroy. + void cancelRequest(HttpOpRequest * op); + +protected: + typedef std::set<HttpOpRequest *> active_set_t; + + /// Simple request handle cache for libcurl. + /// + /// Handle creation is somewhat slow and chunky in libcurl and there's + /// a pretty good speedup to be had from handle re-use. So, a simple + /// vector is kept of 'freed' handles to be reused as needed. When + /// that is empty, the first freed handle is kept as a template for + /// handle duplication. This is still faster than creation from nothing. + /// And when that fails, we init fresh from curl_easy_init(). + /// + /// Handles allocated with getHandle() may be freed with either + /// freeHandle() or curl_easy_cleanup(). Choice may be dictated + /// by thread constraints. + /// + /// Threading: Single-threaded. May only be used by a single thread, + /// typically the worker thread. If freeing requests' handles in an + /// unknown threading context, use curl_easy_cleanup() for safety. + + class HandleCache + { + public: + HandleCache(); + ~HandleCache(); + + private: + HandleCache(const HandleCache &); // Not defined + void operator=(const HandleCache &); // Not defined + + public: + /// Allocate a curl handle for caller. May be freed using + /// either the freeHandle() method or calling curl_easy_cleanup() + /// directly. + /// + /// @return Libcurl handle (CURL *) or NULL on allocation + /// problem. + /// + /// Threading: Single-thread (worker) only. + CURL * getHandle(); + + /// Free a libcurl handle acquired by whatever means. Thread + /// safety is left to the caller. + /// + /// Threading: Single-thread (worker) only. + void freeHandle(CURL * handle); + + protected: + typedef std::vector<CURL *> handle_cache_t; + + protected: + CURL * mHandleTemplate; // Template for duplicating new handles + handle_cache_t mCache; // Cache of old handles + }; // end class HandleCache + +protected: + HttpService * mService; // Simple reference, not owner + HandleCache mHandleCache; // Handle allocator, owner + active_set_t mActiveOps; + int mPolicyCount; + CURLM ** mMultiHandles; // One handle per policy class + int * mActiveHandles; // Active count per policy class + bool * mDirtyPolicy; // Dirty policy update waiting for stall (per pc) + +}; // end class HttpLibcurl + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_LIBCURL_H_ diff --git a/indra/llcorehttp/_httpopcancel.cpp b/indra/llcorehttp/_httpopcancel.cpp new file mode 100755 index 0000000000..c1912eb3db --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.cpp @@ -0,0 +1,73 @@ +/** + * @file _httpopcancel.cpp + * @brief Definitions for internal class HttpOpCancel + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopcancel.h" + +#include "httpcommon.h" +#include "httphandler.h" +#include "httpresponse.h" + +#include "_httpservice.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOpCancel +// ================================== + + +HttpOpCancel::HttpOpCancel(HttpHandle handle) + : HttpOperation(), + mHandle(handle) +{} + + +HttpOpCancel::~HttpOpCancel() +{} + + +// Immediately search for the request on various queues +// and cancel operations if found. Return the status of +// the search and cancel as the status of this request. +// The canceled request will return a canceled status to +// its handler. +void HttpOpCancel::stageFromRequest(HttpService * service) +{ + if (! service->cancel(mHandle)) + { + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + + addAsReply(); +} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h new file mode 100755 index 0000000000..336dfdc573 --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.h @@ -0,0 +1,78 @@ +/** + * @file _httpopcancel.h + * @brief Internal declarations for the HttpOpCancel subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPCANCEL_H_ +#define _LLCORE_HTTP_OPCANCEL_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include "httpcommon.h" + +#include <curl/curl.h> + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpCancel requests that a previously issued request +/// be canceled, if possible. This includes active requests +/// that may be in the middle of an HTTP transaction. Any +/// completed request will not be canceled and will return +/// its final status unchanged and *this* request will complete +/// with an HE_HANDLE_NOT_FOUND error status. + +class HttpOpCancel : public HttpOperation +{ +public: + /// @param handle Handle of previously-issued request to + /// be canceled. + HttpOpCancel(HttpHandle handle); + +protected: + virtual ~HttpOpCancel(); // Use release() + +private: + HttpOpCancel(const HttpOpCancel &); // Not defined + void operator=(const HttpOpCancel &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +public: + // Request data + HttpHandle mHandle; +}; // end class HttpOpCancel + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPCANCEL_H_ + diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp new file mode 100755 index 0000000000..fefe561f80 --- /dev/null +++ b/indra/llcorehttp/_httpoperation.cpp @@ -0,0 +1,256 @@ +/** + * @file _httpoperation.cpp + * @brief Definitions for internal classes based on HttpOperation + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpoperation.h" + +#include "httphandler.h" +#include "httpresponse.h" +#include "httprequest.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httpinternal.h" + +#include "lltimer.h" + + +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + +namespace LLCore +{ + + +// ================================== +// HttpOperation +// ================================== + + +HttpOperation::HttpOperation() + : LLCoreInt::RefCounted(true), + mReplyQueue(NULL), + mUserHandler(NULL), + mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), + mReqPriority(0U), + mTracing(HTTP_TRACE_OFF) +{ + mMetricCreated = totalTime(); +} + + +HttpOperation::~HttpOperation() +{ + setReplyPath(NULL, NULL); +} + + +void HttpOperation::setReplyPath(HttpReplyQueue * reply_queue, + HttpHandler * user_handler) +{ + if (reply_queue != mReplyQueue) + { + if (mReplyQueue) + { + mReplyQueue->release(); + } + + if (reply_queue) + { + reply_queue->addRef(); + } + + mReplyQueue = reply_queue; + } + + // Not refcounted + mUserHandler = user_handler; +} + + + +void HttpOperation::stageFromRequest(HttpService *) +{ + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LL_ERRS(LOG_CORE) << "Default stageFromRequest method may not be called." + << LL_ENDL; +} + + +void HttpOperation::stageFromReady(HttpService *) +{ + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LL_ERRS(LOG_CORE) << "Default stageFromReady method may not be called." + << LL_ENDL; +} + + +void HttpOperation::stageFromActive(HttpService *) +{ + // Default implementation should never be called. This + // indicates an operation making a transition that isn't + // defined. + LL_ERRS(LOG_CORE) << "Default stageFromActive method may not be called." + << LL_ENDL; +} + + +void HttpOperation::visitNotifier(HttpRequest *) +{ + if (mUserHandler) + { + HttpResponse * response = new HttpResponse(); + + response->setStatus(mStatus); + mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); + + response->release(); + } +} + + +HttpStatus HttpOperation::cancel() +{ + HttpStatus status; + + return status; +} + + +void HttpOperation::addAsReply() +{ + if (mTracing > HTTP_TRACE_OFF) + { + LL_INFOS(LOG_CORE) << "TRACE, ToReplyQueue, Handle: " + << static_cast<HttpHandle>(this) + << LL_ENDL; + } + + if (mReplyQueue) + { + addRef(); + mReplyQueue->addOp(this); + } +} + + +// ================================== +// HttpOpStop +// ================================== + + +HttpOpStop::HttpOpStop() + : HttpOperation() +{} + + +HttpOpStop::~HttpOpStop() +{} + + +void HttpOpStop::stageFromRequest(HttpService * service) +{ + // Do operations + service->stopRequested(); + + // Prepare response if needed + addAsReply(); +} + + +// ================================== +// HttpOpNull +// ================================== + + +HttpOpNull::HttpOpNull() + : HttpOperation() +{} + + +HttpOpNull::~HttpOpNull() +{} + + +void HttpOpNull::stageFromRequest(HttpService * service) +{ + // Perform op + // Nothing to perform. This doesn't fall into the libcurl + // ready/active queues, it just bounces over to the reply + // queue directly. + + // Prepare response if needed + addAsReply(); +} + + +// ================================== +// HttpOpSpin +// ================================== + + +HttpOpSpin::HttpOpSpin(int mode) + : HttpOperation(), + mMode(mode) +{} + + +HttpOpSpin::~HttpOpSpin() +{} + + +void HttpOpSpin::stageFromRequest(HttpService * service) +{ + if (0 == mMode) + { + // Spin forever + while (true) + { + ms_sleep(100); + } + } + else + { + ms_sleep(1); // backoff interlock plumbing a bit + this->addRef(); + if (! service->getRequestQueue().addOp(this)) + { + this->release(); + } + } +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h new file mode 100755 index 0000000000..937a61187d --- /dev/null +++ b/indra/llcorehttp/_httpoperation.h @@ -0,0 +1,262 @@ +/** + * @file _httpoperation.h + * @brief Internal declarations for HttpOperation and sub-classes + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPERATION_H_ +#define _LLCORE_HTTP_OPERATION_H_ + + +#include "httpcommon.h" +#include "httprequest.h" +#include "_refcounted.h" + + +namespace LLCore +{ + +class HttpReplyQueue; +class HttpHandler; +class HttpService; + +/// HttpOperation is the base class for all request/reply +/// pairs. +/// +/// Operations are expected to be of two types: immediate +/// and queued. Immediate requests go to the singleton +/// request queue and when picked up by the worker thread +/// are executed immediately and there results placed on +/// the supplied reply queue. Queued requests (namely for +/// HTTP operations), go to the request queue, are picked +/// up and moved to a ready queue where they're ordered by +/// priority and managed by the policy component, are +/// then activated issuing HTTP requests and moved to an +/// active list managed by the transport (libcurl) component +/// and eventually finalized when a response is available +/// and status and data return via reply queue. +/// +/// To manage these transitions, derived classes implement +/// three methods: stageFromRequest, stageFromReady and +/// stageFromActive. Immediate requests will only override +/// stageFromRequest which will perform the operation and +/// return the result by invoking addAsReply() to put the +/// request on a reply queue. Queued requests will involve +/// all three stage methods. +/// +/// Threading: not thread-safe. Base and derived classes +/// provide no locking. Instances move across threads +/// via queue-like interfaces that are thread compatible +/// and those interfaces establish the access rules. + +class HttpOperation : public LLCoreInt::RefCounted +{ +public: + /// Threading: called by consumer thread. + HttpOperation(); + +protected: + /// Threading: called by any thread. + virtual ~HttpOperation(); // Use release() + +private: + HttpOperation(const HttpOperation &); // Not defined + void operator=(const HttpOperation &); // Not defined + +public: + /// Register a reply queue and a handler for completion notifications. + /// + /// Invokers of operations that want to receive notification that an + /// operation has been completed do so by binding a reply queue and + /// a handler object to the request. + /// + /// @param reply_queue Pointer to the reply queue where completion + /// notifications are to be queued (typically + /// by addAsReply()). This will typically be + /// the reply queue referenced by the request + /// object. This method will increment the + /// refcount on the queue holding the queue + /// until delivery is complete. Using a reply_queue + /// even if the handler is NULL has some benefits + /// for memory deallocation by keeping it in the + /// originating thread. + /// + /// @param handler Possibly NULL pointer to a non-refcounted + //// handler object to be invoked (onCompleted) + /// when the operation is finished. Note that + /// the handler object is never dereferenced + /// by the worker thread. This is passible data + /// until notification is performed. + /// + /// Threading: called by consumer thread. + /// + void setReplyPath(HttpReplyQueue * reply_queue, + HttpHandler * handler); + + /// The three possible staging steps in an operation's lifecycle. + /// Asynchronous requests like HTTP operations move from the + /// request queue to the ready queue via stageFromRequest. Then + /// from the ready queue to the active queue by stageFromReady. And + /// when complete, to the reply queue via stageFromActive and the + /// addAsReply utility. + /// + /// Immediate mode operations (everything else) move from the + /// request queue to the reply queue directly via stageFromRequest + /// and addAsReply with no existence on the ready or active queues. + /// + /// These methods will take out a reference count on the request, + /// caller only needs to dispose of its reference when done with + /// the request. + /// + /// Threading: called by worker thread. + /// + virtual void stageFromRequest(HttpService *); + virtual void stageFromReady(HttpService *); + virtual void stageFromActive(HttpService *); + + /// Delivers a notification to a handler object on completion. + /// + /// Once a request is complete and it has been removed from its + /// reply queue, a handler notification may be delivered by a + /// call to HttpRequest::update(). This method does the necessary + /// dispatching. + /// + /// Threading: called by consumer thread. + /// + virtual void visitNotifier(HttpRequest *); + + /// Cancels the operation whether queued or active. + /// Final status of the request becomes canceled (an error) and + /// that will be delivered to caller via notification scheme. + /// + /// Threading: called by worker thread. + /// + virtual HttpStatus cancel(); + +protected: + /// Delivers request to reply queue on completion. After this + /// call, worker thread no longer accesses the object and it + /// is owned by the reply queue. + /// + /// Threading: called by worker thread. + /// + void addAsReply(); + +protected: + HttpReplyQueue * mReplyQueue; // Have refcount + HttpHandler * mUserHandler; // Naked pointer + +public: + // Request Data + HttpRequest::policy_t mReqPolicy; + HttpRequest::priority_t mReqPriority; + + // Reply Data + HttpStatus mStatus; + + // Tracing, debug and metrics + HttpTime mMetricCreated; + int mTracing; +}; // end class HttpOperation + + +/// HttpOpStop requests the servicing thread to shutdown +/// operations, cease pulling requests from the request +/// queue and release shared resources (particularly +/// those shared via reference count). The servicing +/// thread will then exit. The underlying thread object +/// remains so that another thread can join on the +/// servicing thread prior to final cleanup. The +/// request *does* generate a reply on the response +/// queue, if requested. + +class HttpOpStop : public HttpOperation +{ +public: + HttpOpStop(); + +protected: + virtual ~HttpOpStop(); + +private: + HttpOpStop(const HttpOpStop &); // Not defined + void operator=(const HttpOpStop &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +}; // end class HttpOpStop + + +/// HttpOpNull is a do-nothing operation used for testing via +/// a basic loopback pattern. It's executed immediately by +/// the servicing thread which bounces a reply back to the +/// caller without any further delay. + +class HttpOpNull : public HttpOperation +{ +public: + HttpOpNull(); + +protected: + virtual ~HttpOpNull(); + +private: + HttpOpNull(const HttpOpNull &); // Not defined + void operator=(const HttpOpNull &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +}; // end class HttpOpNull + + +/// HttpOpSpin is a test-only request that puts the worker +/// thread into a cpu spin. Used for unit tests and cleanup +/// evaluation. You do not want to use this in production. +class HttpOpSpin : public HttpOperation +{ +public: + // 0 does a hard spin in the operation + // 1 does a soft spin continuously requeuing itself + HttpOpSpin(int mode); + +protected: + virtual ~HttpOpSpin(); + +private: + HttpOpSpin(const HttpOpSpin &); // Not defined + void operator=(const HttpOpSpin &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +protected: + int mMode; +}; // end class HttpOpSpin + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPERATION_H_ + diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp new file mode 100755 index 0000000000..b9632a7921 --- /dev/null +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -0,0 +1,1162 @@ +/** + * @file _httpoprequest.cpp + * @brief Definitions for internal class HttpOpRequest + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpoprequest.h" + +#include <cstdio> +#include <algorithm> + +#include "httpcommon.h" +#include "httphandler.h" +#include "httpresponse.h" +#include "bufferarray.h" +#include "httpheaders.h" +#include "httpoptions.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httppolicy.h" +#include "_httppolicyglobal.h" +#include "_httplibcurl.h" +#include "_httpinternal.h" + +#include "llhttpconstants.h" +#include "llproxy.h" + +// *DEBUG: "[curl:bugs] #1420" problem and testing. +// +// A pipelining problem, https://sourceforge.net/p/curl/bugs/1420/, +// was a source of Core_9 failures. Code related to this can be +// identified and tested by: +// * Looking for '[curl:bugs]' strings in source and following +// instructions there. +// * Set 'QAModeHttpTrace' to 2 or 3 in settings.xml and look for +// 'timed out' events in the log. +// * Enable the HttpRangeRequestsDisable debug setting which causes +// full asset fetches. These slow the pipelines down a bit. +// + +namespace +{ + +// Attempts to parse a 'Content-Range:' header. Caller must already +// have verified that the header tag is present. The 'buffer' argument +// will be processed by strtok_r calls which will modify the buffer. +// +// @return -1 if invalid and response should be dropped, 0 if valid an +// correct, 1 if couldn't be parsed. If 0, the first, last, +// and length arguments are also written. 'length' may be +// 0 if the length wasn't available to the server. +// +int parse_content_range_header(char * buffer, + unsigned int * first, + unsigned int * last, + unsigned int * length); + +// Similar for Retry-After headers. Only parses the delta form +// of the header, HTTP time formats aren't interesting for client +// purposes. +// +// @return 0 if successfully parsed and seconds time delta +// returned in time argument. +// +int parse_retry_after_header(char * buffer, int * time); + + +// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and +// escape and format it for a tracing line in logging. Absolutely +// anything including NULs can be in the data. If @scrub is true, +// non-printing or non-ascii characters are replaced with spaces +// otherwise a %XX form of escaping is used. +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, + std::string & safe_line); + + +// OS-neutral string comparisons of various types. +int os_strcasecmp(const char * s1, const char * s2); +char * os_strtok_r(char * str, const char * delim, char ** saveptr); +char * os_strtrim(char * str); +char * os_strltrim(char * str); +void os_strlower(char * str); + +// Error testing and reporting for libcurl status codes +void check_curl_easy_code(CURLcode code, int curl_setopt_option); + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + +namespace LLCore +{ + + +HttpOpRequest::HttpOpRequest() + : HttpOperation(), + mProcFlags(0U), + mReqMethod(HOR_GET), + mReqBody(NULL), + mReqOffset(0), + mReqLength(0), + mReqHeaders(NULL), + mReqOptions(NULL), + mCurlActive(false), + mCurlHandle(NULL), + mCurlService(NULL), + mCurlHeaders(NULL), + mCurlBodyPos(0), + mCurlTemp(NULL), + mCurlTempLen(0), + mReplyBody(NULL), + mReplyOffset(0), + mReplyLength(0), + mReplyFullLength(0), + mReplyHeaders(NULL), + mPolicyRetries(0), + mPolicy503Retries(0), + mPolicyRetryAt(HttpTime(0)), + mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) +{ + // *NOTE: As members are added, retry initialization/cleanup + // may need to be extended in @see prepareRequest(). +} + + + +HttpOpRequest::~HttpOpRequest() +{ + if (mReqBody) + { + mReqBody->release(); + mReqBody = NULL; + } + + if (mReqOptions) + { + mReqOptions->release(); + mReqOptions = NULL; + } + + if (mReqHeaders) + { + mReqHeaders->release(); + mReqHeaders = NULL; + } + + if (mCurlHandle) + { + // Uncertain of thread context so free using + // safest method. + curl_easy_cleanup(mCurlHandle); + mCurlHandle = NULL; + } + + mCurlService = NULL; + + if (mCurlHeaders) + { + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; + } + + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + + if (mReplyBody) + { + mReplyBody->release(); + mReplyBody = NULL; + } + + if (mReplyHeaders) + { + mReplyHeaders->release(); + mReplyHeaders = NULL; + } +} + + +void HttpOpRequest::stageFromRequest(HttpService * service) +{ + addRef(); + service->getPolicy().addOp(this); // transfers refcount +} + + +void HttpOpRequest::stageFromReady(HttpService * service) +{ + addRef(); + service->getTransport().addOp(this); // transfers refcount +} + + +void HttpOpRequest::stageFromActive(HttpService * service) +{ + if (mReplyLength) + { + // If non-zero, we received and processed a Content-Range + // header with the response. If there is received data + // (and there may not be due to protocol violations, + // HEAD requests, etc., see BUG-2295) Verify that what it + // says is consistent with the received data. + if (mReplyBody && mReplyBody->size() && mReplyLength != mReplyBody->size()) + { + // Not as expected, fail the request + mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + } + + if (mCurlHeaders) + { + // We take these headers out of the request now as they were + // allocated originally in this thread and the notifier doesn't + // need them. This eliminates one source of heap moving across + // threads. + + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; + } + + // Also not needed on the other side + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + + addAsReply(); +} + + +void HttpOpRequest::visitNotifier(HttpRequest * request) +{ + if (mUserHandler) + { + HttpResponse * response = new HttpResponse(); + response->setStatus(mStatus); + response->setBody(mReplyBody); + response->setHeaders(mReplyHeaders); + if (mReplyOffset || mReplyLength) + { + // Got an explicit offset/length in response + response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); + } + response->setContentType(mReplyConType); + response->setRetries(mPolicyRetries, mPolicy503Retries); + + mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); + + response->release(); + } +} + + +HttpStatus HttpOpRequest::cancel() +{ + mStatus = HttpStatus(HttpStatus::LLCORE, HE_OP_CANCELED); + + addAsReply(); + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, NULL, options, headers); + mReqMethod = HOR_GET; + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, NULL, options, headers); + mReqMethod = HOR_GET; + mReqOffset = offset; + mReqLength = len; + if (offset || len) + { + mProcFlags |= PF_SCAN_RANGE_HEADER; + } + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, body, options, headers); + mReqMethod = HOR_POST; + + return HttpStatus(); +} + + +HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + setupCommon(policy_id, priority, url, body, options, headers); + mReqMethod = HOR_PUT; + + return HttpStatus(); +} + + +void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + mProcFlags = 0U; + mReqPolicy = policy_id; + mReqPriority = priority; + mReqURL = url; + if (body) + { + body->addRef(); + mReqBody = body; + } + if (headers && ! mReqHeaders) + { + headers->addRef(); + mReqHeaders = headers; + } + if (options && ! mReqOptions) + { + options->addRef(); + mReqOptions = options; + if (options->getWantHeaders()) + { + mProcFlags |= PF_SAVE_HEADERS; + } + if (options->getUseRetryAfter()) + { + mProcFlags |= PF_USE_RETRY_AFTER; + } + mPolicyRetryLimit = options->getRetries(); + mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); + mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); + } +} + +// Sets all libcurl options and data for a request. +// +// Used both for initial requests and to 'reload' for +// a retry, generally with a different CURL handle. +// Junk may be left around from a failed request and that +// needs to be cleaned out. +// +// *TODO: Move this to _httplibcurl where it belongs. +HttpStatus HttpOpRequest::prepareRequest(HttpService * service) +{ + CURLcode code; + + // Scrub transport and result data for retried op case + mCurlActive = false; + mCurlHandle = NULL; + mCurlService = NULL; + if (mCurlHeaders) + { + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; + } + mCurlBodyPos = 0; + + if (mReplyBody) + { + mReplyBody->release(); + mReplyBody = NULL; + } + mReplyOffset = 0; + mReplyLength = 0; + mReplyFullLength = 0; + if (mReplyHeaders) + { + mReplyHeaders->release(); + mReplyHeaders = NULL; + } + mReplyConType.clear(); + + // *FIXME: better error handling later + HttpStatus status; + + // Get global and class policy options + HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions()); + HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy)); + + mCurlHandle = service->getTransport().getHandle(); + if (! mCurlHandle) + { + // We're in trouble. We'll continue but it won't go well. + LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle. Continuing." + << LL_ENDL; + return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); + } + + code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + check_curl_easy_code(code, CURLOPT_IPRESOLVE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); + check_curl_easy_code(code, CURLOPT_NOSIGNAL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); + check_curl_easy_code(code, CURLOPT_NOPROGRESS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); + check_curl_easy_code(code, CURLOPT_URL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); + check_curl_easy_code(code, CURLOPT_PRIVATE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + check_curl_easy_code(code, CURLOPT_ENCODING); + + // The Linksys WRT54G V5 router has an issue with frequent + // DNS lookups from LAN machines. If they happen too often, + // like for every HTTP request, the router gets annoyed after + // about 700 or so requests and starts issuing TCP RSTs to + // new connections. Reuse the DNS lookups for even a few + // seconds and no RSTs. + code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); + check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT); + code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); + check_curl_easy_code(code, CURLOPT_AUTOREFERER); + code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); + check_curl_easy_code(code, CURLOPT_MAXREDIRS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); + check_curl_easy_code(code, CURLOPT_WRITEFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); + check_curl_easy_code(code, CURLOPT_WRITEDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); + check_curl_easy_code(code, CURLOPT_READFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); + check_curl_easy_code(code, CURLOPT_READDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); + check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER); + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); + check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); + + if (gpolicy.mUseLLProxy) + { + // Use the viewer-based thread-safe API which has a + // fast/safe check for proxy enable. Would like to + // encapsulate this someway... + LLProxy::getInstance()->applyProxySettings(mCurlHandle); + } + else if (gpolicy.mHttpProxy.size()) + { + // *TODO: This is fine for now but get fuller socks5/ + // authentication thing going later.... + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str()); + check_curl_easy_code(code, CURLOPT_PROXY); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + check_curl_easy_code(code, CURLOPT_PROXYTYPE); + } + if (gpolicy.mCAPath.size()) + { + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str()); + check_curl_easy_code(code, CURLOPT_CAPATH); + } + if (gpolicy.mCAFile.size()) + { + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str()); + check_curl_easy_code(code, CURLOPT_CAINFO); + } + + switch (mReqMethod) + { + case HOR_GET: + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + check_curl_easy_code(code, CURLOPT_HTTPGET); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); + break; + + case HOR_POST: + { + code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + check_curl_easy_code(code, CURLOPT_POST); + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + check_curl_easy_code(code, CURLOPT_ENCODING); + long data_size(0); + if (mReqBody) + { + data_size = mReqBody->size(); + } + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); + check_curl_easy_code(code, CURLOPT_POSTFIELDS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); + } + break; + + case HOR_PUT: + { + code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + check_curl_easy_code(code, CURLOPT_UPLOAD); + long data_size(0); + if (mReqBody) + { + data_size = mReqBody->size(); + } + code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + check_curl_easy_code(code, CURLOPT_INFILESIZE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); + check_curl_easy_code(code, CURLOPT_POSTFIELDS); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + // *TODO: Should this be 'Keep-Alive' ? + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); + } + break; + + default: + LL_ERRS(LOG_CORE) << "Invalid HTTP method in request: " + << int(mReqMethod) << ". Can't recover." + << LL_ENDL; + break; + } + + // Tracing + if (mTracing >= HTTP_TRACE_CURL_HEADERS) + { + code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); + check_curl_easy_code(code, CURLOPT_VERBOSE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); + check_curl_easy_code(code, CURLOPT_DEBUGDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION); + } + + // There's a CURLOPT for this now... + if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) + { + static const char * const fmt1("Range: bytes=%lu-%lu"); + static const char * const fmt2("Range: bytes=%lu-"); + + char range_line[64]; + +#if LL_WINDOWS + _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, + (mReqLength ? fmt1 : fmt2), + (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); +#else + if ( mReqLength ) + { + snprintf(range_line, sizeof(range_line), + fmt1, + (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); + } + else + { + snprintf(range_line, sizeof(range_line), + fmt2, + (unsigned long) mReqOffset); + } +#endif // LL_WINDOWS + range_line[sizeof(range_line) - 1] = '\0'; + mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); + } + + mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); + + // Request options + long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); + if (mReqOptions) + { + timeout = mReqOptions->getTimeout(); + timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + xfer_timeout = mReqOptions->getTransferTimeout(); + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + } + if (xfer_timeout == 0L) + { + xfer_timeout = timeout; + } + if (cpolicy.mPipelining > 1L) + { + // Pipelining affects both connection and transfer timeout values. + // Requests that are added to a pipeling immediately have completed + // their connection so the connection delay tends to be less than + // the non-pipelined value. Transfers are the opposite. Transfer + // timeout starts once the connection is established and completion + // can be delayed due to the pipelined requests ahead. So, it's + // a handwave but bump the transfer timeout up by the pipelining + // depth to give some room. + // + // BUG-7698, BUG-7688, BUG-7694 (others). Scylla and Charybdis + // situation. Operating against a CDN having service issues may + // lead to requests stalling for an arbitrarily long time with only + // the CURLOPT_TIMEOUT value leading to a closed connection. Sadly + // for pipelining, libcurl (7.39.0 and earlier, at minimum) starts + // the clock on this value as soon as a request is started down + // the wire. We want a short value to recover and retry from the + // CDN. We need a long value to safely deal with a succession of + // piled-up pipelined requests. + // + // *TODO: Find a better scheme than timeouts to guarantee liveness. + // Progress on the connection is what we really want, not timeouts. + // But we don't have access to that and the request progress indicators + // (various libcurl callbacks) have the same problem TIMEOUT does. + // + // xfer_timeout *= cpolicy.mPipelining; + xfer_timeout *= 2L; + } + // *DEBUG: Enable following override for timeout handling and "[curl:bugs] #1420" tests + // xfer_timeout = 1L; + code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); + check_curl_easy_code(code, CURLOPT_TIMEOUT); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); + check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT); + + // Request headers + if (mReqHeaders) + { + // Caller's headers last to override + mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); + } + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); + check_curl_easy_code(code, CURLOPT_HTTPHEADER); + + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) + { + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); + check_curl_easy_code(code, CURLOPT_HEADERFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); + check_curl_easy_code(code, CURLOPT_HEADERDATA); + } + + if (status) + { + mCurlService = service; + } + return status; +} + + +size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + if (! op->mReplyBody) + { + op->mReplyBody = new BufferArray(); + } + const size_t req_size(size * nmemb); + const size_t write_size(op->mReplyBody->append(static_cast<char *>(data), req_size)); + return write_size; +} + + +size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + if (! op->mReqBody) + { + return 0; + } + const size_t req_size(size * nmemb); + const size_t body_size(op->mReqBody->size()); + if (body_size <= op->mCurlBodyPos) + { + if (body_size < op->mCurlBodyPos) + { + // Warn but continue if the read position moves beyond end-of-body + // for some reason. + LL_WARNS(LOG_CORE) << "Request body position beyond body size. Truncating request body." + << LL_ENDL; + } + return 0; + } + + const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos)); + const size_t read_size(op->mReqBody->read(op->mCurlBodyPos, static_cast<char *>(data), do_size)); + op->mCurlBodyPos += read_size; + return read_size; +} + + +size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + static const char status_line[] = "HTTP/"; + static const size_t status_line_len = sizeof(status_line) - 1; + static const char con_ran_line[] = "content-range"; + static const char con_retry_line[] = "retry-after"; + + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + const size_t hdr_size(size * nmemb); + const char * hdr_data(static_cast<const char *>(data)); // Not null terminated + bool is_header(true); + + if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) + { + // One of possibly several status lines. Reset what we know and start over + // taking results from the last header stanza we receive. + op->mReplyOffset = 0; + op->mReplyLength = 0; + op->mReplyFullLength = 0; + op->mReplyRetryAfter = 0; + op->mStatus = HttpStatus(); + if (op->mReplyHeaders) + { + op->mReplyHeaders->clear(); + } + is_header = false; + } + + // Nothing in here wants a final CR/LF combination. Remove + // it as much as possible. + size_t wanted_hdr_size(hdr_size); + if (wanted_hdr_size && '\n' == hdr_data[wanted_hdr_size - 1]) + { + if (--wanted_hdr_size && '\r' == hdr_data[wanted_hdr_size - 1]) + { + --wanted_hdr_size; + } + } + + // Copy and normalize header fragments for the following + // stages. Would like to modify the data in-place but that + // may not be allowed and we need one byte extra for NUL. + // At the end of this we will have: + // + // If ':' present in header: + // 1. name points to text to left of colon which + // will be ascii lower-cased and left and right + // trimmed of whitespace. + // 2. value points to text to right of colon which + // will be left trimmed of whitespace. + // Otherwise: + // 1. name points to header which will be left + // trimmed of whitespace. + // 2. value is NULL + // Any non-NULL pointer may point to a zero-length string. + // + if (wanted_hdr_size >= op->mCurlTempLen) + { + delete [] op->mCurlTemp; + op->mCurlTempLen = 2 * wanted_hdr_size + 1; + op->mCurlTemp = new char [op->mCurlTempLen]; + } + memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); + op->mCurlTemp[wanted_hdr_size] = '\0'; + char * name(op->mCurlTemp); + char * value(strchr(name, ':')); + if (value) + { + *value++ = '\0'; + os_strlower(name); + name = os_strtrim(name); + value = os_strltrim(value); + } + else + { + // Doesn't look well-formed, do minimal normalization on it + name = os_strltrim(name); + } + + // Normalized, now reject headers with empty names. + if (! *name) + { + // No use continuing + return hdr_size; + } + + // Save header if caller wants them in the response + if (is_header && op->mProcFlags & PF_SAVE_HEADERS) + { + // Save headers in response + if (! op->mReplyHeaders) + { + op->mReplyHeaders = new HttpHeaders; + } + op->mReplyHeaders->append(name, value ? value : ""); + } + + // From this point, header-specific processors are free to + // modify the header value. + + // Detect and parse 'Content-Range' headers + if (is_header + && op->mProcFlags & PF_SCAN_RANGE_HEADER + && value && *value + && ! strcmp(name, con_ran_line)) + { + unsigned int first(0), last(0), length(0); + int status; + + if (! (status = parse_content_range_header(value, &first, &last, &length))) + { + // Success, record the fragment position + op->mReplyOffset = first; + op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; + } + else if (-1 == status) + { + // Response is badly formed and shouldn't be accepted + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + else + { + // Ignore the unparsable. + LL_INFOS_ONCE(LOG_CORE) << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; + } + } + + // Detect and parse 'Retry-After' headers + if (is_header + && op->mProcFlags & PF_USE_RETRY_AFTER + && value && *value + && ! strcmp(name, con_retry_line)) + { + int time(0); + if (! parse_retry_after_header(value, &time)) + { + op->mReplyRetryAfter = time; + } + } + + return hdr_size; +} + + +int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata) +{ + HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata)); + + std::string safe_line; + std::string tag; + bool logit(false); + const size_t log_len((std::min)(len, size_t(256))); // Keep things reasonable in all cases + + switch (info) + { + case CURLINFO_TEXT: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "TEXT"; + escape_libcurl_debug_data(buffer, log_len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_IN: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "HEADERIN"; + escape_libcurl_debug_data(buffer, log_len, true, safe_line); + logit = true; + } + break; + + case CURLINFO_HEADER_OUT: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "HEADEROUT"; + escape_libcurl_debug_data(buffer, log_len, true, safe_line); // Goes out as one line unlike header_in + logit = true; + } + break; + + case CURLINFO_DATA_IN: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "DATAIN"; + logit = true; + if (op->mTracing >= HTTP_TRACE_CURL_BODIES) + { + escape_libcurl_debug_data(buffer, log_len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + case CURLINFO_DATA_OUT: + if (op->mTracing >= HTTP_TRACE_CURL_HEADERS) + { + tag = "DATAOUT"; + logit = true; + if (op->mTracing >= HTTP_TRACE_CURL_BODIES) + { + escape_libcurl_debug_data(buffer, log_len, false, safe_line); + } + else + { + std::ostringstream out; + out << len << " Bytes"; + safe_line = out.str(); + } + } + break; + + default: + logit = false; + break; + } + + if (logit) + { + LL_INFOS(LOG_CORE) << "TRACE, LibcurlDebug, Handle: " + << static_cast<HttpHandle>(op) + << ", Type: " << tag + << ", Data: " << safe_line + << LL_ENDL; + } + + return 0; +} + + +} // end namespace LLCore + + +// ======================================= +// Anonymous Namespace +// ======================================= + +namespace +{ + +int parse_content_range_header(char * buffer, + unsigned int * first, + unsigned int * last, + unsigned int * length) +{ + static const char * const hdr_whitespace(" \t"); + + char * tok_state(NULL), * tok(NULL); + bool match(true); + + if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state))) + match = false; + else + match = (0 == os_strcasecmp("bytes", tok)); + if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) + match = false; + if (match) + { + unsigned int lcl_first(0), lcl_last(0), lcl_len(0); + +#if LL_WINDOWS + if (3 == sscanf_s(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) +#else + if (3 == sscanf(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) +#endif // LL_WINDOWS + { + if (lcl_first > lcl_last || lcl_last >= lcl_len) + return -1; + *first = lcl_first; + *last = lcl_last; + *length = lcl_len; + return 0; + } +#if LL_WINDOWS + if (2 == sscanf_s(tok, "%u-%u/*", &lcl_first, &lcl_last)) +#else + if (2 == sscanf(tok, "%u-%u/*", &lcl_first, &lcl_last)) +#endif // LL_WINDOWS + { + if (lcl_first > lcl_last) + return -1; + *first = lcl_first; + *last = lcl_last; + *length = 0; + return 0; + } + } + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + +int parse_retry_after_header(char * buffer, int * time) +{ + char * endptr(buffer); + long lcl_time(strtol(buffer, &endptr, 10)); + if (*endptr == '\0' && endptr != buffer && lcl_time > 0) + { + *time = lcl_time; + return 0; + } + + // Could attempt to parse HTTP time here but we're not really + // interested in it. Scheduling based on wallclock time on + // user hardware will lead to tears. + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + +void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) +{ + std::string out; + len = (std::min)(len, size_t(200)); + out.reserve(3 * len); + for (int i(0); i < len; ++i) + { + unsigned char uc(static_cast<unsigned char>(buffer[i])); + + if (uc < 32 || uc > 126) + { + if (scrub) + { + out.append(1, ' '); + } + else + { + static const char hex[] = "0123456789ABCDEF"; + char convert[4]; + + convert[0] = '%'; + convert[1] = hex[(uc >> 4) % 16]; + convert[2] = hex[uc % 16]; + convert[3] = '\0'; + out.append(convert); + } + } + else + { + out.append(1, buffer[i]); + } + } + safe_line.swap(out); +} + + + +int os_strcasecmp(const char *s1, const char *s2) +{ +#if LL_WINDOWS + return _stricmp(s1, s2); +#else + return strcasecmp(s1, s2); +#endif // LL_WINDOWS +} + + +char * os_strtok_r(char *str, const char *delim, char ** savestate) +{ +#if LL_WINDOWS + return strtok_s(str, delim, savestate); +#else + return strtok_r(str, delim, savestate); +#endif +} + + +void os_strlower(char * str) +{ + for (char c(0); (c = *str); ++str) + { + *str = tolower(c); + } +} + + +char * os_strtrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + if (*lstr) + { + char * rstr(lstr + strlen(lstr)); + while (lstr < rstr && *--rstr) + { + if (' ' == *rstr || '\t' == *rstr) + { + *rstr = '\0'; + } + } + llassert(lstr <= rstr); + } + return lstr; +} + + +char * os_strltrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + return lstr; +} + + +void check_curl_easy_code(CURLcode code, int curl_setopt_option) +{ + if (CURLE_OK != code) + { + // Comment from old llcurl code which may no longer apply: + // + // linux appears to throw a curl error once per session for a bad initialization + // at a pretty random time (when enabling cookies). + LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code) + << ", curl_easy_setopt option: " << curl_setopt_option + << LL_ENDL; + } +} + +} // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h new file mode 100755 index 0000000000..2f628b5aba --- /dev/null +++ b/indra/llcorehttp/_httpoprequest.h @@ -0,0 +1,223 @@ +/** + * @file _httpoprequest.h + * @brief Internal declarations for the HttpOpRequest subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPREQUEST_H_ +#define _LLCORE_HTTP_OPREQUEST_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include <string> +#include <curl/curl.h> + +#include "httpcommon.h" +#include "httprequest.h" +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +class BufferArray; +class HttpHeaders; +class HttpOptions; + + +/// HttpOpRequest requests a supported HTTP method invocation with +/// option and header overrides. +/// +/// Essentially an RPC to get an HTTP GET, POST or PUT executed +/// asynchronously with options to override behaviors and HTTP +/// headers. +/// +/// Constructor creates a raw object incapable of useful work. +/// A subsequent call to one of the setupXXX() methods provides +/// the information needed to make a working request which can +/// then be enqueued to a request queue. +/// +class HttpOpRequest : public HttpOperation +{ +public: + HttpOpRequest(); + +protected: + virtual ~HttpOpRequest(); // Use release() + +private: + HttpOpRequest(const HttpOpRequest &); // Not defined + void operator=(const HttpOpRequest &); // Not defined + +public: + enum EMethod + { + HOR_GET, + HOR_POST, + HOR_PUT + }; + + virtual void stageFromRequest(HttpService *); + virtual void stageFromReady(HttpService *); + virtual void stageFromActive(HttpService *); + + virtual void visitNotifier(HttpRequest * request); + +public: + /// Setup Methods + /// + /// Basically an RPC setup for each type of HTTP method + /// invocation with one per method type. These are + /// generally invoked right after construction. + /// + /// Threading: called by application thread + /// + HttpStatus setupGet(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus setupGetByteRange(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus setupPost(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus setupPut(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + // Internal method used to setup the libcurl options for a request. + // Does all the libcurl handle setup in one place. + // + // Threading: called by worker thread + // + HttpStatus prepareRequest(HttpService * service); + + virtual HttpStatus cancel(); + +protected: + // Common setup for all the request methods. + // + // Threading: called by application thread + // + void setupCommon(HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + // libcurl operational callbacks + // + // Threading: called by worker thread + // + static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); + static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); + static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); + static int debugCallback(CURL *, curl_infotype info, char * buffer, size_t len, void * userdata); + +protected: + unsigned int mProcFlags; + static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; + static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U; + +public: + // Request data + EMethod mReqMethod; + std::string mReqURL; + BufferArray * mReqBody; + off_t mReqOffset; + size_t mReqLength; + HttpHeaders * mReqHeaders; + HttpOptions * mReqOptions; + + // Transport data + bool mCurlActive; + CURL * mCurlHandle; + HttpService * mCurlService; + curl_slist * mCurlHeaders; + size_t mCurlBodyPos; + char * mCurlTemp; // Scratch buffer for header processing + size_t mCurlTempLen; + + // Result data + HttpStatus mStatus; + BufferArray * mReplyBody; + off_t mReplyOffset; + size_t mReplyLength; + size_t mReplyFullLength; + HttpHeaders * mReplyHeaders; + std::string mReplyConType; + int mReplyRetryAfter; + + // Policy data + int mPolicyRetries; + int mPolicy503Retries; + HttpTime mPolicyRetryAt; + int mPolicyRetryLimit; +}; // end class HttpOpRequest + + +/// HttpOpRequestCompare isn't an operation but a uniform comparison +/// functor for STL containers that order by priority. Mainly +/// used for the ready queue container but defined here. +class HttpOpRequestCompare +{ +public: + bool operator()(const HttpOpRequest * lhs, const HttpOpRequest * rhs) + { + return lhs->mReqPriority > rhs->mReqPriority; + } +}; // end class HttpOpRequestCompare + + +// --------------------------------------- +// Free functions +// --------------------------------------- + +// Internal function to append the contents of an HttpHeaders +// instance to a curl_slist object. +curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPREQUEST_H_ + diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp new file mode 100755 index 0000000000..a5363f9170 --- /dev/null +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -0,0 +1,146 @@ +/** + * @file _httpopsetget.cpp + * @brief Definitions for internal class HttpOpSetGet + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetget.h" + +#include "httpcommon.h" +#include "httprequest.h" + +#include "_httpservice.h" +#include "_httppolicy.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOpSetget +// ================================== + + +HttpOpSetGet::HttpOpSetGet() + : HttpOperation(), + mReqOption(HttpRequest::PO_CONNECTION_LIMIT), + mReqClass(HttpRequest::INVALID_POLICY_ID), + mReqDoSet(false), + mReqLongValue(0L), + mReplyLongValue(0L) +{} + + +HttpOpSetGet::~HttpOpSetGet() +{} + + +HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass) +{ + HttpStatus status; + + mReqOption = opt; + mReqClass = pclass; + return status; +} + + +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value) +{ + HttpStatus status; + + if (! HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqLongValue = value; + + return status; +} + + +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value) +{ + HttpStatus status; + + if (HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqStrValue = value; + + return status; +} + + +void HttpOpSetGet::stageFromRequest(HttpService * service) +{ + if (mReqDoSet) + { + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqLongValue, &mReplyLongValue); + } + else + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqStrValue, &mReplyStrValue); + } + } + else + { + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue); + } + else + { + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue); + } + } + + addAsReply(); +} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h new file mode 100755 index 0000000000..a1e76dd429 --- /dev/null +++ b/indra/llcorehttp/_httpopsetget.h @@ -0,0 +1,90 @@ +/** + * @file _httpopsetget.h + * @brief Internal declarations for the HttpOpSetGet subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPSETGET_H_ +#define _LLCORE_HTTP_OPSETGET_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include "httpcommon.h" + +#include <curl/curl.h> + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetGet requests dynamic changes to policy and +/// configuration settings. +/// +/// *NOTE: Expect this to change. Don't really like it yet. +/// +/// *TODO: Can't return values to caller yet. Need to do +/// something better with HttpResponse and visitNotifier(). +/// +class HttpOpSetGet : public HttpOperation +{ +public: + HttpOpSetGet(); + +protected: + virtual ~HttpOpSetGet(); // Use release() + +private: + HttpOpSetGet(const HttpOpSetGet &); // Not defined + void operator=(const HttpOpSetGet &); // Not defined + +public: + /// Threading: called by application thread + HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value); + + virtual void stageFromRequest(HttpService *); + +public: + // Request data + HttpRequest::EPolicyOption mReqOption; + HttpRequest::policy_t mReqClass; + bool mReqDoSet; + long mReqLongValue; + std::string mReqStrValue; + + // Reply Data + long mReplyLongValue; + std::string mReplyStrValue; +}; // end class HttpOpSetGet + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPSETGET_H_ + diff --git a/indra/llcorehttp/_httpopsetpriority.cpp b/indra/llcorehttp/_httpopsetpriority.cpp new file mode 100755 index 0000000000..d48c7a0b7d --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.cpp @@ -0,0 +1,63 @@ +/** + * @file _httpopsetpriority.cpp + * @brief Definitions for internal classes based on HttpOpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetpriority.h" + +#include "httpresponse.h" +#include "httphandler.h" +#include "_httpservice.h" + + +namespace LLCore +{ + + +HttpOpSetPriority::HttpOpSetPriority(HttpHandle handle, HttpRequest::priority_t priority) + : HttpOperation(), + mHandle(handle), + mPriority(priority) +{} + + +HttpOpSetPriority::~HttpOpSetPriority() +{} + + +void HttpOpSetPriority::stageFromRequest(HttpService * service) +{ + // Do operations + if (! service->changePriority(mHandle, mPriority)) + { + // Request not found, fail the final status + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + + // Move directly to response queue + addAsReply(); +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpopsetpriority.h b/indra/llcorehttp/_httpopsetpriority.h new file mode 100755 index 0000000000..31706b737c --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.h @@ -0,0 +1,73 @@ +/** + * @file _httpsetpriority.h + * @brief Internal declarations for HttpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_SETPRIORITY_H_ +#define _LLCORE_HTTP_SETPRIORITY_H_ + + +#include "httpcommon.h" +#include "httprequest.h" +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetPriority is an immediate request that +/// searches the various queues looking for a given +/// request handle and changing it's priority if +/// found. +/// +/// *NOTE: This will very likely be removed in the near future +/// when priority is removed from the library. + +class HttpOpSetPriority : public HttpOperation +{ +public: + HttpOpSetPriority(HttpHandle handle, HttpRequest::priority_t priority); + +protected: + virtual ~HttpOpSetPriority(); + +private: + HttpOpSetPriority(const HttpOpSetPriority &); // Not defined + void operator=(const HttpOpSetPriority &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + +protected: + // Request Data + HttpHandle mHandle; + HttpRequest::priority_t mPriority; +}; // end class HttpOpSetPriority + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_SETPRIORITY_H_ + diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp new file mode 100755 index 0000000000..e5d6321401 --- /dev/null +++ b/indra/llcorehttp/_httppolicy.cpp @@ -0,0 +1,492 @@ +/** + * @file _httppolicy.cpp + * @brief Internal definitions of the Http policy thread + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "_httppolicy.h" + +#include "_httpoprequest.h" +#include "_httpservice.h" +#include "_httplibcurl.h" +#include "_httppolicyclass.h" + +#include "lltimer.h" + +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + +namespace LLCore +{ + + +// Per-policy-class data for a running system. +// Collection of queues, options and other data +// for a single policy class. +// +// Threading: accessed only by worker thread +struct HttpPolicy::ClassState +{ +public: + ClassState() + : mThrottleEnd(0), + mThrottleLeft(0L), + mRequestCount(0L), + mStallStaging(false) + {} + + HttpReadyQueue mReadyQueue; + HttpRetryQueue mRetryQueue; + + HttpPolicyClass mOptions; + HttpTime mThrottleEnd; + long mThrottleLeft; + long mRequestCount; + bool mStallStaging; +}; + + +HttpPolicy::HttpPolicy(HttpService * service) + : mService(service) +{ + // Create default class + mClasses.push_back(new ClassState()); +} + + +HttpPolicy::~HttpPolicy() +{ + shutdown(); + + for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it) + { + delete (*it); + } + mClasses.clear(); + + mService = NULL; +} + + +HttpRequest::policy_t HttpPolicy::createPolicyClass() +{ + const HttpRequest::policy_t policy_class(mClasses.size()); + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) + { + return HttpRequest::INVALID_POLICY_ID; + } + mClasses.push_back(new ClassState()); + return policy_class; +} + + +void HttpPolicy::shutdown() +{ + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) + { + ClassState & state(*mClasses[policy_class]); + + HttpRetryQueue & retryq(state.mRetryQueue); + while (! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + retryq.pop(); + + op->cancel(); + op->release(); + } + + HttpReadyQueue & readyq(state.mReadyQueue); + while (! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->cancel(); + op->release(); + } + } +} + + +void HttpPolicy::start() +{ +} + + +void HttpPolicy::addOp(HttpOpRequest * op) +{ + const int policy_class(op->mReqPolicy); + + op->mPolicyRetries = 0; + op->mPolicy503Retries = 0; + mClasses[policy_class]->mReadyQueue.push(op); +} + + +void HttpPolicy::retryOp(HttpOpRequest * op) +{ + static const HttpTime retry_deltas[] = + { + 250000, // 1st retry in 0.25 S, etc... + 500000, + 1000000, + 2000000, + 5000000 // ... to every 5.0 S. + }; + static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); + static const HttpStatus error_503(503); + + const HttpTime now(totalTime()); + const int policy_class(op->mReqPolicy); + HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + bool external_delta(false); + + if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) + { + delta = op->mReplyRetryAfter * U64L(1000000); + external_delta = true; + } + op->mPolicyRetryAt = now + delta; + ++op->mPolicyRetries; + if (error_503 == op->mStatus) + { + ++op->mPolicy503Retries; + } + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " retry " << op->mPolicyRetries + << " scheduled in " << (delta / HttpTime(1000)) + << " mS (" << (external_delta ? "external" : "internal") + << "). Status: " << op->mStatus.toTerseString() + << LL_ENDL; + if (op->mTracing > HTTP_TRACE_OFF) + { + LL_INFOS(LOG_CORE) << "TRACE, ToRetryQueue, Handle: " + << static_cast<HttpHandle>(op) + << ", Delta: " << (delta / HttpTime(1000)) + << ", Retries: " << op->mPolicyRetries + << LL_ENDL; + } + mClasses[policy_class]->mRetryQueue.push(op); +} + + +// Attempt to deliver requests to the transport layer. +// +// Tries to find HTTP requests for each policy class with +// available capacity. Starts with the retry queue first +// looking for requests that have waited long enough then +// moves on to the ready queue. +// +// If all queues are empty, will return an indication that +// the worker thread may sleep hard otherwise will ask for +// normal polling frequency. +// +// Implements a client-side request rate throttle as well. +// This is intended to mimic and predict throttling behavior +// of grid services but that is difficult to do with different +// time bases. This also represents a rigid coupling between +// viewer and server that makes it hard to change parameters +// and I hope we can make this go away with pipelining. +// +HttpService::ELoopSpeed HttpPolicy::processReadyQueue() +{ + const HttpTime now(totalTime()); + HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); + HttpLibcurl & transport(mService->getTransport()); + + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) + { + ClassState & state(*mClasses[policy_class]); + HttpRetryQueue & retryq(state.mRetryQueue); + HttpReadyQueue & readyq(state.mReadyQueue); + + if (state.mStallStaging) + { + // Stalling but don't sleep. Need to complete operations + // and get back to servicing queues. Do this test before + // the retryq/readyq test or you'll get stalls until you + // click a setting or an asset request comes in. + result = HttpService::NORMAL; + continue; + } + if (retryq.empty() && readyq.empty()) + { + continue; + } + + const bool throttle_enabled(state.mOptions.mThrottleRate > 0L); + const bool throttle_current(throttle_enabled && now < state.mThrottleEnd); + + if (throttle_current && state.mThrottleLeft <= 0) + { + // Throttled condition, don't serve this class but don't sleep hard. + result = HttpService::NORMAL; + continue; + } + + int active(transport.getActiveCountInClass(policy_class)); + int active_limit(state.mOptions.mPipelining > 1L + ? (state.mOptions.mPerHostConnectionLimit + * state.mOptions.mPipelining) + : state.mOptions.mConnectionLimit); + int needed(active_limit - active); // Expect negatives here + + if (needed > 0) + { + // First see if we have any retries... + while (needed > 0 && ! retryq.empty()) + { + HttpOpRequest * op(retryq.top()); + if (op->mPolicyRetryAt > now) + break; + + retryq.pop(); + + op->stageFromReady(mService); + op->release(); + + ++state.mRequestCount; + --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } + } + + // Now go on to the new requests... + while (needed > 0 && ! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->stageFromReady(mService); + op->release(); + + ++state.mRequestCount; + --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS(LOG_CORE) << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } + } + } + + throttle_on: + + if (! readyq.empty() || ! retryq.empty()) + { + // If anything is ready, continue looping... + result = HttpService::NORMAL; + } + } // end foreach policy_class + + return result; +} + + +bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) +{ + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) + { + ClassState & state(*mClasses[policy_class]); + // We don't scan retry queue because a priority change there + // is meaningless. The request will be issued based on retry + // intervals not priority value, which is now moot. + + // Scan ready queue for requests that match policy + HttpReadyQueue::container_type & c(state.mReadyQueue.get_container()); + for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast<HttpHandle>(*cur) == handle) + { + HttpOpRequest * op(*cur); + c.erase(cur); // All iterators are now invalidated + op->mReqPriority = priority; + state.mReadyQueue.push(op); // Re-insert using adapter class + return true; + } + } + } + + return false; +} + + +bool HttpPolicy::cancel(HttpHandle handle) +{ + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) + { + ClassState & state(*mClasses[policy_class]); + + // Scan retry queue + HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); + for (HttpRetryQueue::container_type::iterator iter(c1.begin()); c1.end() != iter;) + { + HttpRetryQueue::container_type::iterator cur(iter++); + + if (static_cast<HttpHandle>(*cur) == handle) + { + HttpOpRequest * op(*cur); + c1.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + + // Scan ready queue + HttpReadyQueue::container_type & c2(state.mReadyQueue.get_container()); + for (HttpReadyQueue::container_type::iterator iter(c2.begin()); c2.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast<HttpHandle>(*cur) == handle) + { + HttpOpRequest * op(*cur); + c2.erase(cur); // All iterators are now invalidated + op->cancel(); + op->release(); + return true; + } + } + } + + return false; +} + + +bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) +{ + // Retry or finalize + if (! op->mStatus) + { + // *DEBUG: For "[curl:bugs] #1420" tests. This will interfere + // with unit tests due to allocation retention by logging code. + // But you won't be checking this in enabled. +#if 0 + if (op->mStatus == HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT)) + { + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " timed out." + << LL_ENDL; + } +#endif + + // If this failed, we might want to retry. + if (op->mPolicyRetries < op->mPolicyRetryLimit && op->mStatus.isRetryable()) + { + // Okay, worth a retry. + retryOp(op); + return true; // still active/ready + } + } + + // This op is done, finalize it delivering it to the reply queue... + if (! op->mStatus) + { + LL_WARNS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " failed after " << op->mPolicyRetries + << " retries. Reason: " << op->mStatus.toString() + << " (" << op->mStatus.toTerseString() << ")" + << LL_ENDL; + } + else if (op->mPolicyRetries) + { + LL_DEBUGS(LOG_CORE) << "HTTP request " << static_cast<HttpHandle>(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; + } + + op->stageFromActive(mService); + op->release(); + return false; // not active +} + + +HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass) +{ + llassert_always(pclass >= 0 && pclass < mClasses.size()); + + return mClasses[pclass]->mOptions; +} + + +int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const +{ + if (policy_class < mClasses.size()) + { + return (mClasses[policy_class]->mReadyQueue.size() + + mClasses[policy_class]->mRetryQueue.size()); + } + return 0; +} + + +bool HttpPolicy::stallPolicy(HttpRequest::policy_t policy_class, bool stall) +{ + bool ret(false); + + if (policy_class < mClasses.size()) + { + ret = mClasses[policy_class]->mStallStaging; + mClasses[policy_class]->mStallStaging = stall; + } + return ret; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h new file mode 100755 index 0000000000..11cd89bbd1 --- /dev/null +++ b/indra/llcorehttp/_httppolicy.h @@ -0,0 +1,180 @@ +/** + * @file _httppolicy.h + * @brief Declarations for internal class enforcing policy decisions. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_POLICY_H_ +#define _LLCORE_HTTP_POLICY_H_ + + +#include "httprequest.h" +#include "_httpservice.h" +#include "_httpreadyqueue.h" +#include "_httpretryqueue.h" +#include "_httppolicyglobal.h" +#include "_httppolicyclass.h" +#include "_httpinternal.h" + + +namespace LLCore +{ + +class HttpReadyQueue; +class HttpOpRequest; + + +/// Implements class-based queuing policies for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. +class HttpPolicy +{ +public: + HttpPolicy(HttpService *); + virtual ~HttpPolicy(); + +private: + HttpPolicy(const HttpPolicy &); // Not defined + void operator=(const HttpPolicy &); // Not defined + +public: + /// Threading: called by init thread. + HttpRequest::policy_t createPolicyClass(); + + /// Cancel all ready and retry requests sending them to + /// their notification queues. Release state resources + /// making further request handling impossible. + /// + /// Threading: called by worker thread + void shutdown(); + + /// Deliver policy definitions and enable handling of + /// requests. One-time call invoked before starting + /// the worker thread. + /// + /// Threading: called by init thread + void start(); + + /// Give the policy layer some cycles to scan the ready + /// queue promoting higher-priority requests to active + /// as permited. + /// + /// @return Indication of how soon this method + /// should be called again. + /// + /// Threading: called by worker thread + HttpService::ELoopSpeed processReadyQueue(); + + /// Add request to a ready queue. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) + /// + /// OpRequest is owned by the request queue after this call + /// and should not be modified by anyone until retrieved + /// from queue. + /// + /// Threading: called by worker thread + void addOp(HttpOpRequest *); + + /// Similar to addOp, used when a caller wants to retry a + /// request that has failed. It's placed on a special retry + /// queue but ordered by retry time not priority. Otherwise, + /// handling is the same and retried operations are considered + /// before new ones but that doesn't guarantee completion + /// order. + /// + /// Threading: called by worker thread + void retryOp(HttpOpRequest *); + + /// Attempt to change the priority of an earlier request. + /// Request that Shadows HttpService's method + /// + /// Threading: called by worker thread + bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + + /// Attempt to cancel a previous request. + /// Shadows HttpService's method as well + /// + /// Threading: called by worker thread + bool cancel(HttpHandle handle); + + /// When transport is finished with an op and takes it off the + /// active queue, it is delivered here for dispatch. Policy + /// may send it back to the ready/retry queues if it needs another + /// go or we may finalize it and send it on to the reply queue. + /// + /// @return Returns true of the request is still active + /// or ready after staging, false if has been + /// sent on to the reply queue. + /// + /// Threading: called by worker thread + bool stageAfterCompletion(HttpOpRequest * op); + + /// Get a reference to global policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running. + HttpPolicyGlobal & getGlobalOptions() + { + return mGlobalOptions; + } + + /// Get a reference to class policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running and + /// read accesses by other threads are exposed to races at + /// that point. + HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass); + + /// Get ready counts for a particular policy class + /// + /// Threading: called by worker thread + int getReadyCount(HttpRequest::policy_t policy_class) const; + + /// Stall (or unstall) a policy class preventing requests from + /// transitioning to an active state. Used to allow an HTTP + /// request policy to empty prior to changing settings or state + /// that isn't tolerant of changes when work is outstanding. + /// + /// Threading: called by worker thread + bool stallPolicy(HttpRequest::policy_t policy_class, bool stall); + +protected: + struct ClassState; + typedef std::vector<ClassState *> class_list_t; + + HttpPolicyGlobal mGlobalOptions; + class_list_t mClasses; + HttpService * mService; // Naked pointer, not refcounted, not owner +}; // end class HttpPolicy + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_POLICY_H_ diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp new file mode 100755 index 0000000000..2c0f650155 --- /dev/null +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -0,0 +1,125 @@ +/** + * @file _httppolicyclass.cpp + * @brief Definitions for internal class defining class policy option. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httppolicyclass.h" + +#include "_httpinternal.h" + + +namespace LLCore +{ + + +HttpPolicyClass::HttpPolicyClass() + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mPipelining(HTTP_PIPELINING_DEFAULT), + mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT) +{} + + +HttpPolicyClass::~HttpPolicyClass() +{} + + +HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) +{ + if (this != &other) + { + mConnectionLimit = other.mConnectionLimit; + mPerHostConnectionLimit = other.mPerHostConnectionLimit; + mPipelining = other.mPipelining; + mThrottleRate = other.mThrottleRate; + } + return *this; +} + + +HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) + : mConnectionLimit(other.mConnectionLimit), + mPerHostConnectionLimit(other.mPerHostConnectionLimit), + mPipelining(other.mPipelining), + mThrottleRate(other.mThrottleRate) +{} + + +HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) +{ + switch (opt) + { + case HttpRequest::PO_CONNECTION_LIMIT: + mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); + break; + + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: + mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); + break; + + case HttpRequest::PO_PIPELINING_DEPTH: + mPipelining = llclamp(value, 0L, HTTP_PIPELINING_MAX); + break; + + case HttpRequest::PO_THROTTLE_RATE: + mThrottleRate = llclamp(value, 0L, 1000000L); + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + return HttpStatus(); +} + + +HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const +{ + switch (opt) + { + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; + break; + + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: + *value = mPerHostConnectionLimit; + break; + + case HttpRequest::PO_PIPELINING_DEPTH: + *value = mPipelining; + break; + + case HttpRequest::PO_THROTTLE_RATE: + *value = mThrottleRate; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + return HttpStatus(); +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h new file mode 100755 index 0000000000..38f1194ded --- /dev/null +++ b/indra/llcorehttp/_httppolicyclass.h @@ -0,0 +1,71 @@ +/** + * @file _httppolicyclass.h + * @brief Declarations for internal class defining policy class options. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_POLICY_CLASS_H_ +#define _LLCORE_HTTP_POLICY_CLASS_H_ + + +#include "httprequest.h" + + +namespace LLCore +{ + +/// Options struct for per-class policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. +class HttpPolicyClass +{ +public: + HttpPolicyClass(); + ~HttpPolicyClass(); + + HttpPolicyClass & operator=(const HttpPolicyClass &); + HttpPolicyClass(const HttpPolicyClass &); // Not defined + +public: + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; + +public: + long mConnectionLimit; + long mPerHostConnectionLimit; + long mPipelining; + long mThrottleRate; +}; // end class HttpPolicyClass + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_POLICY_CLASS_H_ diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp new file mode 100755 index 0000000000..1dc95f3dce --- /dev/null +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -0,0 +1,157 @@ +/** + * @file _httppolicyglobal.cpp + * @brief Definitions for internal class defining global policy option. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httppolicyglobal.h" + +#include "_httpinternal.h" + + +namespace LLCore +{ + + +HttpPolicyGlobal::HttpPolicyGlobal() + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + mTrace(HTTP_TRACE_OFF), + mUseLLProxy(0) +{} + + +HttpPolicyGlobal::~HttpPolicyGlobal() +{} + + +HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) +{ + if (this != &other) + { + mConnectionLimit = other.mConnectionLimit; + mCAPath = other.mCAPath; + mCAFile = other.mCAFile; + mHttpProxy = other.mHttpProxy; + mTrace = other.mTrace; + mUseLLProxy = other.mUseLLProxy; + } + return *this; +} + + +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value) +{ + switch (opt) + { + case HttpRequest::PO_CONNECTION_LIMIT: + mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); + break; + + case HttpRequest::PO_TRACE: + mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); + break; + + case HttpRequest::PO_LLPROXY: + mUseLLProxy = llclamp(value, 0L, 1L); + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + return HttpStatus(); +} + + +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value) +{ + switch (opt) + { + case HttpRequest::PO_CA_PATH: + mCAPath = value; + break; + + case HttpRequest::PO_CA_FILE: + mCAFile = value; + break; + + case HttpRequest::PO_HTTP_PROXY: + mCAFile = value; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + return HttpStatus(); +} + + +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const +{ + switch (opt) + { + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; + break; + + case HttpRequest::PO_TRACE: + *value = mTrace; + break; + + case HttpRequest::PO_LLPROXY: + *value = mUseLLProxy; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + return HttpStatus(); +} + + +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const +{ + switch (opt) + { + case HttpRequest::PO_CA_PATH: + *value = mCAPath; + break; + + case HttpRequest::PO_CA_FILE: + *value = mCAFile; + break; + + case HttpRequest::PO_HTTP_PROXY: + *value = mHttpProxy; + break; + + default: + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + + return HttpStatus(); +} + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h new file mode 100755 index 0000000000..67c4ba9481 --- /dev/null +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -0,0 +1,77 @@ +/** + * @file _httppolicyglobal.h + * @brief Declarations for internal class defining global policy option. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_POLICY_GLOBAL_H_ +#define _LLCORE_HTTP_POLICY_GLOBAL_H_ + + +#include "httprequest.h" + + +namespace LLCore +{ + +/// Options struct for global policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. +class HttpPolicyGlobal +{ +public: + HttpPolicyGlobal(); + ~HttpPolicyGlobal(); + + HttpPolicyGlobal & operator=(const HttpPolicyGlobal &); + +private: + HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined + +public: + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; + HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const; + +public: + long mConnectionLimit; + std::string mCAPath; + std::string mCAFile; + std::string mHttpProxy; + long mTrace; + long mUseLLProxy; +}; // end class HttpPolicyGlobal + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_POLICY_GLOBAL_H_ diff --git a/indra/llcorehttp/_httpreadyqueue.h b/indra/llcorehttp/_httpreadyqueue.h new file mode 100755 index 0000000000..5f19a9c5f9 --- /dev/null +++ b/indra/llcorehttp/_httpreadyqueue.h @@ -0,0 +1,124 @@ +/** + * @file _httpreadyqueue.h + * @brief Internal declaration for the operation ready queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_READY_QUEUE_H_ +#define _LLCORE_HTTP_READY_QUEUE_H_ + + +#include <queue> + +#include "_httpinternal.h" +#include "_httpoprequest.h" + + +namespace LLCore +{ + +/// HttpReadyQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// If LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY tests true, the class +/// implements a std::priority_queue interface but on std::deque +/// behavior to eliminate sensitivity to priority. In the future, +/// this will likely become the only behavior or it may become +/// a run-time election. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +#if LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + +typedef std::deque<HttpOpRequest *> HttpReadyQueueBase; + +#else + +typedef std::priority_queue<HttpOpRequest *, + std::deque<HttpOpRequest *>, + LLCore::HttpOpRequestCompare> HttpReadyQueueBase; + +#endif // LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + +class HttpReadyQueue : public HttpReadyQueueBase +{ +public: + HttpReadyQueue() + : HttpReadyQueueBase() + {} + + ~HttpReadyQueue() + {} + +protected: + HttpReadyQueue(const HttpReadyQueue &); // Not defined + void operator=(const HttpReadyQueue &); // Not defined + +public: + +#if LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + // Types and methods needed to make a std::deque look + // more like a std::priority_queue, at least for our + // purposes. + typedef HttpReadyQueueBase container_type; + + const_reference top() const + { + return front(); + } + + void pop() + { + pop_front(); + } + + void push(const value_type & v) + { + push_back(v); + } + +#endif // LLCORE_HTTP_READY_QUEUE_IGNORES_PRIORITY + + const container_type & get_container() const + { + return *this; + } + + container_type & get_container() + { + return *this; + } + +}; // end class HttpReadyQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_READY_QUEUE_H_ diff --git a/indra/llcorehttp/_httpreplyqueue.cpp b/indra/llcorehttp/_httpreplyqueue.cpp new file mode 100755 index 0000000000..558b7bdee9 --- /dev/null +++ b/indra/llcorehttp/_httpreplyqueue.cpp @@ -0,0 +1,107 @@ +/** + * @file _httpreplyqueue.cpp + * @brief Internal definitions for the operation reply queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpreplyqueue.h" + + +#include "_mutex.h" +#include "_thread.h" +#include "_httpoperation.h" + +using namespace LLCoreInt; + + +namespace LLCore +{ + + +HttpReplyQueue::HttpReplyQueue() + : RefCounted(true) +{ +} + + +HttpReplyQueue::~HttpReplyQueue() +{ + while (! mQueue.empty()) + { + HttpOperation * op = mQueue.back(); + mQueue.pop_back(); + op->release(); + } +} + + +void HttpReplyQueue::addOp(HttpOperation * op) +{ + { + HttpScopedLock lock(mQueueMutex); + + mQueue.push_back(op); + } + mQueueCV.notify_all(); +} + + +HttpOperation * HttpReplyQueue::fetchOp() +{ + HttpOperation * result(NULL); + + { + HttpScopedLock lock(mQueueMutex); + + if (mQueue.empty()) + return NULL; + + result = mQueue.front(); + mQueue.erase(mQueue.begin()); + } + + // Caller also acquires the reference count + return result; +} + + +void HttpReplyQueue::fetchAll(OpContainer & ops) +{ + // Not valid putting something back on the queue... + llassert_always(ops.empty()); + + { + HttpScopedLock lock(mQueueMutex); + + if (! mQueue.empty()) + { + mQueue.swap(ops); + } + } + + // Caller also acquires the reference counts on each op. + return; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpreplyqueue.h b/indra/llcorehttp/_httpreplyqueue.h new file mode 100755 index 0000000000..4220a09a3b --- /dev/null +++ b/indra/llcorehttp/_httpreplyqueue.h @@ -0,0 +1,108 @@ +/** + * @file _httpreplyqueue.h + * @brief Internal declarations for the operation reply queue. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_REPLY_QUEUE_H_ +#define _LLCORE_HTTP_REPLY_QUEUE_H_ + + +#include "_refcounted.h" +#include "_mutex.h" + + +namespace LLCore +{ + + +class HttpOperation; + + +/// Almost identical to the HttpRequestQueue class but +/// whereas that class is a singleton and is known to the +/// HttpService object, this queue is 1:1 with HttpRequest +/// instances and isn't explicitly referenced by the +/// service object. Instead, HttpOperation objects that +/// want to generate replies back to their creators also +/// keep references to the corresponding HttpReplyQueue. +/// The HttpService plumbing then simply delivers replies +/// to the requested reply queue. +/// +/// One result of that is that the fetch operations do +/// not have a wait forever option. The service object +/// doesn't keep handles on everything it would need to +/// notify so it can't wake up sleepers should it need to +/// shutdown. So only non-blocking or timed-blocking modes +/// are anticipated. These are how most application consumers +/// will be coded anyway so it shouldn't be too much of a +/// burden. + +class HttpReplyQueue : public LLCoreInt::RefCounted +{ +public: + /// Caller acquires a Refcount on construction + HttpReplyQueue(); + +protected: + virtual ~HttpReplyQueue(); // Use release() + +private: + HttpReplyQueue(const HttpReplyQueue &); // Not defined + void operator=(const HttpReplyQueue &); // Not defined + +public: + typedef std::vector<HttpOperation *> OpContainer; + + /// Insert an object at the back of the reply queue. + /// + /// Library also takes possession of one reference count to pass + /// through the queue. + /// + /// Threading: callable by any thread. + void addOp(HttpOperation * op); + + /// Fetch an operation from the head of the queue. Returns + /// NULL if none exists. + /// + /// Caller acquires reference count on returned operation. + /// + /// Threading: callable by any thread. + HttpOperation * fetchOp(); + + /// Caller acquires reference count on each returned operation + /// + /// Threading: callable by any thread. + void fetchAll(OpContainer & ops); + +protected: + OpContainer mQueue; + LLCoreInt::HttpMutex mQueueMutex; + LLCoreInt::HttpConditionVariable mQueueCV; + +}; // end class HttpReplyQueue + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_REPLY_QUEUE_H_ diff --git a/indra/llcorehttp/_httprequestqueue.cpp b/indra/llcorehttp/_httprequestqueue.cpp new file mode 100755 index 0000000000..c16966d078 --- /dev/null +++ b/indra/llcorehttp/_httprequestqueue.cpp @@ -0,0 +1,161 @@ +/** + * @file _httprequestqueue.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httprequestqueue.h" + +#include "_httpoperation.h" +#include "_mutex.h" + + +using namespace LLCoreInt; + +namespace LLCore +{ + +HttpRequestQueue * HttpRequestQueue::sInstance(NULL); + + +HttpRequestQueue::HttpRequestQueue() + : RefCounted(true), + mQueueStopped(false) +{ +} + + +HttpRequestQueue::~HttpRequestQueue() +{ + while (! mQueue.empty()) + { + HttpOperation * op = mQueue.back(); + mQueue.pop_back(); + op->release(); + } +} + + +void HttpRequestQueue::init() +{ + llassert_always(! sInstance); + sInstance = new HttpRequestQueue(); +} + + +void HttpRequestQueue::term() +{ + if (sInstance) + { + sInstance->release(); + sInstance = NULL; + } +} + + +HttpStatus HttpRequestQueue::addOp(HttpOperation * op) +{ + bool wake(false); + { + HttpScopedLock lock(mQueueMutex); + + if (mQueueStopped) + { + // Return op and error to caller + return HttpStatus(HttpStatus::LLCORE, HE_SHUTTING_DOWN); + } + wake = mQueue.empty(); + mQueue.push_back(op); + } + if (wake) + { + mQueueCV.notify_all(); + } + return HttpStatus(); +} + + +HttpOperation * HttpRequestQueue::fetchOp(bool wait) +{ + HttpOperation * result(NULL); + + { + HttpScopedLock lock(mQueueMutex); + + while (mQueue.empty()) + { + if (! wait || mQueueStopped) + return NULL; + mQueueCV.wait(lock); + } + + result = mQueue.front(); + mQueue.erase(mQueue.begin()); + } + + // Caller also acquires the reference count + return result; +} + + +void HttpRequestQueue::fetchAll(bool wait, OpContainer & ops) +{ + // Not valid putting something back on the queue... + llassert_always(ops.empty()); + + { + HttpScopedLock lock(mQueueMutex); + + while (mQueue.empty()) + { + if (! wait || mQueueStopped) + return; + mQueueCV.wait(lock); + } + + mQueue.swap(ops); + } + + // Caller also acquires the reference counts on each op. + return; +} + + +void HttpRequestQueue::wakeAll() +{ + mQueueCV.notify_all(); +} + + +void HttpRequestQueue::stopQueue() +{ + { + HttpScopedLock lock(mQueueMutex); + + mQueueStopped = true; + wakeAll(); + } +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httprequestqueue.h b/indra/llcorehttp/_httprequestqueue.h new file mode 100755 index 0000000000..c9c52b7233 --- /dev/null +++ b/indra/llcorehttp/_httprequestqueue.h @@ -0,0 +1,141 @@ +/** + * @file _httprequestqueue.h + * @brief Internal declaration for the operation request queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_REQUEST_QUEUE_H_ +#define _LLCORE_HTTP_REQUEST_QUEUE_H_ + + +#include <vector> + +#include "httpcommon.h" +#include "_refcounted.h" +#include "_mutex.h" + + +namespace LLCore +{ + + +class HttpOperation; + + +/// Thread-safe queue of HttpOperation objects. Just +/// a simple queue that handles the transfer of operation +/// requests from all HttpRequest instances into the +/// singleton HttpService instance. + +class HttpRequestQueue : public LLCoreInt::RefCounted +{ +protected: + /// Caller acquires a Refcount on construction + HttpRequestQueue(); + +protected: + virtual ~HttpRequestQueue(); // Use release() + +private: + HttpRequestQueue(const HttpRequestQueue &); // Not defined + void operator=(const HttpRequestQueue &); // Not defined + +public: + static void init(); + static void term(); + + /// Threading: callable by any thread once inited. + inline static HttpRequestQueue * instanceOf() + { + return sInstance; + } + +public: + typedef std::vector<HttpOperation *> OpContainer; + + /// Insert an object at the back of the request queue. + /// + /// Caller must provide one refcount to the queue which takes + /// possession of the count on success. + /// + /// @return Standard status. On failure, caller + /// must dispose of the operation with + /// an explicit release() call. + /// + /// Threading: callable by any thread. + HttpStatus addOp(HttpOperation * op); + + /// Return the operation on the front of the queue. If + /// the queue is empty and @wait is false, call returns + /// immediately and a NULL pointer is returned. If true, + /// caller will sleep until explicitly woken. Wakeups + /// can be spurious and callers must expect NULL pointers + /// even if waiting is indicated. + /// + /// Caller acquires reference count any returned operation + /// + /// Threading: callable by any thread. + HttpOperation * fetchOp(bool wait); + + /// Return all queued requests to caller. The @ops argument + /// should be empty when called and will be swap()'d with + /// current contents. Handling of the @wait argument is + /// identical to @fetchOp. + /// + /// Caller acquires reference count on each returned operation + /// + /// Threading: callable by any thread. + void fetchAll(bool wait, OpContainer & ops); + + /// Wake any sleeping threads. Normal queuing operations + /// won't require this but it may be necessary for termination + /// requests. + /// + /// Threading: callable by any thread. + void wakeAll(); + + /// Disallow further request queuing. Callers to @addOp will + /// get a failure status (LLCORE, HE_SHUTTING_DOWN). Callers + /// to @fetchAll or @fetchOp will get requests that are on the + /// queue but the calls will no longer wait. Instead they'll + /// return immediately. Also wakes up all sleepers to send + /// them on their way. + /// + /// Threading: callable by any thread. + void stopQueue(); + +protected: + static HttpRequestQueue * sInstance; + +protected: + OpContainer mQueue; + LLCoreInt::HttpMutex mQueueMutex; + LLCoreInt::HttpConditionVariable mQueueCV; + bool mQueueStopped; + +}; // end class HttpRequestQueue + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_REQUEST_QUEUE_H_ diff --git a/indra/llcorehttp/_httpretryqueue.h b/indra/llcorehttp/_httpretryqueue.h new file mode 100755 index 0000000000..745adec09d --- /dev/null +++ b/indra/llcorehttp/_httpretryqueue.h @@ -0,0 +1,94 @@ +/** + * @file _httpretryqueue.h + * @brief Internal declaration for the operation retry queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_RETRY_QUEUE_H_ +#define _LLCORE_HTTP_RETRY_QUEUE_H_ + + +#include <queue> + +#include "_httpoprequest.h" + + +namespace LLCore +{ + +/// HttpRetryQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +struct HttpOpRetryCompare +{ + bool operator()(const HttpOpRequest * lhs, const HttpOpRequest * rhs) + { + return lhs->mPolicyRetryAt < rhs->mPolicyRetryAt; + } +}; + + +typedef std::priority_queue<HttpOpRequest *, + std::deque<HttpOpRequest *>, + LLCore::HttpOpRetryCompare> HttpRetryQueueBase; + +class HttpRetryQueue : public HttpRetryQueueBase +{ +public: + HttpRetryQueue() + : HttpRetryQueueBase() + {} + + ~HttpRetryQueue() + {} + +protected: + HttpRetryQueue(const HttpRetryQueue &); // Not defined + void operator=(const HttpRetryQueue &); // Not defined + +public: + const container_type & get_container() const + { + return c; + } + + container_type & get_container() + { + return c; + } + +}; // end class HttpRetryQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_RETRY_QUEUE_H_ diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp new file mode 100755 index 0000000000..c673e1be1d --- /dev/null +++ b/indra/llcorehttp/_httpservice.cpp @@ -0,0 +1,494 @@ +/** + * @file _httpservice.cpp + * @brief Internal definitions of the Http service thread + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpservice.h" + +#include <boost/bind.hpp> +#include <boost/function.hpp> + +#include "_httpoperation.h" +#include "_httprequestqueue.h" +#include "_httppolicy.h" +#include "_httplibcurl.h" +#include "_thread.h" +#include "_httpinternal.h" + +#include "lltimer.h" +#include "llthread.h" + + +namespace +{ + +static const char * const LOG_CORE("CoreHttp"); + +} // end anonymous namespace + + +namespace LLCore +{ + +const HttpService::OptionDescriptor HttpService::sOptionDesc[] = +{ // isLong isDynamic isGlobal isClass + { true, true, true, true }, // PO_CONNECTION_LIMIT + { true, true, false, true }, // PO_PER_HOST_CONNECTION_LIMIT + { false, false, true, false }, // PO_CA_PATH + { false, false, true, false }, // PO_CA_FILE + { false, true, true, false }, // PO_HTTP_PROXY + { true, true, true, false }, // PO_LLPROXY + { true, true, true, false }, // PO_TRACE + { true, true, false, true }, // PO_ENABLE_PIPELINING + { true, true, false, true } // PO_THROTTLE_RATE +}; +HttpService * HttpService::sInstance(NULL); +volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); + +HttpService::HttpService() + : mRequestQueue(NULL), + mExitRequested(0U), + mThread(NULL), + mPolicy(NULL), + mTransport(NULL), + mLastPolicy(0) +{} + + +HttpService::~HttpService() +{ + mExitRequested = 1U; + if (RUNNING == sState) + { + // Trying to kill the service object with a running thread + // is a bit tricky. + if (mRequestQueue) + { + mRequestQueue->stopQueue(); + } + + if (mThread) + { + if (! mThread->timedJoin(250)) + { + // Failed to join, expect problems ahead so do a hard termination. + mThread->cancel(); + + LL_WARNS(LOG_CORE) << "Destroying HttpService with running thread. Expect problems." + << LL_ENDL; + } + } + } + + if (mRequestQueue) + { + mRequestQueue->release(); + mRequestQueue = NULL; + } + + delete mTransport; + mTransport = NULL; + + delete mPolicy; + mPolicy = NULL; + + if (mThread) + { + mThread->release(); + mThread = NULL; + } +} + + +void HttpService::init(HttpRequestQueue * queue) +{ + llassert_always(! sInstance); + llassert_always(NOT_INITIALIZED == sState); + sInstance = new HttpService(); + + queue->addRef(); + sInstance->mRequestQueue = queue; + sInstance->mPolicy = new HttpPolicy(sInstance); + sInstance->mTransport = new HttpLibcurl(sInstance); + sState = INITIALIZED; +} + + +void HttpService::term() +{ + if (sInstance) + { + if (RUNNING == sState && sInstance->mThread) + { + // Unclean termination. Thread appears to be running. We'll + // try to give the worker thread a chance to cancel using the + // exit flag... + sInstance->mExitRequested = 1U; + sInstance->mRequestQueue->stopQueue(); + + // And a little sleep + for (int i(0); i < 10 && RUNNING == sState; ++i) + { + ms_sleep(100); + } + } + + delete sInstance; + sInstance = NULL; + } + sState = NOT_INITIALIZED; +} + + +HttpRequest::policy_t HttpService::createPolicyClass() +{ + mLastPolicy = mPolicy->createPolicyClass(); + return mLastPolicy; +} + + +bool HttpService::isStopped() +{ + // What is really wanted here is something like: + // + // HttpService * service = instanceOf(); + // return STOPPED == sState && (! service || ! service->mThread || ! service->mThread->joinable()); + // + // But boost::thread is not giving me a consistent story on joinability + // of a thread after it returns. Debug and non-debug builds are showing + // different behavior on Linux/Etch so we do a weaker test that may + // not be globally correct (i.e. thread *is* stopping, may not have + // stopped but will very soon): + + return STOPPED == sState; +} + + +/// Threading: callable by consumer thread *once*. +void HttpService::startThread() +{ + llassert_always(! mThread || STOPPED == sState); + llassert_always(INITIALIZED == sState || STOPPED == sState); + + if (mThread) + { + mThread->release(); + } + + // Push current policy definitions, enable policy & transport components + mPolicy->start(); + mTransport->start(mLastPolicy + 1); + + mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); + sState = RUNNING; +} + + +/// Threading: callable by worker thread. +void HttpService::stopRequested() +{ + mExitRequested = 1U; +} + + +/// Threading: callable by worker thread. +bool HttpService::changePriority(HttpHandle handle, HttpRequest::priority_t priority) +{ + bool found(false); + + // Skip the request queue as we currently don't leave earlier + // requests sitting there. Start with the ready queue... + found = mPolicy->changePriority(handle, priority); + + // If not there, we could try the transport/active queue but priority + // doesn't really have much effect there so we don't waste cycles. + + return found; +} + + +/// Try to find the given request handle on any of the request +/// queues and cancel the operation. +/// +/// @return True if the request was canceled. +/// +/// Threading: callable by worker thread. +bool HttpService::cancel(HttpHandle handle) +{ + bool canceled(false); + + // Request can't be on request queue so skip that. + + // Check the policy component's queues first + canceled = mPolicy->cancel(handle); + + if (! canceled) + { + // If that didn't work, check transport's. + canceled = mTransport->cancel(handle); + } + + return canceled; +} + + +/// Threading: callable by worker thread. +void HttpService::shutdown() +{ + // Disallow future enqueue of requests + mRequestQueue->stopQueue(); + + // Cancel requests already on the request queue + HttpRequestQueue::OpContainer ops; + mRequestQueue->fetchAll(false, ops); + while (! ops.empty()) + { + HttpOperation * op(ops.front()); + ops.erase(ops.begin()); + + op->cancel(); + op->release(); + } + + // Shutdown transport canceling requests, freeing resources + mTransport->shutdown(); + + // And now policy + mPolicy->shutdown(); +} + + +// Working thread loop-forever method. Gives time to +// each of the request queue, policy layer and transport +// layer pieces and then either sleeps for a small time +// or waits for a request to come in. Repeats until +// requested to stop. +void HttpService::threadRun(LLCoreInt::HttpThread * thread) +{ + boost::this_thread::disable_interruption di; + + LLThread::registerThreadID(); + + ELoopSpeed loop(REQUEST_SLEEP); + while (! mExitRequested) + { + loop = processRequestQueue(loop); + + // Process ready queue issuing new requests as needed + ELoopSpeed new_loop = mPolicy->processReadyQueue(); + loop = (std::min)(loop, new_loop); + + // Give libcurl some cycles + new_loop = mTransport->processTransport(); + loop = (std::min)(loop, new_loop); + + // Determine whether to spin, sleep briefly or sleep for next request + if (REQUEST_SLEEP != loop) + { + ms_sleep(HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS); + } + } + + shutdown(); + sState = STOPPED; +} + + +HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) +{ + HttpRequestQueue::OpContainer ops; + const bool wait_for_req(REQUEST_SLEEP == loop); + + mRequestQueue->fetchAll(wait_for_req, ops); + while (! ops.empty()) + { + HttpOperation * op(ops.front()); + ops.erase(ops.begin()); + + // Process operation + if (! mExitRequested) + { + // Setup for subsequent tracing + long tracing(HTTP_TRACE_OFF); + mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing); + op->mTracing = (std::max)(op->mTracing, int(tracing)); + + if (op->mTracing > HTTP_TRACE_OFF) + { + LL_INFOS(LOG_CORE) << "TRACE, FromRequestQueue, Handle: " + << static_cast<HttpHandle>(op) + << LL_ENDL; + } + + // Stage + op->stageFromRequest(this); + } + + // Done with operation + op->release(); + } + + // Queue emptied, allow polling loop to sleep + return REQUEST_SLEEP; +} + + +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long * ret_value) +{ + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + } + + HttpStatus status; + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return status; + } + + // Only global has string values + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long value, long * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.set(opt, value); + if (status) + { + mTransport->policyUpdated(pclass); + if (ret_value) + { + status = opts.get(opt, ret_value); + } + } + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + const std::string & value, std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + // String values are always global (at this time). + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h new file mode 100755 index 0000000000..cf23f3ab61 --- /dev/null +++ b/indra/llcorehttp/_httpservice.h @@ -0,0 +1,236 @@ +/** + * @file _httpservice.h + * @brief Declarations for internal class providing HTTP service. + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_SERVICE_H_ +#define _LLCORE_HTTP_SERVICE_H_ + + +#include <vector> + +#include "linden_common.h" +#include "llapr.h" +#include "httpcommon.h" +#include "httprequest.h" +#include "_httppolicyglobal.h" +#include "_httppolicyclass.h" + + +namespace LLCoreInt +{ + +class HttpThread; + +} + + +namespace LLCore +{ + + +class HttpRequestQueue; +class HttpPolicy; +class HttpLibcurl; +class HttpOpSetGet; + + +/// The HttpService class does the work behind the request queue. It +/// oversees the HTTP workflow carrying out a number of tasks: +/// - Pulling requests from the global request queue +/// - Executing 'immediate' requests directly +/// - Prioritizing and re-queuing on internal queues the slower requests +/// - Providing cpu cycles to the libcurl plumbing +/// - Overseeing retry operations +/// +/// Note that the service object doesn't have a pointer to any +/// reply queue. These are kept by HttpRequest and HttpOperation +/// only. +/// +/// Service, Policy and Transport +/// +/// HttpService could have been a monolithic class combining a request +/// queue servicer, request policy manager and network transport. +/// Instead, to prevent monolithic growth and allow for easier +/// replacement, it was developed as three separate classes: HttpService, +/// HttpPolicy and HttpLibcurl (transport). These always exist in a +/// 1:1:1 relationship with HttpService managing instances of the other +/// two. So, these classes do not use reference counting to refer +/// to one another, their lifecycles are always managed together. + +class HttpService +{ +protected: + HttpService(); + virtual ~HttpService(); + +private: + HttpService(const HttpService &); // Not defined + void operator=(const HttpService &); // Not defined + +public: + enum EState + { + NOT_INITIALIZED = -1, + INITIALIZED, ///< init() has been called + RUNNING, ///< thread created and running + STOPPED ///< thread has committed to exiting + }; + + // Ordered enumeration of idling strategies available to + // threadRun's loop. Ordered so that std::min on values + // produces the most conservative result of multiple + // requests. + enum ELoopSpeed + { + NORMAL, ///< continuous polling of request, ready, active queues + REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write + }; + + static void init(HttpRequestQueue *); + static void term(); + + /// Threading: callable by any thread once inited. + inline static HttpService * instanceOf() + { + return sInstance; + } + + /// Return the state of the worker thread. Note that the + /// transition from RUNNING to STOPPED is performed by the + /// worker thread itself. This has two weaknesses: + /// - race where the thread hasn't really stopped but will + /// - data ordering between threads where a non-worker thread + /// may see a stale RUNNING status. + /// + /// This transition is generally of interest only to unit tests + /// and these weaknesses shouldn't be any real burden. + /// + /// Threading: callable by any thread with above exceptions. + static EState getState() + { + return sState; + } + + /// Threading: callable by any thread but uses @see getState() and + /// acquires its weaknesses. + static bool isStopped(); + + /// Threading: callable by init thread *once*. + void startThread(); + + /// Threading: callable by worker thread. + void stopRequested(); + + /// Threading: callable by worker thread. + void shutdown(); + + /// Try to find the given request handle on any of the request + /// queues and reset the priority (and queue position) of the + /// request if found. + /// + /// @return True if the request was found somewhere. + /// + /// Threading: callable by worker thread. + bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + + /// Try to find the given request handle on any of the request + /// queues and cancel the operation. + /// + /// @return True if the request was found and canceled. + /// + /// Threading: callable by worker thread. + bool cancel(HttpHandle handle); + + /// Threading: callable by worker thread. + HttpPolicy & getPolicy() + { + return *mPolicy; + } + + /// Threading: callable by worker thread. + HttpLibcurl & getTransport() + { + return *mTransport; + } + + /// Threading: callable by worker thread. + HttpRequestQueue & getRequestQueue() + { + return *mRequestQueue; + } + + /// Threading: callable by consumer thread. + HttpRequest::policy_t createPolicyClass(); + +protected: + void threadRun(LLCoreInt::HttpThread * thread); + + ELoopSpeed processRequestQueue(ELoopSpeed loop); + +protected: + friend class HttpOpSetGet; + friend class HttpRequest; + + // Used internally to describe what operations are allowed + // on each policy option. + struct OptionDescriptor + { + bool mIsLong; + bool mIsDynamic; + bool mIsGlobal; + bool mIsClass; + }; + + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long * ret_value); + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + std::string * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long value, long * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + const std::string & value, std::string * ret_value); + +protected: + static const OptionDescriptor sOptionDesc[HttpRequest::PO_LAST]; + static HttpService * sInstance; + + // === shared data === + static volatile EState sState; + HttpRequestQueue * mRequestQueue; // Refcounted + LLAtomicU32 mExitRequested; + LLCoreInt::HttpThread * mThread; + + // === working-thread-only data === + HttpPolicy * mPolicy; // Simple pointer, has ownership + HttpLibcurl * mTransport; // Simple pointer, has ownership + + // === main-thread-only data === + HttpRequest::policy_t mLastPolicy; + +}; // end class HttpService + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_SERVICE_H_ diff --git a/indra/llcorehttp/_mutex.h b/indra/llcorehttp/_mutex.h new file mode 100755 index 0000000000..4be4d016d4 --- /dev/null +++ b/indra/llcorehttp/_mutex.h @@ -0,0 +1,55 @@ +/** + * @file _mutex.hpp + * @brief mutex type abstraction + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLCOREINT_MUTEX_H_ +#define LLCOREINT_MUTEX_H_ + + +#include <boost/thread.hpp> + + +namespace LLCoreInt +{ + +// MUTEX TYPES + +// unique mutex type +typedef boost::mutex HttpMutex; + +// CONDITION VARIABLES + +// standard condition variable +typedef boost::condition_variable HttpConditionVariable; + +// LOCKS AND FENCES + +// scoped unique lock +typedef boost::unique_lock<HttpMutex> HttpScopedLock; + +} + +#endif // LLCOREINT_MUTEX_H + diff --git a/indra/llcorehttp/_refcounted.cpp b/indra/llcorehttp/_refcounted.cpp new file mode 100755 index 0000000000..e7d0b72741 --- /dev/null +++ b/indra/llcorehttp/_refcounted.cpp @@ -0,0 +1,45 @@ +/** + * @file _refcounted.cpp + * @brief Atomic, thread-safe ref counting and destruction mixin class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_refcounted.h" + + +namespace LLCoreInt +{ + +#if ! LL_WINDOWS + +const S32 RefCounted::NOT_REF_COUNTED; + +#endif // ! LL_WINDOWS + +RefCounted::~RefCounted() +{} + + +} // end namespace LLCoreInt + + diff --git a/indra/llcorehttp/_refcounted.h b/indra/llcorehttp/_refcounted.h new file mode 100755 index 0000000000..402e725152 --- /dev/null +++ b/indra/llcorehttp/_refcounted.h @@ -0,0 +1,126 @@ +/** + * @file _refcounted.h + * @brief Atomic, thread-safe ref counting and destruction mixin class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLCOREINT__REFCOUNTED_H_ +#define LLCOREINT__REFCOUNTED_H_ + + +#include "linden_common.h" + +#include "fix_macros.h" +#include <boost/thread.hpp> + +#include "llapr.h" + + +namespace LLCoreInt +{ + + +class RefCounted +{ +private: + RefCounted(); // Not defined - may not be default constructed + void operator=(const RefCounted &); // Not defined + +public: + explicit RefCounted(bool const implicit) + : mRefCount(implicit) + {} + + // ref-count interface + void addRef() const; + void release() const; + bool isLastRef() const; + S32 getRefCount() const; + void noRef() const; + + static const S32 NOT_REF_COUNTED = -1; + +protected: + virtual ~RefCounted(); + virtual void destroySelf(); + +private: + mutable LLAtomicS32 mRefCount; + +}; // end class RefCounted + + +inline void RefCounted::addRef() const +{ + S32 count(++mRefCount); + llassert_always(count >= 0); +} + + +inline void RefCounted::release() const +{ + S32 count(mRefCount); + llassert_always(count != NOT_REF_COUNTED); + llassert_always(count > 0); + count = --mRefCount; + + // clean ourselves up if that was the last reference + if (0 == count) + { + const_cast<RefCounted *>(this)->destroySelf(); + } +} + + +inline bool RefCounted::isLastRef() const +{ + const S32 count(mRefCount); + llassert_always(count != NOT_REF_COUNTED); + llassert_always(count >= 1); + return (1 == count); +} + + +inline S32 RefCounted::getRefCount() const +{ + const S32 result(mRefCount); + return result; +} + + +inline void RefCounted::noRef() const +{ + llassert_always(mRefCount <= 1); + mRefCount = NOT_REF_COUNTED; +} + + +inline void RefCounted::destroySelf() +{ + delete this; +} + +} // end namespace LLCoreInt + +#endif // LLCOREINT__REFCOUNTED_H_ + diff --git a/indra/llcorehttp/_thread.h b/indra/llcorehttp/_thread.h new file mode 100755 index 0000000000..e058d660e5 --- /dev/null +++ b/indra/llcorehttp/_thread.h @@ -0,0 +1,123 @@ +/** + * @file _thread.h + * @brief thread type abstraction + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LLCOREINT_THREAD_H_ +#define LLCOREINT_THREAD_H_ + +#include "linden_common.h" + +#include <boost/thread.hpp> +#include <boost/function.hpp> +#include <boost/date_time/posix_time/posix_time_types.hpp> + +#include "_refcounted.h" + +namespace LLCoreInt +{ + +class HttpThread : public RefCounted +{ +private: + HttpThread(); // Not defined + void operator=(const HttpThread &); // Not defined + + void at_exit() + { + // the thread function has exited so we need to release our reference + // to ourself so that we will be automagically cleaned up. + release(); + } + + void run() + { // THREAD CONTEXT + + // Take out additional reference for the at_exit handler + addRef(); + boost::this_thread::at_thread_exit(boost::bind(&HttpThread::at_exit, this)); + + // run the thread function + mThreadFunc(this); + + } // THREAD CONTEXT + +protected: + virtual ~HttpThread() + { + delete mThread; + } + +public: + /// Constructs a thread object for concurrent execution but does + /// not start running. Caller receives on refcount on the thread + /// instance. If the thread is started, another will be taken + /// out for the exit handler. + explicit HttpThread(boost::function<void (HttpThread *)> threadFunc) + : RefCounted(true), // implicit reference + mThreadFunc(threadFunc) + { + // this creates a boost thread that will call HttpThread::run on this instance + // and pass it the threadfunc callable... + boost::function<void()> f = boost::bind(&HttpThread::run, this); + + mThread = new boost::thread(f); + } + + inline void join() + { + mThread->join(); + } + + inline bool timedJoin(S32 millis) + { + return mThread->timed_join(boost::posix_time::milliseconds(millis)); + } + + inline bool joinable() const + { + return mThread->joinable(); + } + + // A very hostile method to force a thread to quit + inline void cancel() + { + boost::thread::native_handle_type thread(mThread->native_handle()); +#if LL_WINDOWS + TerminateThread(thread, 0); +#else + pthread_cancel(thread); +#endif + } + +private: + boost::function<void(HttpThread *)> mThreadFunc; + boost::thread * mThread; +}; // end class HttpThread + +} // end namespace LLCoreInt + +#endif // LLCOREINT_THREAD_H_ + + diff --git a/indra/llcorehttp/bufferarray.cpp b/indra/llcorehttp/bufferarray.cpp new file mode 100755 index 0000000000..8eaaeed710 --- /dev/null +++ b/indra/llcorehttp/bufferarray.cpp @@ -0,0 +1,352 @@ +/** + * @file bufferarray.cpp + * @brief Implements the BufferArray scatter/gather buffer + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "bufferarray.h" + + +// BufferArray is a list of chunks, each a BufferArray::Block, of contiguous +// data presented as a single array. Chunks are at least BufferArray::BLOCK_ALLOC_SIZE +// in length and can be larger. Any chunk may be partially filled or even +// empty. +// +// The BufferArray itself is sharable as a RefCounted entity. As shared +// reads don't work with the concept of a current position/seek value, +// none is kept with the object. Instead, the read and write operations +// all take position arguments. Single write/shared read isn't supported +// directly and any such attempts have to be serialized outside of this +// implementation. + +namespace LLCore +{ + + +// ================================== +// BufferArray::Block Declaration +// ================================== + +class BufferArray::Block +{ +public: + ~Block(); + + void operator delete(void *); + void operator delete(void *, size_t len); + +protected: + Block(size_t len); + + Block(const Block &); // Not defined + void operator=(const Block &); // Not defined + + // Allocate the block with the additional space for the + // buffered data at the end of the object. + void * operator new(size_t len, size_t addl_len); + +public: + // Only public entry to get a block. + static Block * alloc(size_t len); + +public: + size_t mUsed; + size_t mAlloced; + + // *NOTE: Must be last member of the object. We'll + // overallocate as requested via operator new and index + // into the array at will. + char mData[1]; +}; + + +// ================================== +// BufferArray Definitions +// ================================== + + +#if ! LL_WINDOWS +const size_t BufferArray::BLOCK_ALLOC_SIZE; +#endif // ! LL_WINDOWS + +BufferArray::BufferArray() + : LLCoreInt::RefCounted(true), + mLen(0) +{} + + +BufferArray::~BufferArray() +{ + for (container_t::iterator it(mBlocks.begin()); + it != mBlocks.end(); + ++it) + { + delete *it; + *it = NULL; + } + mBlocks.clear(); +} + + +size_t BufferArray::append(const void * src, size_t len) +{ + const size_t ret(len); + const char * c_src(static_cast<const char *>(src)); + + // First, try to copy into the last block + if (len && ! mBlocks.empty()) + { + Block & last(*mBlocks.back()); + if (last.mUsed < last.mAlloced) + { + // Some will fit... + const size_t copy_len((std::min)(len, (last.mAlloced - last.mUsed))); + + memcpy(&last.mData[last.mUsed], c_src, copy_len); + last.mUsed += copy_len; + llassert_always(last.mUsed <= last.mAlloced); + mLen += copy_len; + c_src += copy_len; + len -= copy_len; + } + } + + // Then get new blocks as needed + while (len) + { + const size_t copy_len((std::min)(len, BLOCK_ALLOC_SIZE)); + + if (mBlocks.size() >= mBlocks.capacity()) + { + mBlocks.reserve(mBlocks.size() + 5); + } + Block * block = Block::alloc(BLOCK_ALLOC_SIZE); + memcpy(block->mData, c_src, copy_len); + block->mUsed = copy_len; + llassert_always(block->mUsed <= block->mAlloced); + mBlocks.push_back(block); + mLen += copy_len; + c_src += copy_len; + len -= copy_len; + } + return ret; +} + + +void * BufferArray::appendBufferAlloc(size_t len) +{ + // If someone asks for zero-length, we give them a valid pointer. + if (mBlocks.size() >= mBlocks.capacity()) + { + mBlocks.reserve(mBlocks.size() + 5); + } + Block * block = Block::alloc((std::max)(BLOCK_ALLOC_SIZE, len)); + block->mUsed = len; + mBlocks.push_back(block); + mLen += len; + return block->mData; +} + + +size_t BufferArray::read(size_t pos, void * dst, size_t len) +{ + char * c_dst(static_cast<char *>(dst)); + + if (pos >= mLen) + return 0; + size_t len_limit(mLen - pos); + len = (std::min)(len, len_limit); + if (0 == len) + return 0; + + size_t result(0), offset(0); + const int block_limit(mBlocks.size()); + int block_start(findBlock(pos, &offset)); + if (block_start < 0) + return 0; + + do + { + Block & block(*mBlocks[block_start]); + size_t block_limit(block.mUsed - offset); + size_t block_len((std::min)(block_limit, len)); + + memcpy(c_dst, &block.mData[offset], block_len); + result += block_len; + len -= block_len; + c_dst += block_len; + offset = 0; + ++block_start; + } + while (len && block_start < block_limit); + + return result; +} + + +size_t BufferArray::write(size_t pos, const void * src, size_t len) +{ + const char * c_src(static_cast<const char *>(src)); + + if (pos > mLen || 0 == len) + return 0; + + size_t result(0), offset(0); + const int block_limit(mBlocks.size()); + int block_start(findBlock(pos, &offset)); + + if (block_start >= 0) + { + // Some or all of the write will be on top of + // existing data. + do + { + Block & block(*mBlocks[block_start]); + size_t block_limit(block.mUsed - offset); + size_t block_len((std::min)(block_limit, len)); + + memcpy(&block.mData[offset], c_src, block_len); + result += block_len; + c_src += block_len; + len -= block_len; + offset = 0; + ++block_start; + } + while (len && block_start < block_limit); + } + + // Something left, see if it will fit in the free + // space of the last block. + if (len && ! mBlocks.empty()) + { + Block & last(*mBlocks.back()); + if (last.mUsed < last.mAlloced) + { + // Some will fit... + const size_t copy_len((std::min)(len, (last.mAlloced - last.mUsed))); + + memcpy(&last.mData[last.mUsed], c_src, copy_len); + last.mUsed += copy_len; + result += copy_len; + llassert_always(last.mUsed <= last.mAlloced); + mLen += copy_len; + c_src += copy_len; + len -= copy_len; + } + } + + if (len) + { + // Some or all of the remaining write data will + // be an append. + result += append(c_src, len); + } + + return result; +} + + +int BufferArray::findBlock(size_t pos, size_t * ret_offset) +{ + *ret_offset = 0; + if (pos >= mLen) + return -1; // Doesn't exist + + const int block_limit(mBlocks.size()); + for (int i(0); i < block_limit; ++i) + { + if (pos < mBlocks[i]->mUsed) + { + *ret_offset = pos; + return i; + } + pos -= mBlocks[i]->mUsed; + } + + // Shouldn't get here but... + return -1; +} + + +bool BufferArray::getBlockStartEnd(int block, const char ** start, const char ** end) +{ + if (block < 0 || block >= mBlocks.size()) + { + return false; + } + + const Block & b(*mBlocks[block]); + *start = &b.mData[0]; + *end = &b.mData[b.mUsed]; + return true; +} + + +// ================================== +// BufferArray::Block Definitions +// ================================== + + +BufferArray::Block::Block(size_t len) + : mUsed(0), + mAlloced(len) +{ + memset(mData, 0, len); +} + + +BufferArray::Block::~Block() +{ + mUsed = 0; + mAlloced = 0; +} + + +void * BufferArray::Block::operator new(size_t len, size_t addl_len) +{ + void * mem = new char[len + addl_len + sizeof(void *)]; + return mem; +} + + +void BufferArray::Block::operator delete(void * mem) +{ + char * cmem = static_cast<char *>(mem); + delete [] cmem; +} + + +void BufferArray::Block::operator delete(void * mem, size_t) +{ + operator delete(mem); +} + + +BufferArray::Block * BufferArray::Block::alloc(size_t len) +{ + Block * block = new (len) Block(len); + return block; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h new file mode 100755 index 0000000000..1094a435b4 --- /dev/null +++ b/indra/llcorehttp/bufferarray.h @@ -0,0 +1,137 @@ +/** + * @file bufferarray.h + * @brief Public-facing declaration for the BufferArray scatter/gather class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_BUFFER_ARRAY_H_ +#define _LLCORE_BUFFER_ARRAY_H_ + + +#include <cstdlib> +#include <vector> + +#include "_refcounted.h" + + +namespace LLCore +{ + +class BufferArrayStreamBuf; + +/// A very simple scatter/gather type map for bulk data. The motivation +/// for this class is the writedata callback used by libcurl. Response +/// bodies are delivered to the caller in a sequence of sequential write +/// operations and this class captures them without having to reallocate +/// and move data. +/// +/// The interface looks a little like a unix file descriptor but only +/// just. There is a notion of a current position, starting from 0, +/// which is used as the position in the data when performing read and +/// write operations. The position also moves after various operations: +/// - seek(...) +/// - read(...) +/// - write(...) +/// - append(...) +/// - appendBufferAlloc(...) +/// The object also keeps a total length value which is updated after +/// write and append operations and beyond which the current position +/// cannot be set. +/// +/// Threading: not thread-safe +/// +/// Allocation: Refcounted, heap only. Caller of the constructor +/// is given a single refcount. +/// +class BufferArray : public LLCoreInt::RefCounted +{ +public: + // BufferArrayStreamBuf has intimate knowledge of this + // implementation to implement a buffer-free adapter. + // Changes here will likely need to be reflected there. + friend class BufferArrayStreamBuf; + + BufferArray(); + +protected: + virtual ~BufferArray(); // Use release() + +private: + BufferArray(const BufferArray &); // Not defined + void operator=(const BufferArray &); // Not defined + +public: + // Internal magic number, may be used by unit tests. + static const size_t BLOCK_ALLOC_SIZE = 65540; + + /// Appends the indicated data to the BufferArray + /// modifying current position and total size. New + /// position is one beyond the final byte of the buffer. + /// + /// @return Count of bytes copied to BufferArray + size_t append(const void * src, size_t len); + + /// Similar to @see append(), this call guarantees a + /// contiguous block of memory of requested size placed + /// at the current end of the BufferArray. On return, + /// the data in the memory is considered valid whether + /// the caller writes to it or not. + /// + /// @return Pointer to contiguous region at end + /// of BufferArray of 'len' size. + void * appendBufferAlloc(size_t len); + + /// Current count of bytes in BufferArray instance. + size_t size() const + { + return mLen; + } + + /// Copies data from the given position in the instance + /// to the caller's buffer. Will return a short count of + /// bytes copied if the 'len' extends beyond the data. + size_t read(size_t pos, void * dst, size_t len); + + /// Copies data from the caller's buffer to the instance + /// at the current position. May overwrite existing data, + /// append data when current position is equal to the + /// size of the instance or do a mix of both. + size_t write(size_t pos, const void * src, size_t len); + +protected: + int findBlock(size_t pos, size_t * ret_offset); + + bool getBlockStartEnd(int block, const char ** start, const char ** end); + +protected: + class Block; + typedef std::vector<Block *> container_t; + + container_t mBlocks; + size_t mLen; +}; // end class BufferArray + + +} // end namespace LLCore + +#endif // _LLCORE_BUFFER_ARRAY_H_ diff --git a/indra/llcorehttp/bufferstream.cpp b/indra/llcorehttp/bufferstream.cpp new file mode 100755 index 0000000000..678bf5ea9f --- /dev/null +++ b/indra/llcorehttp/bufferstream.cpp @@ -0,0 +1,283 @@ +/** + * @file bufferstream.cpp + * @brief Implements the BufferStream adapter class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "bufferstream.h" + +#include "bufferarray.h" + + +namespace LLCore +{ + +BufferArrayStreamBuf::BufferArrayStreamBuf(BufferArray * array) + : mBufferArray(array), + mReadCurPos(0), + mReadCurBlock(-1), + mReadBegin(NULL), + mReadCur(NULL), + mReadEnd(NULL), + mWriteCurPos(0) +{ + if (array) + { + array->addRef(); + mWriteCurPos = array->mLen; + } +} + + +BufferArrayStreamBuf::~BufferArrayStreamBuf() +{ + if (mBufferArray) + { + mBufferArray->release(); + mBufferArray = NULL; + } +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::underflow() +{ + if (! mBufferArray) + { + return traits_type::eof(); + } + + if (mReadCur == mReadEnd) + { + // Find the next block with actual data or leave + // mCurBlock/mCur/mEnd unchanged if we're at the end + // of any block chain. + const char * new_begin(NULL), * new_end(NULL); + int new_cur_block(mReadCurBlock + 1); + + while (mBufferArray->getBlockStartEnd(new_cur_block, &new_begin, &new_end)) + { + if (new_begin != new_end) + { + break; + } + ++new_cur_block; + } + if (new_begin == new_end) + { + return traits_type::eof(); + } + + mReadCurBlock = new_cur_block; + mReadBegin = mReadCur = new_begin; + mReadEnd = new_end; + } + + return traits_type::to_int_type(*mReadCur); +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::uflow() +{ + const int_type ret(underflow()); + + if (traits_type::eof() != ret) + { + ++mReadCur; + ++mReadCurPos; + } + return ret; +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::pbackfail(int_type ch) +{ + if (! mBufferArray) + { + return traits_type::eof(); + } + + if (mReadCur == mReadBegin) + { + // Find the previous block with actual data or leave + // mCurBlock/mBegin/mCur/mEnd unchanged if we're at the + // beginning of any block chain. + const char * new_begin(NULL), * new_end(NULL); + int new_cur_block(mReadCurBlock - 1); + + while (mBufferArray->getBlockStartEnd(new_cur_block, &new_begin, &new_end)) + { + if (new_begin != new_end) + { + break; + } + --new_cur_block; + } + if (new_begin == new_end) + { + return traits_type::eof(); + } + + mReadCurBlock = new_cur_block; + mReadBegin = new_begin; + mReadEnd = mReadCur = new_end; + } + + if (traits_type::eof() != ch && mReadCur[-1] != ch) + { + return traits_type::eof(); + } + --mReadCurPos; + return traits_type::to_int_type(*--mReadCur); +} + + +std::streamsize BufferArrayStreamBuf::showmanyc() +{ + if (! mBufferArray) + { + return -1; + } + return mBufferArray->mLen - mReadCurPos; +} + + +BufferArrayStreamBuf::int_type BufferArrayStreamBuf::overflow(int c) +{ + if (! mBufferArray || mWriteCurPos > mBufferArray->mLen) + { + return traits_type::eof(); + } + const size_t wrote(mBufferArray->write(mWriteCurPos, &c, 1)); + mWriteCurPos += wrote; + return wrote ? c : traits_type::eof(); +} + + +std::streamsize BufferArrayStreamBuf::xsputn(const char * src, std::streamsize count) +{ + if (! mBufferArray || mWriteCurPos > mBufferArray->mLen) + { + return 0; + } + const size_t wrote(mBufferArray->write(mWriteCurPos, src, count)); + mWriteCurPos += wrote; + return wrote; +} + + +std::streampos BufferArrayStreamBuf::seekoff(std::streamoff off, + std::ios_base::seekdir way, + std::ios_base::openmode which) +{ + std::streampos ret(-1); + + if (! mBufferArray) + { + return ret; + } + + if (std::ios_base::in == which) + { + size_t pos(0); + + switch (way) + { + case std::ios_base::beg: + pos = off; + break; + + case std::ios_base::cur: + pos = mReadCurPos += off; + break; + + case std::ios_base::end: + pos = mBufferArray->mLen - off; + break; + + default: + return ret; + } + + if (pos >= mBufferArray->size()) + { + pos = (std::max)(size_t(0), mBufferArray->size() - 1); + } + size_t ba_offset(0); + int block(mBufferArray->findBlock(pos, &ba_offset)); + if (block < 0) + return ret; + const char * start(NULL), * end(NULL); + if (! mBufferArray->getBlockStartEnd(block, &start, &end)) + return ret; + mReadCurBlock = block; + mReadBegin = start; + mReadCur = start + ba_offset; + mReadEnd = end; + ret = mReadCurPos = pos; + } + else if (std::ios_base::out == which) + { + size_t pos(0); + + switch (way) + { + case std::ios_base::beg: + pos = off; + break; + + case std::ios_base::cur: + pos = mWriteCurPos += off; + break; + + case std::ios_base::end: + pos = mBufferArray->mLen - off; + break; + + default: + return ret; + } + + if (pos > mBufferArray->size()) + { + pos = mBufferArray->size(); + } + ret = mWriteCurPos = pos; + } + + return ret; +} + + +BufferArrayStream::BufferArrayStream(BufferArray * ba) + : std::iostream(&mStreamBuf), + mStreamBuf(ba) +{} + + +BufferArrayStream::~BufferArrayStream() +{} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/bufferstream.h b/indra/llcorehttp/bufferstream.h new file mode 100755 index 0000000000..9327a798aa --- /dev/null +++ b/indra/llcorehttp/bufferstream.h @@ -0,0 +1,153 @@ +/** + * @file bufferstream.h + * @brief Public-facing declaration for the BufferStream adapter class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_BUFFER_STREAM_H_ +#define _LLCORE_BUFFER_STREAM_H_ + + +#include <sstream> +#include <cstdlib> + +#include "bufferarray.h" + + +/// @file bufferstream.h +/// +/// std::streambuf and std::iostream adapters for BufferArray +/// objects. +/// +/// BufferArrayStreamBuf inherits std::streambuf and implements +/// an unbuffered interface for streambuf. This may or may not +/// be the most time efficient implementation and it is a little +/// challenging. +/// +/// BufferArrayStream inherits std::iostream and will be the +/// adapter object most callers will be interested in (though +/// it uses BufferArrayStreamBuf internally). Instances allow +/// for the usual streaming operators ('<<', '>>') and serialization +/// methods. +/// +/// Example of LLSD serialization to a BufferArray: +/// +/// BufferArray * ba = new BufferArray; +/// BufferArrayStream bas(ba); +/// LLSDSerialize::toXML(llsd, bas); +/// operationOnBufferArray(ba); +/// ba->release(); +/// ba = NULL; +/// // operationOnBufferArray and bas are each holding +/// // references to the ba instance at this point. +/// + +namespace LLCore +{ + + +// ===================================================== +// BufferArrayStreamBuf +// ===================================================== + +/// Adapter class to put a std::streambuf interface on a BufferArray +/// +/// Application developers will rarely be interested in anything +/// other than the constructor and even that will rarely be used +/// except indirectly via the @BufferArrayStream class. The +/// choice of interfaces implemented yields a bufferless adapter +/// that doesn't used either the input or output pointer triplets +/// of the more common buffered implementations. This may or may +/// not be faster and that question could stand to be looked at +/// sometime. +/// + +class BufferArrayStreamBuf : public std::streambuf +{ +public: + /// Constructor increments the reference count on the + /// BufferArray argument and calls release() on destruction. + BufferArrayStreamBuf(BufferArray * array); + virtual ~BufferArrayStreamBuf(); + +private: + BufferArrayStreamBuf(const BufferArrayStreamBuf &); // Not defined + void operator=(const BufferArrayStreamBuf &); // Not defined + +public: + // Input interfaces from std::streambuf + int_type underflow(); + int_type uflow(); + int_type pbackfail(int_type ch); + std::streamsize showmanyc(); + + // Output interfaces from std::streambuf + int_type overflow(int c); + std::streamsize xsputn(const char * src, std::streamsize count); + + // Common/misc interfaces from std::streambuf + std::streampos seekoff(std::streamoff off, std::ios_base::seekdir way, std::ios_base::openmode which); + +protected: + BufferArray * mBufferArray; // Ref counted + size_t mReadCurPos; + int mReadCurBlock; + const char * mReadBegin; + const char * mReadCur; + const char * mReadEnd; + size_t mWriteCurPos; + +}; // end class BufferArrayStreamBuf + + +// ===================================================== +// BufferArrayStream +// ===================================================== + +/// Adapter class that supplies streaming operators to BufferArray +/// +/// Provides a streaming adapter to an existing BufferArray +/// instance so that the convenient '<<' and '>>' conversions +/// can be applied to a BufferArray. Very convenient for LLSD +/// serialization and parsing as well. + +class BufferArrayStream : public std::iostream +{ +public: + /// Constructor increments the reference count on the + /// BufferArray argument and calls release() on destruction. + BufferArrayStream(BufferArray * ba); + ~BufferArrayStream(); + +protected: + BufferArrayStream(const BufferArrayStream &); + void operator=(const BufferArrayStream &); + +protected: + BufferArrayStreamBuf mStreamBuf; +}; // end class BufferArrayStream + + +} // end namespace LLCore + +#endif // _LLCORE_BUFFER_STREAM_H_ diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp new file mode 100755 index 0000000000..9d9631b980 --- /dev/null +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -0,0 +1,1061 @@ +/** + * @file http_texture_load.cpp + * @brief Texture download example for core-http library + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include <iostream> +#include <cstdio> +#include <cstdlib> +#include <set> +#include <map> +#if !defined(WIN32) +#include <pthread.h> +#endif + +#include "linden_common.h" + +#include "httpcommon.h" +#include "httprequest.h" +#include "httphandler.h" +#include "httpresponse.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "bufferarray.h" +#include "_mutex.h" + +#include <curl/curl.h> +#include <openssl/crypto.h> + +#include "lltimer.h" + + +void init_curl(); +void term_curl(); +unsigned long ssl_thread_id_callback(void); +void ssl_locking_callback(int mode, int type, const char * file, int line); +void usage(std::ostream & out); + +// Default command line settings +static int concurrency_limit(40); +static int highwater(100); +static int pipeline_depth(0); +static int tracing(0); +static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; + +#if defined(WIN32) + +#define strncpy(_a, _b, _c) strncpy_s(_a, _b, _c) +#define strtok_r(_a, _b, _c) strtok_s(_a, _b, _c) + +int getopt(int argc, char * const argv[], const char *optstring); +char *optarg(NULL); +int optind(1); + +#endif + + +// Mostly just a container for the texture IDs and fetch +// parameters.... +class WorkingSet : public LLCore::HttpHandler +{ +public: + WorkingSet(); + ~WorkingSet(); + + bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *); + + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + void loadAssetUuids(FILE * in); + +public: + struct Spec + { + std::string mUuid; + int mOffset; + int mLength; + }; + typedef std::set<LLCore::HttpHandle> handle_set_t; + typedef std::vector<Spec> asset_list_t; + +public: + bool mVerbose; + bool mRandomRange; + bool mNoRange; + int mRequestLowWater; + int mRequestHighWater; + handle_set_t mHandles; + int mRemaining; + int mLimit; + int mAt; + std::string mUrl; + asset_list_t mAssets; + int mErrorsApi; + int mErrorsHttp; + int mErrorsHttp404; + int mErrorsHttp416; + int mErrorsHttp500; + int mErrorsHttp503; + int mRetries; + int mRetriesHttp503; + int mSuccesses; + long mByteCount; + LLCore::HttpHeaders * mHeaders; +}; + + +// Gather process information while we run. Process +// size, cpu consumed, wallclock time. + +class Metrics +{ +public: + class MetricsImpl; + +public: + Metrics(); + ~Metrics(); + + void init(); + void sample(); + void term(); + +protected: + MetricsImpl * mImpl; + +public: + U64 mMaxVSZ; + U64 mMinVSZ; + U64 mStartWallTime; + U64 mEndWallTime; + U64 mStartUTime; + U64 mEndUTime; + U64 mStartSTime; + U64 mEndSTime; +}; + + +// +// +// +int main(int argc, char** argv) +{ + LLCore::HttpStatus status; + bool do_random(false); + bool do_whole(false); + bool do_verbose(false); + + int option(-1); + while (-1 != (option = getopt(argc, argv, "u:c:h?RwvH:p:t:"))) + { + switch (option) + { + case 'u': + strncpy(url_format, optarg, sizeof(url_format)); + url_format[sizeof(url_format) - 1] = '\0'; + break; + + case 'c': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + concurrency_limit = value; + } + break; + + case 'H': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 200 || *end != '\0') + { + usage(std::cerr); + return 1; + } + highwater = value; + } + break; + + case 'p': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + pipeline_depth = value; + } + break; + + case '5': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value > 3 || *end != '\0') + { + usage(std::cerr); + return 1; + } + tracing = value; + } + break; + + case 'R': + do_random = true; + do_whole = false; + break; + + case 'w': + do_whole = true; + do_random = false; + break; + + case 'v': + do_verbose = true; + break; + + case 'h': + case '?': + usage(std::cout); + return 0; + } + } + + if ((optind + 1) != argc) + { + usage(std::cerr); + return 1; + } + + FILE * uuids(fopen(argv[optind], "r")); + if (! uuids) + { + const char * errstr(strerror(errno)); + + std::cerr << "Couldn't open UUID file '" << argv[optind] << "'. Reason: " + << errstr << std::endl; + return 1; + } + + // Initialization + init_curl(); + LLCore::HttpRequest::createService(); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); + if (pipeline_depth) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + pipeline_depth, + NULL); + } + if (tracing) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + tracing, + NULL); + } + LLCore::HttpRequest::startThread(); + + // Get service point + LLCore::HttpRequest * hr = new LLCore::HttpRequest(); + + // Get request options + LLCore::HttpOptions * opt = new LLCore::HttpOptions(); + opt->setRetries(12); + opt->setUseRetryAfter(true); + + // Get a handler/working set + WorkingSet ws; + + // Fill the working set with work + ws.mUrl = url_format; + ws.loadAssetUuids(uuids); + ws.mRandomRange = do_random; + ws.mNoRange = do_whole; + ws.mVerbose = do_verbose; + ws.mRequestHighWater = highwater; + ws.mRequestLowWater = ws.mRequestHighWater / 2; + + if (! ws.mAssets.size()) + { + std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; + return 1; + } + + // Setup metrics + Metrics metrics; + metrics.init(); + + // Run it + int passes(0); + while (! ws.reload(hr, opt)) + { + hr->update(0); + ms_sleep(2); + if (0 == (++passes % 200)) + { + metrics.sample(); + } + } + metrics.sample(); + metrics.term(); + + // Report + std::cout << "HTTP errors: " << ws.mErrorsHttp << " API errors: " << ws.mErrorsApi + << " Successes: " << ws.mSuccesses << " Byte count: " << ws.mByteCount + << std::endl; + std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416 + << " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503 + << std::endl; + std::cout << "Retries: " << ws.mRetries << " Retries on 503: " << ws.mRetriesHttp503 + << std::endl; + std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime) + << " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime) + << " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime) + << " uS Maximum VSZ: " << metrics.mMaxVSZ + << " Bytes Minimum VSZ: " << metrics.mMinVSZ << " Bytes" + << std::endl; + + // Clean up + hr->requestStopThread(NULL); + ms_sleep(1000); + opt->release(); + opt = NULL; + delete hr; + LLCore::HttpRequest::destroyService(); + term_curl(); + + return 0; +} + + +void usage(std::ostream & out) +{ + out << "\n" + "usage:\thttp_texture_load [options] uuid_file\n" + "\n" + "This is a standalone program to drive the New Platform HTTP Library.\n" + "The program is supplied with a file of texture UUIDs, one per line\n" + "These are fetched sequentially using a pool of concurrent connection\n" + "until all are fetched. The default URL format is only useful from\n" + "within Linden Lab but this can be overriden with a printf-style\n" + "URL formatting string on the command line.\n" + "\n" + "Options:\n" + "\n" + " -u <url_format> printf-style format string for URL generation\n" + " Default: " << url_format << "\n" + " -R Issue GETs with random Range: headers\n" + " -w Issue GETs without Range: headers to get whole object\n" + " -c <limit> Maximum connection concurrency. Range: [1..100]\n" + " Default: " << concurrency_limit << "\n" + " -H <limit> HTTP request highwater (requests fed to llcorehttp).\n" + " Range: [1..200] Default: " << highwater << "\n" + " -p <depth> If <depth> is positive, enables and sets pipelineing\n" + " depth on HTTP requests. Default: " << pipeline_depth << "\n" + " -t <level> If <level> is positive ([1..3]), enables and sets HTTP\n" + " tracing on HTTP requests. Default: " << tracing << "\n" + " -v Verbose mode. Issue some chatter while running\n" + " -h print this help\n" + "\n" + << std::endl; +} + + +WorkingSet::WorkingSet() + : LLCore::HttpHandler(), + mVerbose(false), + mRandomRange(false), + mNoRange(false), + mRemaining(200), + mLimit(200), + mAt(0), + mErrorsApi(0), + mErrorsHttp(0), + mErrorsHttp404(0), + mErrorsHttp416(0), + mErrorsHttp500(0), + mErrorsHttp503(0), + mRetries(0), + mRetriesHttp503(0), + mSuccesses(0), + mByteCount(0L) +{ + mAssets.reserve(30000); + + mHeaders = new LLCore::HttpHeaders; + mHeaders->append("Accept", "image/x-j2c"); +} + + +WorkingSet::~WorkingSet() +{ + if (mHeaders) + { + mHeaders->release(); + mHeaders = NULL; + } +} + + +bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) +{ + if (mRequestLowWater <= mHandles.size()) + { + // Haven't fallen below low-water level yet. + return false; + } + + int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size()))); + + for (int i(0); i < to_do; ++i) + { + char buffer[1024]; +#if defined(WIN32) + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str()); +#else + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); +#endif + int offset(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset)); + int length(mNoRange + ? 0 + : (mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength)); + + LLCore::HttpHandle handle; + if (offset || length) + { + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this); + } + else + { + handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this); + } + if (! handle) + { + // Fatal. Couldn't queue up something. + std::cerr << "Failed to queue work to HTTP Service. Reason: " + << hr->getStatus().toString() << std::endl; + exit(1); + } + else + { + mHandles.insert(handle); + } + mAt++; + mRemaining--; + + if (mVerbose) + { + static int count(0); + ++count; + if (0 == (count %5)) + std::cout << "Queued " << count << std::endl; + } + } + + // Are we done? + return (! mRemaining) && mHandles.empty(); +} + + +void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + handle_set_t::iterator it(mHandles.find(handle)); + if (mHandles.end() == it) + { + // Wha? + std::cerr << "Failed to find handle in request list. Fatal." << std::endl; + exit(1); + } + else + { + LLCore::HttpStatus status(response->getStatus()); + if (status) + { + // More success + LLCore::BufferArray * data(response->getBody()); + mByteCount += data ? data->size() : 0; + ++mSuccesses; + } + else + { + // Something in this library or libcurl + if (status.isHttpStatus()) + { + static const LLCore::HttpStatus hs404(404); + static const LLCore::HttpStatus hs416(416); + static const LLCore::HttpStatus hs500(500); + static const LLCore::HttpStatus hs503(503); + + ++mErrorsHttp; + if (hs404 == status) + { + ++mErrorsHttp404; + } + else if (hs416 == status) + { + ++mErrorsHttp416; + } + else if (hs500 == status) + { + ++mErrorsHttp500; + } + else if (hs503 == status) + { + ++mErrorsHttp503; + } + } + else + { + ++mErrorsApi; + } + } + unsigned int retry(0U), retry_503(0U); + response->getRetries(&retry, &retry_503); + mRetries += int(retry); + mRetriesHttp503 += int(retry_503); + mHandles.erase(it); + } + + if (mVerbose) + { + static int count(0); + ++count; + if (0 == (count %5)) + std::cout << "Handled " << count << std::endl; + } +} + + +void WorkingSet::loadAssetUuids(FILE * in) +{ + char buffer[1024]; + + while (fgets(buffer, sizeof(buffer), in)) + { + WorkingSet::Spec asset; + char * state(NULL); + char * token = strtok_r(buffer, " \t\n,", &state); + if (token && 36 == strlen(token)) + { + // Close enough for this function + asset.mUuid = token; + asset.mOffset = 0; + asset.mLength = 0; + token = strtok_r(buffer, " \t\n,", &state); + if (token) + { + int offset(atoi(token)); + token = strtok_r(buffer, " \t\n,", &state); + if (token) + { + int length(atoi(token)); + asset.mOffset = offset; + asset.mLength = length; + } + } + mAssets.push_back(asset); + } + } + mRemaining = mLimit = mAssets.size(); +} + + +int ssl_mutex_count(0); +LLCoreInt::HttpMutex ** ssl_mutex_list = NULL; + +void init_curl() +{ + curl_global_init(CURL_GLOBAL_ALL); + + ssl_mutex_count = CRYPTO_num_locks(); + if (ssl_mutex_count > 0) + { + ssl_mutex_list = new LLCoreInt::HttpMutex * [ssl_mutex_count]; + + for (int i(0); i < ssl_mutex_count; ++i) + { + ssl_mutex_list[i] = new LLCoreInt::HttpMutex; + } + + CRYPTO_set_locking_callback(ssl_locking_callback); + CRYPTO_set_id_callback(ssl_thread_id_callback); + } +} + + +void term_curl() +{ + CRYPTO_set_locking_callback(NULL); + for (int i(0); i < ssl_mutex_count; ++i) + { + delete ssl_mutex_list[i]; + } + delete [] ssl_mutex_list; +} + + +unsigned long ssl_thread_id_callback(void) +{ +#if defined(WIN32) + return (unsigned long) GetCurrentThread(); +#else + return (unsigned long) pthread_self(); +#endif +} + + +void ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) +{ + if (type >= 0 && type < ssl_mutex_count) + { + if (mode & CRYPTO_LOCK) + { + ssl_mutex_list[type]->lock(); + } + else + { + ssl_mutex_list[type]->unlock(); + } + } +} + + +#if defined(WIN32) + +// Very much a subset of posix functionality. Don't push +// it too hard... +int getopt(int argc, char * const argv[], const char *optstring) +{ + static int pos(0); + while (optind < argc) + { + if (pos == 0) + { + if (argv[optind][0] != '-') + return -1; + pos = 1; + } + if (! argv[optind][pos]) + { + ++optind; + pos = 0; + continue; + } + const char * thing(strchr(optstring, argv[optind][pos])); + if (! thing) + { + ++optind; + return -1; + } + if (thing[1] == ':') + { + optarg = argv[++optind]; + ++optind; + pos = 0; + } + else + { + optarg = NULL; + ++pos; + } + return *thing; + } + return -1; +} + +#endif + + + +#if LL_WINDOWS + +#define PSAPI_VERSION 1 +#include "windows.h" +#include "psapi.h" + +class Metrics::MetricsImpl +{ +public: + MetricsImpl() + {} + + ~MetricsImpl() + {} + + void init(Metrics * metrics) + { + HANDLE self(GetCurrentProcess()); // Does not have to be closed + FILETIME ft_dummy, ft_system, ft_user; + GetProcessTimes(self, &ft_dummy, &ft_dummy, &ft_system, &ft_user); + ULARGE_INTEGER uli; + uli.u.LowPart = ft_system.dwLowDateTime; + uli.u.HighPart = ft_system.dwHighDateTime; + metrics->mStartSTime = uli.QuadPart / U64L(10); // Convert to uS + uli.u.LowPart = ft_user.dwLowDateTime; + uli.u.HighPart = ft_user.dwHighDateTime; + metrics->mStartUTime = uli.QuadPart / U64L(10); + metrics->mStartWallTime = totalTime(); + } + + void sample(Metrics * metrics) + { + PROCESS_MEMORY_COUNTERS_EX counters; + + GetProcessMemoryInfo(GetCurrentProcess(), + (PROCESS_MEMORY_COUNTERS *) &counters, + sizeof(counters)); + // Okay, PrivateUsage isn't truly VSZ but it will be + // a good tracker for leaks and fragmentation. Work on + // a better estimator later... + SIZE_T vsz(counters.PrivateUsage); + metrics->mMaxVSZ = (std::max)(metrics->mMaxVSZ, U64(vsz)); + metrics->mMinVSZ = (std::min)(metrics->mMinVSZ, U64(vsz)); + } + + void term(Metrics * metrics) + { + HANDLE self(GetCurrentProcess()); // Does not have to be closed + FILETIME ft_dummy, ft_system, ft_user; + GetProcessTimes(self, &ft_dummy, &ft_dummy, &ft_system, &ft_user); + ULARGE_INTEGER uli; + uli.u.LowPart = ft_system.dwLowDateTime; + uli.u.HighPart = ft_system.dwHighDateTime; + metrics->mEndSTime = uli.QuadPart / U64L(10); + uli.u.LowPart = ft_user.dwLowDateTime; + uli.u.HighPart = ft_user.dwHighDateTime; + metrics->mEndUTime = uli.QuadPart / U64L(10); + metrics->mEndWallTime = totalTime(); + } + +protected: +}; + +#elif LL_DARWIN + +#include <sys/resource.h> +#include <mach/mach.h> + +class Metrics::MetricsImpl +{ +public: + MetricsImpl() + {} + + ~MetricsImpl() + {} + + void init(Metrics * metrics) + { + U64 utime, stime; + + if (getTimes(&utime, &stime)) + { + metrics->mStartSTime = stime; + metrics->mStartUTime = utime; + } + metrics->mStartWallTime = totalTime(); + sample(metrics); + } + + void sample(Metrics * metrics) + { + U64 vsz; + + if (getVM(&vsz)) + { + metrics->mMaxVSZ = (std::max)(metrics->mMaxVSZ, vsz); + metrics->mMinVSZ = (std::min)(metrics->mMinVSZ, vsz); + } + } + + void term(Metrics * metrics) + { + U64 utime, stime; + + if (getTimes(&utime, &stime)) + { + metrics->mEndSTime = stime; + metrics->mEndUTime = utime; + } + metrics->mEndWallTime = totalTime(); + } + +protected: + bool getVM(U64 * vsz) + { + task_basic_info task_info_block; + mach_msg_type_number_t task_info_count(TASK_BASIC_INFO_COUNT); + + if (KERN_SUCCESS != task_info(mach_task_self(), + TASK_BASIC_INFO, + (task_info_t) &task_info_block, + &task_info_count)) + { + return false; + } + * vsz = task_info_block.virtual_size; + return true; + } + + bool getTimes(U64 * utime, U64 * stime) + { + struct rusage usage; + + if (getrusage(RUSAGE_SELF, &usage)) + { + return false; + } + * utime = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; + * stime = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; + return true; + } + +}; + +#else + +class Metrics::MetricsImpl +{ +public: + MetricsImpl() + : mProcFS(NULL), + mUsecsPerTick(U64L(0)) + {} + + + ~MetricsImpl() + { + if (mProcFS) + { + fclose(mProcFS); + mProcFS = NULL; + } + } + + void init(Metrics * metrics) + { + if (! mProcFS) + { + mProcFS = fopen("/proc/self/stat", "r"); + if (! mProcFS) + { + const int errnum(errno); + LL_ERRS("Main") << "Error opening proc fs: " << strerror(errnum) << LL_ENDL; + } + } + + long ticks_per_sec(sysconf(_SC_CLK_TCK)); + mUsecsPerTick = U64L(1000000) / ticks_per_sec; + U64 usecs_per_sec(mUsecsPerTick * ticks_per_sec); + if (900000 > usecs_per_sec || 1100000 < usecs_per_sec) + { + LL_ERRS("Main") << "Resolution problems using uSecs for ticks" << LL_ENDL; + } + + U64 utime, stime; + if (scanProcFS(&utime, &stime, NULL)) + { + metrics->mStartSTime = stime; + metrics->mStartUTime = utime; + } + metrics->mStartWallTime = totalTime(); + + sample(metrics); + } + + + void sample(Metrics * metrics) + { + U64 vsz; + if (scanProcFS(NULL, NULL, &vsz)) + { + metrics->mMaxVSZ = (std::max)(metrics->mMaxVSZ, vsz); + metrics->mMinVSZ = (std::min)(metrics->mMinVSZ, vsz); + } + } + + + void term(Metrics * metrics) + { + U64 utime, stime; + if (scanProcFS(&utime, &stime, NULL)) + { + metrics->mEndSTime = stime; + metrics->mEndUTime = utime; + } + metrics->mEndWallTime = totalTime(); + + sample(metrics); + + if (mProcFS) + { + fclose(mProcFS); + mProcFS = NULL; + } + } + +protected: + bool scanProcFS(U64 * utime, U64 * stime, U64 * vsz) + { + if (mProcFS) + { + int i_dummy; + unsigned int ui_dummy; + unsigned long ul_dummy, user_ticks, sys_ticks, vsize; + long l_dummy, rss; + unsigned long long ull_dummy; + char c_dummy; + + char buffer[256]; + + static const char * format("%d %*s %c %d %d %d %d %d %u %lu %lu %lu %lu %lu %lu %ld %ld %ld %ld %ld %ld %llu %lu %ld"); + + fseek(mProcFS, 0L, SEEK_SET); + size_t len = fread(buffer, 1, sizeof(buffer) - 1, mProcFS); + if (! len) + { + return false; + } + buffer[len] = '\0'; + if (23 == sscanf(buffer, format, + &i_dummy, // pid + // &s_dummy, // command name + &c_dummy, // state + &i_dummy, // ppid + &i_dummy, // pgrp + &i_dummy, // session + &i_dummy, // terminal + &i_dummy, // terminal group id + &ui_dummy, // flags + &ul_dummy, // minor faults + &ul_dummy, // minor faults in children + &ul_dummy, // major faults + &ul_dummy, // major faults in children + &user_ticks, + &sys_ticks, + &l_dummy, // cutime + &l_dummy, // cstime + &l_dummy, // process priority + &l_dummy, // nice value + &l_dummy, // thread count + &l_dummy, // time to SIGALRM + &ull_dummy, // start time + &vsize, + &rss)) + { + // Looks like we understand the line + if (utime) + { + *utime = user_ticks * mUsecsPerTick; + } + + if (stime) + { + *stime = sys_ticks * mUsecsPerTick; + } + + if (vsz) + { + *vsz = vsize; + } + return true; + } + } + return false; + } + +protected: + FILE * mProcFS; + U64 mUsecsPerTick; + +}; + + +#endif // LL_WINDOWS + +Metrics::Metrics() + : mMaxVSZ(U64(0)), + mMinVSZ(U64L(0xffffffffffffffff)), + mStartWallTime(U64(0)), + mEndWallTime(U64(0)), + mStartUTime(U64(0)), + mEndUTime(U64(0)), + mStartSTime(U64(0)), + mEndSTime(U64(0)) +{ + mImpl = new MetricsImpl(); +} + + +Metrics::~Metrics() +{ + delete mImpl; + mImpl = NULL; +} + + +void Metrics::init() +{ + mImpl->init(this); +} + + +void Metrics::sample() +{ + mImpl->sample(this); +} + + +void Metrics::term() +{ + mImpl->term(this); +} + + diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp new file mode 100755 index 0000000000..7907e958a4 --- /dev/null +++ b/indra/llcorehttp/httpcommon.cpp @@ -0,0 +1,263 @@ +/** + * @file httpcommon.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpcommon.h" + +#include <curl/curl.h> +#include <string> +#include <sstream> + + +namespace LLCore +{ + +HttpStatus::type_enum_t EXT_CURL_EASY; +HttpStatus::type_enum_t EXT_CURL_MULTI; +HttpStatus::type_enum_t LLCORE; + +HttpStatus::operator unsigned long() const +{ + static const int shift(sizeof(unsigned long) * 4); + + unsigned long result(((unsigned long) mType) << shift | (unsigned long) (int) mStatus); + return result; +} + + +std::string HttpStatus::toHex() const +{ + std::ostringstream result; + result.width(8); + result.fill('0'); + result << std::hex << operator unsigned long(); + return result.str(); +} + + +std::string HttpStatus::toString() const +{ + static const char * llcore_errors[] = + { + "", + "HTTP error reply status", + "Services shutting down", + "Operation canceled", + "Invalid Content-Range header encountered", + "Request handle not found", + "Invalid datatype for argument or option", + "Option has not been explicitly set", + "Option is not dynamic and must be set early", + "Invalid HTTP status code received from server", + "Could not allocate required resource" + }; + static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); + + static const struct + { + type_enum_t mCode; + const char * mText; + } + http_errors[] = + { + // Keep sorted by mCode, we binary search this list. + { 100, "Continue" }, + { 101, "Switching Protocols" }, + { 200, "OK" }, + { 201, "Created" }, + { 202, "Accepted" }, + { 203, "Non-Authoritative Information" }, + { 204, "No Content" }, + { 205, "Reset Content" }, + { 206, "Partial Content" }, + { 300, "Multiple Choices" }, + { 301, "Moved Permanently" }, + { 302, "Found" }, + { 303, "See Other" }, + { 304, "Not Modified" }, + { 305, "Use Proxy" }, + { 307, "Temporary Redirect" }, + { 400, "Bad Request" }, + { 401, "Unauthorized" }, + { 402, "Payment Required" }, + { 403, "Forbidden" }, + { 404, "Not Found" }, + { 405, "Method Not Allowed" }, + { 406, "Not Acceptable" }, + { 407, "Proxy Authentication Required" }, + { 408, "Request Time-out" }, + { 409, "Conflict" }, + { 410, "Gone" }, + { 411, "Length Required" }, + { 412, "Precondition Failed" }, + { 413, "Request Entity Too Large" }, + { 414, "Request-URI Too Large" }, + { 415, "Unsupported Media Type" }, + { 416, "Requested range not satisfiable" }, + { 417, "Expectation Failed" }, + { 499, "Linden Catch-All" }, + { 500, "Internal Server Error" }, + { 501, "Not Implemented" }, + { 502, "Bad Gateway" }, + { 503, "Service Unavailable" }, + { 504, "Gateway Time-out" }, + { 505, "HTTP Version not supported" } + }; + static const int http_errors_count(sizeof(http_errors) / sizeof(http_errors[0])); + + if (*this) + { + return std::string(""); + } + switch (mType) + { + case EXT_CURL_EASY: + return std::string(curl_easy_strerror(CURLcode(mStatus))); + + case EXT_CURL_MULTI: + return std::string(curl_multi_strerror(CURLMcode(mStatus))); + + case LLCORE: + if (mStatus >= 0 && mStatus < llcore_errors_count) + { + return std::string(llcore_errors[mStatus]); + } + break; + + default: + if (isHttpStatus()) + { + // Binary search for the error code and string + int bottom(0), top(http_errors_count); + while (true) + { + int at((bottom + top) / 2); + if (mType == http_errors[at].mCode) + { + return std::string(http_errors[at].mText); + } + if (at == bottom) + { + break; + } + else if (mType < http_errors[at].mCode) + { + top = at; + } + else + { + bottom = at; + } + } + } + break; + } + return std::string("Unknown error"); +} + + +std::string HttpStatus::toTerseString() const +{ + std::ostringstream result; + + unsigned int error_value((unsigned short) mStatus); + + switch (mType) + { + case EXT_CURL_EASY: + result << "Easy_"; + break; + + case EXT_CURL_MULTI: + result << "Multi_"; + break; + + case LLCORE: + result << "Core_"; + break; + + default: + if (isHttpStatus()) + { + result << "Http_"; + error_value = mType; + } + else + { + result << "Unknown_"; + } + break; + } + + result << error_value; + return result.str(); +} + + +// Pass true on statuses that might actually be cleared by a +// retry. Library failures, calling problems, etc. aren't +// going to be fixed by squirting bits all over the Net. +// +// HE_INVALID_HTTP_STATUS is special. As of 7.37.0, there are +// some scenarios where response processing in libcurl appear +// to go wrong and response data is corrupted. A side-effect +// of this is that the HTTP status is read as 0 from the library. +// See libcurl bug report 1420 (https://sourceforge.net/p/curl/bugs/1420/) +// for details. +bool HttpStatus::isRetryable() const +{ + static const HttpStatus cant_connect(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + static const HttpStatus cant_res_proxy(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_PROXY); + static const HttpStatus cant_res_host(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_RESOLVE_HOST); + static const HttpStatus send_error(HttpStatus::EXT_CURL_EASY, CURLE_SEND_ERROR); + static const HttpStatus recv_error(HttpStatus::EXT_CURL_EASY, CURLE_RECV_ERROR); + static const HttpStatus upload_failed(HttpStatus::EXT_CURL_EASY, CURLE_UPLOAD_FAILED); + static const HttpStatus op_timedout(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); + static const HttpStatus post_error(HttpStatus::EXT_CURL_EASY, CURLE_HTTP_POST_ERROR); + static const HttpStatus partial_file(HttpStatus::EXT_CURL_EASY, CURLE_PARTIAL_FILE); + static const HttpStatus inv_cont_range(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + static const HttpStatus inv_status(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + + // *DEBUG: For "[curl:bugs] #1420" tests. + // Disable the '*this == inv_status' test and look for 'Core_9' + // failures in log files. + + return ((isHttpStatus() && mType >= 499 && mType <= 599) || // Include special 499 in retryables + *this == cant_connect || // Connection reset/endpoint problems + *this == cant_res_proxy || // DNS problems + *this == cant_res_host || // DNS problems + *this == send_error || // General socket problems + *this == recv_error || // General socket problems + *this == upload_failed || // Transport problem + *this == op_timedout || // Timer expired + *this == post_error || // Transport problem + *this == partial_file || // Data inconsistency in response + // *DEBUG: Comment out 'inv_status' test for [curl:bugs] #1420 testing. + *this == inv_status || // Inv status can reflect internal state problem in libcurl + *this == inv_cont_range); // Short data read disagrees with content-range +} + +} // end namespace LLCore + diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h new file mode 100755 index 0000000000..9601f94125 --- /dev/null +++ b/indra/llcorehttp/httpcommon.h @@ -0,0 +1,411 @@ +/** + * @file httpcommon.h + * @brief Public-facing declarations and definitions of common types + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_COMMON_H_ +#define _LLCORE_HTTP_COMMON_H_ + +/// @package LLCore::HTTP +/// +/// This library implements a high-level, Indra-code-free (somewhat) client +/// interface to HTTP services based on actual patterns found in the viewer +/// and simulator. Interfaces are similar to those supplied by the legacy classes +/// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that +/// allows an application to specify connection behaviors: limits on +/// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. +/// +/// Features of the library include: +/// - Single, private working thread where all transport and processing occurs. +/// - Support for multiple consumers running in multiple threads. +/// - Scatter/gather (a.k.a. buffer array) model for bulk data movement. +/// - Reference counting used for many object instance lifetimes. +/// - Minimal data sharing across threads for correctness and low latency. +/// +/// The public interface is declared in a few key header files: +/// - "llcorehttp/bufferarray.h" +/// - "llcorehttp/httpcommon.h" +/// - "llcorehttp/httphandler.h" +/// - "llcorehttp/httpheaders.h" +/// - "llcorehttp/httpoptions.h" +/// - "llcorehttp/httprequest.h" +/// - "llcorehttp/httpresponse.h" +/// +/// The library is still under development and particular users +/// may need access to internal implementation details that are found +/// in the _*.h header files. But this is a crutch to be avoided if at +/// all possible and probably indicates some interface work is neeeded. +/// +/// Using the library is fairly easy. Global setup needs a few +/// steps: +/// +/// - libcurl initialization including thread-safely callbacks for SSL: +/// . curl_global_init(...) +/// . CRYPTO_set_locking_callback(...) +/// . CRYPTO_set_id_callback(...) +/// - HttpRequest::createService() called to instantiate singletons +/// and support objects. +/// - HttpRequest::startThread() to kick off the worker thread and +/// begin servicing requests. +/// +/// An HTTP consumer in an application, and an application may have many +/// consumers, does a few things: +/// +/// - Instantiate and retain an object based on HttpRequest. This +/// object becomes the portal into runtime services for the consumer. +/// - Derive or mixin the HttpHandler class if you want notification +/// when requests succeed or fail. This object's onCompleted() +/// method is invoked and an instance can be shared across +/// requests. +/// +/// Issuing a request is straightforward: +/// - Construct a suitable URL. +/// - Configure HTTP options for the request. (optional) +/// - Build a list of additional headers. (optional) +/// - Invoke one of the requestXXXX() methods (requestGetByteRange, +/// requestPost, etc.) on the HttpRequest instance supplying the +/// above along with a policy class, a priority and an optional +/// pointer to an HttpHandler instance. Work is then queued to +/// the worker thread and occurs asynchronously. +/// - Periodically invoke the update() method on the HttpRequest +/// instance which performs completion notification to HttpHandler +/// objects. +/// - Do completion processing in your onCompletion() method. +/// +/// Code fragments. +/// +/// Initialization. Rather than a poorly-maintained example in +/// comments, look in the example subdirectory which is a minimal +/// yet functional tool to do GET request performance testing. +/// With four calls: +/// +/// init_curl(); +/// LLCore::HttpRequest::createService(); +/// LLCore::HttpRequest::startThread(); +/// LLCore::HttpRequest * hr = new LLCore::HttpRequest(); +/// +/// the program is basically ready to issue requests. +/// +/// HttpHandler. Having started life as a non-indra library, +/// this code broke away from the classic Responder model and +/// introduced a handler class to represent an interface for +/// request responses. This is a non-reference-counted entity +/// which can be used as a base class or a mixin. An instance +/// of a handler can be used for each request or can be shared +/// among any number of requests. Your choice but expect to +/// code something like the following: +/// +/// class AppHandler : public LLCore::HttpHandler +/// { +/// public: +/// virtual void onCompleted(HttpHandle handle, +/// HttpResponse * response) +/// { +/// ... +/// } +/// ... +/// }; +/// ... +/// handler = new handler(...); +/// +/// +/// Issuing requests. Using 'hr' above, +/// +/// hr->requestGet(HttpRequest::DEFAULT_POLICY_ID, +/// 0, // Priority, not used yet +/// url, +/// NULL, // options +/// NULL, // additional headers +/// handler); +/// +/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID, +/// the request was successfully issued and there will eventally +/// be a status delivered to the handler. If invalid is returnedd, +/// the actual status can be retrieved by calling hr->getStatus(). +/// +/// Completing requests and delivering notifications. Operations +/// are all performed by the worker thread and will be driven to +/// completion regardless of caller actions. Notification of +/// completion (success or failure) is done by calls to +/// HttpRequest::update() which will invoke handlers for completed +/// requests: +/// +/// hr->update(0); +/// // Callbacks into handler->onCompleted() +/// +/// +/// Threads. +/// +/// Threads are supported and used by this library. The various +/// classes, methods and members are documented with thread +/// constraints which programmers must follow and which are +/// defined as follows: +/// +/// consumer Any thread that has instanced HttpRequest and is +/// issuing requests. A particular instance can only +/// be used by one consumer thread but a consumer may +/// have many instances available to it. +/// init Special consumer thread, usually the main thread, +/// involved in setting up the library at startup. +/// worker Thread used internally by the library to perform +/// HTTP operations. Consumers will not have to deal +/// with this thread directly but some APIs are reserved +/// to it. +/// any Consumer or worker thread. +/// +/// For the most part, API users will not have to do much in the +/// way of ensuring thread safely. However, there is a tremendous +/// amount of sharing between threads of read-only data. So when +/// documentation declares that an option or header instance +/// becomes shared between consumer and worker, the consumer must +/// not modify the shared object. +/// +/// Internally, there is almost no thread synchronization. During +/// normal operations (non-init, non-term), only the request queue +/// and the multiple reply queues are shared between threads and +/// only here are mutexes used. +/// + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include <string> + + +namespace LLCore +{ + + +/// All queued requests are represented by an HttpHandle value. +/// The invalid value is returned when a request failed to queue. +/// The actual status for these failures is then fetched with +/// HttpRequest::getStatus(). +/// +/// The handle is valid only for the life of a request. On +/// return from any HttpHandler notification, the handle immediately +/// becomes invalid and may be recycled for other queued requests. + +typedef void * HttpHandle; +#define LLCORE_HTTP_HANDLE_INVALID (NULL) + +/// For internal scheduling and metrics, we use a microsecond +/// timebase compatible with the environment. +typedef U64 HttpTime; + +/// Error codes defined by the library itself as distinct from +/// libcurl (or any other transport provider). +enum HttpError +{ + // Successful value compatible with the libcurl codes. + HE_SUCCESS = 0, + + // Intended for HTTP reply codes 100-999, indicates that + // the reply should be considered an error by the application. + HE_REPLY_ERROR = 1, + + // Service is shutting down and requested operation will + // not be queued or performed. + HE_SHUTTING_DOWN = 2, + + // Operation was canceled by request. + HE_OP_CANCELED = 3, + + // Invalid content range header received. + HE_INV_CONTENT_RANGE_HDR = 4, + + // Request handle not found + HE_HANDLE_NOT_FOUND = 5, + + // Invalid datatype for option/setting + HE_INVALID_ARG = 6, + + // Option hasn't been explicitly set + HE_OPT_NOT_SET = 7, + + // Option not dynamic, must be set during init phase + HE_OPT_NOT_DYNAMIC = 8, + + // Invalid HTTP status code returned by server + HE_INVALID_HTTP_STATUS = 9, + + // Couldn't allocate resource, typically libcurl handle + HE_BAD_ALLOC = 10 + +}; // end enum HttpError + + +/// HttpStatus encapsulates errors from libcurl (easy, multi), HTTP +/// reply status codes and internal errors as well. The encapsulation +/// isn't expected to completely isolate the caller from libcurl but +/// basic operational tests (success or failure) are provided. +/// +/// Non-HTTP status are encoded as (type, status) with type being +/// one of: EXT_CURL_EASY, EXT_CURL_MULTI or LLCORE and status +/// being the success/error code from that domain. HTTP status +/// is encoded as (status, error_flag). Status should be in the +/// range [100, 999] and error_flag is either HE_SUCCESS or +/// HE_REPLY_ERROR to indicate whether this should be treated as +/// a successful status or an error. The application is responsible +/// for making that determination and a range like [200, 299] isn't +/// automatically assumed to be definitive. +/// +/// Examples: +/// +/// 1. Construct a default, successful status code: +/// HttpStatus(); +/// +/// 2. Construct a successful, HTTP 200 status code: +/// HttpStatus(200); +/// +/// 3. Construct a failed, HTTP 404 not-found status code: +/// HttpStatus(404); +/// +/// 4. Construct a failed libcurl couldn't connect status code: +/// HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); +/// +/// 5. Construct an HTTP 301 status code to be treated as success: +/// HttpStatus(301, HE_SUCCESS); +/// + +struct HttpStatus +{ + typedef unsigned short type_enum_t; + + HttpStatus() + : mType(LLCORE), + mStatus(HE_SUCCESS) + {} + + HttpStatus(type_enum_t type, short status) + : mType(type), + mStatus(status) + {} + + HttpStatus(int http_status) + : mType(http_status), + mStatus(http_status >= 200 && http_status <= 299 + ? HE_SUCCESS + : HE_REPLY_ERROR) + { + llassert(http_status >= 100 && http_status <= 999); + } + + HttpStatus(const HttpStatus & rhs) + : mType(rhs.mType), + mStatus(rhs.mStatus) + {} + + HttpStatus & operator=(const HttpStatus & rhs) + { + // Don't care if lhs & rhs are the same object + + mType = rhs.mType; + mStatus = rhs.mStatus; + return *this; + } + + static const type_enum_t EXT_CURL_EASY = 0; ///< mStatus is an error from a curl_easy_*() call + static const type_enum_t EXT_CURL_MULTI = 1; ///< mStatus is an error from a curl_multi_*() call + static const type_enum_t LLCORE = 2; ///< mStatus is an HE_* error code + ///< 100-999 directly represent HTTP status codes + + type_enum_t mType; + short mStatus; + + /// Test for successful status in the code regardless + /// of error source (internal, libcurl). + /// + /// @return 'true' when status is successful. + /// + operator bool() const + { + return 0 == mStatus; + } + + /// Inverse of previous operator. + /// + /// @return 'true' on any error condition + bool operator !() const + { + return 0 != mStatus; + } + + /// Equality and inequality tests to bypass bool conversion + /// which will do the wrong thing in conditional expressions. + bool operator==(const HttpStatus & rhs) const + { + return mType == rhs.mType && mStatus == rhs.mStatus; + } + + bool operator!=(const HttpStatus & rhs) const + { + return ! operator==(rhs); + } + + /// Convert to single numeric representation. Mainly + /// for logging or other informal purposes. Also + /// creates an ambiguous second path to integer conversion + /// which tends to find programming errors such as formatting + /// the status to a stream (operator<<). + operator unsigned long() const; + unsigned long toULong() const + { + return operator unsigned long(); + } + + /// And to convert to a hex string. + std::string toHex() const; + + /// Convert status to a string representation. For + /// success, returns an empty string. For failure + /// statuses, a string as appropriate for the source of + /// the error code (libcurl easy, libcurl multi, or + /// LLCore itself). + std::string toString() const; + + /// Convert status to a compact string representation + /// of the form: "<type>_<value>". The <type> will be + /// one of: Core, Http, Easy, Multi, Unknown. And + /// <value> will be an unsigned integer. More easily + /// interpreted than the hex representation, it's still + /// compact and easily searched. + std::string toTerseString() const; + + /// Returns true if the status value represents an + /// HTTP response status (100 - 999). + bool isHttpStatus() const + { + return mType >= type_enum_t(100) && mType <= type_enum_t(999); + } + + /// Returns true if the status is one that will be retried + /// internally. Provided for external consumption for cases + /// where that logic needs to be replicated. Only applies + /// to failed statuses, successful statuses will return false. + bool isRetryable() const; + +}; // end struct HttpStatus + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_COMMON_H_ diff --git a/indra/llcorehttp/httphandler.h b/indra/llcorehttp/httphandler.h new file mode 100755 index 0000000000..9171e4e7b9 --- /dev/null +++ b/indra/llcorehttp/httphandler.h @@ -0,0 +1,88 @@ +/** + * @file httphandler.h + * @brief Public-facing declarations for the HttpHandler class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_HANDLER_H_ +#define _LLCORE_HTTP_HANDLER_H_ + + +#include "httpcommon.h" + + +namespace LLCore +{ + +class HttpResponse; + + +/// HttpHandler defines an interface used by the library to +/// notify library callers of significant events, currently +/// request completion. Callers must derive or mixin this class +/// then provide an implementation of the @see onCompleted +/// method to receive such notifications. An instance may +/// be shared by any number of requests and across instances +/// of HttpRequest running in the same thread. +/// +/// Threading: HttpHandler itself is pure interface and is +/// tread-compatible. Most derivations, however, will have +/// different constraints. +/// +/// Allocation: Not refcounted, may be stack allocated though +/// that is rarely a good idea. Queued requests and replies keep +/// a naked pointer to the handler and this can result in a +/// dangling pointer if lifetimes aren't managed correctly. + +class HttpHandler +{ +public: + virtual ~HttpHandler() + {} + + /// Method invoked during calls to @see update(). Each invocation + /// represents the completion of some requested operation. Caller + /// can identify the request from the handle and interrogate the + /// response argument for success/failure, data and other information. + /// + /// @param handle Identifier of the request generating + /// the notification. + /// @param response Supplies detailed information about + /// the request including status codes + /// (both programming and HTTP), HTTP body + /// data and encodings, headers, etc. + /// The response object is refcounted and + /// the called code may retain the object + /// by invoking @see addRef() on it. The + /// library itself drops all references to + /// to object on return and never touches + /// it again. + /// + virtual void onCompleted(HttpHandle handle, HttpResponse * response) = 0; + +}; // end class HttpHandler + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_HANDLER_H_ diff --git a/indra/llcorehttp/httpheaders.cpp b/indra/llcorehttp/httpheaders.cpp new file mode 100755 index 0000000000..23ebea361c --- /dev/null +++ b/indra/llcorehttp/httpheaders.cpp @@ -0,0 +1,183 @@ +/** + * @file httpheaders.cpp + * @brief Implementation of the HTTPHeaders class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpheaders.h" + +#include "llstring.h" + + +namespace LLCore +{ + + +HttpHeaders::HttpHeaders() + : RefCounted(true) +{} + + +HttpHeaders::~HttpHeaders() +{} + + +void +HttpHeaders::clear() +{ + mHeaders.clear(); +} + + +void HttpHeaders::append(const std::string & name, const std::string & value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::append(const char * name, const char * value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::appendNormal(const char * header, size_t size) +{ + std::string name; + std::string value; + + int col_pos(0); + for (; col_pos < size; ++col_pos) + { + if (':' == header[col_pos]) + break; + } + + if (col_pos < size) + { + // Looks like a header, split it and normalize. + // Name is everything before the colon, may be zero-length. + name.assign(header, col_pos); + + // Value is everything after the colon, may also be zero-length. + const size_t val_len(size - col_pos - 1); + if (val_len) + { + value.assign(header + col_pos + 1, val_len); + } + + // Clean the strings + LLStringUtil::toLower(name); + LLStringUtil::trim(name); + LLStringUtil::trimHead(value); + } + else + { + // Uncertain what this is, we'll pack it as + // a name without a value. Won't clean as we don't + // know what it is... + name.assign(header, size); + } + + mHeaders.push_back(value_type(name, value)); +} + + +// Find from end to simulate a tradition of using single-valued +// std::map for this in the past. +const std::string * HttpHeaders::find(const char * name) const +{ + const_reverse_iterator iend(rend()); + for (const_reverse_iterator iter(rbegin()); iend != iter; ++iter) + { + if ((*iter).first == name) + { + return &(*iter).second; + } + } + return NULL; +} + + +// Standard Iterators +HttpHeaders::iterator HttpHeaders::begin() +{ + return mHeaders.begin(); +} + + +HttpHeaders::const_iterator HttpHeaders::begin() const +{ + return mHeaders.begin(); +} + + +HttpHeaders::iterator HttpHeaders::end() +{ + return mHeaders.end(); +} + + +HttpHeaders::const_iterator HttpHeaders::end() const +{ + return mHeaders.end(); +} + + +// Standard Reverse Iterators +HttpHeaders::reverse_iterator HttpHeaders::rbegin() +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rbegin() const +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::reverse_iterator HttpHeaders::rend() +{ + return mHeaders.rend(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rend() const +{ + return mHeaders.rend(); +} + + +// Return the raw container to the caller. +// +// To be used FOR UNIT TESTS ONLY. +// +HttpHeaders::container_t & HttpHeaders::getContainerTESTONLY() +{ + return mHeaders; +} + + +} // end namespace LLCore + diff --git a/indra/llcorehttp/httpheaders.h b/indra/llcorehttp/httpheaders.h new file mode 100755 index 0000000000..f70cd898f3 --- /dev/null +++ b/indra/llcorehttp/httpheaders.h @@ -0,0 +1,181 @@ +/** + * @file httpheaders.h + * @brief Public-facing declarations for the HttpHeaders class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_HEADERS_H_ +#define _LLCORE_HTTP_HEADERS_H_ + + +#include <string> + +#include "_refcounted.h" + + +namespace LLCore +{ + +/// +/// Maintains an ordered list of name/value pairs representing +/// HTTP header lines. This is used both to provide additional +/// headers when making HTTP requests and in responses when the +/// caller has asked that headers be returned (not the default +/// option). +/// +/// Class is mostly a thin wrapper around a vector of pairs +/// of strings. Methods provided are few and intended to +/// reflect actual use patterns. These include: +/// - Clearing the list +/// - Appending a name/value pair to the vector +/// - Processing a raw byte string into a normalized name/value +/// pair and appending the result. +/// - Simple case-sensitive find-last-by-name search +/// - Forward and reverse iterators over all pairs +/// +/// Container is ordered and multi-valued. Headers are +/// written in the order in which they are appended and +/// are stored in the order in which they're received from +/// the wire. The same header may appear two or more times +/// in any container. Searches using the simple find() +/// interface will find only the last occurrence (somewhat +/// simulates the use of std::map). Fuller searches require +/// the use of an iterator. Headers received from the wire +/// are only returned from the last request when redirections +/// are involved. +/// +/// Threading: Not intrinsically thread-safe. It *is* expected +/// that callers will build these objects and then share them +/// via reference counting with the worker thread. The implication +/// is that once an HttpHeader instance is handed to a request, +/// the object must be treated as read-only. +/// +/// Allocation: Refcounted, heap only. Caller of the +/// constructor is given a refcount. +/// + +class HttpHeaders : public LLCoreInt::RefCounted +{ +public: + typedef std::pair<std::string, std::string> header_t; + typedef std::vector<header_t> container_t; + typedef container_t::iterator iterator; + typedef container_t::const_iterator const_iterator; + typedef container_t::reverse_iterator reverse_iterator; + typedef container_t::const_reverse_iterator const_reverse_iterator; + typedef container_t::value_type value_type; + typedef container_t::size_type size_type; + +public: + /// @post In addition to the instance, caller has a refcount + /// to the instance. A call to @see release() will destroy + /// the instance. + HttpHeaders(); + +protected: + virtual ~HttpHeaders(); // Use release() + + HttpHeaders(const HttpHeaders &); // Not defined + void operator=(const HttpHeaders &); // Not defined + +public: + // Empty the list of headers. + void clear(); + + // Append a name/value pair supplied as either std::strings + // or NUL-terminated char * to the header list. No normalization + // is performed on the strings. No conformance test is + // performed (names may contain spaces, colons, etc.). + // + void append(const std::string & name, const std::string & value); + void append(const char * name, const char * value); + + // Extract a name/value pair from a raw byte array using + // the first colon character as a separator. Input string + // does not need to be NUL-terminated. Resulting name/value + // pair is appended to the header list. + // + // Normalization is performed on the name/value pair as + // follows: + // - name is lower-cased according to mostly ASCII rules + // - name is left- and right-trimmed of spaces and tabs + // - value is left-trimmed of spaces and tabs + // - either or both of name and value may be zero-length + // + // By convention, headers read from the wire will be normalized + // in this fashion prior to delivery to any HttpHandler code. + // Headers to be written to the wire are left as appended to + // the list. + void appendNormal(const char * header, size_t size); + + // Perform a simple, case-sensitive search of the header list + // returning a pointer to the value of the last matching header + // in the header list. If none is found, a NULL pointer is returned. + // + // Any pointer returned references objects in the container itself + // and will have the same lifetime as this class. If you want + // the value beyond the lifetime of this instance, make a copy. + // + // @arg name C-style string giving the name of a header + // to search. The comparison is case-sensitive + // though list entries may have been normalized + // to lower-case. + // + // @return NULL if the header wasn't found otherwise + // a pointer to a std::string in the container. + // Pointer is valid only for the lifetime of + // the container or until container is modifed. + // + const std::string * find(const char * name) const; + + // Count of headers currently in the list. + size_type size() const + { + return mHeaders.size(); + } + + // Standard std::vector-based forward iterators. + iterator begin(); + const_iterator begin() const; + iterator end(); + const_iterator end() const; + + // Standard std::vector-based reverse iterators. + reverse_iterator rbegin(); + const_reverse_iterator rbegin() const; + reverse_iterator rend(); + const_reverse_iterator rend() const; + +public: + // For unit tests only - not a public API + container_t & getContainerTESTONLY(); + +protected: + container_t mHeaders; + +}; // end class HttpHeaders + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_HEADERS_H_ diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp new file mode 100755 index 0000000000..5bf1ecb4a5 --- /dev/null +++ b/indra/llcorehttp/httpoptions.cpp @@ -0,0 +1,86 @@ +/** + * @file httpoptions.cpp + * @brief Implementation of the HTTPOptions class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpoptions.h" + +#include "_httpinternal.h" + + +namespace LLCore +{ + + +HttpOptions::HttpOptions() + : RefCounted(true), + mWantHeaders(false), + mTracing(HTTP_TRACE_OFF), + mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), + mRetries(HTTP_RETRY_COUNT_DEFAULT), + mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT) +{} + + +HttpOptions::~HttpOptions() +{} + + +void HttpOptions::setWantHeaders(bool wanted) +{ + mWantHeaders = wanted; +} + + +void HttpOptions::setTrace(long level) +{ + mTracing = int(level); +} + + +void HttpOptions::setTimeout(unsigned int timeout) +{ + mTimeout = timeout; +} + + +void HttpOptions::setTransferTimeout(unsigned int timeout) +{ + mTransferTimeout = timeout; +} + + +void HttpOptions::setRetries(unsigned int retries) +{ + mRetries = retries; +} + +void HttpOptions::setUseRetryAfter(bool use_retry) +{ + mUseRetryAfter = use_retry; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h new file mode 100755 index 0000000000..4ab5ff18c4 --- /dev/null +++ b/indra/llcorehttp/httpoptions.h @@ -0,0 +1,125 @@ +/** + * @file httpoptions.h + * @brief Public-facing declarations for the HTTPOptions class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPTIONS_H_ +#define _LLCORE_HTTP_OPTIONS_H_ + + +#include "httpcommon.h" + +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// Really a struct in spirit, it provides options that +/// modify HTTP requests. +/// +/// Sharing instances across requests. It's intended that +/// these be shared across requests: caller can create one +/// of these, set it up as needed and then reference it +/// repeatedly in HTTP operations. But see the Threading +/// note about references. +/// +/// Threading: While this class does nothing to ensure thread +/// safety, it *is* intended to be shared between the application +/// thread and the worker thread. This means that once an instance +/// is delivered to the library in request operations, the +/// option data must not be written until all such requests +/// complete and relinquish their references. +/// +/// Allocation: Refcounted, heap only. Caller of the constructor +/// is given a refcount. +/// +class HttpOptions : public LLCoreInt::RefCounted +{ +public: + HttpOptions(); + +protected: + virtual ~HttpOptions(); // Use release() + + HttpOptions(const HttpOptions &); // Not defined + void operator=(const HttpOptions &); // Not defined + +public: + // Default: false + void setWantHeaders(bool wanted); + bool getWantHeaders() const + { + return mWantHeaders; + } + + // Default: 0 + void setTrace(int long); + int getTrace() const + { + return mTracing; + } + + // Default: 30 + void setTimeout(unsigned int timeout); + unsigned int getTimeout() const + { + return mTimeout; + } + + // Default: 0 + void setTransferTimeout(unsigned int timeout); + unsigned int getTransferTimeout() const + { + return mTransferTimeout; + } + + // Default: 8 + void setRetries(unsigned int retries); + unsigned int getRetries() const + { + return mRetries; + } + + // Default: true + void setUseRetryAfter(bool use_retry); + bool getUseRetryAfter() const + { + return mUseRetryAfter; + } + +protected: + bool mWantHeaders; + int mTracing; + unsigned int mTimeout; + unsigned int mTransferTimeout; + unsigned int mRetries; + bool mUseRetryAfter; +}; // end class HttpOptions + + +} // end namespace HttpOptions + +#endif // _LLCORE_HTTP_OPTIONS_H_ diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp new file mode 100755 index 0000000000..7b1888e3eb --- /dev/null +++ b/indra/llcorehttp/httprequest.cpp @@ -0,0 +1,521 @@ +/** + * @file httprequest.cpp + * @brief Implementation of the HTTPRequest class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httprequest.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httppolicy.h" +#include "_httpoperation.h" +#include "_httpoprequest.h" +#include "_httpopsetpriority.h" +#include "_httpopcancel.h" +#include "_httpopsetget.h" + +#include "lltimer.h" + + +namespace +{ + +bool has_inited(false); + +} + +namespace LLCore +{ + +// ==================================== +// HttpRequest Implementation +// ==================================== + + +HttpRequest::HttpRequest() + : mReplyQueue(NULL), + mRequestQueue(NULL) +{ + mRequestQueue = HttpRequestQueue::instanceOf(); + mRequestQueue->addRef(); + + mReplyQueue = new HttpReplyQueue(); +} + + +HttpRequest::~HttpRequest() +{ + if (mRequestQueue) + { + mRequestQueue->release(); + mRequestQueue = NULL; + } + + if (mReplyQueue) + { + mReplyQueue->release(); + mReplyQueue = NULL; + } +} + + +// ==================================== +// Policy Methods +// ==================================== + + +HttpRequest::policy_t HttpRequest::createPolicyClass() +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return 0; + } + return HttpService::instanceOf()->createPolicyClass(); +} + + +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value) +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); +} + + +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value) +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + long value, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +// ==================================== +// Request Methods +// ==================================== + + +HttpStatus HttpRequest::getStatus() const +{ + return mLastReqStatus; +} + + +HttpHandle HttpRequest::requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupGet(policy_id, priority, url, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestGetByteRange(policy_t policy_id, + priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupGetByteRange(policy_id, priority, url, offset, len, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestPost(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPost(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestPut(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPut(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpNull * op = new HttpOpNull(); + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpStatus HttpRequest::update(long usecs) +{ + HttpOperation * op(NULL); + + if (usecs) + { + const HttpTime limit(totalTime() + HttpTime(usecs)); + while (limit >= totalTime() && (op = mReplyQueue->fetchOp())) + { + // Process operation + op->visitNotifier(this); + + // We're done with the operation + op->release(); + } + } + else + { + // Same as above, just no time limit + HttpReplyQueue::OpContainer replies; + mReplyQueue->fetchAll(replies); + if (! replies.empty()) + { + for (HttpReplyQueue::OpContainer::iterator iter(replies.begin()); + replies.end() != iter; + ++iter) + { + // Swap op pointer for NULL; + op = *iter; *iter = NULL; + + // Process operation + op->visitNotifier(this); + + // We're done with the operation + op->release(); + } + } + } + + return HttpStatus(); +} + + + + +// ==================================== +// Request Management Methods +// ==================================== + +HttpHandle HttpRequest::requestCancel(HttpHandle request, HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpCancel * op = new HttpOpCancel(request); + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return ret_handle; + } + + mLastReqStatus = status; + ret_handle = static_cast<HttpHandle>(op); + + return ret_handle; +} + + +HttpHandle HttpRequest::requestSetPriority(HttpHandle request, priority_t priority, + HttpHandler * handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetPriority * op = new HttpOpSetPriority(request, priority); + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return ret_handle; + } + + mLastReqStatus = status; + ret_handle = static_cast<HttpHandle>(op); + + return ret_handle; +} + + +// ==================================== +// Utility Methods +// ==================================== + +HttpStatus HttpRequest::createService() +{ + HttpStatus status; + + if (! has_inited) + { + HttpRequestQueue::init(); + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + HttpService::init(rq); + has_inited = true; + } + + return status; +} + + +HttpStatus HttpRequest::destroyService() +{ + HttpStatus status; + + if (has_inited) + { + HttpService::term(); + HttpRequestQueue::term(); + has_inited = false; + } + + return status; +} + + +HttpStatus HttpRequest::startThread() +{ + HttpStatus status; + + HttpService::instanceOf()->startThread(); + + return status; +} + + +HttpHandle HttpRequest::requestStopThread(HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpStop * op = new HttpOpStop(); + op->setReplyPath(mReplyQueue, user_handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +HttpHandle HttpRequest::requestSpin(int mode) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSpin * op = new HttpOpSpin(mode); + op->setReplyPath(mReplyQueue, NULL); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast<HttpHandle>(op); + + return handle; +} + + +} // end namespace LLCore + diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h new file mode 100755 index 0000000000..7f23723b0b --- /dev/null +++ b/indra/llcorehttp/httprequest.h @@ -0,0 +1,596 @@ +/** + * @file httprequest.h + * @brief Public-facing declarations for HttpRequest class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_REQUEST_H_ +#define _LLCORE_HTTP_REQUEST_H_ + + +#include "httpcommon.h" +#include "httphandler.h" + + +namespace LLCore +{ + +class HttpRequestQueue; +class HttpReplyQueue; +class HttpService; +class HttpOptions; +class HttpHeaders; +class HttpOperation; +class BufferArray; + +/// HttpRequest supplies the entry into the HTTP transport +/// services in the LLCore libraries. Services provided include: +/// +/// - Some, but not all, global initialization of libcurl. +/// - Starting asynchronous, threaded HTTP requests. +/// - Definition of policy classes affect request handling. +/// - Utilities to control request options and headers +/// +/// Requests +/// +/// The class supports the current HTTP request operations: +/// +/// - requestGetByteRange: GET with Range header for a single range of bytes +/// - requestGet: +/// - requestPost: +/// - requestPut: +/// +/// Policy Classes +/// +/// <TBD> +/// +/// Usage +/// +/// <TBD> +/// +/// Threading: An instance may only be used by one application/ +/// consumer thread. But a thread may have as many instances of +/// this as it likes. +/// +/// Allocation: Not refcounted, may be stack allocated though that +/// hasn't been tested. Queued requests can still run and any +/// queued replies will keep refcounts to the reply queue leading +/// to memory leaks. +/// +/// @pre Before using this class (static or instances), some global +/// initialization is required. See @see httpcommon.h for more information. +/// +/// @nosubgrouping +/// + +class HttpRequest +{ +public: + HttpRequest(); + virtual ~HttpRequest(); + +private: + HttpRequest(const HttpRequest &); // Disallowed + void operator=(const HttpRequest &); // Disallowed + +public: + typedef unsigned int policy_t; + typedef unsigned int priority_t; + +public: + /// @name PolicyMethods + /// @{ + + /// Represents a default, catch-all policy class that guarantees + /// eventual service for any HTTP request. + static const policy_t DEFAULT_POLICY_ID = 0; + static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU; + static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU; + + /// Create a new policy class into which requests can be made. + /// + /// All class creation must occur before threads are started and + /// transport begins. Policy classes are limited to a small value. + /// Currently that limit is the default class + 1. + /// + /// @return If positive, the policy_id used to reference + /// the class in other methods. If 0, requests + /// for classes have exceeded internal limits + /// or caller has tried to create a class after + /// threads have been started. Caller must fallback + /// and recover. + /// + static policy_t createPolicyClass(); + + enum EPolicyOption + { + /// Maximum number of connections the library will use to + /// perform operations. This is somewhat soft as the underlying + /// transport will cache some connections (up to 5). + + /// A long value setting the maximum number of connections + /// allowed over all policy classes. Note that this will be + /// a somewhat soft value. There may be an additional five + /// connections per policy class depending upon runtime + /// behavior. + /// + /// Both global and per-class + PO_CONNECTION_LIMIT, + + /// Limits the number of connections used for a single + /// literal address/port pair within the class. + /// + /// Per-class only + PO_PER_HOST_CONNECTION_LIMIT, + + /// String containing a system-appropriate directory name + /// where SSL certs are stored. + /// + /// Global only + PO_CA_PATH, + + /// String giving a full path to a file containing SSL certs. + /// + /// Global only + PO_CA_FILE, + + /// String of host/port to use as simple HTTP proxy. This is + /// going to change in the future into something more elaborate + /// that may support richer schemes. + /// + /// Global only + PO_HTTP_PROXY, + + /// Long value that if non-zero enables the use of the + /// traditional LLProxy code for http/socks5 support. If + // enabled, has priority over GP_HTTP_PROXY. + /// + /// Global only + PO_LLPROXY, + + /// Long value setting the logging trace level for the + /// library. Possible values are: + /// 0 - No tracing (default) + /// 1 - Basic tracing of request start, stop and major events. + /// 2 - Connection, header and payload size information from + /// HTTP transactions. + /// 3 - Partial logging of payload itself. + /// + /// These values are also used in the trace modes for + /// individual requests in HttpOptions. Also be aware that + /// tracing tends to impact performance of the viewer. + /// + /// Global only + PO_TRACE, + + /// If greater than 1, suitable requests are allowed to + /// pipeline on their connections when they ask for it. + /// Value gives the maximum number of outstanding requests + /// on a connection. + /// + /// There is some interaction between PO_CONNECTION_LIMIT, + /// PO_PER_HOST_CONNECTION_LIMIT, and PO_PIPELINING_DEPTH. + /// When PIPELINING_DEPTH is 0 or 1 (no pipelining), this + /// library manages connection lifecycle and honors the + /// PO_CONNECTION_LIMIT setting as the maximum in-flight + /// request limit. Libcurl itself may be caching additional + /// connections under its connection cache policy. + /// + /// When PIPELINING_DEPTH is 2 or more, libcurl performs + /// connection management and both PO_CONNECTION_LIMIT and + /// PO_PER_HOST_CONNECTION_LIMIT should be set and non-zero. + /// In this case (as of libcurl 7.37.0), libcurl will + /// open new connections in preference to pipelining, up + /// to the above limits at which time pipelining begins. + /// And as usual, an additional cache of open but inactive + /// connections may still be maintained within libcurl. + /// For SL, a good rule-of-thumb is to set + /// PO_PER_HOST_CONNECTION_LIMIT to the user-visible + /// concurrency value and PO_CONNECTION_LIMIT to twice + /// that for baked texture loads and region crossings where + /// additional connection load will be tolerated. If + /// either limit is 0, libcurl will prefer pipelining + /// over connection creation, which is still interesting, + /// but won't be pursued at this time. + /// + /// Per-class only + PO_PIPELINING_DEPTH, + + /// Controls whether client-side throttling should be + /// performed on this policy class. Positive values + /// enable throttling and specify the request rate + /// (requests per second) that should be targetted. + /// A value of zero, the default, specifies no throttling. + /// + /// Per-class only + PO_THROTTLE_RATE, + + PO_LAST // Always at end + }; + + /// Set a policy option for a global or class parameter at + /// startup time (prior to thread start). + /// + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. For globals, specify GLOBAL_POLICY_ID. + /// @param value Desired value of option. + /// @param ret_value Pointer to receive effective set value + /// if successful. May be NULL if effective + /// value not wanted. + /// @return Standard status code. + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value); + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value); + + /// Set a parameter on a class-based policy option. Calls + /// made after the start of the servicing thread are + /// not honored and return an error status. + /// + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. Ignored for globals but recommend + /// using INVALID_POLICY_ID in this case. + /// @param value Desired value of option. + /// @return Handle of dynamic request. Use @see getStatus() if + /// the returned handle is invalid. + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value, + HttpHandler * handler); + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value, + HttpHandler * handler); + + /// @} + + /// @name RequestMethods + /// + /// @{ + + /// Some calls expect to succeed as the normal part of operation and so + /// return a useful value rather than a status. When they do fail, the + /// status is saved and can be fetched with this method. + /// + /// @return Status of the failing method invocation. If the + /// preceding call succeeded or other HttpStatus + /// returning calls immediately preceded this method, + /// the returned value may not be reliable. + /// + HttpStatus getStatus() const; + + /// Queue a full HTTP GET request to be issued for entire entity. + /// The request is queued and serviced by the working thread and + /// notification of completion delivered to the optional HttpHandler + /// argument during @see update() calls. + /// + /// With a valid handle returned, it can be used to reference the + /// request in other requests (like cancellation) and will be an + /// argument when any HttpHandler object is invoked. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-alive: 300 + /// - Host: <stuff> + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-control: + /// - Range: + /// - Transfer-Encoding: + /// - Referer: + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base (U32-type scheme). + /// @param url URL with any encoded query parameters to + /// be accessed. + /// @param options Optional instance of an HttpOptions object + /// to provide additional controls over the request + /// function for this request only. Any such + /// object then becomes shared-read across threads + /// and no code should modify the HttpOptions + /// instance. + /// @param headers Optional instance of an HttpHeaders object + /// to provide additional and/or overridden + /// headers for the request. As with options, + /// the instance becomes shared-read across threads + /// and no code should modify the HttpHeaders + /// instance. + /// @param handler Optional pointer to an HttpHandler instance + /// whose onCompleted() method will be invoked + /// during calls to update(). This is a non- + /// reference-counted object which would be a + /// problem for shutdown and other edge cases but + /// the pointer is only dereferenced during + /// calls to update(). + /// + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestGet(policy_t policy_id, + priority_t priority, + const std::string & url, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a full HTTP GET request to be issued with a 'Range' header. + /// The request is queued and serviced by the working thread and + /// notification of completion delivered to the optional HttpHandler + /// argument during @see update() calls. + /// + /// With a valid handle returned, it can be used to reference the + /// request in other requests (like cancellation) and will be an + /// argument when any HttpHandler object is invoked. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-alive: 300 + /// - Host: <stuff> + /// - Range: <stuff> (will be omitted if offset == 0 and len == 0) + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-control: + /// - Transfer-Encoding: + /// - Referer: + /// + /// @param policy_id @see requestGet() + /// @param priority " + /// @param url " + /// @param offset Offset of first byte into resource to be returned. + /// @param len Count of bytes to be returned + /// @param options @see requestGet() + /// @param headers " + /// @param handler " + /// @return " + /// + HttpHandle requestGetByteRange(policy_t policy_id, + priority_t priority, + const std::string & url, + size_t offset, + size_t len, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a full HTTP POST. Query arguments and body may + /// be provided. Caller is responsible for escaping and + /// encoding and communicating the content types. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-Alive: 300 + /// - Host: <stuff> + /// - Content-Length: <digits> + /// - Content-Type: application/x-www-form-urlencoded + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-Control: + /// - Transfer-Encoding: ... chunked ... + /// - Referer: + /// - Content-Encoding: + /// - Expect: + /// + /// @param policy_id @see requestGet() + /// @param priority " + /// @param url " + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options @see requestGet()K(optional) + /// @param headers " + /// @param handler " + /// @return " + /// + HttpHandle requestPost(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a full HTTP PUT. Query arguments and body may + /// be provided. Caller is responsible for escaping and + /// encoding and communicating the content types. + /// + /// Headers supplied by default: + /// - Connection: keep-alive + /// - Accept: */* + /// - Accept-Encoding: deflate, gzip + /// - Keep-Alive: 300 + /// - Host: <stuff> + /// - Content-Length: <digits> + /// + /// Some headers excluded by default: + /// - Pragma: + /// - Cache-Control: + /// - Transfer-Encoding: ... chunked ... + /// - Referer: + /// - Content-Encoding: + /// - Expect: + /// - Content-Type: + /// + /// @param policy_id @see requestGet() + /// @param priority " + /// @param url " + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options @see requestGet()K(optional) + /// @param headers " + /// @param handler " + /// @return " + /// + HttpHandle requestPut(policy_t policy_id, + priority_t priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + + /// Queue a NoOp request. + /// The request is queued and serviced by the working thread which + /// immediately processes it and returns the request to the reply + /// queue. + /// + /// @param handler @see requestGet() + /// @return " + /// + HttpHandle requestNoOp(HttpHandler * handler); + + /// While all the heavy work is done by the worker thread, notifications + /// must be performed in the context of the application thread. These + /// are done synchronously during calls to this method which gives the + /// library control so notification can be performed. Application handlers + /// are expected to return 'quickly' and do any significant processing + /// outside of the notification callback to onCompleted(). + /// + /// @param usecs Maximum number of wallclock microseconds to + /// spend in the call. As hinted at above, this + /// is partly a function of application code so it's + /// a soft limit. A '0' value will run without + /// time limit until everything queued has been + /// delivered. + /// + /// @return Standard status code. + HttpStatus update(long usecs); + + /// @} + + /// @name RequestMgmtMethods + /// + /// @{ + + HttpHandle requestCancel(HttpHandle request, HttpHandler *); + + /// Request that a previously-issued request be reprioritized. + /// The status of whether the change itself succeeded arrives + /// via notification. + /// + /// @param request Handle of previously-issued request to + /// be changed. + /// @param priority New priority value. + /// @param handler @see requestGet() + /// @return " + /// + HttpHandle requestSetPriority(HttpHandle request, priority_t priority, HttpHandler * handler); + + /// @} + + /// @name UtilityMethods + /// + /// @{ + + /// Initialization method that needs to be called before queueing any + /// requests. Doesn't start the worker thread and may be called befoer + /// or after policy setup. + static HttpStatus createService(); + + /// Mostly clean shutdown of services prior to exit. Caller is expected + /// to have stopped a running worker thread before calling this. + static HttpStatus destroyService(); + + /// Called once after @see createService() to start the worker thread. + /// Stopping the thread is achieved by requesting it via @see requestStopThread(). + /// May be called before or after requests are issued. + static HttpStatus startThread(); + + /// Queues a request to the worker thread to have it stop processing + /// and exit (without exiting the program). When the operation is + /// picked up by the worker thread, it immediately processes it and + /// begins detaching from refcounted resources like request and + /// reply queues and then returns to the host OS. It *does* queue a + /// reply to give the calling application thread a notification that + /// the operation has been performed. + /// + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// As the request cannot be cancelled, the handle + /// is generally not useful. + /// + HttpHandle requestStopThread(HttpHandler * handler); + + /// Queue a Spin request. + /// DEBUG/TESTING ONLY. This puts the worker into a CPU spin for + /// test purposes. + /// + /// @param mode 0 for hard spin, 1 for soft spin + /// @return Standard handle return cases. + /// + HttpHandle requestSpin(int mode); + + /// @} + +protected: + void generateNotification(HttpOperation * op); + +private: + /// @name InstanceData + /// + /// @{ + HttpStatus mLastReqStatus; + HttpReplyQueue * mReplyQueue; + HttpRequestQueue * mRequestQueue; + + /// @} + + // ==================================== + /// @name GlobalState + /// + /// @{ + /// + /// Must be established before any threading is allowed to + /// start. + /// + + /// @} + // End Global State + // ==================================== + +}; // end class HttpRequest + + +} // end namespace LLCore + + + +#endif // _LLCORE_HTTP_REQUEST_H_ diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp new file mode 100755 index 0000000000..c974395b0a --- /dev/null +++ b/indra/llcorehttp/httpresponse.cpp @@ -0,0 +1,93 @@ +/** + * @file httpresponse.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpresponse.h" +#include "bufferarray.h" +#include "httpheaders.h" + + +namespace LLCore +{ + + +HttpResponse::HttpResponse() + : LLCoreInt::RefCounted(true), + mReplyOffset(0U), + mReplyLength(0U), + mReplyFullLength(0U), + mBufferArray(NULL), + mHeaders(NULL), + mRetries(0U), + m503Retries(0U) +{} + + +HttpResponse::~HttpResponse() +{ + setBody(NULL); + setHeaders(NULL); +} + + +void HttpResponse::setBody(BufferArray * ba) +{ + if (mBufferArray == ba) + return; + + if (mBufferArray) + { + mBufferArray->release(); + } + + if (ba) + { + ba->addRef(); + } + + mBufferArray = ba; +} + + +void HttpResponse::setHeaders(HttpHeaders * headers) +{ + if (mHeaders == headers) + return; + + if (mHeaders) + { + mHeaders->release(); + } + + if (headers) + { + headers->addRef(); + } + + mHeaders = headers; +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h new file mode 100755 index 0000000000..aee64e2878 --- /dev/null +++ b/indra/llcorehttp/httpresponse.h @@ -0,0 +1,187 @@ +/** + * @file httpresponse.h + * @brief Public-facing declarations for the HttpResponse class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_RESPONSE_H_ +#define _LLCORE_HTTP_RESPONSE_H_ + + +#include <string> + +#include "httpcommon.h" + +#include "_refcounted.h" + + +namespace LLCore +{ + +class BufferArray; +class HttpHeaders; + +/// HttpResponse is instantiated by the library and handed to +/// the caller during callbacks to the handler. It supplies +/// all the status, header and HTTP body data the caller is +/// interested in. Methods provide simple getters to return +/// individual pieces of the response. +/// +/// Typical usage will have the caller interrogate the object +/// during the handler callback and then simply returning. +/// But instances are refcounted and and callers can add a +/// reference and hold onto the object after the callback. +/// +/// Threading: Not intrinsically thread-safe. +/// +/// Allocation: Refcounted, heap only. Caller of the constructor +/// is given a refcount. +/// +class HttpResponse : public LLCoreInt::RefCounted +{ +public: + HttpResponse(); + +protected: + virtual ~HttpResponse(); // Use release() + + HttpResponse(const HttpResponse &); // Not defined + void operator=(const HttpResponse &); // Not defined + +public: + /// Returns the final status of the requested operation. + /// + HttpStatus getStatus() const + { + return mStatus; + } + + void setStatus(const HttpStatus & status) + { + mStatus = status; + } + + /// Simple getter for the response body returned as a scatter/gather + /// buffer. If the operation doesn't produce data (such as the Null + /// or StopThread operations), this may be NULL. + /// + /// Caller can hold onto the response by incrementing the reference + /// count of the returned object. + BufferArray * getBody() const + { + return mBufferArray; + } + + /// Set the response data in the instance. Will drop the reference + /// count to any existing data and increment the count of that passed + /// in. It is legal to set the data to NULL. + void setBody(BufferArray * ba); + + /// And a getter for the headers. And as with @see getResponse(), + /// if headers aren't available because the operation doesn't produce + /// any or delivery of headers wasn't requested in the options, this + /// will be NULL. + /// + /// Caller can hold onto the headers by incrementing the reference + /// count of the returned object. + HttpHeaders * getHeaders() const + { + return mHeaders; + } + + /// Behaves like @see setResponse() but for header data. + void setHeaders(HttpHeaders * headers); + + /// If a 'Range:' header was used, these methods are involved + /// in setting and returning data about the actual response. + /// If both @offset and @length are returned as 0, we probably + /// didn't get a Content-Range header in the response. This + /// occurs with various Capabilities-based services and the + /// caller is going to have to make assumptions on receipt of + /// a 206 status. The @full value may also be zero in cases of + /// parsing problems or a wild-carded length response. + /// + /// These values will not necessarily agree with the data in + /// the body itself (if present). The BufferArray object + /// is authoritative for actual data length. + void getRange(unsigned int * offset, unsigned int * length, unsigned int * full) const + { + *offset = mReplyOffset; + *length = mReplyLength; + *full = mReplyFullLength; + } + + void setRange(unsigned int offset, unsigned int length, unsigned int full_length) + { + mReplyOffset = offset; + mReplyLength = length; + mReplyFullLength = full_length; + } + + /// + const std::string & getContentType() const + { + return mContentType; + } + + void setContentType(const std::string & con_type) + { + mContentType = con_type; + } + + /// Get and set retry attempt information on the request. + void getRetries(unsigned int * retries, unsigned int * retries_503) const + { + if (retries) + { + *retries = mRetries; + } + if (retries_503) + { + *retries_503 = m503Retries; + } + } + + void setRetries(unsigned int retries, unsigned int retries_503) + { + mRetries = retries; + m503Retries = retries_503; + } + +protected: + // Response data here + HttpStatus mStatus; + unsigned int mReplyOffset; + unsigned int mReplyLength; + unsigned int mReplyFullLength; + BufferArray * mBufferArray; + HttpHeaders * mHeaders; + std::string mContentType; + unsigned int mRetries; + unsigned int m503Retries; +}; + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_RESPONSE_H_ diff --git a/indra/llcorehttp/tests/llcorehttp_test.cpp b/indra/llcorehttp/tests/llcorehttp_test.cpp new file mode 100755 index 0000000000..e863ddd13f --- /dev/null +++ b/indra/llcorehttp/tests/llcorehttp_test.cpp @@ -0,0 +1,175 @@ +/** + * @file llcorehttp_test + * @brief Main test runner + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "llcorehttp_test.h" + +#include <iostream> +#include <sstream> + +// These are not the right way in viewer for some reason: +// #include <tut/tut.hpp> +// #include <tut/tut_reporter.hpp> +// This works: +#include "../test/lltut.h" + +// Pull in each of the test sets +#include "test_bufferarray.hpp" +#include "test_bufferstream.hpp" +#include "test_httpstatus.hpp" +#include "test_refcounted.hpp" +#include "test_httpoperation.hpp" +#include "test_httprequest.hpp" +#include "test_httpheaders.hpp" +#include "test_httprequestqueue.hpp" + +#include "llproxy.h" + +unsigned long ssl_thread_id_callback(void); +void ssl_locking_callback(int mode, int type, const char * file, int line); + +#if 0 // lltut provides main and runner + +namespace tut +{ + test_runner_singleton runner; +} + +int main() +{ + curl_global_init(CURL_GLOBAL_ALL); + + // *FIXME: Need threaded/SSL curl setup here. + + tut::reporter reporter; + + tut::runner.get().set_callback(&reporter); + tut::runner.get().run_tests(); + return !reporter.all_ok(); + + curl_global_cleanup(); +} + +#endif // 0 + +int ssl_mutex_count(0); +LLCoreInt::HttpMutex ** ssl_mutex_list = NULL; + +void init_curl() +{ + curl_global_init(CURL_GLOBAL_ALL); + + ssl_mutex_count = CRYPTO_num_locks(); + if (ssl_mutex_count > 0) + { + ssl_mutex_list = new LLCoreInt::HttpMutex * [ssl_mutex_count]; + + for (int i(0); i < ssl_mutex_count; ++i) + { + ssl_mutex_list[i] = new LLCoreInt::HttpMutex; + } + + CRYPTO_set_locking_callback(ssl_locking_callback); + CRYPTO_set_id_callback(ssl_thread_id_callback); + } + + LLProxy::getInstance(); +} + + +void term_curl() +{ + LLProxy::cleanupClass(); + + CRYPTO_set_locking_callback(NULL); + for (int i(0); i < ssl_mutex_count; ++i) + { + delete ssl_mutex_list[i]; + } + delete [] ssl_mutex_list; +} + + +unsigned long ssl_thread_id_callback(void) +{ +#if defined(WIN32) + return (unsigned long) GetCurrentThread(); +#else + return (unsigned long) pthread_self(); +#endif +} + + +void ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) +{ + if (type >= 0 && type < ssl_mutex_count) + { + if (mode & CRYPTO_LOCK) + { + ssl_mutex_list[type]->lock(); + } + else + { + ssl_mutex_list[type]->unlock(); + } + } +} + + +std::string get_base_url() +{ + const char * env(getenv("LL_TEST_PORT")); + + if (! env) + { + std::cerr << "LL_TEST_PORT environment variable missing." << std::endl; + std::cerr << "Test expects to run in test_llcorehttp_peer.py script." << std::endl; + tut::ensure("LL_TEST_PORT set in environment", NULL != env); + } + + int port(atoi(env)); + std::ostringstream out; + out << "http://localhost:" << port << "/"; + return out.str(); +} + + +void stop_thread(LLCore::HttpRequest * req) +{ + if (req) + { + req->requestStopThread(NULL); + + int count = 0; + int limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + req->update(1000); + usleep(100000); + } + } +} + + diff --git a/indra/llcorehttp/tests/llcorehttp_test.h b/indra/llcorehttp/tests/llcorehttp_test.h new file mode 100755 index 0000000000..a9567435ce --- /dev/null +++ b/indra/llcorehttp/tests/llcorehttp_test.h @@ -0,0 +1,64 @@ +/** + * @file llcorehttp_test.h + * @brief Main test runner + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + + +#ifndef _LLCOREHTTP_TEST_H_ +#define _LLCOREHTTP_TEST_H_ + +#include "linden_common.h" // Modifies curl interfaces + +#include <curl/curl.h> +#include <openssl/crypto.h> +#include <string> + +#include "httprequest.h" + +// Initialization and cleanup for libcurl. Mainly provides +// a mutex callback for SSL and a thread ID hash for libcurl. +// If you don't use these (or equivalent) and do use libcurl, +// you'll see stalls and other anomalies when performing curl +// operations. +extern void init_curl(); +extern void term_curl(); +extern std::string get_base_url(); +extern void stop_thread(LLCore::HttpRequest * req); + +class ScopedCurlInit +{ +public: + ScopedCurlInit() + { + init_curl(); + } + + ~ScopedCurlInit() + { + term_curl(); + } +}; + + +#endif // _LLCOREHTTP_TEST_H_ diff --git a/indra/llcorehttp/tests/test_allocator.cpp b/indra/llcorehttp/tests/test_allocator.cpp new file mode 100755 index 0000000000..ea12dc58eb --- /dev/null +++ b/indra/llcorehttp/tests/test_allocator.cpp @@ -0,0 +1,184 @@ +/** + * @file test_allocator.cpp + * @brief quick and dirty allocator for tracking memory allocations + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "test_allocator.h" + +#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 +#include <libkern/OSAtomic.h> +#elif defined(_MSC_VER) +#include <Windows.h> +#elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ ) > 40100 +// atomic extensions are built into GCC on posix platforms +#endif + +#include <cassert> +#include <cstdlib> +#include <cstring> +#include <vector> +#include <iostream> +#include <new> + +#include <boost/thread.hpp> + + +#if defined(WIN32) +#define THROW_BAD_ALLOC() _THROW1(std::bad_alloc) +#define THROW_NOTHING() _THROW0() +#else +#define THROW_BAD_ALLOC() throw(std::bad_alloc) +#define THROW_NOTHING() throw() +#endif + + +struct BlockHeader +{ + struct Block * next; + std::size_t size; + bool in_use; +}; + +struct Block +{ + BlockHeader hdr; + unsigned char data[1]; +}; + +#define TRACE_MSG(val) std::cout << __FUNCTION__ << "(" << val << ") [" << __FILE__ << ":" << __LINE__ << "]" << std::endl; + +static unsigned char MemBuf[ 4096 * 1024 ]; +Block * pNext = static_cast<Block *>(static_cast<void *>(MemBuf)); +volatile std::size_t MemTotal = 0; + +// cross-platform compare and swap operation +static bool CAS(void * volatile * ptr, void * expected, void * new_value) +{ +#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwapPtr( expected, new_value, ptr ); +#elif defined(_MSC_VER) + return expected == InterlockedCompareExchangePointer( ptr, new_value, expected ); +#elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ ) > 40100 + return __sync_bool_compare_and_swap( ptr, expected, new_value ); +#endif +} + +static void * GetMem(std::size_t size) +{ + // TRACE_MSG(size); + volatile Block * pBlock = NULL; + volatile Block * pNewNext = NULL; + + // do a lock-free update of the global next pointer + do + { + pBlock = pNext; + pNewNext = (volatile Block *)(pBlock->data + size); + + } while(! CAS((void * volatile *) &pNext, (void *) pBlock, (void *) pNewNext)); + + // if we get here, we safely carved out a block of memory in the + // memory pool... + + // initialize our block + pBlock->hdr.next = (Block *)(pBlock->data + size); + pBlock->hdr.size = size; + pBlock->hdr.in_use = true; + memset((void *) pBlock->data, 0, pBlock->hdr.size); + + // do a lock-free update of the global memory total + volatile size_t total = 0; + volatile size_t new_total = 0; + do + { + total = MemTotal; + new_total = total + size; + + } while (! CAS((void * volatile *) &MemTotal, (void *) total, (void *) new_total)); + + return (void *) pBlock->data; +} + + +static void FreeMem(void * p) +{ + // get the pointer to the block record + Block * pBlock = (Block *)((unsigned char *) p - sizeof(BlockHeader)); + + // TRACE_MSG(pBlock->hdr.size); + bool * cur_in_use = &(pBlock->hdr.in_use); + volatile bool in_use = false; + bool new_in_use = false; + do + { + in_use = pBlock->hdr.in_use; + } while (! CAS((void * volatile *) cur_in_use, (void *) in_use, (void *) new_in_use)); + + // do a lock-free update of the global memory total + volatile size_t total = 0; + volatile size_t new_total = 0; + do + { + total = MemTotal; + new_total = total - pBlock->hdr.size; + } while (! CAS((void * volatile *)&MemTotal, (void *) total, (void *) new_total)); +} + + +std::size_t GetMemTotal() +{ + return MemTotal; +} + + +void * operator new(std::size_t size) THROW_BAD_ALLOC() +{ + return GetMem( size ); +} + + +void * operator new[](std::size_t size) THROW_BAD_ALLOC() +{ + return GetMem( size ); +} + + +void operator delete(void * p) THROW_NOTHING() +{ + if (p) + { + FreeMem( p ); + } +} + + +void operator delete[](void * p) THROW_NOTHING() +{ + if (p) + { + FreeMem( p ); + } +} + + diff --git a/indra/llcorehttp/tests/test_allocator.h b/indra/llcorehttp/tests/test_allocator.h new file mode 100755 index 0000000000..3572bbc5c5 --- /dev/null +++ b/indra/llcorehttp/tests/test_allocator.h @@ -0,0 +1,47 @@ +/** + * @file test_allocator.h + * @brief quick and dirty allocator for tracking memory allocations + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef TEST_ALLOCATOR_H +#define TEST_ALLOCATOR_H + +#include <cstdlib> +#include <new> + +size_t GetMemTotal(); +#if defined(WIN32) +void * operator new(std::size_t size) _THROW1(std::bad_alloc); +void * operator new[](std::size_t size) _THROW1(std::bad_alloc); +void operator delete(void * p) _THROW0(); +void operator delete[](void * p) _THROW0(); +#else +void * operator new(std::size_t size) throw (std::bad_alloc); +void * operator new[](std::size_t size) throw (std::bad_alloc); +void operator delete(void * p) throw (); +void operator delete[](void * p) throw (); +#endif + +#endif // TEST_ALLOCATOR_H + diff --git a/indra/llcorehttp/tests/test_bufferarray.hpp b/indra/llcorehttp/tests/test_bufferarray.hpp new file mode 100755 index 0000000000..8a2a64d970 --- /dev/null +++ b/indra/llcorehttp/tests/test_bufferarray.hpp @@ -0,0 +1,432 @@ +/** + * @file test_bufferarray.hpp + * @brief unit tests for the LLCore::BufferArray class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_BUFFER_ARRAY_H_ +#define TEST_LLCORE_BUFFER_ARRAY_H_ + +#include "bufferarray.h" + +#include <iostream> + +#include "test_allocator.h" + + +using namespace LLCore; + + + +namespace tut +{ + +struct BufferArrayTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<BufferArrayTestData> BufferArrayTestGroupType; +typedef BufferArrayTestGroupType::object BufferArrayTestObjectType; +BufferArrayTestGroupType BufferArrayTestGroup("BufferArray Tests"); + +template <> template <> +void BufferArrayTestObjectType::test<1>() +{ + set_test_name("BufferArray construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + ensure("One ref on construction of BufferArray", ba->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + ensure("Nothing in BA", 0 == ba->size()); + + // Try to read + char buffer[20]; + size_t read_len(ba->read(0, buffer, sizeof(buffer))); + ensure("Read returns empty", 0 == read_len); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<2>() +{ + set_test_name("BufferArray single write"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + char buffer[256]; + + size_t len = ba->write(0, str1, strlen(str1)); + ensure("Wrote length correct", strlen(str1) == len); + ensure("Recorded size correct", strlen(str1) == ba->size()); + + // read some data back + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(2, buffer, 2); + ensure("Read length correct", 2 == len); + ensure("Read content correct", 'c' == buffer[0] && 'd' == buffer[1]); + ensure("Read didn't overwrite", 'X' == buffer[2]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferArrayTestObjectType::test<3>() +{ + set_test_name("BufferArray multiple writes"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, strlen(str1)); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // read some data back + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(8, buffer, 4); + ensure("Read length correct", 4 == len); + ensure("Read content correct", 'i' == buffer[0] && 'j' == buffer[1]); + ensure("Read content correct", 'a' == buffer[2] && 'b' == buffer[3]); + ensure("Read didn't overwrite", 'X' == buffer[4]); + + // Read whole thing + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Read length correct", (2 * str1_len) == len); + ensure("Read content correct (3)", 0 == strncmp(buffer, str1, str1_len)); + ensure("Read content correct (4)", 0 == strncmp(&buffer[str1_len], str1, str1_len)); + ensure("Read didn't overwrite (5)", 'X' == buffer[2 * str1_len]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<4>() +{ + set_test_name("BufferArray overwriting"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJ"; + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, strlen(str1)); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // reposition and overwrite + len = ba->write(8, str2, 4); + ensure("Overwrite length correct", 4 == len); + + // Leave position and read verifying content (stale really from seek() days) + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(12, buffer, 4); + ensure("Read length correct", 4 == len); + ensure("Read content correct", 'c' == buffer[0] && 'd' == buffer[1]); + ensure("Read content correct.2", 'e' == buffer[2] && 'f' == buffer[3]); + ensure("Read didn't overwrite", 'X' == buffer[4]); + + // reposition and check + len = ba->read(6, buffer, 8); + ensure("Read length correct.2", 8 == len); + ensure("Read content correct.3", 'g' == buffer[0] && 'h' == buffer[1]); + ensure("Read content correct.4", 'A' == buffer[2] && 'B' == buffer[3]); + ensure("Read content correct.5", 'C' == buffer[4] && 'D' == buffer[5]); + ensure("Read content correct.6", 'c' == buffer[6] && 'd' == buffer[7]); + ensure("Read didn't overwrite.7", 'X' == buffer[8]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<5>() +{ + set_test_name("BufferArray multiple writes - sequential reads"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // read some data back + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(8, buffer, 4); + ensure("Read length correct", 4 == len); + ensure("Read content correct", 'i' == buffer[0] && 'j' == buffer[1]); + ensure("Read content correct.2", 'a' == buffer[2] && 'b' == buffer[3]); + ensure("Read didn't overwrite", 'X' == buffer[4]); + + // Read some more without repositioning + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(12, buffer, sizeof(buffer)); + ensure("Read length correct", (str1_len - 2) == len); + ensure("Read content correct.3", 0 == strncmp(buffer, str1+2, str1_len-2)); + ensure("Read didn't overwrite.2", 'X' == buffer[str1_len-1]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<6>() +{ + set_test_name("BufferArray overwrite spanning blocks and appending"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJKLMNOPQRST"; + size_t str2_len(strlen(str2)); + char buffer[256]; + + size_t len = ba->write(0, str1, str1_len); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", str1_len == ba->size()); + + // again... + len = ba->write(str1_len, str1, strlen(str1)); + ensure("Wrote length correct", str1_len == len); + ensure("Recorded size correct", (2 * str1_len) == ba->size()); + + // reposition and overwrite + len = ba->write(8, str2, str2_len); + ensure("Overwrite length correct", str2_len == len); + + // Leave position and read verifying content + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(8 + str2_len, buffer, 0); + ensure("Read length correct", 0 == len); + ensure("Read didn't overwrite", 'X' == buffer[0]); + + // reposition and check + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Read length correct.2", (str1_len + str2_len - 2) == len); + ensure("Read content correct", 0 == strncmp(buffer, str1, str1_len-2)); + ensure("Read content correct.2", 0 == strncmp(buffer+str1_len-2, str2, str2_len)); + ensure("Read didn't overwrite.2", 'X' == buffer[str1_len + str2_len - 2]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure("All memory released", mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<7>() +{ + set_test_name("BufferArray overwrite spanning blocks and sequential writes"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJKLMNOPQRST"; + size_t str2_len(strlen(str2)); + char buffer[256]; + + // 2x str1 + size_t len = ba->write(0, str1, str1_len); + len = ba->write(str1_len, str1, str1_len); + + // reposition and overwrite + len = ba->write(6, str2, 2); + ensure("Overwrite length correct", 2 == len); + + len = ba->write(8, str2, 2); + ensure("Overwrite length correct.2", 2 == len); + + len = ba->write(10, str2, 2); + ensure("Overwrite length correct.3", 2 == len); + + // append some data + len = ba->append(str2, str2_len); + ensure("Append length correct", str2_len == len); + + // append some more + void * out_buf(ba->appendBufferAlloc(str1_len)); + memcpy(out_buf, str1, str1_len); + + // And some final writes + len = ba->write(3 * str1_len + str2_len, str2, 2); + ensure("Write length correct.2", 2 == len); + + // Check contents + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Final buffer length correct", (3 * str1_len + str2_len + 2) == len); + ensure("Read content correct", 0 == strncmp(buffer, str1, 6)); + ensure("Read content correct.2", 0 == strncmp(buffer + 6, str2, 2)); + ensure("Read content correct.3", 0 == strncmp(buffer + 8, str2, 2)); + ensure("Read content correct.4", 0 == strncmp(buffer + 10, str2, 2)); + ensure("Read content correct.5", 0 == strncmp(buffer + str1_len + 2, str1 + 2, str1_len - 2)); + ensure("Read content correct.6", 0 == strncmp(buffer + str1_len + str1_len, str2, str2_len)); + ensure("Read content correct.7", 0 == strncmp(buffer + str1_len + str1_len + str2_len, str1, str1_len)); + ensure("Read content correct.8", 0 == strncmp(buffer + str1_len + str1_len + str2_len + str1_len, str2, 2)); + ensure("Read didn't overwrite", 'X' == buffer[str1_len + str1_len + str2_len + str1_len + 2]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure("All memory released", mMemTotal == GetMemTotal()); +} + +template <> template <> +void BufferArrayTestObjectType::test<8>() +{ + set_test_name("BufferArray zero-length appendBufferAlloc"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArray * ba = new BufferArray(); + + // write some data to the buffer + char str1[] = "abcdefghij"; + size_t str1_len(strlen(str1)); + char str2[] = "ABCDEFGHIJKLMNOPQRST"; + size_t str2_len(strlen(str2)); + char buffer[256]; + + // 2x str1 + size_t len = ba->write(0, str1, str1_len); + len = ba->write(str1_len, str1, str1_len); + + // zero-length allocate (we allow this with a valid pointer returned) + void * out_buf(ba->appendBufferAlloc(0)); + ensure("Buffer from zero-length appendBufferAlloc non-NULL", NULL != out_buf); + + // Do it again + void * out_buf2(ba->appendBufferAlloc(0)); + ensure("Buffer from zero-length appendBufferAlloc non-NULL.2", NULL != out_buf2); + ensure("Two zero-length appendBufferAlloc buffers distinct", out_buf != out_buf2); + + // And some final writes + len = ba->write(2 * str1_len, str2, str2_len); + + // Check contents + memset(buffer, 'X', sizeof(buffer)); + len = ba->read(0, buffer, sizeof(buffer)); + ensure("Final buffer length correct", (2 * str1_len + str2_len) == len); + ensure("Read content correct.1", 0 == strncmp(buffer, str1, str1_len)); + ensure("Read content correct.2", 0 == strncmp(buffer + str1_len, str1, str1_len)); + ensure("Read content correct.3", 0 == strncmp(buffer + str1_len + str1_len, str2, str2_len)); + ensure("Read didn't overwrite", 'X' == buffer[str1_len + str1_len + str2_len]); + + // release the implicit reference, causing the object to be released + ba->release(); + + // make sure we didn't leak any memory + ensure("All memory released", mMemTotal == GetMemTotal()); +} + +} // end namespace tut + + +#endif // TEST_LLCORE_BUFFER_ARRAY_H_ diff --git a/indra/llcorehttp/tests/test_bufferstream.hpp b/indra/llcorehttp/tests/test_bufferstream.hpp new file mode 100755 index 0000000000..831c901b9d --- /dev/null +++ b/indra/llcorehttp/tests/test_bufferstream.hpp @@ -0,0 +1,304 @@ +/** + * @file test_bufferstream.hpp + * @brief unit tests for the LLCore::BufferArrayStreamBuf/BufferArrayStream classes + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_BUFFER_STREAM_H_ +#define TEST_LLCORE_BUFFER_STREAM_H_ + +#include "bufferstream.h" + +#include <iostream> + +#include "test_allocator.h" +#include "llsd.h" +#include "llsdserialize.h" + + +using namespace LLCore; + + +namespace tut +{ + +struct BufferStreamTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<BufferStreamTestData> BufferStreamTestGroupType; +typedef BufferStreamTestGroupType::object BufferStreamTestObjectType; +BufferStreamTestGroupType BufferStreamTestGroup("BufferStream Tests"); +typedef BufferArrayStreamBuf::traits_type tst_traits_t; + + +template <> template <> +void BufferStreamTestObjectType::test<1>() +{ + set_test_name("BufferArrayStreamBuf construction with NULL BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArrayStreamBuf * bsb = new BufferArrayStreamBuf(NULL); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Not much will work with a NULL + ensure("underflow() on NULL fails", tst_traits_t::eof() == bsb->underflow()); + ensure("uflow() on NULL fails", tst_traits_t::eof() == bsb->uflow()); + ensure("pbackfail() on NULL fails", tst_traits_t::eof() == bsb->pbackfail('c')); + ensure("showmanyc() on NULL fails", bsb->showmanyc() == -1); + ensure("overflow() on NULL fails", tst_traits_t::eof() == bsb->overflow('c')); + ensure("xsputn() on NULL fails", bsb->xsputn("blah", 4) == 0); + ensure("seekoff() on NULL fails", bsb->seekoff(0, std::ios_base::beg, std::ios_base::in) == std::streampos(-1)); + + // release the implicit reference, causing the object to be released + delete bsb; + bsb = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<2>() +{ + set_test_name("BufferArrayStream construction with NULL BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + BufferArrayStream * bas = new BufferArrayStream(NULL); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Not much will work with a NULL here + ensure("eof() is false on NULL", ! bas->eof()); + ensure("fail() is false on NULL", ! bas->fail()); + ensure("good() on NULL", bas->good()); + + // release the implicit reference, causing the object to be released + delete bas; + bas = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<3>() +{ + set_test_name("BufferArrayStreamBuf construction with empty BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + BufferArrayStreamBuf * bsb = new BufferArrayStreamBuf(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // I can release my ref on the BA + ba->release(); + ba = NULL; + + // release the implicit reference, causing the object to be released + delete bsb; + bsb = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<4>() +{ + set_test_name("BufferArrayStream construction with empty BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + + { + // create a new ref counted object with an implicit reference + BufferArrayStream bas(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + } + + // release the implicit reference, causing the object to be released + ba->release(); + ba = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<5>() +{ + set_test_name("BufferArrayStreamBuf construction with real BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + const char * content("This is a string. A fragment."); + const size_t c_len(strlen(content)); + ba->append(content, c_len); + + // Creat an adapter for the BufferArray + BufferArrayStreamBuf * bsb = new BufferArrayStreamBuf(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // I can release my ref on the BA + ba->release(); + ba = NULL; + + // Various static state + ensure("underflow() returns 'T'", bsb->underflow() == 'T'); + ensure("underflow() returns 'T' again", bsb->underflow() == 'T'); + ensure("uflow() returns 'T'", bsb->uflow() == 'T'); + ensure("uflow() returns 'h'", bsb->uflow() == 'h'); + ensure("pbackfail('i') fails", tst_traits_t::eof() == bsb->pbackfail('i')); + ensure("pbackfail('T') fails", tst_traits_t::eof() == bsb->pbackfail('T')); + ensure("pbackfail('h') succeeds", bsb->pbackfail('h') == 'h'); + ensure("showmanyc() is everything but the 'T'", bsb->showmanyc() == (c_len - 1)); + ensure("overflow() appends", bsb->overflow('c') == 'c'); + ensure("showmanyc() reflects append", bsb->showmanyc() == (c_len - 1 + 1)); + ensure("xsputn() appends some more", bsb->xsputn("bla!", 4) == 4); + ensure("showmanyc() reflects 2nd append", bsb->showmanyc() == (c_len - 1 + 5)); + ensure("seekoff() succeeds", bsb->seekoff(0, std::ios_base::beg, std::ios_base::in) == std::streampos(0)); + ensure("seekoff() succeeds 2", bsb->seekoff(4, std::ios_base::cur, std::ios_base::in) == std::streampos(4)); + ensure("showmanyc() picks up seekoff", bsb->showmanyc() == (c_len + 5 - 4)); + ensure("seekoff() succeeds 3", bsb->seekoff(0, std::ios_base::end, std::ios_base::in) == std::streampos(c_len + 4)); + ensure("pbackfail('!') succeeds", tst_traits_t::eof() == bsb->pbackfail('!')); + + // release the implicit reference, causing the object to be released + delete bsb; + bsb = NULL; + + // make sure we didn't leak any memory + ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +template <> template <> +void BufferStreamTestObjectType::test<6>() +{ + set_test_name("BufferArrayStream construction with real BufferArray"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + //const char * content("This is a string. A fragment."); + //const size_t c_len(strlen(content)); + //ba->append(content, strlen(content)); + + { + // Creat an adapter for the BufferArray + BufferArrayStream bas(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Basic operations + bas << "Hello" << 27 << "."; + ensure("BA length 8", ba->size() == 8); + + std::string str; + bas >> str; + ensure("reads correctly", str == "Hello27."); + } + + // release the implicit reference, causing the object to be released + ba->release(); + ba = NULL; + + // make sure we didn't leak any memory + // ensure("Allocated memory returned", mMemTotal == GetMemTotal()); + // static U64 mem = GetMemTotal(); +} + + +template <> template <> +void BufferStreamTestObjectType::test<7>() +{ + set_test_name("BufferArrayStream with LLSD serialization"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted BufferArray with implicit reference + BufferArray * ba = new BufferArray; + + { + // Creat an adapter for the BufferArray + BufferArrayStream bas(ba); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // LLSD + LLSD llsd = LLSD::emptyMap(); + + llsd["int"] = LLSD::Integer(3); + llsd["float"] = LLSD::Real(923289.28992); + llsd["string"] = LLSD::String("aksjdl;ajsdgfjgfal;sdgjakl;sdfjkl;ajsdfkl;ajsdfkl;jaskl;dfj"); + + LLSD llsd_map = LLSD::emptyMap(); + llsd_map["int"] = LLSD::Integer(-2889); + llsd_map["float"] = LLSD::Real(2.37829e32); + llsd_map["string"] = LLSD::String("OHIGODHSPDGHOSDHGOPSHDGP"); + + llsd["map"] = llsd_map; + + // Serialize it + LLSDSerialize::toXML(llsd, bas); + + std::string str; + bas >> str; + // std::cout << "SERIALIZED LLSD: " << str << std::endl; + ensure("Extracted string has reasonable length", str.size() > 60); + } + + // release the implicit reference, causing the object to be released + ba->release(); + ba = NULL; + + // make sure we didn't leak any memory + // ensure("Allocated memory returned", mMemTotal == GetMemTotal()); +} + + +} // end namespace tut + + +#endif // TEST_LLCORE_BUFFER_STREAM_H_ diff --git a/indra/llcorehttp/tests/test_httpheaders.hpp b/indra/llcorehttp/tests/test_httpheaders.hpp new file mode 100755 index 0000000000..668c36dc66 --- /dev/null +++ b/indra/llcorehttp/tests/test_httpheaders.hpp @@ -0,0 +1,433 @@ +/** + * @file test_httpheaders.hpp + * @brief unit tests for the LLCore::HttpHeaders class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_HEADERS_H_ +#define TEST_LLCORE_HTTP_HEADERS_H_ + +#include "httpheaders.h" + +#include <iostream> + +#include "test_allocator.h" + + +using namespace LLCoreInt; + + +namespace tut +{ + +struct HttpHeadersTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<HttpHeadersTestData> HttpHeadersTestGroupType; +typedef HttpHeadersTestGroupType::object HttpHeadersTestObjectType; +HttpHeadersTestGroupType HttpHeadersTestGroup("HttpHeaders Tests"); + +template <> template <> +void HttpHeadersTestObjectType::test<1>() +{ + set_test_name("HttpHeaders construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + ensure("One ref on construction of HttpHeaders", headers->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + ensure("Nothing in headers", 0 == headers->size()); + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<2>() +{ + set_test_name("HttpHeaders construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + // Append a few strings + std::string str1n("Pragma"); + std::string str1v(""); + headers->append(str1n, str1v); + std::string str2n("Accept"); + std::string str2v("application/json"); + headers->append(str2n, str2v); + + ensure("Headers retained", 2 == headers->size()); + HttpHeaders::container_t & c(headers->getContainerTESTONLY()); + + ensure("First name is first name", c[0].first == str1n); + ensure("First value is first value", c[0].second == str1v); + ensure("Second name is second name", c[1].first == str2n); + ensure("Second value is second value", c[1].second == str2v); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<3>() +{ + set_test_name("HttpHeaders basic find"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + // Append a few strings + std::string str1n("Uno"); + std::string str1v("1"); + headers->append(str1n, str1v); + std::string str2n("doS"); + std::string str2v("2-2-2-2"); + headers->append(str2n, str2v); + std::string str3n("TRES"); + std::string str3v("trois gymnopedie"); + headers->append(str3n, str3v); + + ensure("Headers retained", 3 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("TRES"); + ensure("Found the last item", result != NULL); + ensure("Last item is a nice", result != NULL && str3v == *result); + + // appends above are raw and find is case sensitive + result = headers->find("TReS"); + ensure("Last item not found due to case", result == NULL); + + result = headers->find("TRE"); + ensure("Last item not found due to prefixing (1)", result == NULL); + + result = headers->find("TRESS"); + ensure("Last item not found due to prefixing (2)", result == NULL); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<4>() +{ + set_test_name("HttpHeaders normalized header entry"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + ensure("First append worked in some fashion", 1 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("accept"); + ensure("Found 'accept'", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Left-clean on value + static char line2[] = " next : \t\tlinejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + ensure("Second append worked", 2 == headers->size()); + result = headers->find("next"); + ensure("Found 'next'", result != NULL); + ensure("next value is left-clean", result != NULL && + *result == "linejunk \t"); + + // First value unmolested + result = headers->find("accept"); + ensure("Found 'accept' again", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Colons in value are okay + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + ensure("Third append worked", 3 == headers->size()); + result = headers->find("fancy-pants"); + ensure("Found 'fancy-pants'", result != NULL); + ensure("fancy-pants value has colons", result != NULL && *result == line3v); + + // Zero-length value + static char line4[] = "all-talk-no-walk:"; + headers->appendNormal(line4, sizeof(line4) - 1); + ensure("Fourth append worked", 4 == headers->size()); + result = headers->find("all-talk-no-walk"); + ensure("Found 'all-talk'", result != NULL); + ensure("al-talk value is zero-length", result != NULL && result->size() == 0); + + // Zero-length name + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + ensure("Fifth append worked", 5 == headers->size()); + result = headers->find(""); + ensure("Found no-name", result != NULL); + ensure("no-name value is something", result != NULL && *result == line5v); + + // Lone colon is still something + headers->clear(); + static char line6[] = " :"; + headers->appendNormal(line6, sizeof(line6) - 1); + ensure("Sixth append worked", 1 == headers->size()); + result = headers->find(""); + ensure("Found 2nd no-name", result != NULL); + ensure("2nd no-name value is nothing", result != NULL && result->size() == 0); + + // Line without colons is taken as-is and unstripped in name + static char line7[] = " \toskdgioasdghaosdghoowg28342908tg8902hg0hwedfhqew890v7qh0wdebv78q0wdevbhq>?M>BNM<ZV>?NZ? \t"; + headers->appendNormal(line7, sizeof(line7) - 1); + ensure("Seventh append worked", 2 == headers->size()); + result = headers->find(line7); + ensure("Found whatsit line", result != NULL); + ensure("Whatsit line has no value", result != NULL && result->size() == 0); + + // Normaling interface heeds the byte count, doesn't look for NUL-terminator + static char line8[] = "binary:ignorestuffontheendofthis"; + headers->appendNormal(line8, 13); + ensure("Eighth append worked", 3 == headers->size()); + result = headers->find("binary"); + ensure("Found 'binary'", result != NULL); + ensure("binary value was limited to 'ignore'", result != NULL && + *result == "ignore"); + + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Verify forward iterator finds everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<5>() +{ + set_test_name("HttpHeaders iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::iterator end(headers->end()), begin(headers->begin()); + ensure("Empty container has equal begin/end const iterators", end == begin); + HttpHeaders::const_iterator cend(headers->end()), cbegin(headers->begin()); + ensure("Empty container has equal rbegin/rend const iterators", cend == cbegin); + + ensure("Empty container has equal begin/end iterators", headers->end() == headers->begin()); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line1v, + line2v, + line3v, + line4v, + line5v, + line6v + }; + + int i(0); + HttpHeaders::const_iterator cend(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::iterator end(headers->end()); + for (HttpHeaders::iterator it(headers->begin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Reverse iterators find everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<6>() +{ + set_test_name("HttpHeaders reverse iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::reverse_iterator rend(headers->rend()), rbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", rend == rbegin); + HttpHeaders::const_reverse_iterator crend(headers->rend()), crbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", crend == crbegin); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line6v, + line5v, + line4v, + line3v, + line2v, + line1v + }; + + int i(0); + HttpHeaders::const_reverse_iterator cend(headers->rend()); + for (HttpHeaders::const_reverse_iterator it(headers->rbegin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::reverse_iterator end(headers->rend()); + for (HttpHeaders::reverse_iterator it(headers->rbegin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +} // end namespace tut + + +#endif // TEST_LLCORE_HTTP_HEADERS_H_ diff --git a/indra/llcorehttp/tests/test_httpoperation.hpp b/indra/llcorehttp/tests/test_httpoperation.hpp new file mode 100755 index 0000000000..17b1a96878 --- /dev/null +++ b/indra/llcorehttp/tests/test_httpoperation.hpp @@ -0,0 +1,125 @@ +/** + * @file test_httpoperation.hpp + * @brief unit tests for the LLCore::HttpOperation-derived classes + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_OPERATION_H_ +#define TEST_LLCORE_HTTP_OPERATION_H_ + +#include "_httpoperation.h" +#include "httphandler.h" + +#include <iostream> + +#include "test_allocator.h" + + +using namespace LLCoreInt; + + +namespace +{ + +class TestHandler : public LLCore::HttpHandler +{ +public: + virtual void onCompleted(HttpHandle, HttpResponse *) + { + std::cout << "TestHandler::onCompleted() invoked" << std::endl; + } + +}; + + +} // end namespace anonymous + + +namespace tut +{ + struct HttpOperationTestData + { + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; + }; + + typedef test_group<HttpOperationTestData> HttpOperationTestGroupType; + typedef HttpOperationTestGroupType::object HttpOperationTestObjectType; + HttpOperationTestGroupType HttpOperationTestGroup("HttpOperation Tests"); + + template <> template <> + void HttpOperationTestObjectType::test<1>() + { + set_test_name("HttpOpNull construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpOpNull * op = new HttpOpNull(); + ensure(op->getRefCount() == 1); + ensure(mMemTotal < GetMemTotal()); + + // release the implicit reference, causing the object to be released + op->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void HttpOperationTestObjectType::test<2>() + { + set_test_name("HttpOpNull construction with handlers"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // Get some handlers + TestHandler * h1 = new TestHandler(); + + // create a new ref counted object with an implicit reference + HttpOpNull * op = new HttpOpNull(); + + // Add the handlers + op->setReplyPath(NULL, h1); + + // Check ref count + ensure(op->getRefCount() == 1); + + // release the reference, releasing the operation but + // not the handlers. + op->release(); + op = NULL; + ensure(mMemTotal != GetMemTotal()); + + // release the handlers + delete h1; + h1 = NULL; + + ensure(mMemTotal == GetMemTotal()); + } + +} + +#endif // TEST_LLCORE_HTTP_OPERATION_H_ diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp new file mode 100755 index 0000000000..43f7e36da5 --- /dev/null +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -0,0 +1,3330 @@ +/** + * @file test_httprequest.hpp + * @brief unit tests for the LLCore::HttpRequest class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_REQUEST_H_ +#define TEST_LLCORE_HTTP_REQUEST_H_ + +#include "httprequest.h" +#include "bufferarray.h" +#include "httphandler.h" +#include "httpheaders.h" +#include "httpresponse.h" +#include "httpoptions.h" +#include "_httpservice.h" +#include "_httprequestqueue.h" + +#include <curl/curl.h> +#include <boost/regex.hpp> +#include <sstream> + +#include "test_allocator.h" +#include "llcorehttp_test.h" + + +using namespace LLCoreInt; + +// spin/sleep waiting times for client/server exchange tests +// +// These are now fairly generous to try to get around timeout +// ('reasonable time') failures during execution on a heavily- +// loaded system where the unit test is in competition with +// other programs. +static const int LOOP_SLEEP_INTERVAL(10000); +static const int LOOP_COUNT_SHORT(500); // 5-second dwell time +static const int LOOP_COUNT_LONG(3000); // 30-second dwell time + +namespace +{ + +#if defined(WIN32) + +void usleep(unsigned long usec); + +#endif + +} + +namespace tut +{ + +typedef std::vector<std::pair<boost::regex, boost::regex> > regex_container_t; + +struct HttpRequestTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; + int mHandlerCalls; + HttpStatus mStatus; +}; + +class TestHandler2 : public LLCore::HttpHandler +{ +public: + TestHandler2(HttpRequestTestData * state, + const std::string & name) + : mState(state), + mName(name), + mExpectHandle(LLCORE_HTTP_HANDLE_INVALID) + {} + + virtual void onCompleted(HttpHandle handle, HttpResponse * response) + { + if (LLCORE_HTTP_HANDLE_INVALID != mExpectHandle) + { + ensure("Expected handle received in handler", mExpectHandle == handle); + } + ensure("Handler got a response", NULL != response); + if (response && mState) + { + const HttpStatus actual_status(response->getStatus()); + std::ostringstream test; + test << "Expected HttpStatus received in response. Wanted: " + << mState->mStatus.toHex() << " Received: " << actual_status.toHex(); + ensure(test.str().c_str(), actual_status == mState->mStatus); + } + if (mState) + { + mState->mHandlerCalls++; + } + if (! mHeadersRequired.empty() || ! mHeadersDisallowed.empty()) + { + ensure("Response required with header check", response != NULL); + HttpHeaders * header(response->getHeaders()); // Will not hold onto this + ensure("Some quantity of headers returned", header != NULL); + + if (! mHeadersRequired.empty()) + { + for (int i(0); i < mHeadersRequired.size(); ++i) + { + bool found = false; + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; + ++iter) + { + // std::cerr << "Header: " << (*iter).first + // << ": " << (*iter).second << std::endl; + + if (boost::regex_match((*iter).first, + mHeadersRequired[i].first) && + boost::regex_match((*iter).second, + mHeadersRequired[i].second)) + { + found = true; + break; + } + } + std::ostringstream str; + str << "Required header # " << i << " found in response"; + ensure(str.str(), found); + } + } + + if (! mHeadersDisallowed.empty()) + { + for (int i(0); i < mHeadersDisallowed.size(); ++i) + { + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; + ++iter) + { + if (boost::regex_match((*iter).first, + mHeadersDisallowed[i].first) && + boost::regex_match((*iter).second, + mHeadersDisallowed[i].second)) + { + std::ostringstream str; + str << "Disallowed header # " << i << " not found in response"; + ensure(str.str(), false); + } + } + } + } + } + + if (! mCheckContentType.empty()) + { + ensure("Response required with content type check", response != NULL); + std::string con_type(response->getContentType()); + ensure("Content-Type as expected (" + mCheckContentType + ")", + mCheckContentType == con_type); + } + + // std::cout << "TestHandler2::onCompleted() invoked" << std::endl; + } + + HttpRequestTestData * mState; + std::string mName; + HttpHandle mExpectHandle; + std::string mCheckContentType; + regex_container_t mHeadersRequired; + regex_container_t mHeadersDisallowed; +}; + +typedef test_group<HttpRequestTestData> HttpRequestTestGroupType; +typedef HttpRequestTestGroupType::object HttpRequestTestObjectType; +HttpRequestTestGroupType HttpRequestTestGroup("HttpRequest Tests"); + +template <> template <> +void HttpRequestTestObjectType::test<1>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest construction"); + + HttpRequest * req = NULL; + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + try + { + // Get singletons created + HttpRequest::createService(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // release the request object + delete req; + req = NULL; + + HttpRequest::destroyService(); + + // make sure we didn't leak any memory + ensure("Memory returned", mMemTotal == GetMemTotal()); + } + catch (...) + { + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<2>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest and Null Op queued"); + + HttpRequest * req = NULL; + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + try + { + // Get singletons created + HttpRequest::createService(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Issue a NoOp + HttpHandle handle = req->requestNoOp(NULL); + ensure("Request issued", handle != LLCORE_HTTP_HANDLE_INVALID); + + // release the request object + delete req; + req = NULL; + + // We're still holding onto the operation which is + // sitting, unserviced, on the request queue so... + ensure("Memory being used 2", mMemTotal < GetMemTotal()); + + // Request queue should have two references: global singleton & service object + ensure("Two references to request queue", 2 == HttpRequestQueue::instanceOf()->getRefCount()); + + // Okay, tear it down + HttpRequest::destroyService(); + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory returned", mMemTotal == GetMemTotal()); + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<3>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest NoOp + Stop execution"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a NoOp + HttpHandle handle = req->requestNoOp(&handler); + ensure("Valid handle returned for first request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_SHORT); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<4>() +{ + ScopedCurlInit ready; + + set_test_name("2 HttpRequest instances, one thread"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + TestHandler2 handler1(this, "handler1"); + TestHandler2 handler2(this, "handler2"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req1 = NULL; + HttpRequest * req2 = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req1 = new HttpRequest(); + req2 = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue some NoOps + HttpHandle handle = req1->requestNoOp(&handler1); + ensure("Valid handle returned for first request", handle != LLCORE_HTTP_HANDLE_INVALID); + handler1.mExpectHandle = handle; + + handle = req2->requestNoOp(&handler2); + ensure("Valid handle returned for first request", handle != LLCORE_HTTP_HANDLE_INVALID); + handler2.mExpectHandle = handle; + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 2) + { + req1->update(1000000); + req2->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 2); + + // Okay, request a shutdown of the servicing thread + handle = req2->requestStopThread(&handler2); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + handler2.mExpectHandle = handle; + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 3) + { + req1->update(1000000); + req2->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 3); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req1; + req1 = NULL; + delete req2; + req2 = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 3 == mHandlerCalls); + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + } + catch (...) + { + stop_thread(req1); + delete req1; + delete req2; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<5>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest Spin (soft) + NoOp + hard termination"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a Spin + HttpHandle handle = req->requestSpin(1); + ensure("Valid handle returned for spin request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Issue a NoOp + handle = req->requestNoOp(&handler); + ensure("Valid handle returned for no-op request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_SHORT); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("NoOp notification received", mHandlerCalls == 1); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + // Check memory usage + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + // This memory test should work but could give problems as it + // relies on the worker thread picking up a friendly request + // to shutdown. Doing so, it drops references to things and + // we should go back to where we started. If it gives you + // problems, look into the code before commenting things out. + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<6>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest Spin + NoOp + hard termination"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a Spin + HttpHandle handle = req->requestSpin(0); // Hard spin + ensure("Valid handle returned for spin request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Issue a NoOp + handle = req->requestNoOp(&handler); + ensure("Valid handle returned for no-op request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_SHORT); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("No notifications received", mHandlerCalls == 0); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + // Check memory usage + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + // ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); + // This memory test won't work because we're killing the thread + // hard with the hard spinner. There's no opportunity to join + // nicely so many things leak or get destroyed unilaterally. + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<7>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET to dead port + Stop execution"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(1); // Don't try for too long - default retries take about 18S + + // Issue a GET that can't connect + mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + "http://127.0.0.1:2/nothing/here", + 0, + 0, + opts, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if 0 // defined(WIN32) + // Can't do this on any platform anymore, the LL logging system holds + // on to memory and produces what looks like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<8>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<9>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with Range: header to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<10>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest PUT to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + BufferArray * body = new BufferArray; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + static const char * body_text("Now is the time for all good men..."); + body->append(body_text, strlen(body_text)); + mStatus = HttpStatus(200); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + body, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // Lose the request body + body->release(); + body = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if 0 // defined(WIN32) + // Can't do this on any platform anymore, the LL logging system holds + // on to memory and produces what looks like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + if (body) + { + body->release(); + } + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<11>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest POST to real service"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + BufferArray * body = new BufferArray; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + static const char * body_text("Now is the time for all good men..."); + body->append(body_text, strlen(body_text)); + mStatus = HttpStatus(200); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + body, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // Lose the request body + body->release(); + body = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + if (body) + { + body->release(); + } + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<12>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with some tracing"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Enable tracing + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if 0 // defined(WIN32) + // Can't do this on any platform anymore, the LL logging system holds + // on to memory and produces what looks like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<13>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with returned headers"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + handler.mHeadersRequired.reserve(20); // Avoid memory leak test failure + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Enable tracing + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setWantHeaders(true); + + // Issue a GET that succeeds + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back( + regex_container_t::value_type(boost::regex("X-LL-Special", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + opts, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // release options + opts->release(); + opts = NULL; + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +template <> template <> +void HttpRequestTestObjectType::test<14>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET timeout"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + std::string url_base(get_base_url() + "/sleep/"); // path to a 30-second sleep + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(0); // Don't retry + opts->setTimeout(2); + + // Issue a GET that sleeps + mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + 0, + 0, + opts, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + +// Test retrieval of Content-Type/Content-Encoding headers +template <> template <> +void HttpRequestTestObjectType::test<15>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with Content-Type"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // Load and clear the string setting to preload std::string object + // for memory return tests. + handler.mCheckContentType = "application/llsd+xml"; + handler.mCheckContentType.clear(); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + handler.mCheckContentType = "application/llsd+xml"; + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mCheckContentType.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on GET requests +template <> template <> +void HttpRequestTestObjectType::test<16>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest GET"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + options, + NULL, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Do a texture-style fetch + headers = new HttpHeaders; + headers->append("Accept", "image/x-j2c"); + + mStatus = HttpStatus(200); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("image/x-j2c", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("\\W*X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + 0, + 47, + options, + headers, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 2); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 3) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 3); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on POST requests +template <> template <> +void HttpRequestTestObjectType::test<17>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest POST"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // And a buffer array + const char * msg("It was the best of times, it was the worst of times."); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default POST + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer_encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + NULL, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on PUT requests +template <> template <> +void HttpRequestTestObjectType::test<18>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest PUT"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // And a buffer array + const char * msg("It was the best of times, it was the worst of times."); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default PUT + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + NULL, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on GET requests with overrides +template <> template <> +void HttpRequestTestObjectType::test<19>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest GET with header overrides"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // headers + headers = new HttpHeaders; + headers->append("Keep-Alive", "120"); + headers->append("Accept-encoding", "deflate"); + headers->append("Accept", "text/plain"); + + // Issue a GET with modified headers + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("deflate", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + options, + headers, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on POST requests with overrides +template <> template <> +void HttpRequestTestObjectType::test<20>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest POST with header overrides"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // headers + headers = new HttpHeaders(); + headers->append("keep-Alive", "120"); + headers->append("Accept", "text/html"); + headers->append("content-type", "application/llsd+xml"); + headers->append("cache-control", "no-store"); + + // And a buffer array + const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default POST + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex("no-store", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + headers, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +// Test header generation on PUT requests with overrides +template <> template <> +void HttpRequestTestObjectType::test<21>() +{ + ScopedCurlInit ready; + + // Warmup boost::regex to pre-alloc memory for memory size tests + boost::regex warmup("askldjflasdj;f", boost::regex::icase); + boost::regex_match("akl;sjflajfk;ajsk", warmup); + + std::string url_base(get_base_url()); + + set_test_name("Header generation for HttpRequest PUT with header overrides"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * options = NULL; + HttpHeaders * headers = NULL; + BufferArray * ba = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + + // options set + options = new HttpOptions(); + options->setWantHeaders(true); + + // headers + headers = new HttpHeaders; + headers->append("content-type", "text/plain"); + headers->append("content-type", "text/html"); + headers->append("content-type", "application/llsd+xml"); + + // And a buffer array + const char * msg("<xml><llsd><string>It was the best of times, it was the worst of times.</string></llsd></xml>"); + ba = new BufferArray; + ba->append(msg, strlen(msg)); + + // Issue a default PUT + mStatus = HttpStatus(200); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + "reflect/", + ba, + options, + headers, + &handler); + ensure("Valid handle returned for get request", handle != LLCORE_HTTP_HANDLE_INVALID); + ba->release(); + ba = NULL; + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mHeadersRequired.clear(); + handler.mHeadersDisallowed.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options & headers + if (options) + { + options->release(); + } + options = NULL; + + if (headers) + { + headers->release(); + } + headers = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + } + catch (...) + { + stop_thread(req); + if (ba) + { + ba->release(); + ba = NULL; + } + if (options) + { + options->release(); + options = NULL; + } + if (headers) + { + headers->release(); + headers = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + +// BUG-2295 Tests - Content-Range header received but no body +template <> template <> +void HttpRequestTestObjectType::test<22>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("BUG-2295"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpOptions * options = NULL; + HttpRequest * req = NULL; + + try + { + // options set + options = new HttpOptions(); + options->setRetries(1); // Partial_File is retryable and can timeout in here + + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // ====================================== + // Issue bug2295 GETs that will get a 206 + // ====================================== + mStatus = HttpStatus(206); + static const int test_count(3); + for (int i(0); i < test_count; ++i) + { + char buffer[128]; + sprintf(buffer, "/bug2295/%d/", i); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + buffer, + 0, + 25, + options, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + } + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < test_count) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time - ms1", count < limit); + ensure("One handler invocation for each request - ms1", mHandlerCalls == test_count); + + // ====================================== + // Issue bug2295 GETs that will get a libcurl 18 (PARTIAL_FILE) + // ====================================== + mHandlerCalls = 0; + mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_PARTIAL_FILE); + static const int test2_count(1); + for (int i(0); i < test2_count; ++i) + { + char buffer[128]; + sprintf(buffer, "/bug2295/00000012/%d/", i); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + buffer, + 0, + 25, + options, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + } + + // Run the notification pump. + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < test2_count) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time - ms2", count < limit); + ensure("One handler invocation for each request - ms2", mHandlerCalls == test2_count); + + // ====================================== + // Issue bug2295 GETs that will get an llcorehttp HE_INV_CONTENT_RANGE_HDR status + // ====================================== + mHandlerCalls = 0; + mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + static const int test3_count(1); + for (int i(0); i < test3_count; ++i) + { + char buffer[128]; + sprintf(buffer, "/bug2295/inv_cont_range/%d/", i); + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base + buffer, + 0, + 25, + options, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + } + + // Run the notification pump. + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < test3_count) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time - ms3", count < limit); + ensure("One handler invocation for each request - ms3", mHandlerCalls == test3_count); + + // ====================================== + // Okay, request a shutdown of the servicing thread + // ====================================== + mStatus = HttpStatus(); + mHandlerCalls = 0; + HttpHandle handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Shutdown request executed in reasonable time", count < limit); + ensure("Shutdown handler invocation", mHandlerCalls == 1); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + if (options) + { + options->release(); + options = NULL; + } + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + +template <> template <> +void HttpRequestTestObjectType::test<23>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET 503s with 'Retry-After'"); + + // This tests mainly that the code doesn't fall over if + // various well- and mis-formed Retry-After headers are + // sent along with the response. Direct inspection of + // the parsing result isn't supported. + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + std::string url_base(get_base_url() + "/503/"); // path to 503 generators + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(1); // Retry once only + opts->setUseRetryAfter(true); // Try to parse the retry-after header + + // Issue a GET that 503s with valid retry-after + mStatus = HttpStatus(503); + int url_limit(6); + for (int i(0); i < url_limit; ++i) + { + std::ostringstream url; + url << url_base << i << "/"; + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url.str(), + 0, + 0, + opts, + NULL, + &handler); + + std::ostringstream testtag; + testtag << "Valid handle returned for 503 request #" << i; + ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID); + } + + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < url_limit) + { + req->update(0); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == url_limit); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + mHandlerCalls = 0; + HttpHandle handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 1); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + +} // end namespace tut + +namespace +{ + +#if defined(WIN32) + +void usleep(unsigned long usec) +{ + Sleep((DWORD) (usec / 1000UL)); +} + +#endif + +} + +#endif // TEST_LLCORE_HTTP_REQUEST_H_ diff --git a/indra/llcorehttp/tests/test_httprequestqueue.hpp b/indra/llcorehttp/tests/test_httprequestqueue.hpp new file mode 100755 index 0000000000..1de2d8f9ab --- /dev/null +++ b/indra/llcorehttp/tests/test_httprequestqueue.hpp @@ -0,0 +1,186 @@ +/** + * @file test_httprequestqueue.hpp + * @brief unit tests for the LLCore::HttpRequestQueue class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCORE_HTTP_REQUESTQUEUE_H_ +#define TEST_LLCORE_HTTP_REQUESTQUEUE_H_ + +#include "_httprequestqueue.h" + +#include <iostream> + +#include "test_allocator.h" +#include "_httpoperation.h" + + +using namespace LLCoreInt; + + + +namespace tut +{ + +struct HttpRequestqueueTestData +{ + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; +}; + +typedef test_group<HttpRequestqueueTestData> HttpRequestqueueTestGroupType; +typedef HttpRequestqueueTestGroupType::object HttpRequestqueueTestObjectType; +HttpRequestqueueTestGroupType HttpRequestqueueTestGroup("HttpRequestqueue Tests"); + +template <> template <> +void HttpRequestqueueTestObjectType::test<1>() +{ + set_test_name("HttpRequestQueue construction"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + ensure("One ref on construction of HttpRequestQueue", HttpRequestQueue::instanceOf()->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // release the implicit reference, causing the object to be released + HttpRequestQueue::term(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpRequestqueueTestObjectType::test<2>() +{ + set_test_name("HttpRequestQueue refcount works"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + rq->addRef(); + + // release the singleton, hold on to the object + HttpRequestQueue::term(); + + ensure("One ref after term() called", rq->getRefCount() == 1); + ensure("Memory being used", mMemTotal < GetMemTotal()); + + // Drop ref + rq->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpRequestqueueTestObjectType::test<3>() +{ + set_test_name("HttpRequestQueue addOp/fetchOp work"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + + HttpOperation * op = new HttpOpNull(); + + rq->addOp(op); // transfer my refcount + + op = rq->fetchOp(true); // Potentially hangs the test on failure + ensure("One goes in, one comes out", NULL != op); + op->release(); + + op = rq->fetchOp(false); + ensure("Better not be two of them", NULL == op); + + // release the singleton, hold on to the object + HttpRequestQueue::term(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpRequestqueueTestObjectType::test<4>() +{ + set_test_name("HttpRequestQueue addOp/fetchAll work"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpRequestQueue::init(); + + HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); + + HttpOperation * op = new HttpOpNull(); + rq->addOp(op); // transfer my refcount + + op = new HttpOpNull(); + rq->addOp(op); // transfer my refcount + + op = new HttpOpNull(); + rq->addOp(op); // transfer my refcount + + { + HttpRequestQueue::OpContainer ops; + rq->fetchAll(true, ops); // Potentially hangs the test on failure + ensure("Three go in, three come out", 3 == ops.size()); + + op = rq->fetchOp(false); + ensure("Better not be any more of them", NULL == op); + + // release the singleton, hold on to the object + HttpRequestQueue::term(); + + // We're still holding onto the ops. + ensure(mMemTotal < GetMemTotal()); + + // Release them + while (! ops.empty()) + { + HttpOperation * op = ops.front(); + ops.erase(ops.begin()); + op->release(); + } + } + + // Should be clean + ensure("All memory returned", mMemTotal == GetMemTotal()); +} + +} // end namespace tut + + +#endif // TEST_LLCORE_HTTP_REQUESTQUEUE_H_ diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp new file mode 100755 index 0000000000..0b379836c9 --- /dev/null +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -0,0 +1,324 @@ +/** + * @file test_llrefcounted + * @brief unit tests for HttpStatus struct + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012-2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef TEST_HTTP_STATUS_H_ +#define TEST_HTTP_STATUS_H_ + +#include "httpcommon.h" + +#include <curl/curl.h> +#include <curl/multi.h> + +using namespace LLCore; + +namespace tut +{ + +struct HttpStatusTestData +{ + HttpStatusTestData() + {} +}; + +typedef test_group<HttpStatusTestData> HttpStatusTestGroupType; +typedef HttpStatusTestGroupType::object HttpStatusTestObjectType; + +HttpStatusTestGroupType HttpStatusTestGroup("HttpStatus Tests"); + +template <> template <> +void HttpStatusTestObjectType::test<1>() +{ + set_test_name("HttpStatus construction"); + + // auto allocation fine for this + HttpStatus status; + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = 0; + + ensure(bool(status)); + ensure(false == !(status)); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = 0; + + ensure(bool(status)); + ensure(false == !(status)); + + status.mType = HttpStatus::LLCORE; + status.mStatus = HE_SUCCESS; + + ensure(bool(status)); + ensure(false == !(status)); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = -1; + + ensure(false == bool(status)); + ensure(!(status)); + + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = CURLE_BAD_DOWNLOAD_RESUME; + + ensure(false == bool(status)); + ensure(!(status)); +} + + +template <> template <> +void HttpStatusTestObjectType::test<2>() +{ + set_test_name("HttpStatus memory structure"); + + // Require that an HttpStatus object can be trivially + // returned as a function return value in registers. + // One should fit in an int on all platforms. + + ensure(sizeof(HttpStatus) <= sizeof(int)); +} + + +template <> template <> +void HttpStatusTestObjectType::test<3>() +{ + set_test_name("HttpStatus valid status string conversion"); + + HttpStatus status; + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = 0; + std::string msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(msg.empty()); + + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = CURLE_BAD_FUNCTION_ARGUMENT; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = CURLM_OUT_OF_MEMORY; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::LLCORE; + status.mStatus = HE_SHUTTING_DOWN; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); +} + + +template <> template <> +void HttpStatusTestObjectType::test<4>() +{ + set_test_name("HttpStatus invalid status string conversion"); + + HttpStatus status; + status.mType = HttpStatus::EXT_CURL_EASY; + status.mStatus = 32726; + std::string msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::EXT_CURL_MULTI; + status.mStatus = -470; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); + + status.mType = HttpStatus::LLCORE; + status.mStatus = 923; + msg = status.toString(); + // std::cout << "Result: " << msg << std::endl; + ensure(! msg.empty()); +} + +template <> template <> +void HttpStatusTestObjectType::test<5>() +{ + set_test_name("HttpStatus equality/inequality testing"); + + // Make certain equality/inequality tests do not pass + // through the bool conversion. Distinct successful + // and error statuses should compare unequal. + + HttpStatus status1(HttpStatus::LLCORE, HE_SUCCESS); + HttpStatus status2(HttpStatus::EXT_CURL_EASY, HE_SUCCESS); + ensure(status1 != status2); + + status1.mType = HttpStatus::LLCORE; + status1.mStatus = HE_REPLY_ERROR; + status2.mType = HttpStatus::LLCORE; + status2.mStatus= HE_SHUTTING_DOWN; + ensure(status1 != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<6>() +{ + set_test_name("HttpStatus basic HTTP status encoding"); + + HttpStatus status; + status.mType = 200; + status.mStatus = HE_SUCCESS; + std::string msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Normally a success but application says error + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(! msg.empty()); + ensure(! bool(status)); + ensure(status.toULong() > 1UL); // Biggish number, not a bool-to-ulong + + // Same statuses with distinct success/fail are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + HttpStatus status2(200, HE_REPLY_ERROR); + ensure(status != status2); + + // Normally an error but application says okay + status.mType = 406; + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Different statuses but both successful are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + status2.mType = 201; + status2.mStatus = HE_SUCCESS; + ensure(status != status2); + + // Different statuses but both failed are distinct + status.mType = 200; + status.mStatus = HE_REPLY_ERROR; + status2.mType = 201; + status2.mStatus = HE_REPLY_ERROR; + ensure(status != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<7>() +{ + set_test_name("HttpStatus HTTP status text strings"); + + HttpStatus status(100, HE_REPLY_ERROR); + std::string msg(status.toString()); + ensure(! msg.empty()); // Should be something + ensure(msg == "Continue"); + + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); // Success is empty + + status.mType = 199; + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 505; // Last defined string + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "HTTP Version not supported"); + + status.mType = 506; // One beyond + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 999; // Last HTTP status + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); +} + + +template <> template <> +void HttpStatusTestObjectType::test<8>() +{ + set_test_name("HttpStatus toHex() nominal function"); + + HttpStatus status(404); + std::string msg = status.toHex(); + // std::cout << "Result: " << msg << std::endl; + ensure(msg == "01940001"); +} + + +template <> template <> +void HttpStatusTestObjectType::test<9>() +{ + set_test_name("HttpStatus toTerseString() nominal function"); + + HttpStatus status(404); + std::string msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Normal HTTP 404", msg == "Http_404"); + + status = HttpStatus(200); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Normal HTTP 200", msg == "Http_200"); + + status = HttpStatus(200, HE_REPLY_ERROR); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Unsuccessful HTTP 200", msg == "Http_200"); // No distinction for error + + status = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Easy couldn't connect error", msg == "Easy_7"); + + status = HttpStatus(HttpStatus::EXT_CURL_MULTI, CURLM_OUT_OF_MEMORY); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Multi out-of-memory error", msg == "Multi_3"); + + status = HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_SET); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Core option not set error", msg == "Core_7"); + + status = HttpStatus(22000, 1); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Undecodable error", msg == "Unknown_1"); + + status = HttpStatus(22000, -1); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Undecodable error 65535", msg == "Unknown_65535"); +} + +} // end namespace tut + +#endif // TEST_HTTP_STATUS_H + diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py new file mode 100755 index 0000000000..04cde651c4 --- /dev/null +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python +"""\ +@file test_llsdmessage_peer.py +@author Nat Goodspeed +@date 2008-10-09 +@brief This script asynchronously runs the executable (with args) specified on + the command line, returning its result code. While that executable is + running, we provide dummy local services for use by C++ tests. + +$LicenseInfo:firstyear=2008&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2012-2013, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +import os +import sys +import time +import select +import getopt +from threading import Thread +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from SocketServer import ThreadingMixIn + +mydir = os.path.dirname(__file__) # expected to be .../indra/llcorehttp/tests/ +sys.path.insert(0, os.path.join(mydir, os.pardir, os.pardir, "lib", "python")) +from indra.util.fastest_elementtree import parse as xml_parse +from indra.base import llsd +from testrunner import freeport, run, debug, VERBOSE + +class TestHTTPRequestHandler(BaseHTTPRequestHandler): + """This subclass of BaseHTTPRequestHandler is to receive and echo + LLSD-flavored messages sent by the C++ LLHTTPClient. + + Target URLs are fairly free-form and are assembled by + concatinating fragments. Currently defined fragments + are: + - '/reflect/' Request headers are bounced back to caller + after prefixing with 'X-Reflect-' + - '/fail/' Body of request can contain LLSD with + 'reason' string and 'status' integer + which will become response header. + - '/bug2295/' 206 response, no data in body: + -- '/bug2295/0/' "Content-Range: bytes 0-75/2983" + -- '/bug2295/1/' "Content-Range: bytes 0-75/*" + -- '/bug2295/2/' "Content-Range: bytes 0-75/2983", + "Content-Length: 0" + -- '/bug2295/00000018/0/' Generates PARTIAL_FILE (18) error in libcurl. + "Content-Range: bytes 0-75/2983", + "Content-Length: 76" + -- '/bug2295/inv_cont_range/0/' Generates HE_INVALID_CONTENT_RANGE error in llcorehttp. + - '/503/' Generate 503 responses with various kinds + of 'retry-after' headers + -- '/503/0/' "Retry-After: 2" + -- '/503/1/' "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT" + -- '/503/2/' "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT" + -- '/503/3/' "Retry-After: " + -- '/503/4/' "Retry-After: (*#*(@*(@(")" + -- '/503/5/' "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo" + -- '/503/6/' "Retry-After: 1 2 3 4 5 6 7 8 9 10" + + Some combinations make no sense, there's no effort to protect + you from that. + """ + ignore_exceptions = (Exception,) + + def read(self): + # The following logic is adapted from the library module + # SimpleXMLRPCServer.py. + # Get arguments by reading body of request. + # We read this in chunks to avoid straining + # socket.read(); around the 10 or 15Mb mark, some platforms + # begin to have problems (bug #792570). + try: + size_remaining = int(self.headers["content-length"]) + except (KeyError, ValueError): + return "" + max_chunk_size = 10*1024*1024 + L = [] + while size_remaining: + chunk_size = min(size_remaining, max_chunk_size) + chunk = self.rfile.read(chunk_size) + L.append(chunk) + size_remaining -= len(chunk) + return ''.join(L) + # end of swiped read() logic + + def read_xml(self): + # This approach reads the entire POST data into memory first + return llsd.parse(self.read()) +## # This approach attempts to stream in the LLSD XML from self.rfile, +## # assuming that the underlying XML parser reads its input file +## # incrementally. Unfortunately I haven't been able to make it work. +## tree = xml_parse(self.rfile) +## debug("Finished raw parse") +## debug("parsed XML tree %s", tree) +## debug("parsed root node %s", tree.getroot()) +## debug("root node tag %s", tree.getroot().tag) +## return llsd.to_python(tree.getroot()) + + def do_HEAD(self): + self.do_GET(withdata=False) + + def do_GET(self, withdata=True): + # Of course, don't attempt to read data. + try: + self.answer(dict(reply="success", status=200, + reason="Your GET operation worked")) + except self.ignore_exceptions, e: + print >> sys.stderr, "Exception during GET (ignoring): %s" % str(e) + + def do_POST(self): + # Read the provided POST data. + # self.answer(self.read()) + try: + self.answer(dict(reply="success", status=200, + reason=self.read())) + except self.ignore_exceptions, e: + print >> sys.stderr, "Exception during POST (ignoring): %s" % str(e) + + def do_PUT(self): + # Read the provided PUT data. + # self.answer(self.read()) + try: + self.answer(dict(reply="success", status=200, + reason=self.read())) + except self.ignore_exceptions, e: + print >> sys.stderr, "Exception during PUT (ignoring): %s" % str(e) + + def answer(self, data, withdata=True): + debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) + if "/sleep/" in self.path: + time.sleep(30) + + if "/503/" in self.path: + # Tests for various kinds of 'Retry-After' header parsing + body = None + if "/503/0/" in self.path: + self.send_response(503) + self.send_header("retry-after", "2") + elif "/503/1/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT") + elif "/503/2/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT") + elif "/503/3/" in self.path: + self.send_response(503) + self.send_header("retry-after", "") + elif "/503/4/" in self.path: + self.send_response(503) + self.send_header("retry-after", "(*#*(@*(@(") + elif "/503/5/" in self.path: + self.send_response(503) + self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo") + elif "/503/6/" in self.path: + self.send_response(503) + self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10") + else: + # Unknown request + self.send_response(400) + body = "Unknown /503/ path in server" + if "/reflect/" in self.path: + self.reflect_headers() + self.send_header("Content-type", "text/plain") + self.end_headers() + if body: + self.wfile.write(body) + elif "/bug2295/" in self.path: + # Test for https://jira.secondlife.com/browse/BUG-2295 + # + # Client can receive a header indicating data should + # appear in the body without actually getting the body. + # Library needs to defend against this case. + # + body = None + if "/bug2295/0/" in self.path: + self.send_response(206) + self.send_header("Content-Range", "bytes 0-75/2983") + elif "/bug2295/1/" in self.path: + self.send_response(206) + self.send_header("Content-Range", "bytes 0-75/*") + elif "/bug2295/2/" in self.path: + self.send_response(206) + self.send_header("Content-Range", "bytes 0-75/2983") + self.send_header("Content-Length", "0") + elif "/bug2295/00000012/0/" in self.path: + self.send_response(206) + self.send_header("Content-Range", "bytes 0-75/2983") + self.send_header("Content-Length", "76") + elif "/bug2295/inv_cont_range/0/" in self.path: + self.send_response(206) + self.send_header("Content-Range", "bytes 0-75/2983") + body = "Some text, but not enough." + else: + # Unknown request + self.send_response(400) + if "/reflect/" in self.path: + self.reflect_headers() + self.send_header("Content-type", "text/plain") + self.end_headers() + if body: + self.wfile.write(body) + elif "fail" not in self.path: + data = data.copy() # we're going to modify + # Ensure there's a "reply" key in data, even if there wasn't before + data["reply"] = data.get("reply", llsd.LLSD("success")) + response = llsd.format_xml(data) + debug("success: %s", response) + self.send_response(200) + if "/reflect/" in self.path: + self.reflect_headers() + self.send_header("Content-type", "application/llsd+xml") + self.send_header("Content-Length", str(len(response))) + self.send_header("X-LL-Special", "Mememememe"); + self.end_headers() + if withdata: + self.wfile.write(response) + else: # fail requested + status = data.get("status", 500) + # self.responses maps an int status to a (short, long) pair of + # strings. We want the longer string. That's why we pass a string + # pair to get(): the [1] will select the second string, whether it + # came from self.responses or from our default pair. + reason = data.get("reason", + self.responses.get(status, + ("fail requested", + "Your request specified failure status %s " + "without providing a reason" % status))[1]) + debug("fail requested: %s: %r", status, reason) + self.send_error(status, reason) + if "/reflect/" in self.path: + self.reflect_headers() + self.end_headers() + + def reflect_headers(self): + for name in self.headers.keys(): + # print "Header: %s: %s" % (name, self.headers[name]) + self.send_header("X-Reflect-" + name, self.headers[name]) + + if not VERBOSE: + # When VERBOSE is set, skip both these overrides because they exist to + # suppress output. + + def log_request(self, code, size=None): + # For present purposes, we don't want the request splattered onto + # stderr, as it would upset devs watching the test run + pass + + def log_error(self, format, *args): + # Suppress error output as well + pass + +class Server(ThreadingMixIn, HTTPServer): + # This pernicious flag is on by default in HTTPServer. But proper + # operation of freeport() absolutely depends on it being off. + allow_reuse_address = False + + # Override of BaseServer.handle_error(). Not too interested + # in errors and the default handler emits a scary traceback + # to stderr which annoys some. Disable this override to get + # default behavior which *shouldn't* cause the program to return + # a failure status. + def handle_error(self, request, client_address): + print '-'*40 + print 'Ignoring exception during processing of request from', + print client_address + print '-'*40 + +if __name__ == "__main__": + do_valgrind = False + path_search = False + options, args = getopt.getopt(sys.argv[1:], "V", ["valgrind"]) + for option, value in options: + if option == "-V" or option == "--valgrind": + do_valgrind = True + + # Instantiate a Server(TestHTTPRequestHandler) on the first free port + # in the specified port range. Doing this inline is better than in a + # daemon thread: if it blows up here, we'll get a traceback. If it blew up + # in some other thread, the traceback would get eaten and we'd run the + # subject test program anyway. + httpd, port = freeport(xrange(8000, 8020), + lambda port: Server(('127.0.0.1', port), TestHTTPRequestHandler)) + + # Pass the selected port number to the subject test program via the + # environment. We don't want to impose requirements on the test program's + # command-line parsing -- and anyway, for C++ integration tests, that's + # performed in TUT code rather than our own. + os.environ["LL_TEST_PORT"] = str(port) + debug("$LL_TEST_PORT = %s", port) + if do_valgrind: + args = ["valgrind", "--log-file=./valgrind.log"] + args + path_search = True + sys.exit(run(server=Thread(name="httpd", target=httpd.serve_forever), use_path=path_search, *args)) + diff --git a/indra/llcorehttp/tests/test_refcounted.hpp b/indra/llcorehttp/tests/test_refcounted.hpp new file mode 100755 index 0000000000..cb4b50287a --- /dev/null +++ b/indra/llcorehttp/tests/test_refcounted.hpp @@ -0,0 +1,156 @@ +/** + * @file test_refcounted.hpp + * @brief unit tests for the LLCoreInt::RefCounted class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ +#ifndef TEST_LLCOREINT_REF_COUNTED_H_ +#define TEST_LLCOREINT_REF_COUNTED_H_ + +#include "_refcounted.h" + +#include "test_allocator.h" + +using namespace LLCoreInt; + +namespace tut +{ + struct RefCountedTestData + { + // the test objects inherit from this so the member functions and variables + // can be referenced directly inside of the test functions. + size_t mMemTotal; + }; + + typedef test_group<RefCountedTestData> RefCountedTestGroupType; + typedef RefCountedTestGroupType::object RefCountedTestObjectType; + RefCountedTestGroupType RefCountedTestGroup("RefCounted Tests"); + + template <> template <> + void RefCountedTestObjectType::test<1>() + { + set_test_name("RefCounted construction with implicit count"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + RefCounted * rc = new RefCounted(true); + ensure(rc->getRefCount() == 1); + + // release the implicit reference, causing the object to be released + rc->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<2>() + { + set_test_name("RefCounted construction without implicit count"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + RefCounted * rc = new RefCounted(false); + ensure(rc->getRefCount() == 0); + + // add a reference + rc->addRef(); + ensure(rc->getRefCount() == 1); + + // release the implicit reference, causing the object to be released + rc->release(); + + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<3>() + { + set_test_name("RefCounted addRef and release"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + RefCounted * rc = new RefCounted(false); + + for (int i = 0; i < 1024; ++i) + { + rc->addRef(); + } + + ensure(rc->getRefCount() == 1024); + + for (int i = 0; i < 1024; ++i) + { + rc->release(); + } + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<4>() + { + set_test_name("RefCounted isLastRef check"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + RefCounted * rc = new RefCounted(true); + + // with only one reference, isLastRef should be true + ensure(rc->isLastRef()); + + // release it to clean up memory + rc->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); + } + + template <> template <> + void RefCountedTestObjectType::test<5>() + { + set_test_name("RefCounted noRef check"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + RefCounted * rc = new RefCounted(false); + + // set the noRef + rc->noRef(); + + // with only one reference, isLastRef should be true + ensure(rc->getRefCount() == RefCounted::NOT_REF_COUNTED); + + // allow this memory leak, but check that we're leaking a known amount + ensure(mMemTotal == (GetMemTotal() - sizeof(RefCounted))); + } +} + +#endif // TEST_LLCOREINT_REF_COUNTED_H_ diff --git a/indra/llcorehttp/tests/testrunner.py b/indra/llcorehttp/tests/testrunner.py new file mode 100755 index 0000000000..9a2de71142 --- /dev/null +++ b/indra/llcorehttp/tests/testrunner.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python +"""\ +@file testrunner.py +@author Nat Goodspeed +@date 2009-03-20 +@brief Utilities for writing wrapper scripts for ADD_COMM_BUILD_TEST unit tests + +$LicenseInfo:firstyear=2009&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2010, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +from __future__ import with_statement + +import os +import sys +import re +import errno +import socket + +VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "0") # default to quiet +# Support usage such as INTEGRATION_TEST_VERBOSE=off -- distressing to user if +# that construct actually turns on verbosity... +VERBOSE = not re.match(r"(0|off|false|quiet)$", VERBOSE, re.IGNORECASE) + +if VERBOSE: + def debug(fmt, *args): + print fmt % args + sys.stdout.flush() +else: + debug = lambda *args: None + +def freeport(portlist, expr): + """ + Find a free server port to use. Specifically, evaluate 'expr' (a + callable(port)) until it stops raising EADDRINUSE exception. + + Pass: + + portlist: an iterable (e.g. xrange()) of ports to try. If you exhaust the + range, freeport() lets the socket.error exception propagate. If you want + unbounded, you could pass itertools.count(baseport), though of course in + practice the ceiling is 2^16-1 anyway. But it seems prudent to constrain + the range much more sharply: if we're iterating an absurd number of times, + probably something else is wrong. + + expr: a callable accepting a port number, specifically one of the items + from portlist. If calling that callable raises socket.error with + EADDRINUSE, freeport() retrieves the next item from portlist and retries. + + Returns: (expr(port), port) + + port: the value from portlist for which expr(port) succeeded + + Raises: + + Any exception raised by expr(port) other than EADDRINUSE. + + socket.error if, for every item from portlist, expr(port) raises + socket.error. The exception you see is the one from the last item in + portlist. + + StopIteration if portlist is completely empty. + + Example: + + class Server(HTTPServer): + # If you use BaseHTTPServer.HTTPServer, turning off this flag is + # essential for proper operation of freeport()! + allow_reuse_address = False + # ... + server, port = freeport(xrange(8000, 8010), + lambda port: Server(("localhost", port), + MyRequestHandler)) + # pass 'port' to client code + # call server.serve_forever() + """ + try: + # If portlist is completely empty, let StopIteration propagate: that's an + # error because we can't return meaningful values. We have no 'port', + # therefore no 'expr(port)'. + portiter = iter(portlist) + port = portiter.next() + + while True: + try: + # If this value of port works, return as promised. + value = expr(port) + + except socket.error, err: + # Anything other than 'Address already in use', propagate + if err.args[0] != errno.EADDRINUSE: + raise + + # Here we want the next port from portiter. But on StopIteration, + # we want to raise the original exception rather than + # StopIteration. So save the original exc_info(). + type, value, tb = sys.exc_info() + try: + try: + port = portiter.next() + except StopIteration: + raise type, value, tb + finally: + # Clean up local traceback, see docs for sys.exc_info() + del tb + + else: + debug("freeport() returning %s on port %s", value, port) + return value, port + + # Recap of the control flow above: + # If expr(port) doesn't raise, return as promised. + # If expr(port) raises anything but EADDRINUSE, propagate that + # exception. + # If portiter.next() raises StopIteration -- that is, if the port + # value we just passed to expr(port) was the last available -- reraise + # the EADDRINUSE exception. + # If we've actually arrived at this point, portiter.next() delivered a + # new port value. Loop back to pass that to expr(port). + + except Exception, err: + debug("*** freeport() raising %s: %s", err.__class__.__name__, err) + raise + +def run(*args, **kwds): + """All positional arguments collectively form a command line, executed as + a synchronous child process. + In addition, pass server=new_thread_instance as an explicit keyword (to + differentiate it from an additional command-line argument). + new_thread_instance should be an instantiated but not yet started Thread + subclass instance, e.g.: + run("python", "-c", 'print "Hello, world!"', server=TestHTTPServer(name="httpd")) + """ + # If there's no server= keyword arg, don't start a server thread: simply + # run a child process. + try: + thread = kwds.pop("server") + except KeyError: + pass + else: + # Start server thread. Note that this and all other comm server + # threads should be daemon threads: we'll let them run "forever," + # confident that the whole process will terminate when the main thread + # terminates, which will be when the child process terminates. + thread.setDaemon(True) + thread.start() + # choice of os.spawnv(): + # - [v vs. l] pass a list of args vs. individual arguments, + # - [no p] don't use the PATH because we specifically want to invoke the + # executable passed as our first arg, + # - [no e] child should inherit this process's environment. + debug("Running %s...", " ".join(args)) + if kwds.get("use_path", False): + rc = os.spawnvp(os.P_WAIT, args[0], args) + else: + rc = os.spawnv(os.P_WAIT, args[0], args) + debug("%s returned %s", args[0], rc) + return rc + +# **************************************************************************** +# test code -- manual at this point, see SWAT-564 +# **************************************************************************** +def test_freeport(): + # ------------------------------- Helpers -------------------------------- + from contextlib import contextmanager + # helper Context Manager for expecting an exception + # with exc(SomeError): + # raise SomeError() + # raises AssertionError otherwise. + @contextmanager + def exc(exception_class, *args): + try: + yield + except exception_class, err: + for i, expected_arg in enumerate(args): + assert expected_arg == err.args[i], \ + "Raised %s, but args[%s] is %r instead of %r" % \ + (err.__class__.__name__, i, err.args[i], expected_arg) + print "Caught expected exception %s(%s)" % \ + (err.__class__.__name__, ', '.join(repr(arg) for arg in err.args)) + else: + assert False, "Failed to raise " + exception_class.__class__.__name__ + + # helper to raise specified exception + def raiser(exception): + raise exception + + # the usual + def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) + + # ------------------------ Sanity check the above ------------------------ + class SomeError(Exception): pass + # Without extra args, accept any err.args value + with exc(SomeError): + raiser(SomeError("abc")) + # With extra args, accept only the specified value + with exc(SomeError, "abc"): + raiser(SomeError("abc")) + with exc(AssertionError): + with exc(SomeError, "abc"): + raiser(SomeError("def")) + with exc(AssertionError): + with exc(socket.error, errno.EADDRINUSE): + raiser(socket.error(errno.ECONNREFUSED, 'Connection refused')) + + # ----------- freeport() without engaging socket functionality ----------- + # If portlist is empty, freeport() raises StopIteration. + with exc(StopIteration): + freeport([], None) + + assert_equals(freeport([17], str), ("17", 17)) + + # This is the magic exception that should prompt us to retry + inuse = socket.error(errno.EADDRINUSE, 'Address already in use') + # Get the iterator to our ports list so we can check later if we've used all + ports = iter(xrange(5)) + with exc(socket.error, errno.EADDRINUSE): + freeport(ports, lambda port: raiser(inuse)) + # did we entirely exhaust 'ports'? + with exc(StopIteration): + ports.next() + + ports = iter(xrange(2)) + # Any exception but EADDRINUSE should quit immediately + with exc(SomeError): + freeport(ports, lambda port: raiser(SomeError())) + assert_equals(ports.next(), 1) + + # ----------- freeport() with platform-dependent socket stuff ------------ + # This is what we should've had unit tests to begin with (see CHOP-661). + def newbind(port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(('127.0.0.1', port)) + return sock + + bound0, port0 = freeport(xrange(7777, 7780), newbind) + assert_equals(port0, 7777) + bound1, port1 = freeport(xrange(7777, 7780), newbind) + assert_equals(port1, 7778) + bound2, port2 = freeport(xrange(7777, 7780), newbind) + assert_equals(port2, 7779) + with exc(socket.error, errno.EADDRINUSE): + bound3, port3 = freeport(xrange(7777, 7780), newbind) + +if __name__ == "__main__": + test_freeport() |