From 7010459f04177aef1875a110b3d33e10c8ec5cad Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 11 Jul 2012 15:53:57 -0400 Subject: SH-3240 Capture Content-Type and Content-Encoding headers. HttpResponse object now has two strings for these content headers. Either or both may be empty. Tidied up the cross-platform string code and got more defensive about the length of a header line. Integration test for the new response object. --- indra/llcorehttp/_httpoprequest.cpp | 158 ++++++++++++++++++------- indra/llcorehttp/_httpoprequest.h | 4 + indra/llcorehttp/httpresponse.h | 19 ++- indra/llcorehttp/tests/test_httprequest.hpp | 126 ++++++++++++++++++++ indra/llcorehttp/tests/test_llcorehttp_peer.py | 2 +- 5 files changed, 263 insertions(+), 46 deletions(-) (limited to 'indra/llcorehttp') diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index bec82d8449..1854d7ada4 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -74,16 +74,16 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -#if LL_WINDOWS +// OS-neutral string comparisons of various types +int os_strncasecmp(const char *s1, const char *s2, size_t n); +int os_strcasecmp(const char *s1, const char *s2); +char * os_strtok_r(char *str, const char *delim, char **saveptr); -// Not available on windows where the legacy strtok interface -// is thread-safe. -char *strtok_r(char *str, const char *delim, char **saveptr); -#endif // LL_WINDOWS +static const char * const hdr_whitespace(" \t"); +static const char * const hdr_separator(": \t"); - -} +} // end anonymous namespace namespace LLCore @@ -228,7 +228,8 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) // Got an explicit offset/length in response response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } - + response->setContent(mReplyConType, mReplyConEncode); + mUserHandler->onCompleted(static_cast(this), response); response->release(); @@ -315,7 +316,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - mProcFlags = 0; + mProcFlags = PF_SCAN_CONTENT_HEADERS; // Always scan for content headers mReqPolicy = policy_id; mReqPriority = priority; mReqURL = url; @@ -377,6 +378,8 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) mReplyHeaders->release(); mReplyHeaders = NULL; } + mReplyConType.clear(); + mReplyConEncode.clear(); // *FIXME: better error handling later HttpStatus status; @@ -539,7 +542,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_SCAN_CONTENT_HEADERS)) { curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); @@ -598,12 +601,18 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi static const char con_ran_line[] = "content-range:"; static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; + + static const char con_type_line[] = "content-type:"; + static const size_t con_type_line_len = sizeof(con_type_line) - 1; + + static const char con_enc_line[] = "content-encoding:"; + static const size_t con_enc_line_len = sizeof(con_enc_line) - 1; HttpOpRequest * op(static_cast(userdata)); const size_t hdr_size(size * nmemb); const char * hdr_data(static_cast(data)); // Not null terminated - + if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) { // One of possibly several status lines. Reset what we know and start over @@ -611,24 +620,47 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; + op->mReplyConType.clear(); + op->mReplyConEncode.clear(); op->mStatus = HttpStatus(); if (op->mReplyHeaders) { op->mReplyHeaders->mHeaders.clear(); } } - else if (op->mProcFlags & PF_SCAN_RANGE_HEADER) + + // Nothing in here wants a final CR/LF combination. Remove + // it as much as possible. + size_t wanted_hdr_size(hdr_size); + if (wanted_hdr_size && '\n' == hdr_data[wanted_hdr_size - 1]) + { + if (--wanted_hdr_size && '\r' == hdr_data[wanted_hdr_size - 1]) + { + --wanted_hdr_size; + } + } + + // Save header if caller wants them in the response + if (op->mProcFlags & PF_SAVE_HEADERS) + { + // Save headers in response + if (! op->mReplyHeaders) + { + op->mReplyHeaders = new HttpHeaders; + } + op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size)); + } + + // Detect and parse 'Content-Range' headers + if (op->mProcFlags & PF_SCAN_RANGE_HEADER) { char hdr_buffer[128]; // Enough for a reasonable header - size_t frag_size((std::min)(hdr_size, sizeof(hdr_buffer) - 1)); + size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); memcpy(hdr_buffer, hdr_data, frag_size); hdr_buffer[frag_size] = '\0'; -#if LL_WINDOWS - if (! _strnicmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len))) -#else - if (! strncasecmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len))) -#endif // LL_WINDOWS + if (frag_size > con_ran_line_len && + ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) { unsigned int first(0), last(0), length(0); int status; @@ -656,22 +688,43 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi } } - if (op->mProcFlags & PF_SAVE_HEADERS) + // Detect and parse 'Content-Type' and 'Content-Encoding' headers + if (op->mProcFlags & PF_SCAN_CONTENT_HEADERS) { - // Save headers in response - if (! op->mReplyHeaders) + if (wanted_hdr_size > con_type_line_len && + ! os_strncasecmp(hdr_data, con_type_line, con_type_line_len)) { - op->mReplyHeaders = new HttpHeaders; + // Found 'Content-Type:', extract single-token value + std::string rhs(hdr_data + con_type_line_len, wanted_hdr_size - con_type_line_len); + std::string::size_type begin(0), end(rhs.size()), pos; + + if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos) + { + begin = pos; + } + if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos) + { + end = pos; + } + op->mReplyConType.assign(rhs, begin, end - begin); } - size_t wanted_size(hdr_size); - if (wanted_size && '\n' == hdr_data[wanted_size - 1]) + else if (wanted_hdr_size > con_enc_line_len && + ! os_strncasecmp(hdr_data, con_enc_line, con_enc_line_len)) { - if (--wanted_size && '\r' == hdr_data[wanted_size - 1]) + // Found 'Content-Encoding:', extract single-token value + std::string rhs(hdr_data + con_enc_line_len, wanted_hdr_size - con_enc_line_len); + std::string::size_type begin(0), end(rhs.size()), pos; + + if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos) { - --wanted_size; + begin = pos; } + if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos) + { + end = pos; + } + op->mReplyConEncode.assign(rhs, begin, end - begin); } - op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_size)); } return hdr_size; @@ -788,15 +841,11 @@ int parse_content_range_header(char * buffer, char * tok_state(NULL), * tok(NULL); bool match(true); - if (! strtok_r(buffer, ": \t", &tok_state)) + if (! os_strtok_r(buffer, hdr_separator, &tok_state)) match = false; - if (match && (tok = strtok_r(NULL, " \t", &tok_state))) -#if LL_WINDOWS - match = 0 == _stricmp("bytes", tok); -#else - match = 0 == strcasecmp("bytes", tok); -#endif // LL_WINDOWS - if (match && ! (tok = strtok_r(NULL, " \t", &tok_state))) + if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) + match = 0 == os_strcasecmp("bytes", tok); + if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) match = false; if (match) { @@ -834,15 +883,6 @@ int parse_content_range_header(char * buffer, return 1; } -#if LL_WINDOWS - -char *strtok_r(char *str, const char *delim, char ** savestate) -{ - return strtok_s(str, delim, savestate); -} - -#endif // LL_WINDOWS - void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) { @@ -880,6 +920,36 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin } +int os_strncasecmp(const char *s1, const char *s2, size_t n) +{ +#if LL_WINDOWS + return _strnicmp(s1, s2, n); +#else + return strncasecmp(s1, s2, n); +#endif // LL_WINDOWS +} + + +int os_strcasecmp(const char *s1, const char *s2) +{ +#if LL_WINDOWS + return _stricmp(s1, s2); +#else + return strcasecmp(s1, s2); +#endif // LL_WINDOWS +} + + +char * os_strtok_r(char *str, const char *delim, char ** savestate) +{ +#if LL_WINDOWS + return strtok_s(str, delim, savestate); +#else + return strtok_r(str, delim, savestate); +#endif +} + + } // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index a4c5fbb3c2..200b925c4e 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -30,6 +30,7 @@ #include "linden_common.h" // Modifies curl/curl.h interfaces +#include #include #include "httpcommon.h" @@ -137,6 +138,7 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + static const unsigned int PF_SCAN_CONTENT_HEADERS = 0x00000004U; public: // Request data @@ -162,6 +164,8 @@ public: size_t mReplyLength; size_t mReplyFullLength; HttpHeaders * mReplyHeaders; + std::string mReplyConType; + std::string mReplyConEncode; // Policy data int mPolicyRetries; diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 65e403cec8..3d35050c17 100644 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -28,6 +28,8 @@ #define _LLCORE_HTTP_RESPONSE_H_ +#include + #include "httpcommon.h" #include "_refcounted.h" @@ -130,7 +132,20 @@ public: mReplyLength = length; mReplyFullLength = full_length; } - + + /// + void getContent(std::string & con_type, std::string & con_encode) const + { + con_type = mContentType; + con_encode = mContentEncoding; + } + + void setContent(const std::string & con_type, const std::string & con_encode) + { + mContentType = con_type; + mContentEncoding = con_encode; + } + protected: // Response data here HttpStatus mStatus; @@ -139,6 +154,8 @@ protected: unsigned int mReplyFullLength; BufferArray * mBufferArray; HttpHeaders * mHeaders; + std::string mContentType; + std::string mContentEncoding; }; diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index ba7c757af4..9edf3d19ec 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -117,6 +117,15 @@ public: ensure("Special header X-LL-Special in response", found_special); } + if (! mCheckContentType.empty()) + { + ensure("Response required with content type check", response != NULL); + std::string con_type, con_enc; + response->getContent(con_type, con_enc); + ensure("Content-Type as expected (" + mCheckContentType + ")", + mCheckContentType == con_type); + } + // std::cout << "TestHandler2::onCompleted() invoked" << std::endl; } @@ -124,6 +133,7 @@ public: std::string mName; HttpHandle mExpectHandle; bool mCheckHeader; + std::string mCheckContentType; }; typedef test_group HttpRequestTestGroupType; @@ -1502,6 +1512,122 @@ void HttpRequestTestObjectType::test<14>() } } +// Test retrieval of Content-Type/Content-Encoding headers +template <> template <> +void HttpRequestTestObjectType::test<15>() +{ + ScopedCurlInit ready; + + std::string url_base(get_base_url()); + // std::cerr << "Base: " << url_base << std::endl; + + set_test_name("HttpRequest GET with Content-Type"); + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + + // Load and clear the string setting to preload std::string object + // for memory return tests. + handler.mCheckContentType = "application/llsd+xml"; + handler.mCheckContentType.clear(); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + // Issue a GET that *can* connect + mStatus = HttpStatus(200); + handler.mCheckContentType = "application/llsd+xml"; + HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url_base, + NULL, + NULL, + &handler); + ensure("Valid handle returned for ranged request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump. + int count(0); + int limit(10); + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == 1); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + handler.mCheckContentType.clear(); + handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 10; + while (count++ < limit && mHandlerCalls < 2) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 2); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + + ensure("Two handler calls on the way out", 2 == mHandlerCalls); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + delete req; + HttpRequest::destroyService(); + throw; + } +} + + } // end namespace tut namespace diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 9e1847c66e..58b5bedd09 100644 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -9,7 +9,7 @@ $LicenseInfo:firstyear=2008&license=viewerlgpl$ Second Life Viewer Source Code -Copyright (C) 2010, Linden Research, Inc. +Copyright (C) 2012, Linden Research, Inc. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public -- cgit v1.2.3