/**
* @file _httpoprequest.cpp
* @brief Definitions for internal class HttpOpRequest
*
* $LicenseInfo:firstyear=2012&license=viewerlgpl$
* Second Life Viewer Source Code
* Copyright (C) 2012-2014, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License only.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
* $/LicenseInfo$
*/
#include "_httpoprequest.h"
#include <cstdio>
#include <algorithm>
#include "httpcommon.h"
#include "httphandler.h"
#include "httpresponse.h"
#include "bufferarray.h"
#include "httpheaders.h"
#include "httpoptions.h"
#include "_httprequestqueue.h"
#include "_httpreplyqueue.h"
#include "_httpservice.h"
#include "_httppolicy.h"
#include "_httppolicyglobal.h"
#include "_httplibcurl.h"
#include "_httpinternal.h"
#include "llhttpconstants.h"
#include "llproxy.h"
#include "httpstats.h"
// *DEBUG: "[curl:bugs] #1420" problem and testing.
//
// A pipelining problem, https://sourceforge.net/p/curl/bugs/1420/,
// was a source of Core_9 failures. Code related to this can be
// identified and tested by:
// * Looking for '[curl:bugs]' strings in source and following
// instructions there.
// * Set 'QAModeHttpTrace' to 2 or 3 in settings.xml and look for
// 'timed out' events in the log.
// * Enable the HttpRangeRequestsDisable debug setting which causes
// full asset fetches. These slow the pipelines down a bit.
//
namespace
{
// Attempts to parse a 'Content-Range:' header. Caller must already
// have verified that the header tag is present. The 'buffer' argument
// will be processed by strtok_r calls which will modify the buffer.
//
// @return -1 if invalid and response should be dropped, 0 if valid an
// correct, 1 if couldn't be parsed. If 0, the first, last,
// and length arguments are also written. 'length' may be
// 0 if the length wasn't available to the server.
//
int parse_content_range_header(char * buffer,
unsigned int * first,
unsigned int * last,
unsigned int * length);
// Similar for Retry-After headers. Only parses the delta form
// of the header, HTTP time formats aren't interesting for client
// purposes.
//
// @return 0 if successfully parsed and seconds time delta
// returned in time argument.
//
int parse_retry_after_header(char * buffer, int * time);
// Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and
// escape and format it for a tracing line in logging. Absolutely
// anything including NULs can be in the data. If @scrub is true,
// non-printing or non-ascii characters are replaced with spaces
// otherwise a %XX form of escaping is used.
void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,
std::string & safe_line);
// OS-neutral string comparisons of various types.
int os_strcasecmp(const char * s1, const char * s2);
char * os_strtok_r(char * str, const char * delim, char ** saveptr);
char * os_strtrim(char * str);
char * os_strltrim(char * str);
void os_strlower(char * str);
// Error testing and reporting for libcurl status codes
void check_curl_easy_code(CURLcode code, int curl_setopt_option);
// This is a template because different 'option' values require different
// types for 'ARG'. Just pass them through unchanged (by value).
template <typename ARG>
void check_curl_easy_setopt(CURL* handle, CURLoption option, ARG argument)
{
CURLcode code = curl_easy_setopt(handle, option, argument);
check_curl_easy_code(code, option);
}
static const char * const LOG_CORE("CoreHttp");
} // end anonymous namespace
namespace LLCore
{
HttpOpRequest::HttpOpRequest()
: HttpOperation(),
mProcFlags(0U),
mReqMethod(HOR_GET),
mReqBody(NULL),
mReqOffset(0),
mReqLength(0),
mReqHeaders(),
mReqOptions(),
mCurlActive(false),
mCurlHandle(NULL),
mCurlService(NULL),
mCurlHeaders(NULL),
mCurlBodyPos(0),
mCurlTemp(NULL),
mCurlTempLen(0),
mReplyBody(NULL),
mReplyOffset(0),
mReplyLength(0),
mReplyFullLength(0),
mReplyHeaders(),
mPolicyRetries(0),
mPolicy503Retries(0),
mPolicyRetryAt(HttpTime(0)),
mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT),
mPolicyMinRetryBackoff(HttpTime(HTTP_RETRY_BACKOFF_MIN_DEFAULT)),
mPolicyMaxRetryBackoff(HttpTime(HTTP_RETRY_BACKOFF_MAX_DEFAULT)),
mCallbackSSLVerify(NULL)
{
// *NOTE: As members are added, retry initialization/cleanup
// may need to be extended in @see prepareRequest().
}
HttpOpRequest::~HttpOpRequest()
{
if (mReqBody)
{
mReqBody->release();
mReqBody = NULL;
}
if (mCurlHandle)
{
// Uncertain of thread context so free using
// safest method.
curl_easy_cleanup(mCurlHandle);
mCurlHandle = NULL;
}
mCurlService = NULL;
if (mCurlHeaders)
{
curl_slist_free_all(mCurlHeaders);
mCurlHeaders = NULL;
}
delete [] mCurlTemp;
mCurlTemp = NULL;
mCurlTempLen = 0;
if (mReplyBody)
{
mReplyBody->release();
mReplyBody = NULL;
}
}
void HttpOpRequest::stageFromRequest(HttpService * service)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t self(std::dynamic_pointer_cast<HttpOpRequest>(shared_from_this()));
service->getPolicy().addOp(self); // transfers refcount
}
void HttpOpRequest::stageFromReady(HttpService * service)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t self(std::dynamic_pointer_cast<HttpOpRequest>(shared_from_this()));
service->getTransport().addOp(self); // transfers refcount
}
void HttpOpRequest::stageFromActive(HttpService * service)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
if (mReplyLength)
{
// If non-zero, we received and processed a Content-Range
// header with the response. If there is received data
// (and there may not be due to protocol violations,
// HEAD requests, etc., see BUG-2295) Verify that what it
// says is consistent with the received data.
if (mReplyBody && mReplyBody->size() && mReplyLength != mReplyBody->size())
{
// Not as expected, fail the request
mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
}
}
if (mCurlHeaders)
{
// We take these headers out of the request now as they were
// allocated originally in this thread and the notifier doesn't
// need them. This eliminates one source of heap moving across
// threads.
curl_slist_free_all(mCurlHeaders);
mCurlHeaders = NULL;
}
// Also not needed on the other side
delete [] mCurlTemp;
mCurlTemp = NULL;
mCurlTempLen = 0;
addAsReply();
}
void HttpOpRequest::visitNotifier(HttpRequest * request)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
if (mUserHandler)
{
HttpResponse * response = new HttpResponse();
response->setStatus(mStatus);
response->setBody(mReplyBody);
response->setHeaders(mReplyHeaders);
response->setRequestURL(mReqURL);
response->setRequestMethod(methodToString(mReqMethod));
if (mReplyOffset || mReplyLength)
{
// Got an explicit offset/length in response
response->setRange(mReplyOffset, static_cast<unsigned int>(mReplyLength), static_cast<unsigned int>(mReplyFullLength));
}
response->setContentType(mReplyConType);
response->setRetries(mPolicyRetries, mPolicy503Retries);
HttpResponse::TransferStats::ptr_t stats = HttpResponse::TransferStats::ptr_t(new HttpResponse::TransferStats);
curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_DOWNLOAD, &stats->mSizeDownload);
curl_easy_getinfo(mCurlHandle, CURLINFO_TOTAL_TIME, &stats->mTotalTime);
curl_easy_getinfo(mCurlHandle, CURLINFO_SPEED_DOWNLOAD, &stats->mSpeedDownload);
response->setTransferStats(stats);
mUserHandler->onCompleted(this->getHandle(), response);
response->release();
}
}
// /*static*/
// HttpOpRequest::ptr_t HttpOpRequest::fromHandle(HttpHandle handle)
// {
//
// return std::dynamic_pointer_cast<HttpOpRequest>((static_cast<HttpOpRequest *>(handle))->shared_from_this());
// }
HttpStatus HttpOpRequest::cancel()
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
mStatus = HttpStatus(HttpStatus::LLCORE, HE_OP_CANCELED);
addAsReply();
return HttpStatus();
}
HttpStatus HttpOpRequest::setupGet(HttpRequest::policy_t policy_id,
const std::string & url,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t & headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, NULL, options, headers);
mReqMethod = HOR_GET;
return HttpStatus();
}
HttpStatus HttpOpRequest::setupGetByteRange(HttpRequest::policy_t policy_id,
const std::string & url,
size_t offset,
size_t len,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t & headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, NULL, options, headers);
mReqMethod = HOR_GET;
mReqOffset = static_cast<off_t>(offset);
mReqLength = len;
if (offset || len)
{
mProcFlags |= PF_SCAN_RANGE_HEADER;
}
return HttpStatus();
}
HttpStatus HttpOpRequest::setupPost(HttpRequest::policy_t policy_id,
const std::string & url,
BufferArray * body,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t & headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, body, options, headers);
mReqMethod = HOR_POST;
return HttpStatus();
}
HttpStatus HttpOpRequest::setupPut(HttpRequest::policy_t policy_id,
const std::string & url,
BufferArray * body,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t & headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, body, options, headers);
mReqMethod = HOR_PUT;
return HttpStatus();
}
HttpStatus HttpOpRequest::setupDelete(HttpRequest::policy_t policy_id,
const std::string & url,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t & headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, NULL, options, headers);
mReqMethod = HOR_DELETE;
return HttpStatus();
}
HttpStatus HttpOpRequest::setupPatch(HttpRequest::policy_t policy_id,
const std::string & url,
BufferArray * body,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t & headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, body, options, headers);
mReqMethod = HOR_PATCH;
return HttpStatus();
}
HttpStatus HttpOpRequest::setupCopy(HttpRequest::policy_t policy_id,
const std::string & url,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t &headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, NULL, options, headers);
mReqMethod = HOR_COPY;
return HttpStatus();
}
HttpStatus HttpOpRequest::setupMove(HttpRequest::policy_t policy_id,
const std::string & url,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t &headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
setupCommon(policy_id, url, NULL, options, headers);
mReqMethod = HOR_MOVE;
return HttpStatus();
}
void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,
const std::string & url,
BufferArray * body,
const HttpOptions::ptr_t & options,
const HttpHeaders::ptr_t & headers)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
mProcFlags = 0U;
mReqPolicy = policy_id;
mReqURL = url;
if (body)
{
body->addRef();
mReqBody = body;
}
if (headers && ! mReqHeaders)
{
mReqHeaders = headers;
}
if (options && !mReqOptions)
{
mReqOptions = options;
if (options->getWantHeaders())
{
mProcFlags |= PF_SAVE_HEADERS;
}
if (options->getUseRetryAfter())
{
mProcFlags |= PF_USE_RETRY_AFTER;
}
mPolicyRetryLimit = options->getRetries();
mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX);
mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX));
mPolicyMinRetryBackoff = llclamp(options->getMinBackoff(), HttpTime(0), HTTP_RETRY_BACKOFF_MAX);
mPolicyMaxRetryBackoff = llclamp(options->getMaxBackoff(), mPolicyMinRetryBackoff, HTTP_RETRY_BACKOFF_MAX);
}
}
// Sets all libcurl options and data for a request.
//
// Used both for initial requests and to 'reload' for
// a retry, generally with a different CURL handle.
// Junk may be left around from a failed request and that
// needs to be cleaned out.
//
// *TODO: Move this to _httplibcurl where it belongs.
HttpStatus HttpOpRequest::prepareRequest(HttpService * service)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
// Scrub transport and result data for retried op case
mCurlActive = false;
mCurlHandle = NULL;
mCurlService = NULL;
if (mCurlHeaders)
{
curl_slist_free_all(mCurlHeaders);
mCurlHeaders = NULL;
}
mCurlBodyPos = 0;
if (mReplyBody)
{
mReplyBody->release();
mReplyBody = NULL;
}
mReplyOffset = 0;
mReplyLength = 0;
mReplyFullLength = 0;
mReplyHeaders.reset();
mReplyConType.clear();
// *FIXME: better error handling later
HttpStatus status;
// Get global and class policy options
HttpPolicyGlobal & gpolicy(service->getPolicy().getGlobalOptions());
HttpPolicyClass & cpolicy(service->getPolicy().getClassOptions(mReqPolicy));
mCurlHandle = service->getTransport().getHandle();
if (! mCurlHandle)
{
// We're in trouble. We'll continue but it won't go well.
LL_WARNS(LOG_CORE) << "Failed to allocate libcurl easy handle. Continuing."
<< LL_ENDL;
return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC);
}
check_curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
check_curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
check_curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
check_curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str());
check_curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, getHandle());
check_curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
check_curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1);
check_curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT);
check_curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
check_curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, getHandle());
check_curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback);
check_curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, getHandle());
check_curl_easy_setopt(mCurlHandle, CURLOPT_SEEKFUNCTION, seekCallback);
check_curl_easy_setopt(mCurlHandle, CURLOPT_SEEKDATA, getHandle());
check_curl_easy_setopt(mCurlHandle, CURLOPT_COOKIEFILE, "");
if (gpolicy.mSslCtxCallback)
{
check_curl_easy_setopt(mCurlHandle, CURLOPT_SSL_CTX_FUNCTION, curlSslCtxCallback);
check_curl_easy_setopt(mCurlHandle, CURLOPT_SSL_CTX_DATA, getHandle());
mCallbackSSLVerify = gpolicy.mSslCtxCallback;
}
long follow_redirect(1L);
long sslPeerV(0L);
long sslHostV(0L);
long dnsCacheTimeout(-1L);
long nobody(0L);
if (mReqOptions)
{
follow_redirect = mReqOptions->getFollowRedirects() ? 1L : 0L;
sslPeerV = mReqOptions->getSSLVerifyPeer() ? 1L : 0L;
sslHostV = mReqOptions->getSSLVerifyHost() ? 2L : 0L;
dnsCacheTimeout = mReqOptions->getDNSCacheTimeout();
nobody = mReqOptions->getHeadersOnly() ? 1L : 0L;
}
check_curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, follow_redirect);
check_curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, sslPeerV);
check_curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, sslHostV);
check_curl_easy_setopt(mCurlHandle, CURLOPT_NOBODY, nobody);
// The Linksys WRT54G V5 router has an issue with frequent
// DNS lookups from LAN machines. If they happen too often,
// like for every HTTP request, the router gets annoyed after
// about 700 or so requests and starts issuing TCP RSTs to
// new connections. Reuse the DNS lookups for even a few
// seconds and no RSTs.
//
// -1 stores forever
// 0 never stores
// any other positive number specifies seconds
// supposedly curl 7.62.0 can use TTL by default, otherwise default is 60 seconds
check_curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, dnsCacheTimeout);
if (gpolicy.mUseLLProxy)
{
// Use the viewer-based thread-safe API which has a
// fast/safe check for proxy enable. Would like to
// encapsulate this someway...
// Make sure proxy won't be getInstance() from here,
// it is not thread safe
LLProxy::applyProxySettings(mCurlHandle);
}
else if (gpolicy.mHttpProxy.size())
{
// *TODO: This is fine for now but get fuller socks5/
// authentication thing going later....
check_curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, gpolicy.mHttpProxy.c_str());
check_curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
}
if (gpolicy.mCAPath.size())
{
check_curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, gpolicy.mCAPath.c_str());
}
if (gpolicy.mCAFile.size())
{
check_curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, gpolicy.mCAFile.c_str());
}
switch (mReqMethod)
{
case HOR_GET:
if (nobody == 0)
check_curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1);
break;
case HOR_POST:
{
check_curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1);
check_curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
long data_size(0);
if (mReqBody)
{
data_size = static_cast<long>(mReqBody->size());
}
check_curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL));
check_curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
}
break;
case HOR_PATCH:
check_curl_easy_setopt(mCurlHandle, CURLOPT_CUSTOMREQUEST, "PATCH");
// fall through. The rest is the same as PUT
[[fallthrough]];
case HOR_PUT:
{
check_curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1);
long data_size(0);
if (mReqBody)
{
data_size = static_cast<long>(mReqBody->size());
}
check_curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size);
mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");
}
break;
case HOR_DELETE:
check_curl_easy_setopt(mCurlHandle, CURLOPT_CUSTOMREQUEST, "DELETE");
break;
case HOR_COPY:
check_curl_easy_setopt(mCurlHandle, CURLOPT_CUSTOMREQUEST, "COPY");
break;
case HOR_MOVE:
check_curl_easy_setopt(mCurlHandle, CURLOPT_CUSTOMREQUEST, "MOVE");
break;
default:
LL_ERRS(LOG_CORE) << "Invalid HTTP method in request: "
<< int(mReqMethod) << ". Can't recover."
<< LL_ENDL;
break;
}
// *TODO: Should this be 'Keep-Alive' ?
mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");
mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");
// Tracing
if (mTracing >= HTTP_TRACE_CURL_HEADERS)
{
check_curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1);
check_curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this);
check_curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback);
}
// There's a CURLOPT for this now...
if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod)
{
static const char * const fmt1("Range: bytes=%lu-%lu");
static const char * const fmt2("Range: bytes=%lu-");
char range_line[64];
#if LL_WINDOWS
_snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1,
(mReqLength ? fmt1 : fmt2),
(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
#else
if ( mReqLength )
{
snprintf(range_line, sizeof(range_line),
fmt1,
(unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1));
}
else
{
snprintf(range_line, sizeof(range_line),
fmt2,
(unsigned long) mReqOffset);
}
#endif // LL_WINDOWS
range_line[sizeof(range_line) - 1] = '\0';
mCurlHeaders = curl_slist_append(mCurlHeaders, range_line);
}
mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:");
// Request options
long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT);
long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT);
if (mReqOptions)
{
timeout = mReqOptions->getTimeout();
timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
xfer_timeout = mReqOptions->getTransferTimeout();
xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX);
}
if (xfer_timeout == 0L)
{
xfer_timeout = timeout;
}
if (cpolicy.mPipelining > 1L)
{
// Pipelining affects both connection and transfer timeout values.
// Requests that are added to a pipeling immediately have completed
// their connection so the connection delay tends to be less than
// the non-pipelined value. Transfers are the opposite. Transfer
// timeout starts once the connection is established and completion
// can be delayed due to the pipelined requests ahead. So, it's
// a handwave but bump the transfer timeout up by the pipelining
// depth to give some room.
//
// BUG-7698, BUG-7688, BUG-7694 (others). Scylla and Charybdis
// situation. Operating against a CDN having service issues may
// lead to requests stalling for an arbitrarily long time with only
// the CURLOPT_TIMEOUT value leading to a closed connection. Sadly
// for pipelining, libcurl (7.39.0 and earlier, at minimum) starts
// the clock on this value as soon as a request is started down
// the wire. We want a short value to recover and retry from the
// CDN. We need a long value to safely deal with a succession of
// piled-up pipelined requests.
//
// *TODO: Find a better scheme than timeouts to guarantee liveness.
// Progress on the connection is what we really want, not timeouts.
// But we don't have access to that and the request progress indicators
// (various libcurl callbacks) have the same problem TIMEOUT does.
//
// xfer_timeout *= cpolicy.mPipelining;
xfer_timeout *= 2L;
// Also try requesting HTTP/2.
/******************************/
// but for test purposes, only if overriding VIEWERASSET
if (getenv("VIEWERASSET"))
/******************************/
check_curl_easy_setopt(mCurlHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
}
// *DEBUG: Enable following override for timeout handling and "[curl:bugs] #1420" tests
//if (cpolicy.mPipelining)
//{
// xfer_timeout = 1L;
// timeout = 1L;
//}
check_curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout);
check_curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout);
// Request headers
if (mReqHeaders)
{
// Caller's headers last to override
mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);
}
check_curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders);
if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER))
{
check_curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
check_curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this);
}
if (status)
{
mCurlService = service;
}
return status;
}
size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void * userdata)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t op(HttpOpRequest::fromHandle<HttpOpRequest>(userdata));
if (! op->mReplyBody)
{
op->mReplyBody = new BufferArray();
}
const size_t req_size(size * nmemb);
const size_t write_size(op->mReplyBody->append(static_cast<char *>(data), req_size));
HTTPStats::instance().recordDataDown(write_size);
return write_size;
}
size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t op(HttpOpRequest::fromHandle<HttpOpRequest>(userdata));
if (! op->mReqBody)
{
return 0;
}
const size_t req_size(size * nmemb);
const size_t body_size(op->mReqBody->size());
if (body_size <= op->mCurlBodyPos)
{
if (body_size < op->mCurlBodyPos)
{
// Warn but continue if the read position moves beyond end-of-body
// for some reason.
LL_WARNS(LOG_CORE) << "Request body position beyond body size. Truncating request body."
<< LL_ENDL;
}
return 0;
}
const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos));
const size_t read_size(op->mReqBody->read(op->mCurlBodyPos, static_cast<char *>(data), do_size));
// FIXME: singleton's instance() is Thread unsafe! Even if stats accumulators inside are.
HTTPStats::instance().recordDataUp(read_size);
op->mCurlBodyPos += read_size;
return read_size;
}
int HttpOpRequest::seekCallback(void *userdata, curl_off_t offset, int origin)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t op(HttpOpRequest::fromHandle<HttpOpRequest>(userdata));
if (!op->mReqBody)
{
return 0;
}
size_t newPos = 0;
if (origin == SEEK_SET)
newPos = offset;
else if (origin == SEEK_END)
newPos = static_cast<curl_off_t>(op->mReqBody->size()) + offset;
else if (origin == SEEK_CUR)
newPos = static_cast<curl_off_t>(op->mCurlBodyPos) + offset;
else
return 2;
if (newPos >= op->mReqBody->size())
{
LL_WARNS(LOG_CORE) << "Attempt to seek to position outside post body." << LL_ENDL;
return 2;
}
op->mCurlBodyPos = (size_t)newPos;
return 0;
}
size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
static const char status_line[] = "HTTP/";
static const size_t status_line_len = sizeof(status_line) - 1;
static const char con_ran_line[] = "content-range";
static const char con_retry_line[] = "retry-after";
HttpOpRequest::ptr_t op(HttpOpRequest::fromHandle<HttpOpRequest>(userdata));
const size_t hdr_size(size * nmemb);
const char * hdr_data(static_cast<const char *>(data)); // Not null terminated
bool is_header(true);
if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len))
{
// One of possibly several status lines. Reset what we know and start over
// taking results from the last header stanza we receive.
op->mReplyOffset = 0;
op->mReplyLength = 0;
op->mReplyFullLength = 0;
op->mReplyRetryAfter = 0;
op->mStatus = HttpStatus();
if (op->mReplyHeaders)
{
op->mReplyHeaders->clear();
}
is_header = false;
}
// Nothing in here wants a final CR/LF combination. Remove
// it as much as possible.
size_t wanted_hdr_size(hdr_size);
if (wanted_hdr_size && '\n' == hdr_data[wanted_hdr_size - 1])
{
if (--wanted_hdr_size && '\r' == hdr_data[wanted_hdr_size - 1])
{
--wanted_hdr_size;
}
}
// Copy and normalize header fragments for the following
// stages. Would like to modify the data in-place but that
// may not be allowed and we need one byte extra for NUL.
// At the end of this we will have:
//
// If ':' present in header:
// 1. name points to text to left of colon which
// will be ascii lower-cased and left and right
// trimmed of whitespace.
// 2. value points to text to right of colon which
// will be left trimmed of whitespace.
// Otherwise:
// 1. name points to header which will be left
// trimmed of whitespace.
// 2. value is NULL
// Any non-NULL pointer may point to a zero-length string.
//
if (wanted_hdr_size >= op->mCurlTempLen)
{
delete [] op->mCurlTemp;
op->mCurlTempLen = 2 * wanted_hdr_size + 1;
op->mCurlTemp = new char [op->mCurlTempLen];
}
memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size);
op->mCurlTemp[wanted_hdr_size] = '\0';
char * name(op->mCurlTemp);
char * value(strchr(name, ':'));
if (value)
{
*value++ = '\0';
os_strlower(name);
name = os_strtrim(name);
value = os_strltrim(value);
}
else
{
// Doesn't look well-formed, do minimal normalization on it
name = os_strltrim(name);
}
// Normalized, now reject headers with empty names.
if (! *name)
{
// No use continuing
return hdr_size;
}
// Save header if caller wants them in the response
if (is_header && op->mProcFlags & PF_SAVE_HEADERS)
{
// Save headers in response
if (! op->mReplyHeaders)
{
op->mReplyHeaders = HttpHeaders::ptr_t(new HttpHeaders);
}
op->mReplyHeaders->append(name, value ? value : "");
}
// From this point, header-specific processors are free to
// modify the header value.
// Detect and parse 'Content-Range' headers
if (is_header
&& op->mProcFlags & PF_SCAN_RANGE_HEADER
&& value && *value
&& ! strcmp(name, con_ran_line))
{
unsigned int first(0), last(0), length(0);
int status;
if (! (status = parse_content_range_header(value, &first, &last, &length)))
{
// Success, record the fragment position
op->mReplyOffset = first;
op->mReplyLength = last - first + 1;
op->mReplyFullLength = length;
}
else if (-1 == status)
{
// Response is badly formed and shouldn't be accepted
op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR);
}
else
{
// Ignore the unparsable.
LL_INFOS_ONCE(LOG_CORE) << "Problem parsing odd Content-Range header: '"
<< std::string(hdr_data, wanted_hdr_size)
<< "'. Ignoring."
<< LL_ENDL;
}
}
// Detect and parse 'Retry-After' headers
if (is_header
&& op->mProcFlags & PF_USE_RETRY_AFTER
&& value && *value
&& ! strcmp(name, con_retry_line))
{
int time(0);
if (! parse_retry_after_header(value, &time))
{
op->mReplyRetryAfter = time;
}
}
return hdr_size;
}
CURLcode HttpOpRequest::curlSslCtxCallback(CURL *curl, void *sslctx, void *userdata)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t op(HttpOpRequest::fromHandle<HttpOpRequest>(userdata));
if (op->mCallbackSSLVerify)
{
SSL_CTX * ctx = (SSL_CTX *)sslctx;
if (op->mReqOptions && op->mReqOptions->getSSLVerifyPeer())
{
// verification for ssl certs
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL);
}
else
{
// disable any default verification for server certs
// Ex: setting urls (assume non-SL) for parcel media in LLFloaterURLEntry
SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, NULL);
}
// set the verification callback.
SSL_CTX_set_cert_verify_callback(ctx, sslCertVerifyCallback, userdata);
// the calls are void
}
return CURLE_OK;
}
int HttpOpRequest::sslCertVerifyCallback(X509_STORE_CTX *ctx, void *param)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t op(HttpOpRequest::fromHandle<HttpOpRequest>(param));
if (op->mCallbackSSLVerify)
{
op->mStatus = op->mCallbackSSLVerify(op->mReqURL, op->mUserHandler, ctx);
}
return (op->mStatus) ? 1 : 0;
}
int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffer, size_t len, void * userdata)
{
LL_PROFILE_ZONE_SCOPED_CATEGORY_NETWORK;
HttpOpRequest::ptr_t op(HttpOpRequest::fromHandle<HttpOpRequest>(userdata));
std::string safe_line;
std::string tag;
bool logit(false);
const size_t log_len((std::min)(len, size_t(256))); // Keep things reasonable in all cases
switch (info)
{
case CURLINFO_TEXT:
if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
{
tag = "TEXT";
escape_libcurl_debug_data(buffer, log_len, true, safe_line);
logit = true;
}
break;
case CURLINFO_HEADER_IN:
if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
{
tag = "HEADERIN";
escape_libcurl_debug_data(buffer, log_len, true, safe_line);
logit = true;
}
break;
case CURLINFO_HEADER_OUT:
if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
{
tag = "HEADEROUT";
escape_libcurl_debug_data(buffer, log_len, true, safe_line); // Goes out as one line unlike header_in
logit = true;
}
break;
case CURLINFO_DATA_IN:
if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
{
tag = "DATAIN";
logit = true;
if (op->mTracing >= HTTP_TRACE_CURL_BODIES)
{
escape_libcurl_debug_data(buffer, log_len, false, safe_line);
}
else
{
std::ostringstream out;
out << len << " Bytes";
safe_line = out.str();
}
}
break;
case CURLINFO_DATA_OUT:
if (op->mTracing >= HTTP_TRACE_CURL_HEADERS)
{
tag = "DATAOUT";
logit = true;
if (op->mTracing >= HTTP_TRACE_CURL_BODIES)
{
escape_libcurl_debug_data(buffer, log_len, false, safe_line);
}
else
{
std::ostringstream out;
out << len << " Bytes";
safe_line = out.str();
}
}
break;
default:
logit = false;
break;
}
if (logit)
{
LL_INFOS(LOG_CORE) << "TRACE, LibcurlDebug, Handle: "
<< op->getHandle()
<< ", Type: " << tag
<< ", Data: " << safe_line
<< LL_ENDL;
}
return 0;
}
std::string HttpOpRequest::methodToString(const HttpOpRequest::EMethod &e)
{
if (e == HOR_COPY)
return "COPY";
else if (e == HOR_DELETE)
return "DELETE";
else if (e == HOR_GET)
return "GET";
else if (e == HOR_MOVE)
return "MOVE";
else if (e == HOR_PATCH)
return "PATCH";
else if (e == HOR_POST)
return "POST";
else if (e == HOR_PUT)
return "PUT";
return "UNKNOWN";
}
} // end namespace LLCore
// =======================================
// Anonymous Namespace
// =======================================
namespace
{
int parse_content_range_header(char * buffer,
unsigned int * first,
unsigned int * last,
unsigned int * length)
{
static const char * const hdr_whitespace(" \t");
char * tok_state(NULL), * tok(NULL);
bool match(true);
if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state)))
match = false;
else
match = (0 == os_strcasecmp("bytes", tok));
if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))
match = false;
if (match)
{
unsigned int lcl_first(0), lcl_last(0), lcl_len(0);
#if LL_WINDOWS
if (3 == sscanf_s(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len))
#else
if (3 == sscanf(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len))
#endif // LL_WINDOWS
{
if (lcl_first > lcl_last || lcl_last >= lcl_len)
return -1;
*first = lcl_first;
*last = lcl_last;
*length = lcl_len;
return 0;
}
#if LL_WINDOWS
if (2 == sscanf_s(tok, "%u-%u/*", &lcl_first, &lcl_last))
#else
if (2 == sscanf(tok, "%u-%u/*", &lcl_first, &lcl_last))
#endif // LL_WINDOWS
{
if (lcl_first > lcl_last)
return -1;
*first = lcl_first;
*last = lcl_last;
*length = 0;
return 0;
}
}
// Header is there but badly/unexpectedly formed, try to ignore it.
return 1;
}
int parse_retry_after_header(char * buffer, int * time)
{
char * endptr(buffer);
long lcl_time(strtol(buffer, &endptr, 10));
if (*endptr == '\0' && endptr != buffer && lcl_time > 0)
{
*time = lcl_time;
return 0;
}
// Could attempt to parse HTTP time here but we're not really
// interested in it. Scheduling based on wallclock time on
// user hardware will lead to tears.
// Header is there but badly/unexpectedly formed, try to ignore it.
return 1;
}
void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)
{
std::string out;
len = (std::min)(len, size_t(200));
out.reserve(3 * len);
for (int i(0); i < len; ++i)
{
unsigned char uc(static_cast<unsigned char>(buffer[i]));
if (uc < 32 || uc > 126)
{
if (scrub)
{
out.append(1, ' ');
}
else
{
static const char hex[] = "0123456789ABCDEF";
char convert[4];
convert[0] = '%';
convert[1] = hex[(uc >> 4) % 16];
convert[2] = hex[uc % 16];
convert[3] = '\0';
out.append(convert);
}
}
else
{
out.append(1, buffer[i]);
}
}
safe_line.swap(out);
}
int os_strcasecmp(const char *s1, const char *s2)
{
#if LL_WINDOWS
return _stricmp(s1, s2);
#else
return strcasecmp(s1, s2);
#endif // LL_WINDOWS
}
char * os_strtok_r(char *str, const char *delim, char ** savestate)
{
#if LL_WINDOWS
return strtok_s(str, delim, savestate);
#else
return strtok_r(str, delim, savestate);
#endif
}
void os_strlower(char * str)
{
for (char c(0); (c = *str); ++str)
{
*str = tolower(c);
}
}
char * os_strtrim(char * lstr)
{
while (' ' == *lstr || '\t' == *lstr)
{
++lstr;
}
if (*lstr)
{
char * rstr(lstr + strlen(lstr));
while (lstr < rstr && *--rstr)
{
if (' ' == *rstr || '\t' == *rstr)
{
*rstr = '\0';
}
}
llassert(lstr <= rstr);
}
return lstr;
}
char * os_strltrim(char * lstr)
{
while (' ' == *lstr || '\t' == *lstr)
{
++lstr;
}
return lstr;
}
void check_curl_easy_code(CURLcode code, int curl_setopt_option)
{
if (CURLE_OK != code)
{
// Comment from old llcurl code which may no longer apply:
//
// linux appears to throw a curl error once per session for a bad initialization
// at a pretty random time (when enabling cookies).
LL_WARNS(LOG_CORE) << "libcurl error detected: " << curl_easy_strerror(code)
<< ", curl_easy_setopt option: " << curl_setopt_option
<< LL_ENDL;
}
}
} // end anonymous namespace