/** * @file llappcorehttp.cpp * @brief * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2012-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ #include "llviewerprecompiledheaders.h" #include "llappcorehttp.h" #include "llappviewer.h" #include "llviewercontrol.h" #include "llexception.h" #include "stringize.h" #include #include #include "llsecapi.h" #include #include "llcorehttputil.h" #include "httpstats.h" // Here is where we begin to get our connection usage under control. // This establishes llcorehttp policy classes that, among other // things, limit the maximum number of connections to outside // services. Each of the entries below maps to a policy class and // has a limit, sometimes configurable, of how many connections can // be open at a time. const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); const long LLAppCoreHttp::PIPELINING_DEPTH(5L); // Default and dynamic values for classes static const struct { U32 mDefault; U32 mMin; U32 mMax; U32 mRate; bool mPipelined; std::string mKey; const char * mUsage; } init_data[LLAppCoreHttp::AP_COUNT] = { { // AP_DEFAULT 8, 8, 8, 0, false, "", "other" }, { // AP_TEXTURE 8, 1, 12, 0, true, "TextureFetchConcurrency", "texture fetch" }, { // AP_MESH1 32, 1, 128, 0, false, "MeshMaxConcurrentRequests", "mesh fetch" }, { // AP_MESH2 8, 1, 32, 0, true, "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, { // AP_LARGE_MESH 2, 1, 8, 0, false, "", "large mesh fetch" }, { // AP_UPLOADS 2, 1, 8, 0, false, "", "asset upload" }, { // AP_LONG_POLL 32, 32, 32, 0, false, "", "long poll" }, { // AP_INVENTORY 4, 1, 4, 0, false, "", "inventory" }, { // AP_MATERIALS 2, 1, 8, 0, false, "RenderMaterials", "material manager requests" }, { // AP_AGENT 2, 1, 32, 0, false, "Agent", "Agent requests" } }; static void setting_changed(); static void ssl_verification_changed(); LLAppCoreHttp::HttpClass::HttpClass() : mPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID), mConnLimit(0U), mPipelined(false) {} LLAppCoreHttp::LLAppCoreHttp() : mRequest(NULL), mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), mStopped(false), mPipelined(true) {} LLAppCoreHttp::~LLAppCoreHttp() { delete mRequest; mRequest = NULL; } void LLAppCoreHttp::init() { LLCoreHttpUtil::setPropertyMethods( boost::bind(&LLControlGroup::getBOOL, boost::ref(gSavedSettings), _1), boost::bind(&LLControlGroup::declareBOOL, boost::ref(gSavedSettings), _1, _2, _3, LLControlVariable::PERSIST_NONDFT)); LLCore::LLHttp::initialize(); LLCore::HttpStatus status = LLCore::HttpRequest::createService(); if (! status) { LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " << status.toString() << LL_ENDL; } // Point to our certs or SSH/https: will fail on connect std::string ca_file = gDirUtilp->getCAFile(); if ( LLFile::isfile(ca_file) ) { LL_DEBUGS("Init") << "Setting CA File to " << ca_file << LL_ENDL; status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, LLCore::HttpRequest::GLOBAL_POLICY_ID, ca_file, NULL); } else { LLError::LLUserWarningMsg::showMissingFiles(); LL_ERRS("Init") << "Missing CA File; should be at " << ca_file << LL_ENDL; } if (! status) { LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " << status.toString() << LL_ENDL; } // Establish HTTP Proxy, if desired. status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_LLPROXY, LLCore::HttpRequest::GLOBAL_POLICY_ID, 1, NULL); if (! status) { LL_WARNS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() << LL_ENDL; } // Set up SSL Verification call back. status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_SSL_VERIFY_CALLBACK, LLCore::HttpRequest::GLOBAL_POLICY_ID, sslVerify, NULL); if (!status) { LL_WARNS("Init") << "Failed to set SSL Verification. Reason: " << status.toString() << LL_ENDL; } // Set up Default SSL Verification option. const std::string no_verify_ssl("NoVerifySSLCert"); if (gSavedSettings.controlExists(no_verify_ssl)) { LLPointer cntrl_ptr = gSavedSettings.getControl(no_verify_ssl); if (cntrl_ptr.isNull()) { LL_WARNS("Init") << "Unable to set signal on global setting '" << no_verify_ssl << "'" << LL_ENDL; } else { mSSLNoVerifySignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&ssl_verification_changed)); LLCore::HttpOptions::setDefaultSSLVerifyPeer(!cntrl_ptr->getValue().asBoolean()); } } // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): // 0 - None // 1 - Basic start, stop simple transitions // 2 - libcurl CURLOPT_VERBOSE mode with brief lines // 3 - with partial data content static const std::string http_trace("QAModeHttpTrace"); if (gSavedSettings.controlExists(http_trace)) { long trace_level(0L); trace_level = long(gSavedSettings.getU32(http_trace)); status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, LLCore::HttpRequest::GLOBAL_POLICY_ID, trace_level, NULL); } // Setup default policy and constrain if directed to mHttpClasses[AP_DEFAULT].mPolicy = LLCore::HttpRequest::DEFAULT_POLICY_ID; // Setup additional policies based on table and some special rules llassert(LL_ARRAY_SIZE(init_data) == AP_COUNT); for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { const EAppPolicy app_policy(static_cast(i)); if (AP_DEFAULT == app_policy) { // Pre-created continue; } mHttpClasses[app_policy].mPolicy = LLCore::HttpRequest::createPolicyClass(); // We have run out of available HTTP policies. Adjust HTTP_POLICY_CLASS_LIMIT in _httpinternal.h llassert(mHttpClasses[app_policy].mPolicy != LLCore::HttpRequest::INVALID_POLICY_ID); if (! mHttpClasses[app_policy].mPolicy) { // Use default policy (but don't accidentally modify default) LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage << ". Using default policy." << LL_ENDL; mHttpClasses[app_policy].mPolicy = mHttpClasses[AP_DEFAULT].mPolicy; continue; } } // Need a request object to handle dynamic options before setting them mRequest = new LLCore::HttpRequest; // Apply initial settings refreshSettings(true); // Kick the thread status = LLCore::HttpRequest::startThread(); if (! status) { LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " << status.toString() << LL_ENDL; } // Global pipelining setting static const std::string http_pipelining("HttpPipelining"); if (gSavedSettings.controlExists(http_pipelining)) { // Default to true (in ctor) if absent. mPipelined = gSavedSettings.getBOOL(http_pipelining); LL_INFOS("Init") << "HTTP Pipelining " << (mPipelined ? "enabled" : "disabled") << "!" << LL_ENDL; } // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { const EAppPolicy app_policy(static_cast(i)); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { LLPointer cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); if (cntrl_ptr.isNull()) { LL_WARNS("Init") << "Unable to set signal on global setting '" << init_data[i].mKey << "'" << LL_ENDL; } else { mHttpClasses[app_policy].mSettingsSignal = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); } } } } void setting_changed() { LLAppViewer::instance()->getAppCoreHttp().refreshSettings(false); } void ssl_verification_changed() { LLCore::HttpOptions::setDefaultSSLVerifyPeer(!gSavedSettings.getBOOL("NoVerifySSLCert")); } namespace { // The NoOpDeletor is used when wrapping LLAppCoreHttp in a smart pointer below for // passage into the LLCore::Http libararies. When the smart pointer is destroyed, // no action will be taken since we do not in this case want the entire LLAppCoreHttp object // to be destroyed at the end of the call. // // *NOTE$: Yes! It is "Deletor" // http://english.stackexchange.com/questions/4733/what-s-the-rule-for-adding-er-vs-or-when-nouning-a-verb // "delete" derives from Latin "deletus" void NoOpDeletor(LLCore::HttpHandler *) { /*NoOp*/ } } void LLAppCoreHttp::requestStop() { llassert_always(mRequest); mStopHandle = mRequest->requestStopThread(LLCore::HttpHandler::ptr_t(this, NoOpDeletor)); if (LLCORE_HTTP_HANDLE_INVALID != mStopHandle) { mStopRequested = LLTimer::getTotalSeconds(); } } void LLAppCoreHttp::cleanup() { LLCore::HTTPStats::instance().dumpStats(); if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) { // Should have been started already... requestStop(); } if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) { LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services without thread shutdown" << LL_ENDL; } else { while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) { mRequest->update(200000); ms_sleep(50); } if (! mStopped) { LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services with thread shutdown incomplete" << LL_ENDL; } } for (int i(0); i < LL_ARRAY_SIZE(mHttpClasses); ++i) { mHttpClasses[i].mSettingsSignal.disconnect(); } mSSLNoVerifySignal.disconnect(); mPipelinedSignal.disconnect(); delete mRequest; mRequest = NULL; LLCore::HttpStatus status = LLCore::HttpRequest::destroyService(); if (! status) { LL_WARNS("Cleanup") << "Failed to shutdown HTTP services, continuing. Reason: " << status.toString() << LL_ENDL; } } void LLAppCoreHttp::refreshSettings(bool initial) { LLCore::HttpStatus status; for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { const EAppPolicy app_policy(static_cast(i)); if (initial) { // Init-time only settings, can use the static setters here if (init_data[i].mRate) { // Set any desired throttle status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, mHttpClasses[app_policy].mPolicy, init_data[i].mRate, NULL); if (! status) { LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage << " throttle rate. Reason: " << status.toString() << LL_ENDL; } } } // Init- or run-time settings. Must use the queued request API. // Pipelining changes if (initial) { const bool to_pipeline(mPipelined && init_data[i].mPipelined); if (to_pipeline != mHttpClasses[app_policy].mPipelined) { // Pipeline election changing, set dynamic option via request LLCore::HttpHandle handle; const long new_depth(to_pipeline ? PIPELINING_DEPTH : 0); handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PIPELINING_DEPTH, mHttpClasses[app_policy].mPolicy, new_depth, LLCore::HttpHandler::ptr_t()); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage << " pipelining. Reason: " << status.toString() << LL_ENDL; } else { LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage << " pipelining. New value: " << new_depth << LL_ENDL; mHttpClasses[app_policy].mPipelined = to_pipeline; } } } // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); if (new_setting) { // Treat zero settings as an ask for default setting = llclamp(new_setting, init_data[i].mMin, init_data[i].mMax); } } if (initial || setting != mHttpClasses[app_policy].mConnLimit) { // Set it and report. Strategies depend on pipelining: // // No Pipelining. Llcorehttp manages connections itself based // on the PO_CONNECTION_LIMIT setting. Set both limits to the // same value for logical consistency. In the future, may // hand over connection management to libcurl after the // connection cache has been better vetted. // // Pipelining. Libcurl is allowed to manage connections to a // great degree. Steady state will connection limit based on // the per-host setting. Transitions (region crossings, new // avatars, etc.) can request additional outbound connections // to other servers via 2X total connection limit. // LLCore::HttpHandle handle; handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, mHttpClasses[app_policy].mPolicy, (mHttpClasses[app_policy].mPipelined ? 2 * setting : setting), LLCore::HttpHandler::ptr_t()); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage << " concurrency. Reason: " << status.toString() << LL_ENDL; } else { handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_PER_HOST_CONNECTION_LIMIT, mHttpClasses[app_policy].mPolicy, setting, LLCore::HttpHandler::ptr_t()); if (LLCORE_HTTP_HANDLE_INVALID == handle) { status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage << " per-host concurrency. Reason: " << status.toString() << LL_ENDL; } else { LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage << " concurrency. New value: " << setting << LL_ENDL; mHttpClasses[app_policy].mConnLimit = setting; if (initial && setting != init_data[i].mDefault) { LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage << " concurrency. New value: " << setting << LL_ENDL; } } } } } } LLCore::HttpStatus LLAppCoreHttp::sslVerify(const std::string &url, const LLCore::HttpHandler::ptr_t &handler, void *appdata) { if (gDisconnected) { return LLCore::HttpStatus(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_OPERATION_TIMEDOUT); } LLCore::HttpStatus result; try { X509_STORE_CTX *ctx = static_cast(appdata); LLPointer store = gSecAPIHandler->getCertificateStore(""); LLPointer chain = gSecAPIHandler->getCertificateChain(ctx); LLSD validation_params = LLSD::emptyMap(); LLURI uri(url); validation_params[CERT_HOSTNAME] = uri.hostName(); // *TODO: In the case of an exception while validating the cert, we need a way // to pass the offending(?) cert back out. *Rider* // don't validate hostname. Let libcurl do it instead. That way, it'll handle redirects store->validate(VALIDATION_POLICY_SSL & (~VALIDATION_POLICY_HOSTNAME), chain, validation_params); } catch (LLCertValidationTrustException &cert_exception) { // this exception is is handled differently than the general cert // exceptions, as we allow the user to actually add the certificate // for trust. // therefore we pass back a different error code // NOTE: We're currently 'wired' to pass around CURL error codes. This is // somewhat clumsy, as we may run into errors that do not map directly to curl // error codes. Should be refactored with login refactoring, perhaps. result = LLCore::HttpStatus(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_SSL_CACERT); result.setMessage(cert_exception.what()); LLSD certdata = cert_exception.getCertData(); result.setErrorData(certdata); // We should probably have a more generic way of passing information // back to the error handlers. } catch (LLCertException &cert_exception) { result = LLCore::HttpStatus(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_SSL_PEER_CERTIFICATE); result.setMessage(cert_exception.what()); LLSD certdata = cert_exception.getCertData(); result.setErrorData(certdata); } catch (...) { LOG_UNHANDLED_EXCEPTION(STRINGIZE("('" << url << "')")); // any other odd error, we just handle as a connect error. result = LLCore::HttpStatus(LLCore::HttpStatus::EXT_CURL_EASY, CURLE_SSL_CONNECT_ERROR); } return result; } void LLAppCoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) { mStopped = true; }