diff options
| -rwxr-xr-x | indra/llmessage/CMakeLists.txt | 2 | ||||
| -rwxr-xr-x | indra/llmessage/llassetstorage.cpp | 1 | ||||
| -rwxr-xr-x | indra/llmessage/llassetstorage.h | 1 | ||||
| -rwxr-xr-x | indra/llmessage/llhttpassetstorage.cpp | 1454 | ||||
| -rwxr-xr-x | indra/llmessage/llhttpassetstorage.h | 159 | ||||
| -rwxr-xr-x | indra/newview/CMakeLists.txt | 12 | ||||
| -rwxr-xr-x | indra/newview/llfloatermarketplacelistings.cpp | 2 | ||||
| -rwxr-xr-x | indra/newview/llfloateroutbox.cpp | 4 | ||||
| -rwxr-xr-x | indra/newview/llfloatertranslationsettings.cpp | 64 | ||||
| -rwxr-xr-x | indra/newview/llfloatertranslationsettings.h | 2 | ||||
| -rwxr-xr-x | indra/newview/llmarketplacefunctions.cpp | 42 | ||||
| -rwxr-xr-x | indra/newview/llmarketplacefunctions.h | 14 | ||||
| -rwxr-xr-x | indra/newview/lltranslate.cpp | 384 | ||||
| -rwxr-xr-x | indra/newview/lltranslate.h | 215 | ||||
| -rwxr-xr-x | indra/newview/llviewermessage.cpp | 64 |
15 files changed, 387 insertions, 2033 deletions
diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index 12fc1bbcfc..41b618f8c3 100755 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -48,7 +48,6 @@ set(llmessage_SOURCE_FILES llexperiencecache.cpp llfiltersd2xmlrpc.cpp llhost.cpp - llhttpassetstorage.cpp llhttpclient.cpp llhttpconstants.cpp llhttpnode.cpp @@ -141,7 +140,6 @@ set(llmessage_HEADER_FILES llfiltersd2xmlrpc.h llfollowcamparams.h llhost.h - llhttpassetstorage.h llhttpclient.h llhttpclientinterface.h llhttpconstants.h diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index 5740b8f7da..0d8aaf2269 100755 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -1609,3 +1609,4 @@ void LLAssetStorage::markAssetToxic( const LLUUID& uuid ) } } + diff --git a/indra/llmessage/llassetstorage.h b/indra/llmessage/llassetstorage.h index 1bb4acea9e..c3d31ba84c 100755 --- a/indra/llmessage/llassetstorage.h +++ b/indra/llmessage/llassetstorage.h @@ -27,7 +27,6 @@ #ifndef LL_LLASSETSTORAGE_H #define LL_LLASSETSTORAGE_H - #include <string> #include "lluuid.h" diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp deleted file mode 100755 index e202154445..0000000000 --- a/indra/llmessage/llhttpassetstorage.cpp +++ /dev/null @@ -1,1454 +0,0 @@ -/** - * @file llhttpassetstorage.cpp - * @brief Subclass capable of loading asset data to/from an external - * source. Currently, a web server accessed via curl - * - * $LicenseInfo:firstyear=2003&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#include "linden_common.h" - -#include "llhttpassetstorage.h" - -#include <sys/stat.h> - -#include "indra_constants.h" -#include "message.h" -#include "llproxy.h" -#include "llvfile.h" -#include "llvfs.h" -#include "llxfer.h" - -#ifdef LL_USESYSTEMLIBS -# include <zlib.h> -#else -# include "zlib/zlib.h" -#endif - -const char* const LOCAL_ASSET_URL_FORMAT = "http://%s:12041/asset"; - -const U32 MAX_RUNNING_REQUESTS = 1; - -// Try for 30 minutes for now. -const F32 GET_URL_TO_FILE_TIMEOUT = 1800.0f; - -const S32 COMPRESSED_INPUT_BUFFER_SIZE = 4096; - -///////////////////////////////////////////////////////////////////////////////// -// LLTempAssetData -// An asset not stored on central asset store, but on a simulator node somewhere. -///////////////////////////////////////////////////////////////////////////////// -struct LLTempAssetData -{ - LLUUID mAssetID; - LLUUID mAgentID; - std::string mHostName; -}; - -///////////////////////////////////////////////////////////////////////////////// -// LLHTTPAssetRequest -///////////////////////////////////////////////////////////////////////////////// - -class LLHTTPAssetRequest : public LLAssetRequest -{ -public: - LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, - LLAssetType::EType type, LLAssetStorage::ERequestType rt, - const std::string& url, CURLM *curl_multi); - virtual ~LLHTTPAssetRequest(); - - void setupCurlHandle(); - void cleanupCurlHandle(); - - void prepareCompressedUpload(); - void finishCompressedUpload(); - size_t readCompressedData(void* data, size_t size); - - static size_t curlCompressedUploadCallback( - void *data, size_t size, size_t nmemb, void *user_data); - - virtual LLSD getTerseDetails() const; - virtual LLSD getFullDetails() const; - -public: - LLHTTPAssetStorage *mAssetStoragep; - - CURL *mCurlHandle; - CURLM *mCurlMultiHandle; - std::string mURLBuffer; - struct curl_slist *mHTTPHeaders; - LLVFile *mVFile; - LLUUID mTmpUUID; - LLAssetStorage::ERequestType mRequestType; - - bool mZInitialized; - z_stream mZStream; - char* mZInputBuffer; - bool mZInputExhausted; - - FILE *mFP; -}; - - -LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, - const LLUUID &uuid, - LLAssetType::EType type, - LLAssetStorage::ERequestType rt, - const std::string& url, - CURLM *curl_multi) - : LLAssetRequest(uuid, type), - mZInitialized(false) -{ - memset(&mZStream, 0, sizeof(mZStream)); // we'll initialize this later, but for now zero the whole C-style struct to avoid debug/coverity noise - mAssetStoragep = asp; - mCurlHandle = NULL; - mCurlMultiHandle = curl_multi; - mVFile = NULL; - mRequestType = rt; - mHTTPHeaders = NULL; - mFP = NULL; - mZInputBuffer = NULL; - mZInputExhausted = false; - - mURLBuffer = url; -} - -LLHTTPAssetRequest::~LLHTTPAssetRequest() -{ - // Cleanup/cancel the request - if (mCurlHandle) - { - curl_multi_remove_handle(mCurlMultiHandle, mCurlHandle); - cleanupCurlHandle(); - } - if (mHTTPHeaders) - { - curl_slist_free_all(mHTTPHeaders); - } - delete mVFile; - finishCompressedUpload(); -} - -// virtual -LLSD LLHTTPAssetRequest::getTerseDetails() const -{ - LLSD sd = LLAssetRequest::getTerseDetails(); - - sd["url"] = mURLBuffer; - - return sd; -} - -// virtual -LLSD LLHTTPAssetRequest::getFullDetails() const -{ - LLSD sd = LLAssetRequest::getFullDetails(); - - if (mCurlHandle) - { - long curl_response = -1; - long curl_connect = -1; - double curl_total_time = -1.0f; - double curl_size_upload = -1.0f; - double curl_size_download = -1.0f; - double curl_content_length_upload = -1.0f; - double curl_content_length_download = -1.0f; - long curl_request_size = -1; - const char* curl_content_type = NULL; - - curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CODE, &curl_response); - curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CONNECTCODE, &curl_connect); - curl_easy_getinfo(mCurlHandle, CURLINFO_TOTAL_TIME, &curl_total_time); - curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_UPLOAD, &curl_size_upload); - curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_DOWNLOAD, &curl_size_download); - curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_UPLOAD, &curl_content_length_upload); - curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &curl_content_length_download); - curl_easy_getinfo(mCurlHandle, CURLINFO_REQUEST_SIZE, &curl_request_size); - curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_TYPE, &curl_content_type); - - sd["curl_response_code"] = (int) curl_response; - sd["curl_http_connect_code"] = (int) curl_connect; - sd["curl_total_time"] = curl_total_time; - sd["curl_size_upload"] = curl_size_upload; - sd["curl_size_download"] = curl_size_download; - sd["curl_content_length_upload"] = curl_content_length_upload; - sd["curl_content_length_download"] = curl_content_length_download; - sd["curl_request_size"] = (int) curl_request_size; - if (curl_content_type) - { - sd["curl_content_type"] = curl_content_type; - } - else - { - sd["curl_content_type"] = ""; - } - } - - sd["temp_id"] = mTmpUUID; - sd["request_type"] = LLAssetStorage::getRequestName(mRequestType); - sd["z_initialized"] = mZInitialized; - sd["z_input_exhausted"] = mZInputExhausted; - - S32 file_size = -1; - if (mFP) - { - struct stat file_stat; - int file_desc = fileno(mFP); - if ( fstat(file_desc, &file_stat) == 0) - { - file_size = file_stat.st_size; - } - } - sd["file_size"] = file_size; - - return sd; -} - - -void LLHTTPAssetRequest::setupCurlHandle() -{ - // *NOTE: Similar code exists in mapserver/llcurlutil.cpp JC - mCurlHandle = LLCurl::newEasyHandle(); - llassert_always(mCurlHandle != NULL) ; - - // Apply proxy settings if configured to do so - LLProxy::getInstance()->applyProxySettings(mCurlHandle); - - curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer.c_str()); - curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); - if (LLAssetStorage::RT_DOWNLOAD == mRequestType) - { - curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); - // only do this on downloads, as uploads - // to some apache configs (like our test grids) - // mistakenly claim the response is gzip'd if the resource - // name ends in .gz, even though in a PUT, the response is - // just plain HTML saying "created" - } - /* Remove the Pragma: no-cache header that libcurl inserts by default; - we want the cached version, if possible. */ - if (mZInitialized) - { - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, ""); - // disable use of proxy, which can't handle chunked transfers - } - mHTTPHeaders = curl_slist_append(mHTTPHeaders, "Pragma:"); - - // bug in curl causes DNS to be cached for too long a time, 0 sets it to never cache DNS results internally (to curl) - curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); - - // resist the temptation to explicitly add the Transfer-Encoding: chunked - // header here - invokes a libCURL bug - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mHTTPHeaders); - if (mAssetStoragep) - { - // Set the appropriate pending upload or download flag - mAssetStoragep->addRunningRequest(mRequestType, this); - } - else - { - LL_ERRS() << "LLHTTPAssetRequest::setupCurlHandle - No asset storage associated with this request!" << LL_ENDL; - } -} - -void LLHTTPAssetRequest::cleanupCurlHandle() -{ - LLCurl::deleteEasyHandle(mCurlHandle); - if (mAssetStoragep) - { - // Terminating a request. Thus upload or download is no longer pending. - mAssetStoragep->removeRunningRequest(mRequestType, this); - } - else - { - LL_ERRS() << "LLHTTPAssetRequest::~LLHTTPAssetRequest - No asset storage associated with this request!" << LL_ENDL; - } - mCurlHandle = NULL; -} - -void LLHTTPAssetRequest::prepareCompressedUpload() -{ - mZStream.next_in = Z_NULL; - mZStream.avail_in = 0; - mZStream.zalloc = Z_NULL; - mZStream.zfree = Z_NULL; - mZStream.opaque = Z_NULL; - - int r = deflateInit2(&mZStream, - 1, // compression level - Z_DEFLATED, // the only method defined - 15 + 16, // the default windowBits + gzip header flag - 8, // the default memLevel - Z_DEFAULT_STRATEGY); - - if (r != Z_OK) - { - LL_ERRS() << "LLHTTPAssetRequest::prepareCompressedUpload defalateInit2() failed" << LL_ENDL; - } - - mZInitialized = true; - mZInputBuffer = new char[COMPRESSED_INPUT_BUFFER_SIZE]; - mZInputExhausted = false; - - mVFile = new LLVFile(gAssetStorage->mVFS, - getUUID(), getType(), LLVFile::READ); -} - -void LLHTTPAssetRequest::finishCompressedUpload() -{ - if (mZInitialized) - { - LL_INFOS() << "LLHTTPAssetRequest::finishCompressedUpload: " - << "read " << mZStream.total_in << " byte asset file, " - << "uploaded " << mZStream.total_out << " byte compressed asset" - << LL_ENDL; - - deflateEnd(&mZStream); - delete[] mZInputBuffer; - } -} - -size_t LLHTTPAssetRequest::readCompressedData(void* data, size_t size) -{ - llassert(mZInitialized); - - mZStream.next_out = (Bytef*)data; - mZStream.avail_out = size; - - while (mZStream.avail_out > 0) - { - if (mZStream.avail_in == 0 && !mZInputExhausted) - { - S32 to_read = llmin(COMPRESSED_INPUT_BUFFER_SIZE, - (S32)(mVFile->getSize() - mVFile->tell())); - - if ( to_read > 0 ) - { - mVFile->read((U8*)mZInputBuffer, to_read); /*Flawfinder: ignore*/ - mZStream.next_in = (Bytef*)mZInputBuffer; - mZStream.avail_in = mVFile->getLastBytesRead(); - } - - mZInputExhausted = mZStream.avail_in == 0; - } - - int r = deflate(&mZStream, - mZInputExhausted ? Z_FINISH : Z_NO_FLUSH); - - if (r == Z_STREAM_END || r < 0 || mZInputExhausted) - { - if (r < 0) - { - LL_WARNS() << "LLHTTPAssetRequest::readCompressedData: deflate returned error code " - << (S32) r << LL_ENDL; - } - break; - } - } - - return size - mZStream.avail_out; -} - -//static -size_t LLHTTPAssetRequest::curlCompressedUploadCallback( - void *data, size_t size, size_t nmemb, void *user_data) -{ - size_t num_read = 0; - - if (gAssetStorage) - { - CURL *curl_handle = (CURL *)user_data; - LLHTTPAssetRequest *req = NULL; - curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req); - if (req) - { - num_read = req->readCompressedData(data, size * nmemb); - } - } - - return num_read; -} - -///////////////////////////////////////////////////////////////////////////////// -// LLHTTPAssetStorage -///////////////////////////////////////////////////////////////////////////////// - - -LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, - LLVFS *vfs, LLVFS *static_vfs, - const LLHost &upstream_host, - const std::string& web_host, - const std::string& local_web_host, - const std::string& host_name) - : LLAssetStorage(msg, xfer, vfs, static_vfs, upstream_host) -{ - _init(web_host, local_web_host, host_name); -} - -LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, - LLVFS *vfs, - LLVFS *static_vfs, - const std::string& web_host, - const std::string& local_web_host, - const std::string& host_name) - : LLAssetStorage(msg, xfer, vfs, static_vfs) -{ - _init(web_host, local_web_host, host_name); -} - -void LLHTTPAssetStorage::_init(const std::string& web_host, const std::string& local_web_host, const std::string& host_name) -{ - mBaseURL = web_host; - mLocalBaseURL = local_web_host; - mHostName = host_name; - - // curl_global_init moved to LLCurl::initClass() - - mCurlMultiHandle = LLCurl::newMultiHandle() ; - llassert_always(mCurlMultiHandle != NULL) ; -} - -LLHTTPAssetStorage::~LLHTTPAssetStorage() -{ - LLCurl::deleteMultiHandle(mCurlMultiHandle); - mCurlMultiHandle = NULL; - - // curl_global_cleanup moved to LLCurl::initClass() -} - -// storing data is simpler than getting it, so we just overload the whole method -void LLHTTPAssetStorage::storeAssetData( - const LLUUID& uuid, - LLAssetType::EType type, - LLAssetStorage::LLStoreAssetCallback callback, - void* user_data, - bool temp_file, - bool is_priority, - bool store_local, - const LLUUID& requesting_agent_id, - bool user_waiting, - F64Seconds timeout) -{ - if (mVFS->getExists(uuid, type)) // VFS treats nonexistant and zero-length identically - { - LLAssetRequest *req = new LLAssetRequest(uuid, type); - req->mUpCallback = callback; - req->mUserData = user_data; - req->mRequestingAgentID = requesting_agent_id; - req->mIsUserWaiting = user_waiting; - req->mTimeout = timeout; - - // LLAssetStorage metric: Successful Request - S32 size = mVFS->getSize(uuid, type); - const char *message; - if( store_local ) - { - message = "Added to local upload queue"; - } - else - { - message = "Added to upload queue"; - } - reportMetric( uuid, type, LLStringUtil::null, requesting_agent_id, size, MR_OKAY, __FILE__, __LINE__, message ); - - // this will get picked up and transmitted in checkForTimeouts - if(store_local) - { - mPendingLocalUploads.push_back(req); - } - else if(is_priority) - { - mPendingUploads.push_front(req); - } - else - { - mPendingUploads.push_back(req); - } - } - else - { - LL_WARNS() << "AssetStorage: attempt to upload non-existent vfile " << uuid << ":" << LLAssetType::lookup(type) << LL_ENDL; - if (callback) - { - // LLAssetStorage metric: Zero size VFS - reportMetric( uuid, type, LLStringUtil::null, requesting_agent_id, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file didn't exist or was zero length (VFS - can't tell which)" ); - callback(uuid, user_data, LL_ERR_ASSET_REQUEST_NONEXISTENT_FILE, LL_EXSTAT_NONEXISTENT_FILE); - } - } -} - -// virtual -void LLHTTPAssetStorage::storeAssetData( - const std::string& filename, - const LLUUID& asset_id, - LLAssetType::EType asset_type, - LLStoreAssetCallback callback, - void* user_data, - bool temp_file, - bool is_priority, - bool user_waiting, - F64Seconds timeout) -{ - LL_INFOS() << "LLAssetStorage::storeAssetData (legacy)" << asset_id << ":" << LLAssetType::lookup(asset_type) << LL_ENDL; - - LLLegacyAssetRequest *legacy = new LLLegacyAssetRequest; - - legacy->mUpCallback = callback; - legacy->mUserData = user_data; - - FILE *fp = LLFile::fopen(filename, "rb"); /*Flawfinder: ignore*/ - S32 size = 0; - if (fp) - { - fseek(fp, 0, SEEK_END); - size = ftell(fp); - fseek(fp, 0, SEEK_SET); - } - - if( size ) - { - LLVFile file(mVFS, asset_id, asset_type, LLVFile::WRITE); - - file.setMaxSize(size); - - const S32 buf_size = 65536; - U8 copy_buf[buf_size]; - while ((size = (S32)fread(copy_buf, 1, buf_size, fp))) - { - file.write(copy_buf, size); - } - fclose(fp); - - // if this upload fails, the caller needs to setup a new tempfile for us - if (temp_file) - { - LLFile::remove(filename); - } - - // LLAssetStorage metric: Success not needed; handled in the overloaded method here: - storeAssetData( - asset_id, - asset_type, - legacyStoreDataCallback, - (void**)legacy, - temp_file, - is_priority, - false, - LLUUID::null, - user_waiting, - timeout); - } - else // !size - { - if( fp ) - { - // LLAssetStorage metric: Zero size - reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_ZERO_SIZE, __FILE__, __LINE__, "The file was zero length" ); - fclose( fp ); - } - else - { - // LLAssetStorage metric: Missing File - reportMetric( asset_id, asset_type, filename, LLUUID::null, 0, MR_FILE_NONEXIST, __FILE__, __LINE__, "The file didn't exist" ); - } - if (callback) - { - callback(LLUUID::null, user_data, LL_ERR_CANNOT_OPEN_FILE, LL_EXSTAT_BLOCKED_FILE); - } - delete legacy; - } -} - -// virtual -LLSD LLHTTPAssetStorage::getPendingDetails(LLAssetStorage::ERequestType rt, - LLAssetType::EType asset_type, - const std::string& detail_prefix) const -{ - LLSD sd = LLAssetStorage::getPendingDetails(rt, asset_type, detail_prefix); - const request_list_t* running = getRunningList(rt); - if (running) - { - // Loop through the pending requests sd, and add extra info about its running status. - S32 num_pending = sd["requests"].size(); - S32 i; - for (i = 0; i < num_pending; ++i) - { - LLSD& pending = sd["requests"][i]; - // See if this pending request is running. - const LLAssetRequest* req = findRequest(running, - LLAssetType::lookup(pending["type"].asString()), - pending["asset_id"]); - if (req) - { - // Keep the detail_url so we don't have to rebuild it. - LLURI detail_url = pending["detail"]; - pending = req->getTerseDetails(); - pending["detail"] = detail_url; - pending["is_running"] = true; - } - else - { - pending["is_running"] = false; - } - } - } - return sd; -} - -// virtual -LLSD LLHTTPAssetStorage::getPendingRequest(LLAssetStorage::ERequestType rt, - LLAssetType::EType asset_type, - const LLUUID& asset_id) const -{ - // Look for this asset in the running list first. - const request_list_t* running = getRunningList(rt); - if (running) - { - LLSD sd = LLAssetStorage::getPendingRequestImpl(running, asset_type, asset_id); - if (sd) - { - sd["is_running"] = true; - return sd; - } - } - LLSD sd = LLAssetStorage::getPendingRequest(rt, asset_type, asset_id); - if (sd) - { - sd["is_running"] = false; - } - return sd; -} - -// virtual -bool LLHTTPAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt, - LLAssetType::EType asset_type, - const LLUUID& asset_id) -{ - // Try removing this from the running list first. - request_list_t* running = getRunningList(rt); - if (running) - { - LLAssetRequest* req = findRequest(running, asset_type, asset_id); - if (req) - { - // Remove this request from the running list to get it out of curl. - running->remove(req); - - // Find this request in the pending list, so we can move it to the end of the line. - request_list_t* pending = getRequestList(rt); - if (pending) - { - request_list_t::iterator result = std::find_if(pending->begin(), pending->end(), - std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req)); - if (pending->end() != result) - { - // This request was found in the pending list. Move it to the end! - LLAssetRequest* pending_req = *result; - pending->remove(pending_req); - - if (!pending_req->mIsUserWaiting) //A user is waiting on this request. Toss it. - { - pending->push_back(pending_req); - } - else - { - if (pending_req->mUpCallback) //Clean up here rather than _callUploadCallbacks because this request is already cleared the req. - { - pending_req->mUpCallback(pending_req->getUUID(), pending_req->mUserData, -1, LL_EXSTAT_REQUEST_DROPPED); - } - - } - - LL_INFOS() << "Asset " << getRequestName(rt) << " request for " - << asset_id << "." << LLAssetType::lookup(asset_type) - << " removed from curl and placed at the end of the pending queue." - << LL_ENDL; - } - else - { - LL_WARNS() << "Unable to find pending " << getRequestName(rt) << " request for " - << asset_id << "." << LLAssetType::lookup(asset_type) << LL_ENDL; - } - } - delete req; - - return true; - } - } - return LLAssetStorage::deletePendingRequest(rt, asset_type, asset_id); -} - -// internal requester, used by getAssetData in superclass -void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType type, - void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32, LLExtStat), - void *user_data, BOOL duplicate, - BOOL is_priority) -{ - // stash the callback info so we can find it after we get the response message - LLAssetRequest *req = new LLAssetRequest(uuid, type); - req->mDownCallback = callback; - req->mUserData = user_data; - req->mIsPriority = is_priority; - - // this will get picked up and downloaded in checkForTimeouts - - // - // HAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACK! Asset requests were taking too long and timing out. - // Since texture requests are the LEAST sensitive (on the simulator) to being delayed, add - // non-texture requests to the front, and add texture requests to the back. The theory is - // that we always want them first, even if they're out of order. - // - - if (req->getType() == LLAssetType::AT_TEXTURE) - { - mPendingDownloads.push_back(req); - } - else - { - mPendingDownloads.push_front(req); - } -} - -LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list_t& pending, - LLAssetStorage::request_list_t& running) -{ - // Early exit if the running list is full, or we don't have more pending than running. - if (running.size() >= MAX_RUNNING_REQUESTS - || pending.size() <= running.size()) return NULL; - - // Look for the first pending request that is not already running. - request_list_t::iterator running_begin = running.begin(); - request_list_t::iterator running_end = running.end(); - - request_list_t::iterator pending_iter = pending.begin(); - - // Loop over all pending requests until we miss finding it in the running list. - for (; pending_iter != pending.end(); ++pending_iter) - { - LLAssetRequest* req = *pending_iter; - // Look for this pending request in the running list. - if (running_end == std::find_if(running_begin, running_end, - std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req))) - { - // It isn't running! Return it. - return req; - } - } - return NULL; -} - -// overloaded to additionally move data to/from the webserver -void LLHTTPAssetStorage::checkForTimeouts() -{ - CURLMcode mcode; - LLAssetRequest *req; - while ( (req = findNextRequest(mPendingDownloads, mRunningDownloads)) ) - { - // Setup this curl download request - // We need to generate a new request here - // since the one in the list could go away - std::string tmp_url; - std::string uuid_str; - req->getUUID().toString(uuid_str); - std::string base_url = getBaseURL(req->getUUID(), req->getType()); - tmp_url = llformat("%s/%36s.%s", base_url.c_str() , uuid_str.c_str(), LLAssetType::lookup(req->getType())); - - LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), - req->getType(), RT_DOWNLOAD, tmp_url, mCurlMultiHandle); - new_req->mTmpUUID.generate(); - - // Sets pending download flag internally - new_req->setupCurlHandle(); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &curlDownCallback); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEDATA, new_req->mCurlHandle); - - mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); - if (mcode > CURLM_OK) - { - // Failure. Deleting the pending request will remove it from the running - // queue, and push it to the end of the pending queue. - new_req->cleanupCurlHandle(); - deletePendingRequest(RT_DOWNLOAD, new_req->getType(), new_req->getUUID()); - break; - } - else - { - LL_INFOS() << "Requesting " << new_req->mURLBuffer << LL_ENDL; - } - } - - while ( (req = findNextRequest(mPendingUploads, mRunningUploads)) ) - { - // setup this curl upload request - - bool do_compress = req->getType() == LLAssetType::AT_OBJECT; - - std::string tmp_url; - std::string uuid_str; - req->getUUID().toString(uuid_str); - tmp_url = mBaseURL + "/" + uuid_str + "." + LLAssetType::lookup(req->getType()); - if (do_compress) tmp_url += ".gz"; - - LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), - req->getType(), RT_UPLOAD, tmp_url, mCurlMultiHandle); - - if (req->mIsUserWaiting) //If a user is waiting on a realtime response, we want to perserve information across upload attempts. - { - new_req->mTime = req->mTime; - new_req->mTimeout = req->mTimeout; - new_req->mIsUserWaiting = req->mIsUserWaiting; - } - - if (do_compress) - { - new_req->prepareCompressedUpload(); - } - - // Sets pending upload flag internally - new_req->setupCurlHandle(); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_UPLOAD, 1); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback); - - if (do_compress) - { - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, - &LLHTTPAssetRequest::curlCompressedUploadCallback); - } - else - { - LLVFile file(mVFS, req->getUUID(), req->getType()); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize()); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, - &curlUpCallback); - } - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); - - mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); - if (mcode > CURLM_OK) - { - // Failure. Deleting the pending request will remove it from the running - // queue, and push it to the end of the pending queue. - new_req->cleanupCurlHandle(); - deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID()); - break; - } - else - { - // Get the uncompressed file size. - LLVFile file(mVFS,new_req->getUUID(),new_req->getType()); - S32 size = file.getSize(); - LL_INFOS() << "Requesting PUT " << new_req->mURLBuffer << ", asset size: " << size << " bytes" << LL_ENDL; - if (size == 0) - { - LL_WARNS() << "Rejecting zero size PUT request!" << LL_ENDL; - new_req->cleanupCurlHandle(); - deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID()); - } - } - // Pending upload will have been flagged by the request - } - - while ( (req = findNextRequest(mPendingLocalUploads, mRunningLocalUploads)) ) - { - // setup this curl upload request - LLVFile file(mVFS, req->getUUID(), req->getType()); - - std::string tmp_url; - std::string uuid_str; - req->getUUID().toString(uuid_str); - - // KLW - All temporary uploads are saved locally "http://localhost:12041/asset" - tmp_url = llformat("%s/%36s.%s", mLocalBaseURL.c_str(), uuid_str.c_str(), LLAssetType::lookup(req->getType())); - - LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), - req->getType(), RT_LOCALUPLOAD, tmp_url, mCurlMultiHandle); - new_req->mRequestingAgentID = req->mRequestingAgentID; - - // Sets pending upload flag internally - new_req->setupCurlHandle(); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_PUT, 1); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize()); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, &curlUpCallback); - curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle); - - mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle); - if (mcode > CURLM_OK) - { - // Failure. Deleting the pending request will remove it from the running - // queue, and push it to the end of the pending queue. - new_req->cleanupCurlHandle(); - deletePendingRequest(RT_LOCALUPLOAD, new_req->getType(), new_req->getUUID()); - break; - } - else - { - // Get the uncompressed file size. - S32 size = file.getSize(); - - LL_INFOS() << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!" - << " Requesting PUT " << new_req->mURLBuffer << ", asset size: " << size << " bytes" << LL_ENDL; - if (size == 0) - { - - LL_WARNS() << "Rejecting zero size PUT request!" << LL_ENDL; - new_req->cleanupCurlHandle(); - deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID()); - } - - } - // Pending upload will have been flagged by the request - } - S32 count = 0; - int queue_length; - do - { - mcode = curl_multi_perform(mCurlMultiHandle, &queue_length); - count++; - } while (mcode == CURLM_CALL_MULTI_PERFORM && (count < 5)); - - CURLMsg *curl_msg; - do - { - curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length); - if (curl_msg && curl_msg->msg == CURLMSG_DONE) - { - long curl_result = 0; - S32 xfer_result = LL_ERR_NOERR; - - LLHTTPAssetRequest *req = NULL; - curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &req); - - // TODO: Throw curl_result at all callbacks. - curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result); - if (RT_UPLOAD == req->mRequestType || RT_LOCALUPLOAD == req->mRequestType) - { - if (curl_msg->data.result == CURLE_OK && - ( curl_result == HTTP_OK - || curl_result == HTTP_CREATED - || curl_result == HTTP_NO_CONTENT)) - { - LL_INFOS() << "Success uploading " << req->getUUID() << " to " << req->mURLBuffer << LL_ENDL; - if (RT_LOCALUPLOAD == req->mRequestType) - { - addTempAssetData(req->getUUID(), req->mRequestingAgentID, mHostName); - } - } - else if (curl_msg->data.result == CURLE_COULDNT_CONNECT || - curl_msg->data.result == CURLE_OPERATION_TIMEOUTED || - curl_result == HTTP_BAD_GATEWAY || - curl_result == HTTP_SERVICE_UNAVAILABLE) - { - LL_WARNS() << "Re-requesting upload for " << req->getUUID() << ". Received upload error to " << req->mURLBuffer << - " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << LL_ENDL; - - ////HACK (probably) I am sick of this getting requeued and driving me mad. - //if (req->mIsUserWaiting) - //{ - // deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID()); - //} - } - else - { - LL_WARNS() << "Failure uploading " << req->getUUID() << " to " << req->mURLBuffer << - " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << LL_ENDL; - - xfer_result = LL_ERR_ASSET_REQUEST_FAILED; - } - - if (!(curl_msg->data.result == CURLE_COULDNT_CONNECT || - curl_msg->data.result == CURLE_OPERATION_TIMEOUTED || - curl_result == HTTP_BAD_GATEWAY || - curl_result == HTTP_SERVICE_UNAVAILABLE)) - { - // shared upload finished callback - // in the base class, this is called from processUploadComplete - _callUploadCallbacks(req->getUUID(), req->getType(), (xfer_result == 0), LL_EXSTAT_CURL_RESULT | curl_result); - // Pending upload flag will get cleared when the request is deleted - } - } - else if (RT_DOWNLOAD == req->mRequestType) - { - if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK) - { - if (req->mVFile && req->mVFile->getSize() > 0) - { - LL_INFOS() << "Success downloading " << req->mURLBuffer << ", size " << req->mVFile->getSize() << LL_ENDL; - - req->mVFile->rename(req->getUUID(), req->getType()); - } - else - { - // *TODO: if this actually indicates a bad asset on the server - // (not certain at this point), then delete it - LL_WARNS() << "Found " << req->mURLBuffer << " to be zero size" << LL_ENDL; - xfer_result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE; - } - } - else - { - // KLW - TAT See if an avatar owns this texture, and if so request re-upload. - LL_WARNS() << "Failure downloading " << req->mURLBuffer << - " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << LL_ENDL; - - xfer_result = (curl_result == HTTP_NOT_FOUND) ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED; - - if (req->mVFile) - { - req->mVFile->remove(); - } - } - - // call the static callback for transfer completion - // this will cleanup all requests for this asset, including ours - downloadCompleteCallback( - xfer_result, - req->getUUID(), - req->getType(), - (void *)req, - LL_EXSTAT_CURL_RESULT | curl_result); - // Pending download flag will get cleared when the request is deleted - } - else - { - // nothing, just axe this request - // currently this can only mean an asset delete - } - - // Deleting clears the pending upload/download flag if it's set and the request is transferring - delete req; - req = NULL; - } - - } while (curl_msg && queue_length > 0); - - - // Cleanup - // We want to bump to the back of the line any running uploads that have timed out. - bumpTimedOutUploads(); - - LLAssetStorage::checkForTimeouts(); -} - -void LLHTTPAssetStorage::bumpTimedOutUploads() -{ - bool user_waiting=FALSE; - - F64Seconds mt_secs = LLMessageSystem::getMessageTimeSeconds(); - - if (mPendingUploads.size()) - { - request_list_t::iterator it = mPendingUploads.begin(); - LLAssetRequest* req = *it; - user_waiting=req->mIsUserWaiting; - } - - // No point bumping currently running uploads if there are no others in line. - if (!(mPendingUploads.size() > mRunningUploads.size()) && !user_waiting) - { - return; - } - - // deletePendingRequest will modify the mRunningUploads list so we don't want to iterate over it. - request_list_t temp_running = mRunningUploads; - - request_list_t::iterator it = temp_running.begin(); - request_list_t::iterator end = temp_running.end(); - for ( ; it != end; ++it) - { - //request_list_t::iterator curiter = iter++; - LLAssetRequest* req = *it; - - if ( req->mTimeout < (mt_secs - req->mTime) ) - { - LL_WARNS() << "Asset upload request timed out for " - << req->getUUID() << "." - << LLAssetType::lookup(req->getType()) - << ", bumping to the back of the line!" << LL_ENDL; - - deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID()); - } - } -} - -// static -size_t LLHTTPAssetStorage::curlDownCallback(void *data, size_t size, size_t nmemb, void *user_data) -{ - if (!gAssetStorage) - { - LL_WARNS() << "Missing gAssetStorage, aborting curl download callback!" << LL_ENDL; - return 0; - } - S32 bytes = (S32)(size * nmemb); - CURL *curl_handle = (CURL *)user_data; - LLHTTPAssetRequest *req = NULL; - curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req); - - if (! req->mVFile) - { - req->mVFile = new LLVFile(gAssetStorage->mVFS, req->mTmpUUID, LLAssetType::AT_NONE, LLVFile::APPEND); - } - - double content_length = 0.0; - curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &content_length); - - // sanitize content_length, reconcile w/ actual data - S32 file_length = llmax(0, (S32)llmin(content_length, 20000000.0), bytes + req->mVFile->getSize()); - - req->mVFile->setMaxSize(file_length); - req->mVFile->write((U8*)data, bytes); - - return nmemb; -} - -// static -size_t LLHTTPAssetStorage::curlUpCallback(void *data, size_t size, size_t nmemb, void *user_data) -{ - if (!gAssetStorage) - { - LL_WARNS() << "Missing gAssetStorage, aborting curl download callback!" << LL_ENDL; - return 0; - } - CURL *curl_handle = (CURL *)user_data; - LLHTTPAssetRequest *req = NULL; - curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req); - - if (! req->mVFile) - { - req->mVFile = new LLVFile(gAssetStorage->mVFS, req->getUUID(), req->getType(), LLVFile::READ); - } - - S32 bytes = llmin((S32)(size * nmemb), (S32)(req->mVFile->getSize() - req->mVFile->tell())); - - req->mVFile->read((U8*)data, bytes);/*Flawfinder: ignore*/ - - return req->mVFile->getLastBytesRead(); -} - -// static -size_t LLHTTPAssetStorage::nullOutputCallback(void *data, size_t size, size_t nmemb, void *user_data) -{ - // do nothing, this is here to soak up script output so it doesn't end up on stdout - - return nmemb; -} - - - -// blocking asset fetch which bypasses the VFS -// this is a very limited function for use by the simstate loader and other one-offs -S32 LLHTTPAssetStorage::getURLToFile(const LLUUID& uuid, LLAssetType::EType asset_type, const std::string &url, const std::string& filename, progress_callback callback, void *userdata) -{ - // *NOTE: There is no guarantee that the uuid and the asset_type match - // - not that it matters. - Doug - LL_DEBUGS() << "LLHTTPAssetStorage::getURLToFile() - " << url << LL_ENDL; - - FILE *fp = LLFile::fopen(filename, "wb"); /*Flawfinder: ignore*/ - if (! fp) - { - LL_WARNS() << "Failed to open " << filename << " for writing" << LL_ENDL; - return LL_ERR_ASSET_REQUEST_FAILED; - } - - // make sure we use the normal curl setup, even though we don't really need a request object - LLHTTPAssetRequest req(this, uuid, asset_type, RT_DOWNLOAD, url, mCurlMultiHandle); - req.mFP = fp; - - req.setupCurlHandle(); - curl_easy_setopt(req.mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE); - curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEFUNCTION, &curlFileDownCallback); - curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEDATA, req.mCurlHandle); - - curl_multi_add_handle(mCurlMultiHandle, req.mCurlHandle); - LL_INFOS() << "Requesting as file " << req.mURLBuffer << LL_ENDL; - - // braindead curl loop - int queue_length; - CURLMsg *curl_msg; - LLTimer timeout; - timeout.setTimerExpirySec(GET_URL_TO_FILE_TIMEOUT); - bool success = false; - S32 xfer_result = 0; - do - { - curl_multi_perform(mCurlMultiHandle, &queue_length); - curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length); - - if (callback) - { - callback(userdata); - } - - if ( curl_msg && (CURLMSG_DONE == curl_msg->msg) ) - { - success = true; - } - else if (timeout.hasExpired()) - { - LL_WARNS() << "Request for " << url << " has timed out." << LL_ENDL; - success = false; - xfer_result = LL_ERR_ASSET_REQUEST_FAILED; - break; - } - } while (!success); - - if (success) - { - long curl_result = 0; - curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result); - - if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK) - { - S32 size = ftell(req.mFP); - if (size > 0) - { - // everything seems to be in order - LL_INFOS() << "Success downloading " << req.mURLBuffer << " to file, size " << size << LL_ENDL; - } - else - { - LL_WARNS() << "Found " << req.mURLBuffer << " to be zero size" << LL_ENDL; - xfer_result = LL_ERR_ASSET_REQUEST_FAILED; - } - } - else - { - xfer_result = curl_result == HTTP_NOT_FOUND ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED; - LL_INFOS() << "Failure downloading " << req.mURLBuffer << - " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << LL_ENDL; - } - } - - fclose(fp); - if (xfer_result) - { - LLFile::remove(filename); - } - return xfer_result; -} - - -// static -size_t LLHTTPAssetStorage::curlFileDownCallback(void *data, size_t size, size_t nmemb, void *user_data) -{ - CURL *curl_handle = (CURL *)user_data; - LLHTTPAssetRequest *req = NULL; - curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req); - - if (! req->mFP) - { - LL_WARNS() << "Missing mFP, aborting curl file download callback!" << LL_ENDL; - return 0; - } - - return fwrite(data, size, nmemb, req->mFP); -} - -LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) -{ - switch (rt) - { - case RT_DOWNLOAD: - return &mRunningDownloads; - case RT_UPLOAD: - return &mRunningUploads; - case RT_LOCALUPLOAD: - return &mRunningLocalUploads; - default: - return NULL; - } -} - -const LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) const -{ - switch (rt) - { - case RT_DOWNLOAD: - return &mRunningDownloads; - case RT_UPLOAD: - return &mRunningUploads; - case RT_LOCALUPLOAD: - return &mRunningLocalUploads; - default: - return NULL; - } -} - - -void LLHTTPAssetStorage::addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request) -{ - request_list_t* requests = getRunningList(rt); - if (requests) - { - requests->push_back(request); - } - else - { - LL_ERRS() << "LLHTTPAssetStorage::addRunningRequest - Request is not an upload OR download, this is bad!" << LL_ENDL; - } -} - -void LLHTTPAssetStorage::removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request) -{ - request_list_t* requests = getRunningList(rt); - if (requests) - { - requests->remove(request); - } - else - { - LL_ERRS() << "LLHTTPAssetStorage::removeRunningRequest - Destroyed request is not an upload OR download, this is bad!" << LL_ENDL; - } -} - -// virtual -void LLHTTPAssetStorage::addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name) -{ - if (agent_id.isNull() || asset_id.isNull()) - { - LL_WARNS() << "TAT: addTempAssetData bad id's asset_id: " << asset_id << " agent_id: " << agent_id << LL_ENDL; - return; - } - - LLTempAssetData temp_asset_data; - temp_asset_data.mAssetID = asset_id; - temp_asset_data.mAgentID = agent_id; - temp_asset_data.mHostName = host_name; - - mTempAssets[asset_id] = temp_asset_data; -} - -// virtual -BOOL LLHTTPAssetStorage::hasTempAssetData(const LLUUID& texture_id) const -{ - uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id); - BOOL found = (citer != mTempAssets.end()); - return found; -} - -// virtual -std::string LLHTTPAssetStorage::getTempAssetHostName(const LLUUID& texture_id) const -{ - uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id); - if (citer != mTempAssets.end()) - { - return citer->second.mHostName; - } - else - { - return std::string(); - } -} - -// virtual -LLUUID LLHTTPAssetStorage::getTempAssetAgentID(const LLUUID& texture_id) const -{ - uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id); - if (citer != mTempAssets.end()) - { - return citer->second.mAgentID; - } - else - { - return LLUUID::null; - } -} - -// virtual -void LLHTTPAssetStorage::removeTempAssetData(const LLUUID& asset_id) -{ - mTempAssets.erase(asset_id); -} - -// virtual -void LLHTTPAssetStorage::removeTempAssetDataByAgentID(const LLUUID& agent_id) -{ - uuid_tempdata_map::iterator it = mTempAssets.begin(); - uuid_tempdata_map::iterator end = mTempAssets.end(); - - while (it != end) - { - const LLTempAssetData& asset_data = it->second; - if (asset_data.mAgentID == agent_id) - { - mTempAssets.erase(it++); - } - else - { - ++it; - } - } -} - -std::string LLHTTPAssetStorage::getBaseURL(const LLUUID& asset_id, LLAssetType::EType asset_type) -{ - if (LLAssetType::AT_TEXTURE == asset_type) - { - uuid_tempdata_map::const_iterator citer = mTempAssets.find(asset_id); - if (citer != mTempAssets.end()) - { - const std::string& host_name = citer->second.mHostName; - std::string url = llformat(LOCAL_ASSET_URL_FORMAT, host_name.c_str()); - return url; - } - } - - return mBaseURL; -} - -void LLHTTPAssetStorage::dumpTempAssetData(const LLUUID& avatar_id) const -{ - uuid_tempdata_map::const_iterator it = mTempAssets.begin(); - uuid_tempdata_map::const_iterator end = mTempAssets.end(); - S32 count = 0; - for ( ; it != end; ++it) - { - const LLTempAssetData& temp_asset_data = it->second; - if (avatar_id.isNull() - || avatar_id == temp_asset_data.mAgentID) - { - LL_INFOS() << "TAT: dump agent " << temp_asset_data.mAgentID - << " texture " << temp_asset_data.mAssetID - << " host " << temp_asset_data.mHostName - << LL_ENDL; - count++; - } - } - - if (avatar_id.isNull()) - { - LL_INFOS() << "TAT: dumped " << count << " entries for all avatars" << LL_ENDL; - } - else - { - LL_INFOS() << "TAT: dumped " << count << " entries for avatar " << avatar_id << LL_ENDL; - } -} - -void LLHTTPAssetStorage::clearTempAssetData() -{ - LL_INFOS() << "TAT: Clearing temp asset data map" << LL_ENDL; - mTempAssets.clear(); -} diff --git a/indra/llmessage/llhttpassetstorage.h b/indra/llmessage/llhttpassetstorage.h deleted file mode 100755 index 783e95cac6..0000000000 --- a/indra/llmessage/llhttpassetstorage.h +++ /dev/null @@ -1,159 +0,0 @@ -/** - * @file llhttpassetstorage.h - * @brief Class for loading asset data to/from an external source over http. - * - * $LicenseInfo:firstyear=2003&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LLHTTPASSETSTORAGE_H -#define LLHTTPASSETSTORAGE_H - -#include "llassetstorage.h" -#include "curl/curl.h" - -class LLVFile; -class LLHTTPAssetRequest; -typedef void (*progress_callback)(void* userdata); - -struct LLTempAssetData; - -typedef std::map<LLUUID,LLTempAssetData> uuid_tempdata_map; - -class LLHTTPAssetStorage : public LLAssetStorage -{ -public: - LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, - LLVFS *vfs, LLVFS *static_vfs, - const LLHost &upstream_host, - const std::string& web_host, - const std::string& local_web_host, - const std::string& host_name); - - LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, - LLVFS *vfs, LLVFS *static_vfs, - const std::string& web_host, - const std::string& local_web_host, - const std::string& host_name); - - - virtual ~LLHTTPAssetStorage(); - - using LLAssetStorage::storeAssetData; // Unhiding virtuals... - - virtual void storeAssetData( - const LLUUID& uuid, - LLAssetType::EType atype, - LLStoreAssetCallback callback, - void* user_data, - bool temp_file = false, - bool is_priority = false, - bool store_local = false, - const LLUUID& requesting_agent_id = LLUUID::null, - bool user_waiting=FALSE, - F64Seconds timeout=LL_ASSET_STORAGE_TIMEOUT); - - virtual void storeAssetData( - const std::string& filename, - const LLUUID& asset_id, - LLAssetType::EType atype, - LLStoreAssetCallback callback, - void* user_data, - bool temp_file, - bool is_priority, - bool user_waiting=FALSE, - F64Seconds timeout=LL_ASSET_STORAGE_TIMEOUT); - - virtual LLSD getPendingDetails(ERequestType rt, - LLAssetType::EType asset_type, - const std::string& detail_prefix) const; - - virtual LLSD getPendingRequest(ERequestType rt, - LLAssetType::EType asset_type, - const LLUUID& asset_id) const; - - virtual bool deletePendingRequest(ERequestType rt, - LLAssetType::EType asset_type, - const LLUUID& asset_id); - - // Hack. One off curl download an URL to a file. Probably should be elsewhere. - // Only used by lldynamicstate. The API is broken, and should be replaced with - // a generic HTTP file fetch - Doug 9/25/06 - S32 getURLToFile(const LLUUID& uuid, LLAssetType::EType asset_type, const std::string &url, const std::string& filename, progress_callback callback, void *userdata); - - LLAssetRequest* findNextRequest(request_list_t& pending, request_list_t& running); - - void checkForTimeouts(); - - static size_t curlDownCallback(void *data, size_t size, size_t nmemb, void *user_data); - static size_t curlFileDownCallback(void *data, size_t size, size_t nmemb, void *user_data); - static size_t curlUpCallback(void *data, size_t size, size_t nmemb, void *user_data); - static size_t nullOutputCallback(void *data, size_t size, size_t nmemb, void *user_data); - - // Should only be used by the LLHTTPAssetRequest - void addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request); - void removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request); - - request_list_t* getRunningList(ERequestType rt); - const request_list_t* getRunningList(ERequestType rt) const; - - // Temp assets are stored on sim nodes, they have agent ID and location data associated with them. - virtual void addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name); - virtual BOOL hasTempAssetData(const LLUUID& texture_id) const; - virtual std::string getTempAssetHostName(const LLUUID& texture_id) const; - virtual LLUUID getTempAssetAgentID(const LLUUID& texture_id) const; - virtual void removeTempAssetData(const LLUUID& asset_id); - virtual void removeTempAssetDataByAgentID(const LLUUID& agent_id); - - // Pass LLUUID::null for all - virtual void dumpTempAssetData(const LLUUID& avatar_id) const; - virtual void clearTempAssetData(); - -protected: - void _queueDataRequest(const LLUUID& uuid, LLAssetType::EType type, - void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32, LLExtStat), - void *user_data, BOOL duplicate, BOOL is_priority); - -private: - void _init(const std::string& web_host, const std::string& local_web_host, const std::string& host_name); - - // This will return the correct base URI for any http asset request - std::string getBaseURL(const LLUUID& asset_id, LLAssetType::EType asset_type); - - // Check for running uploads that have timed out - // Bump these to the back of the line to let other uploads complete. - void bumpTimedOutUploads(); - -protected: - std::string mBaseURL; - std::string mLocalBaseURL; - std::string mHostName; - - CURLM *mCurlMultiHandle; - - request_list_t mRunningDownloads; - request_list_t mRunningUploads; - request_list_t mRunningLocalUploads; - - uuid_tempdata_map mTempAssets; -}; - -#endif diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index 979b182662..7eb4174b7f 100755 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -2224,7 +2224,7 @@ if (LL_TESTS) # llmediadataclient.cpp lllogininstance.cpp # llremoteparcelrequest.cpp - lltranslate.cpp +# lltranslate.cpp llviewerhelputil.cpp llversioninfo.cpp llworldmap.cpp @@ -2245,11 +2245,11 @@ if (LL_TESTS) ${CURL_LIBRARIES} ) - set_source_files_properties( - lltranslate.cpp - PROPERTIES - LL_TEST_ADDITIONAL_LIBRARIES "${test_libs}" - ) +# set_source_files_properties( +# lltranslate.cpp +# PROPERTIES +# LL_TEST_ADDITIONAL_LIBRARIES "${test_libs}" +# ) set_source_files_properties( llmediadataclient.cpp diff --git a/indra/newview/llfloatermarketplacelistings.cpp b/indra/newview/llfloatermarketplacelistings.cpp index b2d36479cd..18f0bc4498 100755 --- a/indra/newview/llfloatermarketplacelistings.cpp +++ b/indra/newview/llfloatermarketplacelistings.cpp @@ -572,7 +572,7 @@ void LLFloaterMarketplaceListings::updateView() std::string title; std::string tooltip; - const LLSD& subs = getMarketplaceStringSubstitutions(); + const LLSD& subs = LLMarketplaceData::getMarketplaceStringSubstitutions(); // Update the top message or flip to the tabs and folders view // *TODO : check those messages and create better appropriate ones in strings.xml diff --git a/indra/newview/llfloateroutbox.cpp b/indra/newview/llfloateroutbox.cpp index b7b1634a5f..f61b50003d 100755 --- a/indra/newview/llfloateroutbox.cpp +++ b/indra/newview/llfloateroutbox.cpp @@ -397,7 +397,7 @@ void LLFloaterOutbox::updateView() std::string outbox_title; std::string outbox_tooltip; - const LLSD& subs = getMarketplaceStringSubstitutions(); + const LLSD& subs = LLMarketplaceData::getMarketplaceStringSubstitutions(); U32 mkt_status = LLMarketplaceInventoryImporter::getInstance()->getMarketPlaceStatus(); if (mOutboxId.notNull()) @@ -544,7 +544,7 @@ void LLFloaterOutbox::importReportResults(U32 status, const LLSD& content) } else if (status == MarketplaceErrorCodes::IMPORT_DONE_WITH_ERRORS) { - const LLSD& subs = getMarketplaceStringSubstitutions(); + const LLSD& subs = LLMarketplaceData::getMarketplaceStringSubstitutions(); LLNotificationsUtil::add("OutboxImportHadErrors", subs); } diff --git a/indra/newview/llfloatertranslationsettings.cpp b/indra/newview/llfloatertranslationsettings.cpp index 965d29b09c..b1316e386d 100755 --- a/indra/newview/llfloatertranslationsettings.cpp +++ b/indra/newview/llfloatertranslationsettings.cpp @@ -42,41 +42,6 @@ #include "llnotificationsutil.h" #include "llradiogroup.h" -class EnteredKeyVerifier : public LLTranslate::KeyVerificationReceiver -{ -public: - EnteredKeyVerifier(LLTranslate::EService service, bool alert) - : LLTranslate::KeyVerificationReceiver(service) - , mAlert(alert) - { - } - -private: - /*virtual*/ void setVerificationStatus(bool ok) - { - LLFloaterTranslationSettings* floater = - LLFloaterReg::getTypedInstance<LLFloaterTranslationSettings>("prefs_translation"); - - if (!floater) - { - LL_WARNS() << "Cannot find translation settings floater" << LL_ENDL; - return; - } - - switch (getService()) - { - case LLTranslate::SERVICE_BING: - floater->setBingVerified(ok, mAlert); - break; - case LLTranslate::SERVICE_GOOGLE: - floater->setGoogleVerified(ok, mAlert); - break; - } - } - - bool mAlert; -}; - LLFloaterTranslationSettings::LLFloaterTranslationSettings(const LLSD& key) : LLFloater(key) , mMachineTranslationCB(NULL) @@ -231,11 +196,34 @@ void LLFloaterTranslationSettings::updateControlsEnabledState() mOKBtn->setEnabled(!on || service_verified); } +/*static*/ +void LLFloaterTranslationSettings::setVerificationStatus(int service, bool ok, bool alert) +{ + LLFloaterTranslationSettings* floater = + LLFloaterReg::getTypedInstance<LLFloaterTranslationSettings>("prefs_translation"); + + if (!floater) + { + LL_WARNS() << "Cannot find translation settings floater" << LL_ENDL; + return; + } + + switch (service) + { + case LLTranslate::SERVICE_BING: + floater->setBingVerified(ok, alert); + break; + case LLTranslate::SERVICE_GOOGLE: + floater->setGoogleVerified(ok, alert); + break; + } +} + + void LLFloaterTranslationSettings::verifyKey(int service, const std::string& key, bool alert) { - LLTranslate::KeyVerificationReceiverPtr receiver = - new EnteredKeyVerifier((LLTranslate::EService) service, alert); - LLTranslate::verifyKey(receiver, key); + LLTranslate::verifyKey(static_cast<LLTranslate::EService>(service), key, + boost::bind(&LLFloaterTranslationSettings::setVerificationStatus, _1, _2, alert)); } void LLFloaterTranslationSettings::onEditorFocused(LLFocusableElement* control) diff --git a/indra/newview/llfloatertranslationsettings.h b/indra/newview/llfloatertranslationsettings.h index b9bfd6265a..2a15eacded 100755 --- a/indra/newview/llfloatertranslationsettings.h +++ b/indra/newview/llfloatertranslationsettings.h @@ -61,6 +61,8 @@ private: void onBtnGoogleVerify(); void onBtnOK(); + static void setVerificationStatus(int service, bool alert, bool ok); + LLCheckBoxCtrl* mMachineTranslationCB; LLComboBox* mLanguageCombo; LLLineEditor* mBingAPIKeyEditor; diff --git a/indra/newview/llmarketplacefunctions.cpp b/indra/newview/llmarketplacefunctions.cpp index f2f18cd6a2..cff8446545 100755 --- a/indra/newview/llmarketplacefunctions.cpp +++ b/indra/newview/llmarketplacefunctions.cpp @@ -146,28 +146,6 @@ namespace { } -LLSD getMarketplaceStringSubstitutions() -{ - std::string marketplace_url = getMarketplaceURL("MarketplaceURL"); - std::string marketplace_url_create = getMarketplaceURL("MarketplaceURL_CreateStore"); - std::string marketplace_url_dashboard = getMarketplaceURL("MarketplaceURL_Dashboard"); - std::string marketplace_url_imports = getMarketplaceURL("MarketplaceURL_Imports"); - std::string marketplace_url_info = getMarketplaceURL("MarketplaceURL_LearnMore"); - - LLSD marketplace_sub_map; - - marketplace_sub_map["[MARKETPLACE_URL]"] = marketplace_url; - marketplace_sub_map["[MARKETPLACE_CREATE_STORE_URL]"] = marketplace_url_create; - marketplace_sub_map["[MARKETPLACE_LEARN_MORE_URL]"] = marketplace_url_info; - marketplace_sub_map["[MARKETPLACE_DASHBOARD_URL]"] = marketplace_url_dashboard; - marketplace_sub_map["[MARKETPLACE_IMPORTS_URL]"] = marketplace_url_imports; - - return marketplace_sub_map; -} - - -// SLM Responders End -/////////////////////////////////////////////////////////////////////////////// #if 1 namespace LLMarketplaceImport @@ -720,6 +698,26 @@ LLMarketplaceData::~LLMarketplaceData() gInventory.removeObserver(mInventoryObserver); } + +LLSD LLMarketplaceData::getMarketplaceStringSubstitutions() +{ + std::string marketplace_url = getMarketplaceURL("MarketplaceURL"); + std::string marketplace_url_create = getMarketplaceURL("MarketplaceURL_CreateStore"); + std::string marketplace_url_dashboard = getMarketplaceURL("MarketplaceURL_Dashboard"); + std::string marketplace_url_imports = getMarketplaceURL("MarketplaceURL_Imports"); + std::string marketplace_url_info = getMarketplaceURL("MarketplaceURL_LearnMore"); + + LLSD marketplace_sub_map; + + marketplace_sub_map["[MARKETPLACE_URL]"] = marketplace_url; + marketplace_sub_map["[MARKETPLACE_CREATE_STORE_URL]"] = marketplace_url_create; + marketplace_sub_map["[MARKETPLACE_LEARN_MORE_URL]"] = marketplace_url_info; + marketplace_sub_map["[MARKETPLACE_DASHBOARD_URL]"] = marketplace_url_dashboard; + marketplace_sub_map["[MARKETPLACE_IMPORTS_URL]"] = marketplace_url_imports; + + return marketplace_sub_map; +} + void LLMarketplaceData::initializeSLM(const status_updated_signal_t::slot_type& cb) { if (mStatusUpdatedSignal == NULL) diff --git a/indra/newview/llmarketplacefunctions.h b/indra/newview/llmarketplacefunctions.h index f9e2ac98d0..9d795c6ced 100755 --- a/indra/newview/llmarketplacefunctions.h +++ b/indra/newview/llmarketplacefunctions.h @@ -37,8 +37,6 @@ #include "llstring.h" -LLSD getMarketplaceStringSubstitutions(); - namespace MarketplaceErrorCodes { @@ -183,6 +181,8 @@ class LLSLMDeleteListingsResponder; class LLMarketplaceData : public LLSingleton<LLMarketplaceData> { + friend class LLSingleton < LLMarketplaceData > ; + public: friend class LLSLMGetMerchantResponder; friend class LLSLMGetListingsResponder; @@ -192,9 +192,8 @@ public: friend class LLSLMAssociateListingsResponder; friend class LLSLMDeleteListingsResponder; - LLMarketplaceData(); - virtual ~LLMarketplaceData(); - + static LLSD getMarketplaceStringSubstitutions(); + // Public SLM API : Initialization and status typedef boost::signals2::signal<void ()> status_updated_signal_t; void initializeSLM(const status_updated_signal_t::slot_type& cb); @@ -241,8 +240,11 @@ public: // Used to decide when to run a validation on listing folders void setValidationWaiting(const LLUUID& folder_id, S32 count); void decrementValidationWaiting(const LLUUID& folder_id, S32 count = 1); - + private: + LLMarketplaceData(); + virtual ~LLMarketplaceData(); + // Modify Marketplace data set : each method returns true if the function succeeds, false if error // Used internally only by SLM Responders when data are received from the SLM Server bool addListing(const LLUUID& folder_id, S32 listing_id, const LLUUID& version_id, bool is_listed, const std::string& edit_url, S32 count); diff --git a/indra/newview/lltranslate.cpp b/indra/newview/lltranslate.cpp index c0ba0a1f39..1ca2011a70 100755 --- a/indra/newview/lltranslate.cpp +++ b/indra/newview/lltranslate.cpp @@ -35,31 +35,254 @@ #include "llui.h" #include "llversioninfo.h" #include "llviewercontrol.h" - +#include "llcoros.h" #include "reader.h" +#include "llcorehttputil.h" + + +/** +* Handler of an HTTP machine translation service. +* +* Derived classes know the service URL +* and how to parse the translation result. +*/ +class LLTranslationAPIHandler +{ +public: + typedef std::pair<std::string, std::string> LanguagePair_t; + + /** + * Get URL for translation of the given string. + * + * Sending HTTP GET request to the URL will initiate translation. + * + * @param[out] url Place holder for the result. + * @param from_lang Source language. Leave empty for auto-detection. + * @param to_lang Target language. + * @param text Text to translate. + */ + virtual std::string getTranslateURL( + const std::string &from_lang, + const std::string &to_lang, + const std::string &text) const = 0; + + /** + * Get URL to verify the given API key. + * + * Sending request to the URL verifies the key. + * Positive HTTP response (code 200) means that the key is valid. + * + * @param[out] url Place holder for the URL. + * @param[in] key Key to verify. + */ + virtual std::string getKeyVerificationURL( + const std::string &key) const = 0; + + /** + * Parse translation response. + * + * @param[in,out] status HTTP status. May be modified while parsing. + * @param body Response text. + * @param[out] translation Translated text. + * @param[out] detected_lang Detected source language. May be empty. + * @param[out] err_msg Error message (in case of error). + */ + virtual bool parseResponse( + int& status, + const std::string& body, + std::string& translation, + std::string& detected_lang, + std::string& err_msg) const = 0; + + /** + * @return if the handler is configured to function properly + */ + virtual bool isConfigured() const = 0; + + virtual void verifyKey(const std::string &key, LLTranslate::KeyVerificationResult_fn fnc) = 0; + virtual void translateMessage(LanguagePair_t fromTo, std::string msg, LLTranslate::TranslationSuccess_fn success, LLTranslate::TranslationFailure_fn failure); + + + virtual ~LLTranslationAPIHandler() {} + + void verifyKeyCoro(LLTranslate::EService service, std::string key, LLTranslate::KeyVerificationResult_fn fnc); + void translateMessageCoro(LanguagePair_t fromTo, std::string msg, LLTranslate::TranslationSuccess_fn success, LLTranslate::TranslationFailure_fn failure); +}; + +void LLTranslationAPIHandler::translateMessage(LanguagePair_t fromTo, std::string msg, LLTranslate::TranslationSuccess_fn success, LLTranslate::TranslationFailure_fn failure) +{ + LLCoros::instance().launch("Translation", boost::bind(&LLTranslationAPIHandler::translateMessageCoro, + this, fromTo, msg, success, failure)); + +} + + +void LLTranslationAPIHandler::verifyKeyCoro(LLTranslate::EService service, std::string key, LLTranslate::KeyVerificationResult_fn fnc) +{ + LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t + httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("getMerchantStatusCoro", httpPolicy)); + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); + LLCore::HttpHeaders::ptr_t httpHeaders(new LLCore::HttpHeaders); + + + std::string user_agent = llformat("%s %d.%d.%d (%d)", + LLVersionInfo::getChannel().c_str(), + LLVersionInfo::getMajor(), + LLVersionInfo::getMinor(), + LLVersionInfo::getPatch(), + LLVersionInfo::getBuild()); + + httpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_TEXT_PLAIN); + httpHeaders->append(HTTP_OUT_HEADER_USER_AGENT, user_agent); + httpOpts->setFollowRedirects(true); + + std::string url = this->getKeyVerificationURL(key); + if (url.empty()) + { + LL_INFOS("Translate") << "No translation URL" << LL_ENDL; + return; + } + + LLSD result = httpAdapter->getAndYield(httpRequest, url, httpOpts, httpHeaders); + + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; + LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); + + bool bOk = true; + if (!status) + bOk = false; + + if (!fnc.empty()) + fnc(service, bOk); +} + +void LLTranslationAPIHandler::translateMessageCoro(LanguagePair_t fromTo, std::string msg, + LLTranslate::TranslationSuccess_fn success, LLTranslate::TranslationFailure_fn failure) +{ + LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t + httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("getMerchantStatusCoro", httpPolicy)); + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); + LLCore::HttpHeaders::ptr_t httpHeaders(new LLCore::HttpHeaders); + + + std::string user_agent = llformat("%s %d.%d.%d (%d)", + LLVersionInfo::getChannel().c_str(), + LLVersionInfo::getMajor(), + LLVersionInfo::getMinor(), + LLVersionInfo::getPatch(), + LLVersionInfo::getBuild()); + + httpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_TEXT_PLAIN); + httpHeaders->append(HTTP_OUT_HEADER_USER_AGENT, user_agent); + + std::string url = this->getTranslateURL(fromTo.first, fromTo.second, msg); + if (url.empty()) + { + LL_INFOS("Translate") << "No translation URL" << LL_ENDL; + return; + } + + LLSD result = httpAdapter->getRawAndYield(httpRequest, url, httpOpts, httpHeaders); + + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; + LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); + + std::string translation, detected_lang, err_msg; + + int parseResult = status.getType(); + if (this->parseResponse(parseResult, result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS_RAW].asString(), + translation, detected_lang, err_msg)) + { + // Fix up the response + LLStringUtil::replaceString(translation, "<", "<"); + LLStringUtil::replaceString(translation, ">", ">"); + LLStringUtil::replaceString(translation, """, "\""); + LLStringUtil::replaceString(translation, "'", "'"); + LLStringUtil::replaceString(translation, "&", "&"); + LLStringUtil::replaceString(translation, "'", "'"); + + if (!success.empty()) + success(translation, detected_lang); + } + else + { + if (err_msg.empty()) + { + err_msg = LLTrans::getString("TranslationResponseParseError"); + } + + LL_WARNS() << "Translation request failed: " << err_msg << LL_ENDL; + if (!failure.empty()) + failure(status, err_msg); + } + + +} + +//========================================================================= +/// Google Translate v2 API handler. +class LLGoogleTranslationHandler : public LLTranslationAPIHandler +{ + LOG_CLASS(LLGoogleTranslationHandler); + +public: + /*virtual*/ std::string getTranslateURL( + const std::string &from_lang, + const std::string &to_lang, + const std::string &text) const; + /*virtual*/ std::string getKeyVerificationURL( + const std::string &key) const; + /*virtual*/ bool parseResponse( + int& status, + const std::string& body, + std::string& translation, + std::string& detected_lang, + std::string& err_msg) const; + /*virtual*/ bool isConfigured() const; + + /*virtual*/ void verifyKey(const std::string &key, LLTranslate::KeyVerificationResult_fn fnc); + +private: + static void parseErrorResponse( + const Json::Value& root, + int& status, + std::string& err_msg); + static bool parseTranslation( + const Json::Value& root, + std::string& translation, + std::string& detected_lang); + static std::string getAPIKey(); + +}; + +//------------------------------------------------------------------------- // virtual -void LLGoogleTranslationHandler::getTranslateURL( - std::string &url, +std::string LLGoogleTranslationHandler::getTranslateURL( const std::string &from_lang, const std::string &to_lang, const std::string &text) const { - url = std::string("https://www.googleapis.com/language/translate/v2?key=") + std::string url = std::string("https://www.googleapis.com/language/translate/v2?key=") + getAPIKey() + "&q=" + LLURI::escape(text) + "&target=" + to_lang; if (!from_lang.empty()) { url += "&source=" + from_lang; } + return url; } // virtual -void LLGoogleTranslationHandler::getKeyVerificationURL( - std::string& url, +std::string LLGoogleTranslationHandler::getKeyVerificationURL( const std::string& key) const { - url = std::string("https://www.googleapis.com/language/translate/v2/languages?key=") + std::string url = std::string("https://www.googleapis.com/language/translate/v2/languages?key=") + key + "&target=en"; + return url; } // virtual @@ -154,28 +377,66 @@ std::string LLGoogleTranslationHandler::getAPIKey() return gSavedSettings.getString("GoogleTranslateAPIKey"); } +/*virtual*/ +void LLGoogleTranslationHandler::verifyKey(const std::string &key, LLTranslate::KeyVerificationResult_fn fnc) +{ + LLCoros::instance().launch("Google /Verify Key", boost::bind(&LLTranslationAPIHandler::verifyKeyCoro, + this, LLTranslate::SERVICE_GOOGLE, key, fnc)); +} + + +//========================================================================= +/// Microsoft Translator v2 API handler. +class LLBingTranslationHandler : public LLTranslationAPIHandler +{ + LOG_CLASS(LLBingTranslationHandler); + +public: + /*virtual*/ std::string getTranslateURL( + const std::string &from_lang, + const std::string &to_lang, + const std::string &text) const; + /*virtual*/ std::string getKeyVerificationURL( + const std::string &key) const; + /*virtual*/ bool parseResponse( + int& status, + const std::string& body, + std::string& translation, + std::string& detected_lang, + std::string& err_msg) const; + /*virtual*/ bool isConfigured() const; + + /*virtual*/ void verifyKey(const std::string &key, LLTranslate::KeyVerificationResult_fn fnc); +private: + static std::string getAPIKey(); + static std::string getAPILanguageCode(const std::string& lang); + +}; + +//------------------------------------------------------------------------- // virtual -void LLBingTranslationHandler::getTranslateURL( - std::string &url, +std::string LLBingTranslationHandler::getTranslateURL( const std::string &from_lang, const std::string &to_lang, const std::string &text) const { - url = std::string("http://api.microsofttranslator.com/v2/Http.svc/Translate?appId=") + std::string url = std::string("http://api.microsofttranslator.com/v2/Http.svc/Translate?appId=") + getAPIKey() + "&text=" + LLURI::escape(text) + "&to=" + getAPILanguageCode(to_lang); if (!from_lang.empty()) { url += "&from=" + getAPILanguageCode(from_lang); } + return url; } + // virtual -void LLBingTranslationHandler::getKeyVerificationURL( - std::string& url, +std::string LLBingTranslationHandler::getKeyVerificationURL( const std::string& key) const { - url = std::string("http://api.microsofttranslator.com/v2/Http.svc/GetLanguagesForTranslate?appId=") + std::string url = std::string("http://api.microsofttranslator.com/v2/Http.svc/GetLanguagesForTranslate?appId=") + key; + return url; } // virtual @@ -242,96 +503,31 @@ std::string LLBingTranslationHandler::getAPILanguageCode(const std::string& lang return lang == "zh" ? "zh-CHT" : lang; // treat Chinese as Traditional Chinese } -LLTranslate::TranslationReceiver::TranslationReceiver(const std::string& from_lang, const std::string& to_lang) -: mFromLang(from_lang) -, mToLang(to_lang) -, mHandler(LLTranslate::getPreferredHandler()) +/*virtual*/ +void LLBingTranslationHandler::verifyKey(const std::string &key, LLTranslate::KeyVerificationResult_fn fnc) { + LLCoros::instance().launch("Bing /Verify Key", boost::bind(&LLTranslationAPIHandler::verifyKeyCoro, + this, LLTranslate::SERVICE_BING, key, fnc)); } -// virtual -void LLTranslate::TranslationReceiver::completedRaw( - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) +//========================================================================= +/*static*/ +void LLTranslate::translateMessage(const std::string &from_lang, const std::string &to_lang, + const std::string &mesg, TranslationSuccess_fn success, TranslationFailure_fn failure) { - LLBufferStream istr(channels, buffer.get()); - std::stringstream strstrm; - strstrm << istr.rdbuf(); - - const std::string body = strstrm.str(); - std::string translation, detected_lang, err_msg; - int status = getStatus(); - LL_DEBUGS("Translate") << "HTTP status: " << status << " " << getReason() << LL_ENDL; - LL_DEBUGS("Translate") << "Response body: " << body << LL_ENDL; - if (mHandler.parseResponse(status, body, translation, detected_lang, err_msg)) - { - // Fix up the response - LLStringUtil::replaceString(translation, "<", "<"); - LLStringUtil::replaceString(translation, ">",">"); - LLStringUtil::replaceString(translation, ""","\""); - LLStringUtil::replaceString(translation, "'","'"); - LLStringUtil::replaceString(translation, "&","&"); - LLStringUtil::replaceString(translation, "'","'"); - - handleResponse(translation, detected_lang); - } - else - { - if (err_msg.empty()) - { - err_msg = LLTrans::getString("TranslationResponseParseError"); - } + LLTranslationAPIHandler& handler = getPreferredHandler(); - LL_WARNS() << "Translation request failed: " << err_msg << LL_ENDL; - handleFailure(status, err_msg); - } + handler.translateMessage(LLTranslationAPIHandler::LanguagePair_t(from_lang, to_lang), mesg, success, failure); } -LLTranslate::KeyVerificationReceiver::KeyVerificationReceiver(EService service) -: mService(service) +/*static*/ +void LLTranslate::verifyKey(EService service, const std::string &key, KeyVerificationResult_fn fnc) { -} + LLTranslationAPIHandler& handler = getHandler(service); -LLTranslate::EService LLTranslate::KeyVerificationReceiver::getService() const -{ - return mService; + handler.verifyKey(key, fnc); } -// virtual -void LLTranslate::KeyVerificationReceiver::completedRaw( - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) -{ - bool ok = (getStatus() == HTTP_OK); - setVerificationStatus(ok); -} - -//static -void LLTranslate::translateMessage( - TranslationReceiverPtr &receiver, - const std::string &from_lang, - const std::string &to_lang, - const std::string &mesg) -{ - std::string url; - receiver->mHandler.getTranslateURL(url, from_lang, to_lang, mesg); - - LL_DEBUGS("Translate") << "Sending translation request: " << url << LL_ENDL; - sendRequest(url, receiver); -} - -// static -void LLTranslate::verifyKey( - KeyVerificationReceiverPtr& receiver, - const std::string& key) -{ - std::string url; - const LLTranslationAPIHandler& handler = getHandler(receiver->getService()); - handler.getKeyVerificationURL(url, key); - - LL_DEBUGS("Translate") << "Sending key verification request: " << url << LL_ENDL; - sendRequest(url, receiver); -} //static std::string LLTranslate::getTranslateLanguage() @@ -352,7 +548,7 @@ bool LLTranslate::isTranslationConfigured() } // static -const LLTranslationAPIHandler& LLTranslate::getPreferredHandler() +LLTranslationAPIHandler& LLTranslate::getPreferredHandler() { EService service = SERVICE_BING; @@ -366,7 +562,7 @@ const LLTranslationAPIHandler& LLTranslate::getPreferredHandler() } // static -const LLTranslationAPIHandler& LLTranslate::getHandler(EService service) +LLTranslationAPIHandler& LLTranslate::getHandler(EService service) { static LLGoogleTranslationHandler google; static LLBingTranslationHandler bing; diff --git a/indra/newview/lltranslate.h b/indra/newview/lltranslate.h index 972274714a..49a0105dbb 100755 --- a/indra/newview/lltranslate.h +++ b/indra/newview/lltranslate.h @@ -29,134 +29,14 @@ #include "llhttpclient.h" #include "llbufferstream.h" +#include <boost/function.hpp> namespace Json { class Value; } -/** - * Handler of an HTTP machine translation service. - * - * Derived classes know the service URL - * and how to parse the translation result. - */ -class LLTranslationAPIHandler -{ -public: - /** - * Get URL for translation of the given string. - * - * Sending HTTP GET request to the URL will initiate translation. - * - * @param[out] url Place holder for the result. - * @param from_lang Source language. Leave empty for auto-detection. - * @param to_lang Target language. - * @param text Text to translate. - */ - virtual void getTranslateURL( - std::string &url, - const std::string &from_lang, - const std::string &to_lang, - const std::string &text) const = 0; - - /** - * Get URL to verify the given API key. - * - * Sending request to the URL verifies the key. - * Positive HTTP response (code 200) means that the key is valid. - * - * @param[out] url Place holder for the URL. - * @param[in] key Key to verify. - */ - virtual void getKeyVerificationURL( - std::string &url, - const std::string &key) const = 0; - - /** - * Parse translation response. - * - * @param[in,out] status HTTP status. May be modified while parsing. - * @param body Response text. - * @param[out] translation Translated text. - * @param[out] detected_lang Detected source language. May be empty. - * @param[out] err_msg Error message (in case of error). - */ - virtual bool parseResponse( - int& status, - const std::string& body, - std::string& translation, - std::string& detected_lang, - std::string& err_msg) const = 0; - - /** - * @return if the handler is configured to function properly - */ - virtual bool isConfigured() const = 0; - - virtual ~LLTranslationAPIHandler() {} -}; - -/// Google Translate v2 API handler. -class LLGoogleTranslationHandler : public LLTranslationAPIHandler -{ - LOG_CLASS(LLGoogleTranslationHandler); - -public: - /*virtual*/ void getTranslateURL( - std::string &url, - const std::string &from_lang, - const std::string &to_lang, - const std::string &text) const; - /*virtual*/ void getKeyVerificationURL( - std::string &url, - const std::string &key) const; - /*virtual*/ bool parseResponse( - int& status, - const std::string& body, - std::string& translation, - std::string& detected_lang, - std::string& err_msg) const; - /*virtual*/ bool isConfigured() const; - -private: - static void parseErrorResponse( - const Json::Value& root, - int& status, - std::string& err_msg); - static bool parseTranslation( - const Json::Value& root, - std::string& translation, - std::string& detected_lang); - static std::string getAPIKey(); -}; - -/// Microsoft Translator v2 API handler. -class LLBingTranslationHandler : public LLTranslationAPIHandler -{ - LOG_CLASS(LLBingTranslationHandler); - -public: - /*virtual*/ void getTranslateURL( - std::string &url, - const std::string &from_lang, - const std::string &to_lang, - const std::string &text) const; - /*virtual*/ void getKeyVerificationURL( - std::string &url, - const std::string &key) const; - /*virtual*/ bool parseResponse( - int& status, - const std::string& body, - std::string& translation, - std::string& detected_lang, - std::string& err_msg) const; - /*virtual*/ bool isConfigured() const; -private: - static std::string getAPIKey(); - static std::string getAPILanguageCode(const std::string& lang); -}; - +class LLTranslationAPIHandler; /** * Entry point for machine translation services. * @@ -180,84 +60,9 @@ public : SERVICE_GOOGLE, } EService; - /** - * Subclasses are supposed to handle translation results (e.g. show them in chat) - */ - class TranslationReceiver: public LLHTTPClient::Responder - { - public: - - /** - * Using mHandler, parse incoming response. - * - * Calls either handleResponse() or handleFailure() - * depending on the HTTP status code and parsing success. - * - * @see handleResponse() - * @see handleFailure() - * @see mHandler - */ - /*virtual*/ void completedRaw( - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - - protected: - friend class LLTranslate; - - /// Remember source and target languages for subclasses to be able to filter inappropriate results. - TranslationReceiver(const std::string& from_lang, const std::string& to_lang); - - /// Override point to handle successful translation. - virtual void handleResponse(const std::string &translation, const std::string &recognized_lang) = 0; - - /// Override point to handle unsuccessful translation. - virtual void handleFailure(int status, const std::string& err_msg) = 0; - - std::string mFromLang; - std::string mToLang; - const LLTranslationAPIHandler& mHandler; - }; - - /** - * Subclasses are supposed to handle API key verification result. - */ - class KeyVerificationReceiver: public LLHTTPClient::Responder - { - public: - EService getService() const; - - protected: - /** - * Save the translation service the key belongs to. - * - * Subclasses need to know it. - * - * @see getService() - */ - KeyVerificationReceiver(EService service); - - /** - * Parse verification response. - * - * Calls setVerificationStatus() with the verification status, - * which is true if HTTP status code is 200. - * - * @see setVerificationStatus() - */ - /*virtual*/ void completedRaw( - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - - /** - * Override point for subclasses to handle key verification status. - */ - virtual void setVerificationStatus(bool ok) = 0; - - EService mService; - }; - - typedef LLPointer<TranslationReceiver> TranslationReceiverPtr; - typedef LLPointer<KeyVerificationReceiver> KeyVerificationReceiverPtr; + typedef boost::function<void(EService, bool)> KeyVerificationResult_fn; + typedef boost::function<void(std::string , std::string )> TranslationSuccess_fn; + typedef boost::function<void(int, std::string)> TranslationFailure_fn; /** * Translate given text. @@ -267,15 +72,15 @@ public : * @param to_lang Target language. * @param mesg Text to translate. */ - static void translateMessage(TranslationReceiverPtr &receiver, const std::string &from_lang, const std::string &to_lang, const std::string &mesg); + static void translateMessage(const std::string &from_lang, const std::string &to_lang, const std::string &mesg, TranslationSuccess_fn success, TranslationFailure_fn failure); - /** + /** * Verify given API key of a translation service. * * @param receiver Object to pass verification result to. * @param key Key to verify. */ - static void verifyKey(KeyVerificationReceiverPtr& receiver, const std::string& key); + static void verifyKey(EService service, const std::string &key, KeyVerificationResult_fn fnc); /** * @return translation target language @@ -288,8 +93,8 @@ public : static bool isTranslationConfigured(); private: - static const LLTranslationAPIHandler& getPreferredHandler(); - static const LLTranslationAPIHandler& getHandler(EService service); + static LLTranslationAPIHandler& getPreferredHandler(); + static LLTranslationAPIHandler& getHandler(EService service); static void sendRequest(const std::string& url, LLHTTPClient::ResponderPtr responder); }; diff --git a/indra/newview/llviewermessage.cpp b/indra/newview/llviewermessage.cpp index 0643f7b1e3..4062228ae5 100755 --- a/indra/newview/llviewermessage.cpp +++ b/indra/newview/llviewermessage.cpp @@ -3492,53 +3492,29 @@ void process_decline_callingcard(LLMessageSystem* msg, void**) LLNotificationsUtil::add("CallingCardDeclined"); } -class ChatTranslationReceiver : public LLTranslate::TranslationReceiver +void translateSuccess(LLChat chat, LLSD toastArgs, std::string originalMsg, std::string expectLang, std::string translation, const std::string detected_language) { -public : - ChatTranslationReceiver(const std::string &from_lang, const std::string &to_lang, const std::string &mesg, - const LLChat &chat, const LLSD &toast_args) - : LLTranslate::TranslationReceiver(from_lang, to_lang), - m_chat(chat), - m_toastArgs(toast_args), - m_origMesg(mesg) - { - } - - static ChatTranslationReceiver* build(const std::string &from_lang, const std::string &to_lang, const std::string &mesg, const LLChat &chat, const LLSD &toast_args) - { - return new ChatTranslationReceiver(from_lang, to_lang, mesg, chat, toast_args); - } - -protected: - void handleResponse(const std::string &translation, const std::string &detected_language) - { - // filter out non-interesting responeses - if ( !translation.empty() - && (mToLang != detected_language) - && (LLStringUtil::compareInsensitive(translation, m_origMesg) != 0) ) - { - m_chat.mText += " (" + translation + ")"; - } + // filter out non-interesting responses + if (!translation.empty() + && (expectLang != detected_language) + && (LLStringUtil::compareInsensitive(translation, originalMsg) != 0)) + { + chat.mText += " (" + translation + ")"; + } - LLNotificationsUI::LLNotificationManager::instance().onChat(m_chat, m_toastArgs); - } + LLNotificationsUI::LLNotificationManager::instance().onChat(chat, toastArgs); +} - void handleFailure(int status, const std::string& err_msg) - { - LL_WARNS() << "Translation failed for mesg " << m_origMesg << " toLang " << mToLang << " fromLang " << mFromLang << LL_ENDL; +void translateFailure(LLChat chat, LLSD toastArgs, int status, const std::string err_msg) +{ + std::string msg = LLTrans::getString("TranslationFailed", LLSD().with("[REASON]", err_msg)); + LLStringUtil::replaceString(msg, "\n", " "); // we want one-line error messages + chat.mText += " (" + msg + ")"; - std::string msg = LLTrans::getString("TranslationFailed", LLSD().with("[REASON]", err_msg)); - LLStringUtil::replaceString(msg, "\n", " "); // we want one-line error messages - m_chat.mText += " (" + msg + ")"; + LLNotificationsUI::LLNotificationManager::instance().onChat(chat, toastArgs); +} - LLNotificationsUI::LLNotificationManager::instance().onChat(m_chat, m_toastArgs); - } -private: - LLChat m_chat; - std::string m_origMesg; - LLSD m_toastArgs; -}; void process_chat_from_simulator(LLMessageSystem *msg, void **user_data) { LLChat chat; @@ -3774,8 +3750,10 @@ void process_chat_from_simulator(LLMessageSystem *msg, void **user_data) const std::string from_lang = ""; // leave empty to trigger autodetect const std::string to_lang = LLTranslate::getTranslateLanguage(); - LLTranslate::TranslationReceiverPtr result = ChatTranslationReceiver::build(from_lang, to_lang, mesg, chat, args); - LLTranslate::translateMessage(result, from_lang, to_lang, mesg); + LLTranslate::translateMessage(from_lang, to_lang, mesg, + boost::bind(&translateSuccess, chat, args, mesg, from_lang, _1, _2), + boost::bind(&translateFailure, chat, args, _1, _2)); + } else { |