From 5611cb6d476540e6a1c654c1f9acdce2787b3505 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Apr 2012 16:19:39 -0400 Subject: Okay, imported the core-http library and got it compiling suspiciously easily. The unit/integration tests don't work yet as I'm still battling cmake/autobuild as usual but first milestone passed. --- indra/llcorehttp/bufferarray.h | 168 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 indra/llcorehttp/bufferarray.h (limited to 'indra/llcorehttp/bufferarray.h') diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h new file mode 100644 index 0000000000..b26ad1b297 --- /dev/null +++ b/indra/llcorehttp/bufferarray.h @@ -0,0 +1,168 @@ +/** + * @file bufferarray.h + * @brief Public-facing declaration for the BufferArray scatter/gather class + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_BUFFER_ARRAY_H_ +#define _LLCORE_BUFFER_ARRAY_H_ + + +#include +#include + +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// A very simple scatter/gather type map for bulk data. The motivation +/// for this class is the writedata callback used by libcurl. Response +/// bodies are delivered to the caller in a sequence of sequential write +/// operations and this class captures them without having to reallocate +/// and move data. +/// +/// The interface looks a little like a unix file descriptor but only +/// just. There is a notion of a current position, starting from 0, +/// which is used as the position in the data when performing read and +/// write operations. The position also moves after various operations: +/// - seek(...) +/// - read(...) +/// - write(...) +/// - append(...) +/// - appendBufferAlloc(...) +/// The object also keeps a total length value which is updated after +/// write and append operations and beyond which the current position +/// cannot be set. +/// +/// Threading: not thread-safe +/// +/// Allocation: Refcounted, heap only. Caller of the constructor +/// is given a single refcount. +/// +class BufferArray : public LLCoreInt::RefCounted +{ +public: + BufferArray(); + virtual ~BufferArray(); + +private: + BufferArray(const BufferArray &); // Not defined + void operator=(const BufferArray &); // Not defined + +public: + /// Appends the indicated data to the BufferArray + /// modifying current position and total size. New + /// position is one beyond the final byte of the buffer. + /// + /// @return Count of bytes copied to BufferArray + size_t append(const char * src, size_t len); + + /// Similar to @see append(), this call guarantees a + /// contiguous block of memory of requested size placed + /// at the current end of the BufferArray. On return, + /// the data in the memory is considered valid whether + /// the caller writes to it or not. + /// + /// @return Pointer to contiguous region at end + /// of BufferArray of 'len' size. + char * appendBufferAlloc(size_t len); + + /// Current count of bytes in BufferArray instance. + size_t size() const + { + return mLen; + } + + /// Set the current position for subsequent read and + /// write operations. 'pos' values before the beginning + /// or greater than the size of the buffer are coerced + /// to a value within the buffer. + /// + /// @return Actual current position after seek. + size_t seek(size_t pos); + + /// Copies data from the current position in the instance + /// to the caller's buffer. Will return a short count of + /// bytes copied if the 'len' extends beyond the data. + size_t read(char * dst, size_t len); + + /// Copies data from the caller's buffer to the instance + /// at the current position. May overwrite existing data, + /// append data when current position is equal to the + /// size of the instance or do a mix of both. + size_t write(const char * src, size_t len); + +protected: + int findBlock(size_t pos, size_t * ret_offset); + +protected: + class Block; + typedef std::vector container_t; + + container_t mBlocks; + size_t mPos; + size_t mLen; +}; // end class BufferArray + + +#if 0 + +// Conceptual for now. Another possibility is going with +// something like Boost::asio's buffers interface. They're +// trying to achieve the same thing above and below.... + +class BufferStream : public std::streambuf +{ +public: + BufferStream(BufferArray * buffer); + virtual ~BufferStream(); + +private: + BufferStream(const BufferStream &); // Not defined + void operator=(const BufferStream &); // Not defined + +public: + // Types + typedef std::streambuf::pos_type pos_type; + typedef std::streambuf::off_type off_type; + + virtual int underflow(); + + virtual int overflow(int c); + + virtual int sync(); + + virtual pos_type seekoff(off_type off, std::ios::seekdir way, std::ios::openmode which); + +protected: + BufferArray * mBufferArray; +}; // end class BufferStream + +#endif // 0 + +} // end namespace LLCore + +#endif // _LLCORE_BUFFER_ARRAY_H_ -- cgit v1.2.3 From 4155301015525a242a79b9b3134e66d366bc0ebd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 21:30:45 -0400 Subject: Do some work on BufferArray to make it a bit less naive about chunking data. Remove the stateful use of a seek pointer so that shared read is possible (though maybe not interesting). --- indra/llcorehttp/bufferarray.h | 53 +++++------------------------------------- 1 file changed, 6 insertions(+), 47 deletions(-) (limited to 'indra/llcorehttp/bufferarray.h') diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h index b26ad1b297..9ccd85d4f8 100644 --- a/indra/llcorehttp/bufferarray.h +++ b/indra/llcorehttp/bufferarray.h @@ -73,6 +73,9 @@ private: void operator=(const BufferArray &); // Not defined public: + // Internal magic number, may be used by unit tests. + static const size_t BLOCK_ALLOC_SIZE = 1504; + /// Appends the indicated data to the BufferArray /// modifying current position and total size. New /// position is one beyond the final byte of the buffer. @@ -96,24 +99,16 @@ public: return mLen; } - /// Set the current position for subsequent read and - /// write operations. 'pos' values before the beginning - /// or greater than the size of the buffer are coerced - /// to a value within the buffer. - /// - /// @return Actual current position after seek. - size_t seek(size_t pos); - - /// Copies data from the current position in the instance + /// Copies data from the given position in the instance /// to the caller's buffer. Will return a short count of /// bytes copied if the 'len' extends beyond the data. - size_t read(char * dst, size_t len); + size_t read(size_t pos, char * dst, size_t len); /// Copies data from the caller's buffer to the instance /// at the current position. May overwrite existing data, /// append data when current position is equal to the /// size of the instance or do a mix of both. - size_t write(const char * src, size_t len); + size_t write(size_t pos, const char * src, size_t len); protected: int findBlock(size_t pos, size_t * ret_offset); @@ -123,46 +118,10 @@ protected: typedef std::vector container_t; container_t mBlocks; - size_t mPos; size_t mLen; }; // end class BufferArray -#if 0 - -// Conceptual for now. Another possibility is going with -// something like Boost::asio's buffers interface. They're -// trying to achieve the same thing above and below.... - -class BufferStream : public std::streambuf -{ -public: - BufferStream(BufferArray * buffer); - virtual ~BufferStream(); - -private: - BufferStream(const BufferStream &); // Not defined - void operator=(const BufferStream &); // Not defined - -public: - // Types - typedef std::streambuf::pos_type pos_type; - typedef std::streambuf::off_type off_type; - - virtual int underflow(); - - virtual int overflow(int c); - - virtual int sync(); - - virtual pos_type seekoff(off_type off, std::ios::seekdir way, std::ios::openmode which); - -protected: - BufferArray * mBufferArray; -}; // end class BufferStream - -#endif // 0 - } // end namespace LLCore #endif // _LLCORE_BUFFER_ARRAY_H_ -- cgit v1.2.3 From 267ab5b417eaef64a170d69ad83334df9d566ed9 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 11 Jun 2012 17:47:04 -0400 Subject: Convert BufferArray interfaces to void * (not char *). HttpRequest::update() honor time limit. Generally, opaque data operations are expected to be over 'void *' and have now converted interfaces to do that. Update() method honors millisecond limit to dwell time. Might want to homologate the millis/uSecs mix later.... --- indra/llcorehttp/bufferarray.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'indra/llcorehttp/bufferarray.h') diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h index 9ccd85d4f8..72c3e1c669 100644 --- a/indra/llcorehttp/bufferarray.h +++ b/indra/llcorehttp/bufferarray.h @@ -81,7 +81,7 @@ public: /// position is one beyond the final byte of the buffer. /// /// @return Count of bytes copied to BufferArray - size_t append(const char * src, size_t len); + size_t append(const void * src, size_t len); /// Similar to @see append(), this call guarantees a /// contiguous block of memory of requested size placed @@ -91,7 +91,7 @@ public: /// /// @return Pointer to contiguous region at end /// of BufferArray of 'len' size. - char * appendBufferAlloc(size_t len); + void * appendBufferAlloc(size_t len); /// Current count of bytes in BufferArray instance. size_t size() const @@ -102,13 +102,13 @@ public: /// Copies data from the given position in the instance /// to the caller's buffer. Will return a short count of /// bytes copied if the 'len' extends beyond the data. - size_t read(size_t pos, char * dst, size_t len); + size_t read(size_t pos, void * dst, size_t len); /// Copies data from the caller's buffer to the instance /// at the current position. May overwrite existing data, /// append data when current position is equal to the /// size of the instance or do a mix of both. - size_t write(size_t pos, const char * src, size_t len); + size_t write(size_t pos, const void * src, size_t len); protected: int findBlock(size_t pos, size_t * ret_offset); -- cgit v1.2.3 From 6193ee6a331e3dfd562400a32a961bad0b8bed12 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sat, 16 Jun 2012 15:50:48 -0400 Subject: First round of basic tuning work (shorter sleeps, larger BufferArray blocks). Beefed up the metrics gathering in http_texture_load to get memory sizes and cpu consumption on windows (still need to implement that on Mac & linux). Ran runs with various idle loops with sleeps from 20 ms down to pure spinning, varied Block allocation size from 1504 to 2^20 bytes. 2ms/2ms/65540 appears to be a good spot under the test conditions (Win7, danu grid, client in Boston). --- indra/llcorehttp/bufferarray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcorehttp/bufferarray.h') diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h index 72c3e1c669..d3862b45e1 100644 --- a/indra/llcorehttp/bufferarray.h +++ b/indra/llcorehttp/bufferarray.h @@ -74,7 +74,7 @@ private: public: // Internal magic number, may be used by unit tests. - static const size_t BLOCK_ALLOC_SIZE = 1504; + static const size_t BLOCK_ALLOC_SIZE = 65540; /// Appends the indicated data to the BufferArray /// modifying current position and total size. New -- cgit v1.2.3 From 46662a3010b2c920ae60b4ca246d56e3caee6f6f Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 18 Jun 2012 11:16:58 -0400 Subject: Move dtors for refcounted objects to protected access. --- indra/llcorehttp/bufferarray.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/llcorehttp/bufferarray.h') diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h index d3862b45e1..d0c51d3c73 100644 --- a/indra/llcorehttp/bufferarray.h +++ b/indra/llcorehttp/bufferarray.h @@ -66,7 +66,9 @@ class BufferArray : public LLCoreInt::RefCounted { public: BufferArray(); - virtual ~BufferArray(); + +protected: + virtual ~BufferArray(); // Use release() private: BufferArray(const BufferArray &); // Not defined -- cgit v1.2.3 From 4da93b6ad91dff1de98c3c8dd3674c0544f2958b Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 21 Jun 2012 19:45:40 -0400 Subject: SH-3177 Add streambuf/iostream adapters to BufferArray object. Initial version that should have enough of the plumbing to produce a working adapter. Memory test is showing 8 bytes held after one of the tests so I'm going to revisit that later. But basic functionality is there going by the unit tests. --- indra/llcorehttp/bufferarray.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'indra/llcorehttp/bufferarray.h') diff --git a/indra/llcorehttp/bufferarray.h b/indra/llcorehttp/bufferarray.h index d0c51d3c73..1094a435b4 100644 --- a/indra/llcorehttp/bufferarray.h +++ b/indra/llcorehttp/bufferarray.h @@ -37,6 +37,7 @@ namespace LLCore { +class BufferArrayStreamBuf; /// A very simple scatter/gather type map for bulk data. The motivation /// for this class is the writedata callback used by libcurl. Response @@ -65,6 +66,11 @@ namespace LLCore class BufferArray : public LLCoreInt::RefCounted { public: + // BufferArrayStreamBuf has intimate knowledge of this + // implementation to implement a buffer-free adapter. + // Changes here will likely need to be reflected there. + friend class BufferArrayStreamBuf; + BufferArray(); protected: @@ -114,6 +120,8 @@ public: protected: int findBlock(size_t pos, size_t * ret_offset); + + bool getBlockStartEnd(int block, const char ** start, const char ** end); protected: class Block; -- cgit v1.2.3