Add a small block recycling optimisation.

asio/include/Makefile.am

@@ -182,7 +182,9 @@ nobase_include_HEADERS = \
 	asio/detail/task_io_service_fwd.hpp \
 	asio/detail/task_io_service.hpp \
 	asio/detail/task_io_service_operation.hpp \
+	asio/detail/task_io_service_thread_info.hpp \
 	asio/detail/thread.hpp \
+	asio/detail/thread_info_base.hpp \
 	asio/detail/throw_error.hpp \
 	asio/detail/timer_queue_base.hpp \
 	asio/detail/timer_queue_fwd.hpp \
@@ -216,6 +218,7 @@ nobase_include_HEADERS = \
 	asio/detail/win_iocp_socket_send_op.hpp \
 	asio/detail/win_iocp_socket_service_base.hpp \
 	asio/detail/win_iocp_socket_service.hpp \
+	asio/detail/win_iocp_thread_info.hpp \
 	asio/detail/win_mutex.hpp \
 	asio/detail/win_object_handle_service.hpp \
 	asio/detail/winsock_init.hpp \
@@ -232,6 +235,7 @@ nobase_include_HEADERS = \
 	asio/impl/connect.hpp \
 	asio/impl/error_code.ipp \
 	asio/impl/error.ipp \
+	asio/impl/handler_alloc_hook.ipp \
 	asio/impl/io_service.hpp \
 	asio/impl/io_service.ipp \
 	asio/impl/read_at.hpp \

asio/include/asio/detail/call_stack.hpp

@@ -101,6 +101,13 @@ public:
     return 0;
   }
 
+  // Obtain the value at the top of the stack.
+  static Value* top()
+  {
+    context* elem = top_;
+    return elem ? elem->value_ : 0;
+  }
+
 private:
   // The top of the stack of calls for the current thread.
   static tss_ptr<context> top_;

asio/include/asio/detail/impl/task_io_service.ipp

@@ -23,20 +23,13 @@
 #include "asio/detail/event.hpp"
 #include "asio/detail/reactor.hpp"
 #include "asio/detail/task_io_service.hpp"
+#include "asio/detail/task_io_service_thread_info.hpp"
 
 #include "asio/detail/push_options.hpp"
 
 namespace asio {
 namespace detail {
 
-struct task_io_service::thread_info
-{
-  event* wakeup_event;
-  op_queue<operation> private_op_queue;
-  long private_outstanding_work;
-  thread_info* next;
-};
-
 struct task_io_service::task_cleanup
 {
   ~task_cleanup()

asio/include/asio/detail/impl/win_iocp_io_service.hpp

@@ -32,7 +32,7 @@ namespace detail {
 template <typename Handler>
 void win_iocp_io_service::dispatch(Handler handler)
 {
-  if (call_stack<win_iocp_io_service>::contains(this))
+  if (thread_call_stack::contains(this))
   {
     fenced_block b(fenced_block::full);
     asio_handler_invoke_helpers::invoke(handler, handler);

asio/include/asio/detail/impl/win_iocp_io_service.ipp

@@ -153,7 +153,8 @@ size_t win_iocp_io_service::run(asio::error_code& ec)
     return 0;
   }
 
-  call_stack<win_iocp_io_service>::context ctx(this);
+  win_iocp_thread_info this_thread;
+  thread_call_stack::context ctx(this, this_thread);
 
   size_t n = 0;
   while (do_one(true, ec))
@@ -171,7 +172,8 @@ size_t win_iocp_io_service::run_one(asio::error_code& ec)
     return 0;
   }
 
-  call_stack<win_iocp_io_service>::context ctx(this);
+  win_iocp_thread_info this_thread;
+  thread_call_stack::context ctx(this, this_thread);
 
   return do_one(true, ec);
 }
@@ -185,7 +187,8 @@ size_t win_iocp_io_service::poll(asio::error_code& ec)
     return 0;
   }
 
-  call_stack<win_iocp_io_service>::context ctx(this);
+  win_iocp_thread_info this_thread;
+  thread_call_stack::context ctx(this, this_thread);
 
   size_t n = 0;
   while (do_one(false, ec))
@@ -203,7 +206,8 @@ size_t win_iocp_io_service::poll_one(asio::error_code& ec)
     return 0;
   }
 
-  call_stack<win_iocp_io_service>::context ctx(this);
+  win_iocp_thread_info this_thread;
+  thread_call_stack::context ctx(this, this_thread);
 
   return do_one(false, ec);
 }

asio/include/asio/detail/task_io_service.hpp

@@ -127,7 +127,7 @@ public:
 
 private:
   // Structure containing information about an idle thread.
-  struct thread_info;
+  typedef task_io_service_thread_info thread_info;
 
   // Request invocation of the given operation, avoiding the thread-private
   // queue, and return immediately. Assumes that work_started() has not yet

asio/include/asio/detail/task_io_service_fwd.hpp

@@ -19,6 +19,8 @@ namespace asio {
 namespace detail {
 
 class task_io_service;
+class task_io_service_operation;
+struct task_io_service_thread_info;
 
 } // namespace detail
 } // namespace asio

asio/include/asio/detail/task_io_service_thread_info.hpp

@@ -0,0 +1,41 @@
+//
+// detail/task_io_service_thread_info.hpp
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Copyright (c) 2003-2012 Christopher M. Kohlhoff (chris at kohlhoff dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef ASIO_DETAIL_TASK_IO_SERVICE_THREAD_INFO_HPP
+#define ASIO_DETAIL_TASK_IO_SERVICE_THREAD_INFO_HPP
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1200)
+# pragma once
+#endif // defined(_MSC_VER) && (_MSC_VER >= 1200)
+
+#include "asio/detail/event.hpp"
+#include "asio/detail/op_queue.hpp"
+#include "asio/detail/task_io_service_fwd.hpp"
+#include "asio/detail/thread_info_base.hpp"
+
+#include "asio/detail/push_options.hpp"
+
+namespace asio {
+namespace detail {
+
+struct task_io_service_thread_info : public thread_info_base
+{
+  event* wakeup_event;
+  op_queue<task_io_service_operation> private_op_queue;
+  long private_outstanding_work;
+  task_io_service_thread_info* next;
+};
+
+} // namespace detail
+} // namespace asio
+
+#include "asio/detail/pop_options.hpp"
+
+#endif // ASIO_DETAIL_TASK_IO_SERVICE_THREAD_INFO_HPP

asio/include/asio/detail/thread_info_base.hpp

@@ -0,0 +1,90 @@
+//
+// detail/thread_info_base.hpp
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Copyright (c) 2003-2012 Christopher M. Kohlhoff (chris at kohlhoff dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef ASIO_DETAIL_THREAD_INFO_BASE_HPP
+#define ASIO_DETAIL_THREAD_INFO_BASE_HPP
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1200)
+# pragma once
+#endif // defined(_MSC_VER) && (_MSC_VER >= 1200)
+
+#include <cstddef>
+#include "asio/detail/noncopyable.hpp"
+
+#include "asio/detail/push_options.hpp"
+
+namespace asio {
+namespace detail {
+
+class thread_info_base
+  : private noncopyable
+{
+public:
+  thread_info_base()
+    : reusable_memory_(0)
+  {
+  }
+
+  ~thread_info_base()
+  {
+    if (reusable_memory_)
+      ::operator delete(reusable_memory_);
+  }
+
+  static void* allocate(thread_info_base* this_thread, std::size_t size)
+  {
+    if (this_thread && this_thread->reusable_memory_)
+    {
+      void* const pointer = this_thread->reusable_memory_;
+      this_thread->reusable_memory_ = 0;
+
+      unsigned char* const mem = static_cast<unsigned char*>(pointer);
+      if (static_cast<std::size_t>(mem[0]) >= size)
+      {
+        mem[size] = mem[0];
+        return pointer;
+      }
+
+      ::operator delete(pointer);
+    }
+
+    void* const pointer = ::operator new(size + 1);
+    unsigned char* const mem = static_cast<unsigned char*>(pointer);
+    mem[size] = (size <= UCHAR_MAX) ? static_cast<unsigned char>(size) : 0;
+    return pointer;
+  }
+
+  static void deallocate(thread_info_base* this_thread,
+      void* pointer, std::size_t size)
+  {
+    if (size <= UCHAR_MAX)
+    {
+      if (this_thread && this_thread->reusable_memory_ == 0)
+      {
+        unsigned char* const mem = static_cast<unsigned char*>(pointer);
+        mem[0] = mem[size];
+        this_thread->reusable_memory_ = pointer;
+        return;
+      }
+    }
+
+    ::operator delete(pointer);
+  }
+
+private:
+  void* reusable_memory_;
+};
+
+} // namespace detail
+} // namespace asio
+
+#include "asio/detail/pop_options.hpp"
+
+#endif // ASIO_DETAIL_THREAD_INFO_BASE_HPP

asio/include/asio/detail/win_iocp_io_service.hpp

@@ -33,6 +33,7 @@
 #include "asio/detail/wait_op.hpp"
 #include "asio/detail/win_iocp_io_service_fwd.hpp"
 #include "asio/detail/win_iocp_operation.hpp"
+#include "asio/detail/win_iocp_thread_info.hpp"
 
 #include "asio/detail/push_options.hpp"
 
@@ -106,7 +107,7 @@ public:
   // Return whether a handler can be dispatched immediately.
   bool can_dispatch()
   {
-    return call_stack<win_iocp_io_service>::contains(this) != 0;
+    return thread_call_stack::contains(this) != 0;
   }
 
   // Request invocation of the given handler.
@@ -289,6 +290,10 @@ private:
 
   // The operations that are ready to dispatch.
   op_queue<win_iocp_operation> completed_ops_;
+
+  // Per-thread call stack to track the state of each thread in the io_service.
+  typedef call_stack<win_iocp_io_service,
+      win_iocp_thread_info> thread_call_stack;
 };
 
 } // namespace detail

asio/include/asio/detail/win_iocp_thread_info.hpp

@@ -0,0 +1,34 @@
+//
+// detail/win_iocp_thread_info.hpp
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Copyright (c) 2003-2012 Christopher M. Kohlhoff (chris at kohlhoff dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef ASIO_DETAIL_WIN_IOCP_THREAD_INFO_HPP
+#define ASIO_DETAIL_WIN_IOCP_THREAD_INFO_HPP
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1200)
+# pragma once
+#endif // defined(_MSC_VER) && (_MSC_VER >= 1200)
+
+#include "asio/detail/thread_info_base.hpp"
+
+#include "asio/detail/push_options.hpp"
+
+namespace asio {
+namespace detail {
+
+struct win_iocp_thread_info : public thread_info_base
+{
+};
+
+} // namespace detail
+} // namespace asio
+
+#include "asio/detail/pop_options.hpp"
+
+#endif // ASIO_DETAIL_WIN_IOCP_THREAD_INFO_HPP

asio/include/asio/handler_alloc_hook.hpp

@@ -31,10 +31,8 @@ namespace asio {
  * Implement asio_handler_allocate and asio_handler_deallocate for your own
  * handlers to provide custom allocation for these temporary objects.
  *
- * This default implementation is simply:
- * @code
- * return ::operator new(size);
- * @endcode
+ * The default implementation of these allocation hooks uses <tt>::operator
+ * new</tt> and <tt>::operator delete</tt>.
  *
  * @note All temporary objects associated with a handler will be deallocated
  * before the upcall to the handler is performed. This allows the same memory to
@@ -56,31 +54,28 @@ namespace asio {
  * }
  * @endcode
  */
-inline void* asio_handler_allocate(std::size_t size, ...)
-{
-  return ::operator new(size);
-}
+ASIO_DECL void* asio_handler_allocate(
+    std::size_t size, ...);
 
 /// Default deallocation function for handlers.
 /**
  * Implement asio_handler_allocate and asio_handler_deallocate for your own
  * handlers to provide custom allocation for the associated temporary objects.
  *
- * This default implementation is simply:
- * @code
- * ::operator delete(pointer);
- * @endcode
+ * The default implementation of these allocation hooks uses <tt>::operator
+ * new</tt> and <tt>::operator delete</tt>.
  *
  * @sa asio_handler_allocate.
  */
-inline void asio_handler_deallocate(void* pointer, std::size_t size, ...)
-{
-  (void)(size);
-  ::operator delete(pointer);
-}
+ASIO_DECL void asio_handler_deallocate(
+    void* pointer, std::size_t size, ...);
 
 } // namespace asio
 
 #include "asio/detail/pop_options.hpp"
 
+#if defined(ASIO_HEADER_ONLY)
+# include "asio/impl/handler_alloc_hook.ipp"
+#endif // defined(ASIO_HEADER_ONLY)
+
 #endif // ASIO_HANDLER_ALLOC_HOOK_HPP

asio/include/asio/impl/handler_alloc_hook.ipp

@@ -0,0 +1,74 @@
+//
+// impl/handler_alloc_hook.ipp
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Copyright (c) 2003-2012 Christopher M. Kohlhoff (chris at kohlhoff dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef ASIO_IMPL_HANDLER_ALLOC_HOOK_IPP
+#define ASIO_IMPL_HANDLER_ALLOC_HOOK_IPP
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1200)
+# pragma once
+#endif // defined(_MSC_VER) && (_MSC_VER >= 1200)
+
+#include "asio/detail/config.hpp"
+#include "asio/detail/call_stack.hpp"
+#include "asio/handler_alloc_hook.hpp"
+
+#if !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+# if defined(ASIO_HAS_IOCP)
+#  include "asio/detail/win_iocp_io_service_fwd.hpp"
+#  include "asio/detail/win_iocp_thread_info.hpp"
+# else // defined(ASIO_HAS_IOCP)
+#  include "asio/detail/task_io_service_thread_info.hpp"
+# endif // defined(ASIO_HAS_IOCP)
+#endif // !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+
+#include "asio/detail/push_options.hpp"
+
+namespace asio {
+
+void* asio_handler_allocate(std::size_t size, ...)
+{
+#if !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+# if defined(ASIO_HAS_IOCP)
+  typedef detail::win_iocp_io_service io_service_impl;
+  typedef detail::win_iocp_thread_info thread_info;
+# else // defined(ASIO_HAS_IOCP)
+  typedef detail::task_io_service io_service_impl;
+  typedef detail::task_io_service_thread_info thread_info;
+# endif // defined(ASIO_HAS_IOCP)
+  typedef detail::call_stack<io_service_impl, thread_info> call_stack;
+  return thread_info::allocate(call_stack::top(), size);
+#else // !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+  return ::operator new(size);
+#endif // !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+}
+
+void asio_handler_deallocate(void* pointer, std::size_t size, ...)
+{
+#if !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+# if defined(ASIO_HAS_IOCP)
+  typedef detail::win_iocp_io_service io_service_impl;
+  typedef detail::win_iocp_thread_info thread_info;
+# else // defined(ASIO_HAS_IOCP)
+  typedef detail::task_io_service io_service_impl;
+  typedef detail::task_io_service_thread_info thread_info;
+# endif // defined(ASIO_HAS_IOCP)
+  typedef detail::call_stack<io_service_impl, thread_info> call_stack;
+  thread_info::deallocate(call_stack::top(), pointer, size);
+#else // !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+  (void)size;
+  ::operator delete(pointer);
+#endif // !defined(ASIO_DISABLE_SMALL_BLOCK_RECYCLING)
+}
+
+} // namespace asio
+
+#include "asio/detail/pop_options.hpp"
+
+#endif // ASIO_IMPL_HANDLER_ALLOC_HOOK_IPP

asio/include/asio/impl/src.hpp

@@ -21,6 +21,7 @@
 
 #include "asio/impl/error.ipp"
 #include "asio/impl/error_code.ipp"
+#include "asio/impl/handler_alloc_hook.ipp"
 #include "asio/impl/io_service.ipp"
 #include "asio/impl/serial_port_base.ipp"
 #include "asio/detail/impl/descriptor_ops.ipp"