Add FIFO thread pool, use it for most things.

We like a LIFO default thread pool in tools like DM for better
memory/time locality... the bots use less memory this way, and
generally run faster.

But most use cases want a FIFO queue, so that they can get going
on the next parts of early work while later work is still running.

This splits the implementation into one using SkTArray and pop_back
for LIFO, and a new one using std::deque and pop_front for FIFO.

Change-Id: Ief203b6869a00f1f8084019431a781d15fc63750
Reviewed-on: https://skia-review.googlesource.com/41849
Commit-Queue: Mike Klein <mtklein@chromium.org>
Commit-Queue: Brian Osman <brianosman@google.com>
Reviewed-by: Yuqian Li <liyuqian@google.com>
Reviewed-by: Brian Osman <brianosman@google.com>

dm/DMSrcSink.cpp

@@ -1520,7 +1520,7 @@ GPUThreadTestingSink::GPUThreadTestingSink(GrContextFactory::ContextType ct,
                                            const GrContextOptions& grCtxOptions)
         : INHERITED(ct, overrides, samples, diText, colorType, alphaType, std::move(colorSpace),
                     threaded, grCtxOptions)
-        , fExecutor(SkExecutor::MakeThreadPool(FLAGS_gpuThreads)) {
+        , fExecutor(SkExecutor::MakeFIFOThreadPool(FLAGS_gpuThreads)) {
     SkASSERT(fExecutor);
 }
 

include/core/SkExecutor.h

@@ -16,7 +16,8 @@ public:
     virtual ~SkExecutor();
 
     // Create a thread pool SkExecutor with a fixed thread count, by default the number of cores.
-    static std::unique_ptr<SkExecutor> MakeThreadPool(int threads = 0);
+    static std::unique_ptr<SkExecutor> MakeFIFOThreadPool(int threads = 0);
+    static std::unique_ptr<SkExecutor> MakeLIFOThreadPool(int threads = 0);
 
     // There is always a default SkExecutor available by calling SkExecutor::GetDefault().
     static SkExecutor& GetDefault();

src/core/SkExecutor.cpp

@@ -12,6 +12,7 @@
 #include "SkSpinlock.h"
 #include "SkTArray.h"
 #include "SkThreadUtils.h"
+#include <deque>
 
 #if defined(SK_BUILD_FOR_WIN32)
     #include <windows.h>
@@ -46,7 +47,20 @@ void SkExecutor::SetDefault(SkExecutor* executor) {
     gDefaultExecutor = executor ? executor : &gTrivial;
 }
 
+// We'll always push_back() new work, but pop from the front of deques or the back of SkTArray.
+static inline std::function<void(void)> pop(std::deque<std::function<void(void)>>* list) {
+    std::function<void(void)> fn = std::move(list->front());
+    list->pop_front();
+    return fn;
+}
+static inline std::function<void(void)> pop(SkTArray<std::function<void(void)>>* list) {
+    std::function<void(void)> fn = std::move(list->back());
+    list->pop_back();
+    return fn;
+}
+
 // An SkThreadPool is an executor that runs work on a fixed pool of OS threads.
+template <typename WorkList>
 class SkThreadPool final : public SkExecutor {
 public:
     explicit SkThreadPool(int threads) {
@@ -91,8 +105,7 @@ private:
         {
             SkAutoExclusive lock(fWorkLock);
             SkASSERT(!fWork.empty());        // TODO: if (fWork.empty()) { return true; } ?
-            work = std::move(fWork.back());
-            fWork.pop_back();
+            work = pop(&fWork);
         }
 
         if (!work) {
@@ -114,11 +127,16 @@ private:
     using Lock = SkMutex;
 
     SkTArray<std::unique_ptr<SkThread>> fThreads;
-    SkTArray<std::function<void(void)>> fWork;
+    WorkList                            fWork;
     Lock                                fWorkLock;
     SkSemaphore                         fWorkAvailable;
 };
 
-std::unique_ptr<SkExecutor> SkExecutor::MakeThreadPool(int threads) {
-    return skstd::make_unique<SkThreadPool>(threads > 0 ? threads : num_cores());
+std::unique_ptr<SkExecutor> SkExecutor::MakeFIFOThreadPool(int threads) {
+    using WorkList = std::deque<std::function<void(void)>>;
+    return skstd::make_unique<SkThreadPool<WorkList>>(threads > 0 ? threads : num_cores());
+}
+std::unique_ptr<SkExecutor> SkExecutor::MakeLIFOThreadPool(int threads) {
+    using WorkList = SkTArray<std::function<void(void)>>;
+    return skstd::make_unique<SkThreadPool<WorkList>>(threads > 0 ? threads : num_cores());
 }

src/core/SkTaskGroup.cpp

@@ -45,7 +45,7 @@ void SkTaskGroup::wait() {
 
 SkTaskGroup::Enabler::Enabler(int threads) {
     if (threads) {
-        fThreadPool = SkExecutor::MakeThreadPool(threads);
+        fThreadPool = SkExecutor::MakeLIFOThreadPool(threads);
         SkExecutor::SetDefault(fThreadPool.get());
     }
 }

src/core/SkThreadedBMPDevice.cpp

@@ -218,7 +218,7 @@ SkThreadedBMPDevice::SkThreadedBMPDevice(const SkBitmap& bitmap,
         , fThreadCnt(threads <= 0 ? tiles : threads)
 {
     if (executor == nullptr) {
-        fInternalExecutor = SkExecutor::MakeThreadPool(fThreadCnt);
+        fInternalExecutor = SkExecutor::MakeFIFOThreadPool(fThreadCnt);
         executor = fInternalExecutor.get();
     }
     fExecutor = executor;

tests/GrContextFactoryTest.cpp

@@ -147,7 +147,7 @@ DEF_GPUTEST(GrContextFactory_executorAndTaskGroup, reporter, /*factory*/) {
     contextOptions.fExecutor = nullptr;
     GrContextFactory serialFactory(contextOptions);
 
-    std::unique_ptr<SkExecutor> threadPool = SkExecutor::MakeThreadPool(1);
+    std::unique_ptr<SkExecutor> threadPool = SkExecutor::MakeFIFOThreadPool(1);
     contextOptions.fExecutor = threadPool.get();
     GrContextFactory threadedFactory(contextOptions);
 

tools/flags/SkCommonFlags.cpp

@@ -131,6 +131,6 @@ bool CollectImages(SkCommandLineFlags::StringArray images, SkTArray<SkString>* o
 
 SkExecutor* GpuExecutorForTools() {
     static std::unique_ptr<SkExecutor> gGpuExecutor = (0 != FLAGS_gpuThreads)
-        ? SkExecutor::MakeThreadPool(FLAGS_gpuThreads) : nullptr;
+        ? SkExecutor::MakeFIFOThreadPool(FLAGS_gpuThreads) : nullptr;
     return gGpuExecutor.get();
 }