Revert "Perform Vulkan resets in a background thread"

This reverts commit 30e6343b35.

Reason for revert: occasional failures

Original change's description:
> Perform Vulkan resets in a background thread
> 
> Bug: skia:
> Change-Id: I90783dc9ac2fcae560cd54e6c8c6df11d7195ac0
> Reviewed-on: https://skia-review.googlesource.com/c/168488
> Commit-Queue: Ethan Nicholas <ethannicholas@google.com>
> Reviewed-by: Greg Daniel <egdaniel@google.com>

TBR=egdaniel@google.com,ethannicholas@google.com

Change-Id: Iab0d65331c44fa8976bd35e5efea4b42174836b0
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: skia:
Reviewed-on: https://skia-review.googlesource.com/c/177061
Reviewed-by: Ethan Nicholas <ethannicholas@google.com>
Commit-Queue: Ethan Nicholas <ethannicholas@google.com>

gn/gpu.gni

@@ -584,8 +584,6 @@ skia_vk_sources = [
   "$_src/gpu/vk/GrVkCaps.h",
   "$_src/gpu/vk/GrVkCommandBuffer.cpp",
   "$_src/gpu/vk/GrVkCommandBuffer.h",
-  "$_src/gpu/vk/GrVkCommandPool.cpp",
-  "$_src/gpu/vk/GrVkCommandPool.h",
   "$_src/gpu/vk/GrVkCopyManager.cpp",
   "$_src/gpu/vk/GrVkCopyManager.h",
   "$_src/gpu/vk/GrVkCopyPipeline.cpp",

src/gpu/vk/GrVkBuffer.cpp

@@ -102,7 +102,7 @@ void GrVkBuffer::addMemoryBarrier(const GrVkGpu* gpu,
     gpu->addBufferMemoryBarrier(srcStageMask, dstStageMask, byRegion, &bufferMemoryBarrier);
 }
 
-void GrVkBuffer::Resource::freeGPUData(GrVkGpu* gpu) const {
+void GrVkBuffer::Resource::freeGPUData(const GrVkGpu* gpu) const {
     SkASSERT(fBuffer);
     SkASSERT(fAlloc.fMemory);
     VK_CALL(gpu, DestroyBuffer(gpu->device(), fBuffer, nullptr));

src/gpu/vk/GrVkBuffer.h

@@ -71,7 +71,7 @@ protected:
         Type               fType;
 
     private:
-        void freeGPUData(GrVkGpu* gpu) const override;
+        void freeGPUData(const GrVkGpu* gpu) const override;
 
         void onRecycle(GrVkGpu* gpu) const override { this->unref(gpu); }
 

src/gpu/vk/GrVkBufferView.cpp

@@ -33,6 +33,6 @@ const GrVkBufferView* GrVkBufferView::Create(const GrVkGpu* gpu, VkBuffer buffer
     return new GrVkBufferView(bufferView);
 }
 
-void GrVkBufferView::freeGPUData(GrVkGpu* gpu) const {
+void GrVkBufferView::freeGPUData(const GrVkGpu* gpu) const {
     GR_VK_CALL(gpu->vkInterface(), DestroyBufferView(gpu->device(), fBufferView, nullptr));
 }

src/gpu/vk/GrVkBufferView.h

@@ -29,7 +29,7 @@ public:
 private:
     GrVkBufferView(VkBufferView bufferView) : INHERITED(), fBufferView(bufferView) {}
 
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
 
     VkBufferView  fBufferView;
 

src/gpu/vk/GrVkCommandBuffer.cpp

@@ -7,7 +7,6 @@
 
 #include "GrVkCommandBuffer.h"
 
-#include "GrVkCommandPool.h"
 #include "GrVkGpu.h"
 #include "GrVkFramebuffer.h"
 #include "GrVkImage.h"
@@ -41,7 +40,7 @@ void GrVkCommandBuffer::invalidateState() {
     }
 }
 
-void GrVkCommandBuffer::freeGPUData(GrVkGpu* gpu) const {
+void GrVkCommandBuffer::freeGPUData(const GrVkGpu* gpu) const {
     SkASSERT(!fIsActive);
     for (int i = 0; i < fTrackedResources.count(); ++i) {
         fTrackedResources[i]->unref(gpu);
@@ -55,14 +54,13 @@ void GrVkCommandBuffer::freeGPUData(GrVkGpu* gpu) const {
         fTrackedRecordingResources[i]->unref(gpu);
     }
 
-    GR_VK_CALL(gpu->vkInterface(), FreeCommandBuffers(gpu->device(), fCmdPool->vkCommandPool(), 1,
+    GR_VK_CALL(gpu->vkInterface(), FreeCommandBuffers(gpu->device(), gpu->cmdPool(),
-                                                      &fCmdBuffer));
+                                                      1, &fCmdBuffer));
 
     this->onFreeGPUData(gpu);
 }
 
 void GrVkCommandBuffer::abandonGPUData() const {
-    SkDEBUGCODE(fResourcesReleased = true;)
     for (int i = 0; i < fTrackedResources.count(); ++i) {
         fTrackedResources[i]->unrefAndAbandon();
     }
@@ -75,12 +73,9 @@ void GrVkCommandBuffer::abandonGPUData() const {
     for (int i = 0; i < fTrackedRecordingResources.count(); ++i) {
         fTrackedRecordingResources[i]->unrefAndAbandon();
     }
-
-    this->onAbandonGPUData();
 }
 
-void GrVkCommandBuffer::releaseResources(GrVkGpu* gpu) {
+void GrVkCommandBuffer::reset(GrVkGpu* gpu) {
-    SkDEBUGCODE(fResourcesReleased = true;)
     SkASSERT(!fIsActive);
     for (int i = 0; i < fTrackedResources.count(); ++i) {
         fTrackedResources[i]->unref(gpu);
@@ -107,9 +102,14 @@ void GrVkCommandBuffer::releaseResources(GrVkGpu* gpu) {
         fTrackedRecordingResources.rewind();
     }
 
+
     this->invalidateState();
 
-    this->onReleaseResources(gpu);
+    // we will retain resources for later use
+    VkCommandBufferResetFlags flags = 0;
+    GR_VK_CALL(gpu->vkInterface(), ResetCommandBuffer(fCmdBuffer, flags));
+
+    this->onReset(gpu);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -358,11 +358,11 @@ GrVkPrimaryCommandBuffer::~GrVkPrimaryCommandBuffer() {
 }
 
 GrVkPrimaryCommandBuffer* GrVkPrimaryCommandBuffer::Create(const GrVkGpu* gpu,
-                                                           GrVkCommandPool* cmdPool) {
+                                                           VkCommandPool cmdPool) {
     const VkCommandBufferAllocateInfo cmdInfo = {
         VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,   // sType
         nullptr,                                          // pNext
-        cmdPool->vkCommandPool(),                         // commandPool
+        cmdPool,                                          // commandPool
         VK_COMMAND_BUFFER_LEVEL_PRIMARY,                  // level
         1                                                 // bufferCount
     };
@@ -374,7 +374,7 @@ GrVkPrimaryCommandBuffer* GrVkPrimaryCommandBuffer::Create(const GrVkGpu* gpu,
     if (err) {
         return nullptr;
     }
-    return new GrVkPrimaryCommandBuffer(cmdBuffer, cmdPool);
+    return new GrVkPrimaryCommandBuffer(cmdBuffer);
 }
 
 void GrVkPrimaryCommandBuffer::begin(const GrVkGpu* gpu) {
@@ -391,7 +391,7 @@ void GrVkPrimaryCommandBuffer::begin(const GrVkGpu* gpu) {
     fIsActive = true;
 }
 
-void GrVkPrimaryCommandBuffer::end(GrVkGpu* gpu) {
+void GrVkPrimaryCommandBuffer::end(const GrVkGpu* gpu) {
     SkASSERT(fIsActive);
     SkASSERT(!fActiveRenderPass);
     GR_VK_CALL_ERRCHECK(gpu->vkInterface(), EndCommandBuffer(fCmdBuffer));
@@ -445,10 +445,6 @@ void GrVkPrimaryCommandBuffer::endRenderPass(const GrVkGpu* gpu) {
 
 void GrVkPrimaryCommandBuffer::executeCommands(const GrVkGpu* gpu,
                                                GrVkSecondaryCommandBuffer* buffer) {
-    // The Vulkan spec allows secondary command buffers to be executed on a primary command buffer
-    // if the command pools both were created from were created with the same queue family. However,
-    // we currently always create them from the same pool.
-    SkASSERT(buffer->commandPool() == fCmdPool);
     SkASSERT(fIsActive);
     SkASSERT(!buffer->fIsActive);
     SkASSERT(fActiveRenderPass);
@@ -569,7 +565,6 @@ void GrVkPrimaryCommandBuffer::submitToQueue(
 }
 
 bool GrVkPrimaryCommandBuffer::finished(const GrVkGpu* gpu) const {
-    SkASSERT(!fIsActive);
     if (VK_NULL_HANDLE == fSubmitFence) {
         return true;
     }
@@ -591,16 +586,9 @@ bool GrVkPrimaryCommandBuffer::finished(const GrVkGpu* gpu) const {
     return false;
 }
 
-void GrVkPrimaryCommandBuffer::onReleaseResources(GrVkGpu* gpu) {
+void GrVkPrimaryCommandBuffer::onReset(GrVkGpu* gpu) {
     for (int i = 0; i < fSecondaryCommandBuffers.count(); ++i) {
-        fSecondaryCommandBuffers[i]->releaseResources(gpu);
+        gpu->resourceProvider().recycleSecondaryCommandBuffer(fSecondaryCommandBuffers[i]);
-    }
-}
-
-void GrVkPrimaryCommandBuffer::recycleSecondaryCommandBuffers() {
-    for (int i = 0; i < fSecondaryCommandBuffers.count(); ++i) {
-        SkASSERT(fSecondaryCommandBuffers[i]->commandPool() == fCmdPool);
-        fCmdPool->recycleSecondaryCommandBuffer(fSecondaryCommandBuffers[i]);
     }
     fSecondaryCommandBuffers.reset();
 }
@@ -802,22 +790,12 @@ void GrVkPrimaryCommandBuffer::resolveImage(GrVkGpu* gpu,
                                                    regions));
 }
 
-void GrVkPrimaryCommandBuffer::onFreeGPUData(GrVkGpu* gpu) const {
+void GrVkPrimaryCommandBuffer::onFreeGPUData(const GrVkGpu* gpu) const {
     SkASSERT(!fActiveRenderPass);
     // Destroy the fence, if any
     if (VK_NULL_HANDLE != fSubmitFence) {
         GR_VK_CALL(gpu->vkInterface(), DestroyFence(gpu->device(), fSubmitFence, nullptr));
     }
-    for (GrVkSecondaryCommandBuffer* buffer : fSecondaryCommandBuffers) {
-        buffer->unref(gpu);
-    }
-}
-
-void GrVkPrimaryCommandBuffer::onAbandonGPUData() const {
-    SkASSERT(!fActiveRenderPass);
-    for (GrVkSecondaryCommandBuffer* buffer : fSecondaryCommandBuffers) {
-        buffer->unrefAndAbandon();
-    }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -825,11 +803,11 @@ void GrVkPrimaryCommandBuffer::onAbandonGPUData() const {
 ////////////////////////////////////////////////////////////////////////////////
 
 GrVkSecondaryCommandBuffer* GrVkSecondaryCommandBuffer::Create(const GrVkGpu* gpu,
-                                                               GrVkCommandPool* cmdPool) {
+                                                               VkCommandPool cmdPool) {
     const VkCommandBufferAllocateInfo cmdInfo = {
         VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,   // sType
         nullptr,                                          // pNext
-        cmdPool->vkCommandPool(),                         // commandPool
+        cmdPool,                                          // commandPool
         VK_COMMAND_BUFFER_LEVEL_SECONDARY,                // level
         1                                                 // bufferCount
     };
@@ -841,7 +819,7 @@ GrVkSecondaryCommandBuffer* GrVkSecondaryCommandBuffer::Create(const GrVkGpu* gp
     if (err) {
         return nullptr;
     }
-    return new GrVkSecondaryCommandBuffer(cmdBuffer, cmdPool);
+    return new GrVkSecondaryCommandBuffer(cmdBuffer);
 }
 
 
@@ -875,9 +853,10 @@ void GrVkSecondaryCommandBuffer::begin(const GrVkGpu* gpu, const GrVkFramebuffer
     fIsActive = true;
 }
 
-void GrVkSecondaryCommandBuffer::end(GrVkGpu* gpu) {
+void GrVkSecondaryCommandBuffer::end(const GrVkGpu* gpu) {
     SkASSERT(fIsActive);
     GR_VK_CALL_ERRCHECK(gpu->vkInterface(), EndCommandBuffer(fCmdBuffer));
     this->invalidateState();
     fIsActive = false;
 }
+

src/gpu/vk/GrVkCommandBuffer.h

@@ -71,8 +71,6 @@ public:
                             uint32_t dynamicOffsetCount,
                             const uint32_t* dynamicOffsets);
 
-    GrVkCommandPool* commandPool() { return fCmdPool; }
-
     void setViewport(const GrVkGpu* gpu,
                      uint32_t firstViewport,
                      uint32_t viewportCount,
@@ -126,15 +124,13 @@ public:
         fTrackedRecordingResources.append(1, &resource);
     }
 
-    void releaseResources(GrVkGpu* gpu);
+    void reset(GrVkGpu* gpu);
 
 protected:
-        GrVkCommandBuffer(VkCommandBuffer cmdBuffer, GrVkCommandPool* cmdPool,
+        GrVkCommandBuffer(VkCommandBuffer cmdBuffer, const GrVkRenderPass* rp = VK_NULL_HANDLE)
-                          const GrVkRenderPass* rp = VK_NULL_HANDLE)
             : fIsActive(false)
             , fActiveRenderPass(rp)
             , fCmdBuffer(cmdBuffer)
-            , fCmdPool(cmdPool)
             , fNumResets(0) {
             fTrackedResources.setReserve(kInitialTrackedResourcesCount);
             fTrackedRecycledResources.setReserve(kInitialTrackedResourcesCount);
@@ -157,19 +153,14 @@ protected:
 
         VkCommandBuffer           fCmdBuffer;
 
-        // Raw pointer, not refcounted. The command pool controls the command buffer's lifespan, so
-        // it's guaranteed to outlive us.
-        GrVkCommandPool*          fCmdPool;
-
 private:
     static const int kInitialTrackedResourcesCount = 32;
 
-    void freeGPUData(GrVkGpu* gpu) const final override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
-    virtual void onFreeGPUData(GrVkGpu* gpu) const = 0;
+    virtual void onFreeGPUData(const GrVkGpu* gpu) const = 0;
-    void abandonGPUData() const final override;
+    void abandonGPUData() const override;
-    virtual void onAbandonGPUData() const = 0;
 
-    virtual void onReleaseResources(GrVkGpu* gpu) {}
+    virtual void onReset(GrVkGpu* gpu) {}
 
     static constexpr uint32_t kMaxInputBuffers = 2;
 
@@ -187,10 +178,6 @@ private:
     VkViewport fCachedViewport;
     VkRect2D   fCachedScissor;
     float      fCachedBlendConstant[4];
-
-#ifdef SK_DEBUG
-    mutable bool fResourcesReleased = false;
-#endif
 };
 
 class GrVkSecondaryCommandBuffer;
@@ -199,10 +186,10 @@ class GrVkPrimaryCommandBuffer : public GrVkCommandBuffer {
 public:
     ~GrVkPrimaryCommandBuffer() override;
 
-    static GrVkPrimaryCommandBuffer* Create(const GrVkGpu* gpu, GrVkCommandPool* cmdPool);
+    static GrVkPrimaryCommandBuffer* Create(const GrVkGpu* gpu, VkCommandPool cmdPool);
 
     void begin(const GrVkGpu* gpu);
-    void end(GrVkGpu* gpu);
+    void end(const GrVkGpu* gpu);
 
     // Begins render pass on this command buffer. The framebuffer from GrVkRenderTarget will be used
     // in the render pass.
@@ -296,8 +283,6 @@ public:
                        SkTArray<GrVkSemaphore::Resource*>& waitSemaphores);
     bool finished(const GrVkGpu* gpu) const;
 
-    void recycleSecondaryCommandBuffers();
-
 #ifdef SK_TRACE_VK_RESOURCES
     void dumpInfo() const override {
         SkDebugf("GrVkPrimaryCommandBuffer: %d (%d refs)\n", fCmdBuffer, this->getRefCnt());
@@ -305,15 +290,13 @@ public:
 #endif
 
 private:
-    explicit GrVkPrimaryCommandBuffer(VkCommandBuffer cmdBuffer, GrVkCommandPool* cmdPool)
+    explicit GrVkPrimaryCommandBuffer(VkCommandBuffer cmdBuffer)
-        : INHERITED(cmdBuffer, cmdPool)
+        : INHERITED(cmdBuffer)
         , fSubmitFence(VK_NULL_HANDLE) {}
 
-    void onFreeGPUData(GrVkGpu* gpu) const override;
+    void onFreeGPUData(const GrVkGpu* gpu) const override;
 
-    void onAbandonGPUData() const override;
+    void onReset(GrVkGpu* gpu) override;
-
-    void onReleaseResources(GrVkGpu* gpu) override;
 
     SkTArray<GrVkSecondaryCommandBuffer*, true> fSecondaryCommandBuffers;
     VkFence                                     fSubmitFence;
@@ -323,11 +306,11 @@ private:
 
 class GrVkSecondaryCommandBuffer : public GrVkCommandBuffer {
 public:
-    static GrVkSecondaryCommandBuffer* Create(const GrVkGpu* gpu, GrVkCommandPool* cmdPool);
+    static GrVkSecondaryCommandBuffer* Create(const GrVkGpu* gpu, VkCommandPool cmdPool);
 
     void begin(const GrVkGpu* gpu, const GrVkFramebuffer* framebuffer,
                const GrVkRenderPass* compatibleRenderPass);
-    void end(GrVkGpu* gpu);
+    void end(const GrVkGpu* gpu);
 
     VkCommandBuffer vkCommandBuffer() { return fCmdBuffer; }
 
@@ -338,12 +321,11 @@ public:
 #endif
 
 private:
-    explicit GrVkSecondaryCommandBuffer(VkCommandBuffer cmdBuffer, GrVkCommandPool* cmdPool)
+    explicit GrVkSecondaryCommandBuffer(VkCommandBuffer cmdBuffer)
-        : INHERITED(cmdBuffer, cmdPool) {}
+        : INHERITED(cmdBuffer) {
+    }
 
-    void onFreeGPUData(GrVkGpu* gpu) const override {}
+    void onFreeGPUData(const GrVkGpu* gpu) const override {}
-
-    void onAbandonGPUData() const override {}
 
     friend class GrVkPrimaryCommandBuffer;
 

src/gpu/vk/GrVkCommandPool.cpp

@@ -1,84 +0,0 @@
-/*
- * Copyright 2018 Google Inc.
- *
- * Use of this source code is governed by a BSD-style license that can be
- * found in the LICENSE file.
- */
-
-#include "GrVkCommandPool.h"
-
-#include "GrContextPriv.h"
-#include "GrVkCommandBuffer.h"
-#include "GrVkGpu.h"
-
-GrVkCommandPool* GrVkCommandPool::Create(const GrVkGpu* gpu) {
-    const VkCommandPoolCreateInfo cmdPoolInfo = {
-        VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,      // sType
-        nullptr,                                         // pNext
-        VK_COMMAND_POOL_CREATE_TRANSIENT_BIT |
-        VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, // CmdPoolCreateFlags
-        gpu->queueIndex(),                              // queueFamilyIndex
-    };
-    VkCommandPool pool;
-    GR_VK_CALL_ERRCHECK(gpu->vkInterface(), CreateCommandPool(gpu->device(), &cmdPoolInfo,
-                                                               nullptr, &pool));
-    return new GrVkCommandPool(gpu, pool);
-}
-
-GrVkCommandPool::GrVkCommandPool(const GrVkGpu* gpu, VkCommandPool commandPool)
-        : fCommandPool(commandPool) {
-    fPrimaryCommandBuffer = GrVkPrimaryCommandBuffer::Create(gpu, this);
-}
-
-GrVkSecondaryCommandBuffer* GrVkCommandPool::findOrCreateSecondaryCommandBuffer(GrVkGpu* gpu) {
-    if (fAvailableSecondaryBuffers.count()) {
-        GrVkSecondaryCommandBuffer* result = fAvailableSecondaryBuffers.back();
-        fAvailableSecondaryBuffers.pop_back();
-        return result;
-    }
-    return GrVkSecondaryCommandBuffer::Create(gpu, this);
-}
-
-void GrVkCommandPool::recycleSecondaryCommandBuffer(GrVkSecondaryCommandBuffer* buffer) {
-    SkASSERT(buffer->commandPool() == this);
-    fAvailableSecondaryBuffers.push_back(buffer);
-}
-
-void GrVkCommandPool::close() {
-    fOpen = false;
-}
-
-void GrVkCommandPool::reset(GrVkGpu* gpu) {
-    SkASSERT(!fOpen);
-    fOpen = true;
-    fPrimaryCommandBuffer->recycleSecondaryCommandBuffers();
-    GR_VK_CALL_ERRCHECK(gpu->vkInterface(), ResetCommandPool(gpu->device(), fCommandPool, 0));
-}
-
-void GrVkCommandPool::releaseResources(GrVkGpu* gpu) {
-    SkASSERT(!fOpen);
-    fPrimaryCommandBuffer->releaseResources(gpu);
-    for (GrVkSecondaryCommandBuffer* buffer : fAvailableSecondaryBuffers) {
-        buffer->releaseResources(gpu);
-    }
-}
-
-void GrVkCommandPool::abandonGPUData() const {
-    fPrimaryCommandBuffer->unrefAndAbandon();
-    for (GrVkSecondaryCommandBuffer* buffer : fAvailableSecondaryBuffers) {
-        SkASSERT(buffer->unique());
-        buffer->unrefAndAbandon();
-    }
-}
-
-void GrVkCommandPool::freeGPUData(GrVkGpu* gpu) const {
-    fPrimaryCommandBuffer->unref(gpu);
-    for (GrVkSecondaryCommandBuffer* buffer : fAvailableSecondaryBuffers) {
-        SkASSERT(buffer->unique());
-        buffer->unref(gpu);
-    }
-    if (fCommandPool != VK_NULL_HANDLE) {
-        GR_VK_CALL(gpu->vkInterface(),
-                   DestroyCommandPool(gpu->device(), fCommandPool, nullptr));
-    }
-}

src/gpu/vk/GrVkCommandPool.h

@@ -1,70 +0,0 @@
-/*
- * Copyright 2018 Google Inc.
- *
- * Use of this source code is governed by a BSD-style license that can be
- * found in the LICENSE file.
- */
-
-#ifndef GrVkCommandPool_DEFINED
-#define GrVkCommandPool_DEFINED
-
-#include "GrVkGpuCommandBuffer.h"
-#include "GrVkInterface.h"
-#include "GrVkResource.h"
-#include "GrVkResourceProvider.h"
-
-class GrVkPrimaryCommandBuffer;
-class GrVkSecondaryCommandBuffer;
-class GrVkGpu;
-
-class GrVkCommandPool : public GrVkResource {
-public:
-    static GrVkCommandPool* Create(const GrVkGpu* gpu);
-
-    VkCommandPool vkCommandPool() const {
-        return fCommandPool;
-    }
-
-    void reset(GrVkGpu* gpu);
-
-    void releaseResources(GrVkGpu* gpu);
-
-    GrVkPrimaryCommandBuffer* getPrimaryCommandBuffer() { return fPrimaryCommandBuffer; }
-
-    GrVkSecondaryCommandBuffer* findOrCreateSecondaryCommandBuffer(GrVkGpu* gpu);
-
-    void recycleSecondaryCommandBuffer(GrVkSecondaryCommandBuffer* buffer);
-
-    // marks that we are finished with this command pool; it is not legal to continue creating or
-    // writing to command buffers in a closed pool
-    void close();
-
-    // returns true if close() has not been called
-    bool isOpen() const { return fOpen; }
-
-#ifdef SK_DEBUG
-    void dumpInfo() const override {
-        SkDebugf("GrVkCommandPool: %p (%d refs)\n", fCommandPool, this->getRefCnt());
-    }
-#endif
-
-private:
-    GrVkCommandPool() = delete;
-
-    GrVkCommandPool(const GrVkGpu* gpu, VkCommandPool commandPool);
-
-    void abandonGPUData() const override;
-
-    void freeGPUData(GrVkGpu* gpu) const override;
-
-    bool fOpen = true;
-
-    VkCommandPool fCommandPool;
-
-    GrVkPrimaryCommandBuffer* fPrimaryCommandBuffer;
-
-    // Array of available secondary command buffers that are not in flight
-    SkSTArray<4, GrVkSecondaryCommandBuffer*, true> fAvailableSecondaryBuffers;
-};
-
-#endif

src/gpu/vk/GrVkCopyManager.cpp

@@ -13,7 +13,6 @@
 #include "GrSurface.h"
 #include "GrTexturePriv.h"
 #include "GrVkCommandBuffer.h"
-#include "GrVkCommandPool.h"
 #include "GrVkCopyPipeline.h"
 #include "GrVkDescriptorSet.h"
 #include "GrVkGpu.h"
@@ -352,7 +351,8 @@ bool GrVkCopyManager::copySurfaceAsDraw(GrVkGpu* gpu,
     GrVkPrimaryCommandBuffer* cmdBuffer = gpu->currentCommandBuffer();
     cmdBuffer->beginRenderPass(gpu, renderPass, nullptr, *rt, bounds, true);
 
-    GrVkSecondaryCommandBuffer* secondary = gpu->cmdPool()->findOrCreateSecondaryCommandBuffer(gpu);
+    GrVkSecondaryCommandBuffer* secondary =
+                                       gpu->resourceProvider().findOrCreateSecondaryCommandBuffer();
     if (!secondary) {
         return false;
     }

src/gpu/vk/GrVkDescriptorPool.cpp

@@ -44,7 +44,7 @@ void GrVkDescriptorPool::reset(const GrVkGpu* gpu) {
     GR_VK_CALL_ERRCHECK(gpu->vkInterface(), ResetDescriptorPool(gpu->device(), fDescPool, 0));
 }
 
-void GrVkDescriptorPool::freeGPUData(GrVkGpu* gpu) const {
+void GrVkDescriptorPool::freeGPUData(const GrVkGpu* gpu) const {
     // Destroying the VkDescriptorPool will automatically free and delete any VkDescriptorSets
     // allocated from the pool.
     GR_VK_CALL(gpu->vkInterface(), DestroyDescriptorPool(gpu->device(), fDescPool, nullptr));

src/gpu/vk/GrVkDescriptorPool.h

@@ -39,7 +39,7 @@ public:
 #endif
 
 private:
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
 
     VkDescriptorType     fType;
     uint32_t             fCount;

src/gpu/vk/GrVkDescriptorSet.cpp

@@ -20,7 +20,7 @@ GrVkDescriptorSet::GrVkDescriptorSet(VkDescriptorSet descSet,
     fPool->ref();
 }
 
-void GrVkDescriptorSet::freeGPUData(GrVkGpu* gpu) const {
+void GrVkDescriptorSet::freeGPUData(const GrVkGpu* gpu) const {
     fPool->unref(gpu);
 }
 

src/gpu/vk/GrVkDescriptorSet.h

@@ -33,7 +33,7 @@ public:
 #endif
 
 private:
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
     void abandonGPUData() const override;
     void onRecycle(GrVkGpu* gpu) const override;
 

src/gpu/vk/GrVkDescriptorSetManager.cpp

@@ -97,7 +97,7 @@ void GrVkDescriptorSetManager::recycleDescriptorSet(const GrVkDescriptorSet* des
     fFreeSets.push_back(descSet);
 }
 
-void GrVkDescriptorSetManager::release(GrVkGpu* gpu) {
+void GrVkDescriptorSetManager::release(const GrVkGpu* gpu) {
     fPoolManager.freeGPUResources(gpu);
 
     for (int i = 0; i < fFreeSets.count(); ++i) {
@@ -310,7 +310,7 @@ void GrVkDescriptorSetManager::DescriptorPoolManager::getNewDescriptorSet(GrVkGp
                                                                    ds));
 }
 
-void GrVkDescriptorSetManager::DescriptorPoolManager::freeGPUResources(GrVkGpu* gpu) {
+void GrVkDescriptorSetManager::DescriptorPoolManager::freeGPUResources(const GrVkGpu* gpu) {
     if (fDescLayout) {
         GR_VK_CALL(gpu->vkInterface(), DestroyDescriptorSetLayout(gpu->device(), fDescLayout,
                                                                   nullptr));

src/gpu/vk/GrVkDescriptorSetManager.h

@@ -37,7 +37,7 @@ public:
     ~GrVkDescriptorSetManager() {}
 
     void abandon();
-    void release(GrVkGpu* gpu);
+    void release(const GrVkGpu* gpu);
 
     VkDescriptorSetLayout layout() const { return fPoolManager.fDescLayout; }
 
@@ -63,7 +63,7 @@ private:
 
         void getNewDescriptorSet(GrVkGpu* gpu, VkDescriptorSet* ds);
 
-        void freeGPUResources(GrVkGpu* gpu);
+        void freeGPUResources(const GrVkGpu* gpu);
         void abandonGPUResources();
 
         VkDescriptorSetLayout  fDescLayout;

src/gpu/vk/GrVkFramebuffer.cpp

@@ -51,7 +51,7 @@ GrVkFramebuffer* GrVkFramebuffer::Create(GrVkGpu* gpu,
     return new GrVkFramebuffer(framebuffer);
 }
 
-void GrVkFramebuffer::freeGPUData(GrVkGpu* gpu) const {
+void GrVkFramebuffer::freeGPUData(const GrVkGpu* gpu) const {
     SkASSERT(fFramebuffer);
     GR_VK_CALL(gpu->vkInterface(), DestroyFramebuffer(gpu->device(), fFramebuffer, nullptr));
 }

src/gpu/vk/GrVkFramebuffer.h

@@ -39,7 +39,7 @@ private:
     GrVkFramebuffer(const GrVkFramebuffer&);
     GrVkFramebuffer& operator=(const GrVkFramebuffer&);
 
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
 
     VkFramebuffer  fFramebuffer;
 

src/gpu/vk/GrVkGpu.cpp

@@ -7,7 +7,6 @@
 
 #include "GrVkGpu.h"
 
-#include "GrContextPriv.h"
 #include "GrBackendSemaphore.h"
 #include "GrBackendSurface.h"
 #include "GrContextOptions.h"
@@ -19,7 +18,6 @@
 #include "GrTexturePriv.h"
 #include "GrVkAMDMemoryAllocator.h"
 #include "GrVkCommandBuffer.h"
-#include "GrVkCommandPool.h"
 #include "GrVkGpuCommandBuffer.h"
 #include "GrVkImage.h"
 #include "GrVkIndexBuffer.h"
@@ -170,20 +168,31 @@ GrVkGpu::GrVkGpu(GrContext* context, const GrContextOptions& options,
     VK_CALL(GetPhysicalDeviceProperties(backendContext.fPhysicalDevice, &fPhysDevProps));
     VK_CALL(GetPhysicalDeviceMemoryProperties(backendContext.fPhysicalDevice, &fPhysDevMemProps));
 
-    fCmdPool = fResourceProvider.findOrCreateCommandPool();
+    const VkCommandPoolCreateInfo cmdPoolInfo = {
-    fCurrentCmdBuffer = fCmdPool->getPrimaryCommandBuffer();
+        VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,      // sType
-    SkASSERT(fCurrentCmdBuffer);
+        nullptr,                                         // pNext
+        VK_COMMAND_POOL_CREATE_TRANSIENT_BIT |
+        VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, // CmdPoolCreateFlags
+        backendContext.fGraphicsQueueIndex,              // queueFamilyIndex
+    };
+    GR_VK_CALL_ERRCHECK(this->vkInterface(), CreateCommandPool(fDevice, &cmdPoolInfo, nullptr,
+                                                               &fCmdPool));
+
+    // must call this after creating the CommandPool
     fResourceProvider.init();
+    fCurrentCmdBuffer = fResourceProvider.findOrCreatePrimaryCommandBuffer();
+    SkASSERT(fCurrentCmdBuffer);
     fCurrentCmdBuffer->begin(this);
 }
 
 void GrVkGpu::destroyResources() {
-    if (fCmdPool) {
+    if (fCurrentCmdBuffer) {
-        fCmdPool->getPrimaryCommandBuffer()->end(this);
+        fCurrentCmdBuffer->end(this);
-        fCmdPool->close();
+        fCurrentCmdBuffer->unref(this);
     }
 
     // wait for all commands to finish
+    fResourceProvider.checkCommandBuffers();
     VkResult res = VK_CALL(QueueWaitIdle(fQueue));
 
     // On windows, sometimes calls to QueueWaitIdle return before actually signalling the fences
@@ -204,11 +213,6 @@ void GrVkGpu::destroyResources() {
     SkASSERT(VK_SUCCESS == res || VK_ERROR_DEVICE_LOST == res);
 #endif
 
-    if (fCmdPool) {
-        fCmdPool->unref(this);
-        fCmdPool = nullptr;
-    }
-
     for (int i = 0; i < fSemaphoresToWaitOn.count(); ++i) {
         fSemaphoresToWaitOn[i]->unref(this);
     }
@@ -225,6 +229,10 @@ void GrVkGpu::destroyResources() {
     // must call this just before we destroy the command pool and VkDevice
     fResourceProvider.destroyResources(VK_ERROR_DEVICE_LOST == res);
 
+    if (fCmdPool != VK_NULL_HANDLE) {
+        VK_CALL(DestroyCommandPool(fDevice, fCmdPool, nullptr));
+    }
+
     fMemoryAllocator.reset();
 
     fQueue = VK_NULL_HANDLE;
@@ -246,9 +254,8 @@ void GrVkGpu::disconnect(DisconnectType type) {
         if (DisconnectType::kCleanup == type) {
             this->destroyResources();
         } else {
-            if (fCmdPool) {
+            if (fCurrentCmdBuffer) {
-                fCmdPool->unrefAndAbandon();
+                fCurrentCmdBuffer->unrefAndAbandon();
-                fCmdPool = nullptr;
             }
             for (int i = 0; i < fSemaphoresToWaitOn.count(); ++i) {
                 fSemaphoresToWaitOn[i]->unrefAndAbandon();
@@ -266,6 +273,7 @@ void GrVkGpu::disconnect(DisconnectType type) {
         fSemaphoresToWaitOn.reset();
         fSemaphoresToSignal.reset();
         fCurrentCmdBuffer = nullptr;
+        fCmdPool = VK_NULL_HANDLE;
         fDisconnected = true;
     }
 }
@@ -296,7 +304,7 @@ GrGpuTextureCommandBuffer* GrVkGpu::getCommandBuffer(GrTexture* texture, GrSurfa
 void GrVkGpu::submitCommandBuffer(SyncQueue sync) {
     SkASSERT(fCurrentCmdBuffer);
     fCurrentCmdBuffer->end(this);
-    fCmdPool->close();
+
     fCurrentCmdBuffer->submitToQueue(this, fQueue, sync, fSemaphoresToSignal, fSemaphoresToWaitOn);
 
     // We must delete and drawables that have been waitint till submit for us to destroy.
@@ -311,11 +319,13 @@ void GrVkGpu::submitCommandBuffer(SyncQueue sync) {
     }
     fSemaphoresToSignal.reset();
 
-    // Release old command pool and create a new one
-    fCmdPool->unref(this);
     fResourceProvider.checkCommandBuffers();
-    fCmdPool = fResourceProvider.findOrCreateCommandPool();
+
-    fCurrentCmdBuffer = fCmdPool->getPrimaryCommandBuffer();
+    // Release old command buffer and create a new one
+    fCurrentCmdBuffer->unref(this);
+    fCurrentCmdBuffer = fResourceProvider.findOrCreatePrimaryCommandBuffer();
+    SkASSERT(fCurrentCmdBuffer);
+
     fCurrentCmdBuffer->begin(this);
 }
 
@@ -1211,7 +1221,7 @@ bool GrVkGpu::createTestingOnlyVkImage(GrPixelConfig config, int w, int h, bool
     const VkCommandBufferAllocateInfo cmdInfo = {
         VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,   // sType
         nullptr,                                          // pNext
-        fCmdPool->vkCommandPool(),                        // commandPool
+        fCmdPool,                                         // commandPool
         VK_COMMAND_BUFFER_LEVEL_PRIMARY,                  // level
         1                                                 // bufferCount
     };
@@ -1280,7 +1290,7 @@ bool GrVkGpu::createTestingOnlyVkImage(GrPixelConfig config, int w, int h, bool
         GrVkMemory::FreeImageMemory(this, false, alloc);
         VK_CALL(DestroyImage(fDevice, image, nullptr));
         VK_CALL(EndCommandBuffer(cmdBuffer));
-        VK_CALL(FreeCommandBuffers(fDevice, fCmdPool->vkCommandPool(), 1, &cmdBuffer));
+        VK_CALL(FreeCommandBuffers(fDevice, fCmdPool, 1, &cmdBuffer));
         return false;
     }
 
@@ -1290,7 +1300,7 @@ bool GrVkGpu::createTestingOnlyVkImage(GrPixelConfig config, int w, int h, bool
         VK_CALL(DestroyImage(fDevice, image, nullptr));
         VK_CALL(DestroyBuffer(fDevice, buffer, nullptr));
         VK_CALL(EndCommandBuffer(cmdBuffer));
-        VK_CALL(FreeCommandBuffers(fDevice, fCmdPool->vkCommandPool(), 1, &cmdBuffer));
+        VK_CALL(FreeCommandBuffers(fDevice, fCmdPool, 1, &cmdBuffer));
         return false;
     }
 
@@ -1307,7 +1317,7 @@ bool GrVkGpu::createTestingOnlyVkImage(GrPixelConfig config, int w, int h, bool
             GrVkMemory::FreeBufferMemory(this, GrVkBuffer::kCopyRead_Type, bufferAlloc);
             VK_CALL(DestroyBuffer(fDevice, buffer, nullptr));
             VK_CALL(EndCommandBuffer(cmdBuffer));
-            VK_CALL(FreeCommandBuffers(fDevice, fCmdPool->vkCommandPool(), 1, &cmdBuffer));
+            VK_CALL(FreeCommandBuffers(fDevice, fCmdPool, 1, &cmdBuffer));
             return false;
         }
         currentWidth = SkTMax(1, currentWidth / 2);
@@ -1411,7 +1421,7 @@ bool GrVkGpu::createTestingOnlyVkImage(GrPixelConfig config, int w, int h, bool
         VK_CALL(DestroyImage(fDevice, image, nullptr));
         GrVkMemory::FreeBufferMemory(this, GrVkBuffer::kCopyRead_Type, bufferAlloc);
         VK_CALL(DestroyBuffer(fDevice, buffer, nullptr));
-        VK_CALL(FreeCommandBuffers(fDevice, fCmdPool->vkCommandPool(), 1, &cmdBuffer));
+        VK_CALL(FreeCommandBuffers(fDevice, fCmdPool, 1, &cmdBuffer));
         VK_CALL(DestroyFence(fDevice, fence, nullptr));
         SkDebugf("Fence failed to signal: %d\n", err);
         SK_ABORT("failing");
@@ -1423,7 +1433,7 @@ bool GrVkGpu::createTestingOnlyVkImage(GrPixelConfig config, int w, int h, bool
         GrVkMemory::FreeBufferMemory(this, GrVkBuffer::kCopyRead_Type, bufferAlloc);
         VK_CALL(DestroyBuffer(fDevice, buffer, nullptr));
     }
-    VK_CALL(FreeCommandBuffers(fDevice, fCmdPool->vkCommandPool(), 1, &cmdBuffer));
+    VK_CALL(FreeCommandBuffers(fDevice, fCmdPool, 1, &cmdBuffer));
     VK_CALL(DestroyFence(fDevice, fence, nullptr));
 
     info->fImage = image;

src/gpu/vk/GrVkGpu.h

@@ -24,7 +24,6 @@
 class GrPipeline;
 
 class GrVkBufferImpl;
-class GrVkCommandPool;
 class GrVkGpuRTCommandBuffer;
 class GrVkGpuTextureCommandBuffer;
 class GrVkMemoryAllocator;
@@ -57,7 +56,7 @@ public:
     VkDevice device() const { return fDevice; }
     VkQueue  queue() const { return fQueue; }
     uint32_t  queueIndex() const { return fQueueIndex; }
-    GrVkCommandPool* cmdPool() const { return fCmdPool; }
+    VkCommandPool cmdPool() const { return fCmdPool; }
     VkPhysicalDeviceProperties physicalDeviceProperties() const {
         return fPhysDevProps;
     }
@@ -260,10 +259,8 @@ private:
 
     // Created by GrVkGpu
     GrVkResourceProvider                                  fResourceProvider;
+    VkCommandPool                                         fCmdPool;
 
-    GrVkCommandPool*                                      fCmdPool;
-
-    // just a raw pointer; object's lifespan is managed by fCmdPool
     GrVkPrimaryCommandBuffer*                             fCurrentCmdBuffer;
 
     SkSTArray<1, GrVkSemaphore::Resource*>                fSemaphoresToWaitOn;

src/gpu/vk/GrVkGpuCommandBuffer.cpp

@@ -15,7 +15,6 @@
 #include "GrRenderTargetPriv.h"
 #include "GrTexturePriv.h"
 #include "GrVkCommandBuffer.h"
-#include "GrVkCommandPool.h"
 #include "GrVkGpu.h"
 #include "GrVkPipeline.h"
 #include "GrVkRenderPass.h"
@@ -123,7 +122,7 @@ void GrVkGpuRTCommandBuffer::init() {
         cbInfo.fLoadStoreState = LoadStoreState::kStartsWithDiscard;
     }
 
-    cbInfo.fCommandBuffers.push_back(fGpu->cmdPool()->findOrCreateSecondaryCommandBuffer(fGpu));
+    cbInfo.fCommandBuffers.push_back(fGpu->resourceProvider().findOrCreateSecondaryCommandBuffer());
     cbInfo.currentCmdBuf()->begin(fGpu, vkRT->framebuffer(), cbInfo.fRenderPass);
 }
 
@@ -462,7 +461,7 @@ void GrVkGpuRTCommandBuffer::addAdditionalCommandBuffer() {
 
     CommandBufferInfo& cbInfo = fCommandBufferInfos[fCurrentCmdInfo];
     cbInfo.currentCmdBuf()->end(fGpu);
-    cbInfo.fCommandBuffers.push_back(fGpu->cmdPool()->findOrCreateSecondaryCommandBuffer(fGpu));
+    cbInfo.fCommandBuffers.push_back(fGpu->resourceProvider().findOrCreateSecondaryCommandBuffer());
     cbInfo.currentCmdBuf()->begin(fGpu, vkRT->framebuffer(), cbInfo.fRenderPass);
 }
 
@@ -492,7 +491,7 @@ void GrVkGpuRTCommandBuffer::addAdditionalRenderPass() {
     }
     cbInfo.fLoadStoreState = LoadStoreState::kLoadAndStore;
 
-    cbInfo.fCommandBuffers.push_back(fGpu->cmdPool()->findOrCreateSecondaryCommandBuffer(fGpu));
+    cbInfo.fCommandBuffers.push_back(fGpu->resourceProvider().findOrCreateSecondaryCommandBuffer());
     // It shouldn't matter what we set the clear color to here since we will assume loading of the
     // attachment.
     memset(&cbInfo.fColorClearValue, 0, sizeof(VkClearValue));

src/gpu/vk/GrVkImage.cpp

@@ -216,7 +216,7 @@ GrVkImage::~GrVkImage() {
     SkASSERT(!fResource);
 }
 
-void GrVkImage::releaseImage(GrVkGpu* gpu) {
+void GrVkImage::releaseImage(const GrVkGpu* gpu) {
     if (fInfo.fCurrentQueueFamily != fInitialQueueFamily) {
         this->setImageLayout(gpu, this->currentLayout(), 0, 0, false, true);
     }
@@ -238,14 +238,14 @@ void GrVkImage::setResourceRelease(sk_sp<GrReleaseProcHelper> releaseHelper) {
     fResource->setRelease(std::move(releaseHelper));
 }
 
-void GrVkImage::Resource::freeGPUData(GrVkGpu* gpu) const {
+void GrVkImage::Resource::freeGPUData(const GrVkGpu* gpu) const {
     SkASSERT(!fReleaseHelper);
     VK_CALL(gpu, DestroyImage(gpu->device(), fImage, nullptr));
     bool isLinear = (VK_IMAGE_TILING_LINEAR == fImageTiling);
     GrVkMemory::FreeImageMemory(gpu, isLinear, fAlloc);
 }
 
-void GrVkImage::BorrowedResource::freeGPUData(GrVkGpu* gpu) const {
+void GrVkImage::BorrowedResource::freeGPUData(const GrVkGpu* gpu) const {
     this->invokeReleaseProc();
 }
 

src/gpu/vk/GrVkImage.h

@@ -115,7 +115,7 @@ public:
     static VkAccessFlags LayoutToSrcAccessMask(const VkImageLayout layout);
 
 protected:
-    void releaseImage(GrVkGpu* gpu);
+    void releaseImage(const GrVkGpu* gpu);
     void abandonImage();
 
     void setNewResource(VkImage image, const GrVkAlloc& alloc, VkImageTiling tiling);
@@ -155,7 +155,7 @@ private:
         mutable sk_sp<GrReleaseProcHelper> fReleaseHelper;
 
     private:
-        void freeGPUData(GrVkGpu* gpu) const override;
+        void freeGPUData(const GrVkGpu* gpu) const override;
         void abandonGPUData() const override {
             SkASSERT(!fReleaseHelper);
         }
@@ -182,7 +182,7 @@ private:
             }
         }
 
-        void freeGPUData(GrVkGpu* gpu) const override;
+        void freeGPUData(const GrVkGpu* gpu) const override;
         void abandonGPUData() const override;
     };
 

src/gpu/vk/GrVkImageView.cpp

@@ -61,7 +61,7 @@ const GrVkImageView* GrVkImageView::Create(GrVkGpu* gpu, VkImage image, VkFormat
     return new GrVkImageView(imageView, ycbcrConversion);
 }
 
-void GrVkImageView::freeGPUData(GrVkGpu* gpu) const {
+void GrVkImageView::freeGPUData(const GrVkGpu* gpu) const {
     GR_VK_CALL(gpu->vkInterface(), DestroyImageView(gpu->device(), fImageView, nullptr));
 
     if (fYcbcrConversion) {

src/gpu/vk/GrVkImageView.h

@@ -42,7 +42,7 @@ private:
     GrVkImageView(const GrVkImageView&);
     GrVkImageView& operator=(const GrVkImageView&);
 
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
     void abandonGPUData() const override;
 
     VkImageView  fImageView;

src/gpu/vk/GrVkPipeline.cpp

@@ -569,7 +569,7 @@ GrVkPipeline* GrVkPipeline::Create(GrVkGpu* gpu, const GrPrimitiveProcessor& pri
     return new GrVkPipeline(vkPipeline);
 }
 
-void GrVkPipeline::freeGPUData(GrVkGpu* gpu) const {
+void GrVkPipeline::freeGPUData(const GrVkGpu* gpu) const {
     GR_VK_CALL(gpu->vkInterface(), DestroyPipeline(gpu->device(), fPipeline, nullptr));
 }
 

src/gpu/vk/GrVkPipeline.h

@@ -56,7 +56,7 @@ protected:
     VkPipeline  fPipeline;
 
 private:
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
 
     typedef GrVkResource INHERITED;
 };

src/gpu/vk/GrVkPipelineLayout.cpp

@@ -9,6 +9,6 @@
 #include "GrVkGpu.h"
 #include "GrVkUtil.h"
 
-void GrVkPipelineLayout::freeGPUData(GrVkGpu* gpu) const {
+void GrVkPipelineLayout::freeGPUData(const GrVkGpu* gpu) const {
     GR_VK_CALL(gpu->vkInterface(), DestroyPipelineLayout(gpu->device(), fPipelineLayout, nullptr));
 }

src/gpu/vk/GrVkPipelineLayout.h

@@ -29,7 +29,7 @@ private:
     GrVkPipelineLayout(const GrVkPipelineLayout&);
     GrVkPipelineLayout& operator=(const GrVkPipelineLayout&);
 
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
 
     VkPipelineLayout  fPipelineLayout;
 

src/gpu/vk/GrVkPipelineState.cpp

@@ -75,7 +75,7 @@ GrVkPipelineState::~GrVkPipelineState() {
     SkASSERT(!fPipelineLayout);
 }
 
-void GrVkPipelineState::freeGPUResources(GrVkGpu* gpu) {
+void GrVkPipelineState::freeGPUResources(const GrVkGpu* gpu) {
     if (fPipeline) {
         fPipeline->unref(gpu);
         fPipeline = nullptr;

src/gpu/vk/GrVkPipelineState.h

@@ -71,7 +71,7 @@ public:
 
     void addUniformResources(GrVkCommandBuffer&, GrVkSampler*[], GrVkTexture*[], int numTextures);
 
-    void freeGPUResources(GrVkGpu* gpu);
+    void freeGPUResources(const GrVkGpu* gpu);
 
     void abandonGPUResources();
 

src/gpu/vk/GrVkRenderPass.cpp

@@ -163,7 +163,7 @@ void GrVkRenderPass::init(const GrVkGpu* gpu,
     this->init(gpu, colorOp, stencilOp);
 }
 
-void GrVkRenderPass::freeGPUData(GrVkGpu* gpu) const {
+void GrVkRenderPass::freeGPUData(const GrVkGpu* gpu) const {
     GR_VK_CALL(gpu->vkInterface(), DestroyRenderPass(gpu->device(), fRenderPass, nullptr));
 }
 

src/gpu/vk/GrVkRenderPass.h

@@ -125,7 +125,7 @@ private:
 
     bool isCompatible(const AttachmentsDescriptor&, const AttachmentFlags&) const;
 
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
 
     VkRenderPass          fRenderPass;
     AttachmentFlags       fAttachmentFlags;

src/gpu/vk/GrVkResource.h

@@ -55,14 +55,8 @@ public:
             });
             SkASSERT(0 == fHashSet.count());
         }
-
+        void add(const GrVkResource* r) { fHashSet.add(r); }
-        void add(const GrVkResource* r) {
+        void remove(const GrVkResource* r) { fHashSet.remove(r); }
-            fHashSet.add(r);
-        }
-
-        void remove(const GrVkResource* r) {
-            fHashSet.remove(r);
-        }
 
     private:
         SkTHashSet<const GrVkResource*, GrVkResource::Hash> fHashSet;
@@ -109,9 +103,8 @@ public:
         Must be balanced by a call to unref() or unrefAndFreeResources().
      */
     void ref() const {
-        // No barrier required.
+        SkASSERT(this->getRefCnt() > 0);
-        SkDEBUGCODE(int newRefCount = )fRefCnt.fetch_add(+1, std::memory_order_relaxed);
+        (void)fRefCnt.fetch_add(+1, std::memory_order_relaxed);  // No barrier required.
-        SkASSERT(newRefCount >= 1);
     }
 
     /** Decrement the reference count. If the reference count is 1 before the
@@ -119,12 +112,11 @@ public:
         the object needs to have been allocated via new, and not on the stack.
         Any GPU data associated with this resource will be freed before it's deleted.
      */
-    void unref(GrVkGpu* gpu) const {
+    void unref(const GrVkGpu* gpu) const {
+        SkASSERT(this->getRefCnt() > 0);
         SkASSERT(gpu);
         // A release here acts in place of all releases we "should" have been doing in ref().
-        int newRefCount = fRefCnt.fetch_add(-1, std::memory_order_acq_rel);
+        if (1 == fRefCnt.fetch_add(-1, std::memory_order_acq_rel)) {
-        SkASSERT(newRefCount >= 0);
-        if (newRefCount == 1) {
             // Like unique(), the acquire is only needed on success, to make sure
             // code in internal_dispose() doesn't happen before the decrement.
             this->internal_dispose(gpu);
@@ -135,9 +127,7 @@ public:
     void unrefAndAbandon() const {
         SkASSERT(this->getRefCnt() > 0);
         // A release here acts in place of all releases we "should" have been doing in ref().
-        int newRefCount = fRefCnt.fetch_add(-1, std::memory_order_acq_rel);
+        if (1 == fRefCnt.fetch_add(-1, std::memory_order_acq_rel)) {
-        SkASSERT(newRefCount >= 0);
-        if (newRefCount == 1) {
             // Like unique(), the acquire is only needed on success, to make sure
             // code in internal_dispose() doesn't happen before the decrement.
             this->internal_dispose();
@@ -167,7 +157,7 @@ private:
     /** Must be implemented by any subclasses.
      *  Deletes any Vk data associated with this resource
      */
-    virtual void freeGPUData(GrVkGpu* gpu) const = 0;
+    virtual void freeGPUData(const GrVkGpu* gpu) const = 0;
 
     /**
      * Called from unrefAndAbandon. Resources should do any necessary cleanup without freeing
@@ -179,7 +169,7 @@ private:
     /**
      *  Called when the ref count goes to 0. Will free Vk resources.
      */
-    void internal_dispose(GrVkGpu* gpu) const {
+    void internal_dispose(const GrVkGpu* gpu) const {
         this->freeGPUData(gpu);
 #ifdef SK_TRACE_VK_RESOURCES
         GetTrace()->remove(this);

src/gpu/vk/GrVkResourceProvider.cpp

@@ -7,17 +7,14 @@
 
 #include "GrVkResourceProvider.h"
 
-#include "GrContextPriv.h"
 #include "GrSamplerState.h"
 #include "GrVkCommandBuffer.h"
-#include "GrVkCommandPool.h"
 #include "GrVkCopyPipeline.h"
 #include "GrVkGpu.h"
 #include "GrVkPipeline.h"
 #include "GrVkRenderTarget.h"
 #include "GrVkUniformBuffer.h"
 #include "GrVkUtil.h"
-#include "SkTaskGroup.h"
 
 #ifdef SK_TRACE_VK_RESOURCES
 std::atomic<uint32_t> GrVkResource::fKeyCounter{0};
@@ -277,40 +274,47 @@ void GrVkResourceProvider::recycleDescriptorSet(const GrVkDescriptorSet* descSet
     fDescriptorSetManagers[managerIdx]->recycleDescriptorSet(descSet);
 }
 
-GrVkCommandPool* GrVkResourceProvider::findOrCreateCommandPool() {
+GrVkPrimaryCommandBuffer* GrVkResourceProvider::findOrCreatePrimaryCommandBuffer() {
-    std::unique_lock<std::recursive_mutex> lock(fBackgroundMutex);
+    GrVkPrimaryCommandBuffer* cmdBuffer = nullptr;
-    GrVkCommandPool* result;
+    int count = fAvailableCommandBuffers.count();
-    if (fAvailableCommandPools.count()) {
+    if (count > 0) {
-        result = fAvailableCommandPools.back();
+        cmdBuffer = fAvailableCommandBuffers[count - 1];
-        fAvailableCommandPools.pop_back();
+        SkASSERT(cmdBuffer->finished(fGpu));
+        fAvailableCommandBuffers.removeShuffle(count - 1);
     } else {
-        result = GrVkCommandPool::Create(fGpu);
+        cmdBuffer = GrVkPrimaryCommandBuffer::Create(fGpu, fGpu->cmdPool());
     }
-    SkASSERT(result->unique());
+    fActiveCommandBuffers.push_back(cmdBuffer);
-    SkDEBUGCODE(
+    cmdBuffer->ref();
-        for (const GrVkCommandPool* pool : fActiveCommandPools) {
+    return cmdBuffer;
-            SkASSERT(pool != result);
-        }
-        for (const GrVkCommandPool* pool : fAvailableCommandPools) {
-            SkASSERT(pool != result);
-        }
-    );
-    fActiveCommandPools.push_back(result);
-    result->ref();
-    return result;
 }
 
 void GrVkResourceProvider::checkCommandBuffers() {
-    for (int i = fActiveCommandPools.count() - 1; i >= 0; --i) {
+    for (int i = fActiveCommandBuffers.count()-1; i >= 0; --i) {
-        GrVkCommandPool* pool = fActiveCommandPools[i];
+        if (fActiveCommandBuffers[i]->finished(fGpu)) {
-        if (!pool->isOpen()) {
+            GrVkPrimaryCommandBuffer* cmdBuffer = fActiveCommandBuffers[i];
-            GrVkPrimaryCommandBuffer* buffer = pool->getPrimaryCommandBuffer();
+            cmdBuffer->reset(fGpu);
-            if (buffer->finished(fGpu)) {
+            fAvailableCommandBuffers.push_back(cmdBuffer);
-                fActiveCommandPools.removeShuffle(i);
+            fActiveCommandBuffers.removeShuffle(i);
-                this->backgroundReset(pool);
         }
     }
+}
+
+GrVkSecondaryCommandBuffer* GrVkResourceProvider::findOrCreateSecondaryCommandBuffer() {
+    GrVkSecondaryCommandBuffer* cmdBuffer = nullptr;
+    int count = fAvailableSecondaryCommandBuffers.count();
+    if (count > 0) {
+        cmdBuffer = fAvailableSecondaryCommandBuffers[count-1];
+        fAvailableSecondaryCommandBuffers.removeShuffle(count - 1);
+    } else {
+        cmdBuffer = GrVkSecondaryCommandBuffer::Create(fGpu, fGpu->cmdPool());
     }
+    return cmdBuffer;
+}
+
+void GrVkResourceProvider::recycleSecondaryCommandBuffer(GrVkSecondaryCommandBuffer* cb) {
+    cb->reset(fGpu);
+    fAvailableSecondaryCommandBuffers.push_back(cb);
 }
 
 const GrVkResource* GrVkResourceProvider::findOrCreateStandardUniformBufferResource() {
@@ -330,6 +334,29 @@ void GrVkResourceProvider::recycleStandardUniformBufferResource(const GrVkResour
 }
 
 void GrVkResourceProvider::destroyResources(bool deviceLost) {
+    // release our active command buffers
+    for (int i = 0; i < fActiveCommandBuffers.count(); ++i) {
+        SkASSERT(deviceLost || fActiveCommandBuffers[i]->finished(fGpu));
+        SkASSERT(fActiveCommandBuffers[i]->unique());
+        fActiveCommandBuffers[i]->reset(fGpu);
+        fActiveCommandBuffers[i]->unref(fGpu);
+    }
+    fActiveCommandBuffers.reset();
+    // release our available command buffers
+    for (int i = 0; i < fAvailableCommandBuffers.count(); ++i) {
+        SkASSERT(deviceLost || fAvailableCommandBuffers[i]->finished(fGpu));
+        SkASSERT(fAvailableCommandBuffers[i]->unique());
+        fAvailableCommandBuffers[i]->unref(fGpu);
+    }
+    fAvailableCommandBuffers.reset();
+
+    // release our available secondary command buffers
+    for (int i = 0; i < fAvailableSecondaryCommandBuffers.count(); ++i) {
+        SkASSERT(fAvailableSecondaryCommandBuffers[i]->unique());
+        fAvailableSecondaryCommandBuffers[i]->unref(fGpu);
+    }
+    fAvailableSecondaryCommandBuffers.reset();
+
     // Release all copy pipelines
     for (int i = 0; i < fCopyPipelines.count(); ++i) {
         fCopyPipelines[i]->unref(fGpu);
@@ -353,18 +380,6 @@ void GrVkResourceProvider::destroyResources(bool deviceLost) {
     GR_VK_CALL(fGpu->vkInterface(), DestroyPipelineCache(fGpu->device(), fPipelineCache, nullptr));
     fPipelineCache = VK_NULL_HANDLE;
 
-    for (GrVkCommandPool* pool : fActiveCommandPools) {
-        SkASSERT(pool->unique());
-        pool->unref(fGpu);
-    }
-    fActiveCommandPools.reset();
-
-    for (GrVkCommandPool* pool : fAvailableCommandPools) {
-        SkASSERT(pool->unique());
-        pool->unref(fGpu);
-    }
-    fAvailableCommandPools.reset();
-
     // We must release/destroy all command buffers and pipeline states before releasing the
     // GrVkDescriptorSetManagers
     for (int i = 0; i < fDescriptorSetManagers.count(); ++i) {
@@ -381,17 +396,25 @@ void GrVkResourceProvider::destroyResources(bool deviceLost) {
 }
 
 void GrVkResourceProvider::abandonResources() {
-    // Abandon all command pools
+    // release our active command buffers
-    for (int i = 0; i < fActiveCommandPools.count(); ++i) {
+    for (int i = 0; i < fActiveCommandBuffers.count(); ++i) {
-        SkASSERT(fActiveCommandPools[i]->unique());
+        SkASSERT(fActiveCommandBuffers[i]->unique());
-        fActiveCommandPools[i]->unrefAndAbandon();
+        fActiveCommandBuffers[i]->unrefAndAbandon();
     }
-    fActiveCommandPools.reset();
+    fActiveCommandBuffers.reset();
-    for (int i = 0; i < fAvailableCommandPools.count(); ++i) {
+    // release our available command buffers
-        SkASSERT(fAvailableCommandPools[i]->unique());
+    for (int i = 0; i < fAvailableCommandBuffers.count(); ++i) {
-        fAvailableCommandPools[i]->unrefAndAbandon();
+        SkASSERT(fAvailableCommandBuffers[i]->unique());
+        fAvailableCommandBuffers[i]->unrefAndAbandon();
     }
-    fAvailableCommandPools.reset();
+    fAvailableCommandBuffers.reset();
+
+    // release our available secondary command buffers
+    for (int i = 0; i < fAvailableSecondaryCommandBuffers.count(); ++i) {
+        SkASSERT(fAvailableSecondaryCommandBuffers[i]->unique());
+        fAvailableSecondaryCommandBuffers[i]->unrefAndAbandon();
+    }
+    fAvailableSecondaryCommandBuffers.reset();
 
     // Abandon all copy pipelines
     for (int i = 0; i < fCopyPipelines.count(); ++i) {
@@ -430,26 +453,6 @@ void GrVkResourceProvider::abandonResources() {
     fAvailableUniformBufferResources.reset();
 }
 
-void GrVkResourceProvider::backgroundReset(GrVkCommandPool* pool) {
-    SkASSERT(pool->unique());
-    pool->releaseResources(fGpu);
-    SkTaskGroup* taskGroup = fGpu->getContext()->contextPriv().getTaskGroup();
-    if (taskGroup) {
-        taskGroup->add([this, pool]() {
-            this->reset(pool);
-        });
-    } else {
-        this->reset(pool);
-    }
-}
-
-void GrVkResourceProvider::reset(GrVkCommandPool* pool) {
-    SkASSERT(pool->unique());
-    pool->reset(fGpu);
-    std::unique_lock<std::recursive_mutex> providerLock(fBackgroundMutex);
-    fAvailableCommandPools.push_back(pool);
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 
 GrVkResourceProvider::CompatibleRenderPassSet::CompatibleRenderPassSet(
@@ -485,7 +488,7 @@ GrVkRenderPass* GrVkResourceProvider::CompatibleRenderPassSet::getRenderPass(
     return renderPass;
 }
 
-void GrVkResourceProvider::CompatibleRenderPassSet::releaseResources(GrVkGpu* gpu) {
+void GrVkResourceProvider::CompatibleRenderPassSet::releaseResources(const GrVkGpu* gpu) {
     for (int i = 0; i < fRenderPasses.count(); ++i) {
         if (fRenderPasses[i]) {
             fRenderPasses[i]->unref(gpu);

src/gpu/vk/GrVkResourceProvider.h

@@ -24,10 +24,9 @@
 #include "SkTDynamicHash.h"
 #include "SkTInternalLList.h"
 
-#include <mutex>
+class GrPipeline;
-#include <thread>
+class GrPrimitiveProcessor;
-
+class GrSamplerState;
-class GrVkCommandPool;
 class GrVkCopyPipeline;
 class GrVkGpu;
 class GrVkPipeline;
@@ -85,10 +84,12 @@ public:
                                          const GrVkRenderPass::LoadStoreOps& colorOps,
                                          const GrVkRenderPass::LoadStoreOps& stencilOps);
 
-    GrVkCommandPool* findOrCreateCommandPool();
+    GrVkPrimaryCommandBuffer* findOrCreatePrimaryCommandBuffer();
-
     void checkCommandBuffers();
 
+    GrVkSecondaryCommandBuffer* findOrCreateSecondaryCommandBuffer();
+    void recycleSecondaryCommandBuffer(GrVkSecondaryCommandBuffer* cb);
+
     // Finds or creates a compatible GrVkDescriptorPool for the requested type and count.
     // The refcount is incremented and a pointer returned.
     // TODO: Currently this will just create a descriptor pool without holding onto a ref itself
@@ -167,12 +168,7 @@ public:
     // resource usages.
     void abandonResources();
 
-    void backgroundReset(GrVkCommandPool* pool);
-
-    void reset(GrVkCommandPool* pool);
-
 private:
-
 #ifdef SK_DEBUG
 #define GR_PIPELINE_STATE_CACHE_STATS
 #endif
@@ -235,7 +231,7 @@ private:
                                       const GrVkRenderPass::LoadStoreOps& colorOps,
                                       const GrVkRenderPass::LoadStoreOps& stencilOps);
 
-        void releaseResources(GrVkGpu* gpu);
+        void releaseResources(const GrVkGpu* gpu);
         void abandonResources();
 
     private:
@@ -253,11 +249,13 @@ private:
 
     SkSTArray<4, CompatibleRenderPassSet> fRenderPassArray;
 
-    // Array of command pools that we are waiting on
+    // Array of PrimaryCommandBuffers that are currently in flight
-    SkSTArray<4, GrVkCommandPool*, true> fActiveCommandPools;
+    SkSTArray<4, GrVkPrimaryCommandBuffer*, true> fActiveCommandBuffers;
+    // Array of available primary command buffers that are not in flight
+    SkSTArray<4, GrVkPrimaryCommandBuffer*, true> fAvailableCommandBuffers;
 
-    // Array of available command pools that are not in flight
+    // Array of available secondary command buffers
-    SkSTArray<4, GrVkCommandPool*, true> fAvailableCommandPools;
+    SkSTArray<16, GrVkSecondaryCommandBuffer*, true> fAvailableSecondaryCommandBuffers;
 
     // Array of available uniform buffer resources
     SkSTArray<16, const GrVkResource*, true> fAvailableUniformBufferResources;
@@ -275,8 +273,6 @@ private:
     SkSTArray<4, std::unique_ptr<GrVkDescriptorSetManager>> fDescriptorSetManagers;
 
     GrVkDescriptorSetManager::Handle fUniformDSHandle;
-
-    std::recursive_mutex fBackgroundMutex;
 };
 
 #endif

src/gpu/vk/GrVkSampler.cpp

@@ -112,7 +112,7 @@ GrVkSampler* GrVkSampler::Create(GrVkGpu* gpu, const GrSamplerState& samplerStat
     return new GrVkSampler(sampler, ycbcrConversion, GenerateKey(samplerState, ycbcrInfo));
 }
 
-void GrVkSampler::freeGPUData(GrVkGpu* gpu) const {
+void GrVkSampler::freeGPUData(const GrVkGpu* gpu) const {
     SkASSERT(fSampler);
     GR_VK_CALL(gpu->vkInterface(), DestroySampler(gpu->device(), fSampler, nullptr));
     if (fYcbcrConversion) {

src/gpu/vk/GrVkSampler.h

@@ -67,7 +67,7 @@ private:
             , fKey(key)
             , fUniqueID(GenID()) {}
 
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
     void abandonGPUData() const override;
 
     static uint32_t GenID() {

src/gpu/vk/GrVkSamplerYcbcrConversion.cpp

@@ -68,7 +68,7 @@ GrVkSamplerYcbcrConversion* GrVkSamplerYcbcrConversion::Create(
 #endif
 }
 
-void GrVkSamplerYcbcrConversion::freeGPUData(GrVkGpu* gpu) const {
+void GrVkSamplerYcbcrConversion::freeGPUData(const GrVkGpu* gpu) const {
     SkASSERT(fYcbcrConversion);
     GR_VK_CALL(gpu->vkInterface(), DestroySamplerYcbcrConversion(gpu->device(), fYcbcrConversion,
                                                                  nullptr));

src/gpu/vk/GrVkSamplerYcbcrConversion.h

@@ -62,7 +62,7 @@ private:
             , fYcbcrConversion(ycbcrConversion)
             , fKey(key) {}
 
-    void freeGPUData(GrVkGpu* gpu) const override;
+    void freeGPUData(const GrVkGpu* gpu) const override;
 
     VkSamplerYcbcrConversion fYcbcrConversion;
     Key                      fKey;

src/gpu/vk/GrVkSemaphore.cpp

@@ -51,7 +51,7 @@ GrVkSemaphore::GrVkSemaphore(GrVkGpu* gpu, VkSemaphore semaphore, bool prohibitS
 
 void GrVkSemaphore::onRelease() {
     if (fResource) {
-        fResource->unref(static_cast<GrVkGpu*>(this->getGpu()));
+        fResource->unref(static_cast<const GrVkGpu*>(this->getGpu()));
         fResource = nullptr;
     }
     INHERITED::onRelease();
@@ -65,7 +65,7 @@ void GrVkSemaphore::onAbandon() {
     INHERITED::onAbandon();
 }
 
-void GrVkSemaphore::Resource::freeGPUData(GrVkGpu* gpu) const {
+void GrVkSemaphore::Resource::freeGPUData(const GrVkGpu* gpu) const {
     if (fIsOwned) {
         GR_VK_CALL(gpu->vkInterface(),
                    DestroySemaphore(gpu->device(), fSemaphore, nullptr));

src/gpu/vk/GrVkSemaphore.h

@@ -71,7 +71,7 @@ public:
         }
 #endif
     private:
-        void freeGPUData(GrVkGpu* gpu) const override;
+        void freeGPUData(const GrVkGpu* gpu) const override;
 
         static SkMutex* GetMutex() {
             static SkMutex kMutex;

tests/SurfaceSemaphoreTest.cpp

@@ -20,7 +20,6 @@
 #include "gl/GrGLUtil.h"
 
 #ifdef SK_VULKAN
-#include "vk/GrVkCommandPool.h"
 #include "vk/GrVkGpu.h"
 #include "vk/GrVkTypes.h"
 #include "vk/GrVkUtil.h"
@@ -246,14 +245,14 @@ DEF_GPUTEST_FOR_RENDERING_CONTEXTS(EmptySurfaceSemaphoreTest, reporter, ctxInfo)
         const GrVkInterface* interface = gpu->vkInterface();
         VkDevice device = gpu->device();
         VkQueue queue = gpu->queue();
-        GrVkCommandPool* cmdPool = gpu->cmdPool();
+        VkCommandPool cmdPool = gpu->cmdPool();
         VkCommandBuffer cmdBuffer;
 
         // Create Command Buffer
         const VkCommandBufferAllocateInfo cmdInfo = {
             VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,   // sType
             nullptr,                                          // pNext
-            cmdPool->vkCommandPool(),                         // commandPool
+            cmdPool,                                          // commandPool
             VK_COMMAND_BUFFER_LEVEL_PRIMARY,                  // level
             1                                                 // bufferCount
         };