Hook up SkHeifCodec for ImageDecoder animation

bug: 78868457
bug: 120414514

test: local test with OpenGL Rendrerer Tests animation
demo and heifs files

Change-Id: I09a7667a57f545927dbe9ac24c1a6b405ff0006d
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/232839
Commit-Queue: Leon Scroggins <scroggo@google.com>
Reviewed-by: Leon Scroggins <scroggo@google.com>
Reviewed-by: Derek Sollenberger <djsollen@google.com>
Auto-Submit: Chong Zhang <chz@google.com>

RELEASE_NOTES.txt

@@ -26,3 +26,6 @@ Milestone 78
 
  * Vulkan backend now supports YCbCr sampler for I420 Vulkan images that are not
    backed by external images.
+
+ * Add SkCodec::SelectionPolicy for distinguishing between decoding a still image
+   or an image sequence for a container format that has both (e.g. HEIF).

include/codec/SkCodec.h

@@ -111,6 +111,24 @@ public:
      */
     static const char* ResultToString(Result);
 
+    /**
+     * For container formats that contain both still images and image sequences,
+     * instruct the decoder how the output should be selected. (Refer to comments
+     * for each value for more details.)
+     */
+    enum class SelectionPolicy {
+        /**
+         *  If the container format contains both still images and image sequences,
+         *  SkCodec should choose one of the still images. This is the default.
+         */
+        kPreferStillImage,
+        /**
+         *  If the container format contains both still images and image sequences,
+         *  SkCodec should choose one of the image sequences for animation.
+         */
+        kPreferAnimation,
+    };
+
     /**
      *  If this stream represents an encoded image that we know how to decode,
      *  return an SkCodec that can decode it. Otherwise return NULL.
@@ -145,8 +163,10 @@ public:
      *  If NULL is returned, the stream is deleted immediately. Otherwise, the
      *  SkCodec takes ownership of it, and will delete it when done with it.
      */
-    static std::unique_ptr<SkCodec> MakeFromStream(std::unique_ptr<SkStream>, Result* = nullptr,
+    static std::unique_ptr<SkCodec> MakeFromStream(
-                                                   SkPngChunkReader* = nullptr);
+            std::unique_ptr<SkStream>, Result* = nullptr,
+            SkPngChunkReader* = nullptr,
+            SelectionPolicy selectionPolicy = SelectionPolicy::kPreferStillImage);
 
     /**
      *  If this data represents an encoded image that we know how to decode,

src/codec/SkCodec.cpp

@@ -53,9 +53,6 @@ static std::vector<DecoderProc>* decoders() {
     #endif
         { SkBmpCodec::IsBmp, SkBmpCodec::MakeFromStream },
         { SkWbmpCodec::IsWbmp, SkWbmpCodec::MakeFromStream },
-    #ifdef SK_HAS_HEIF_LIBRARY
-        { SkHeifCodec::IsHeif, SkHeifCodec::MakeFromStream },
-    #endif
     };
     return decoders;
 }
@@ -66,9 +63,9 @@ void SkCodec::Register(
     decoders()->push_back(DecoderProc{peek, make});
 }
 
-
+std::unique_ptr<SkCodec> SkCodec::MakeFromStream(
-std::unique_ptr<SkCodec> SkCodec::MakeFromStream(std::unique_ptr<SkStream> stream,
+        std::unique_ptr<SkStream> stream, Result* outResult,
-                                                 Result* outResult, SkPngChunkReader* chunkReader) {
+        SkPngChunkReader* chunkReader, SelectionPolicy selectionPolicy) {
     Result resultStorage;
     if (!outResult) {
         outResult = &resultStorage;
@@ -79,6 +76,12 @@ std::unique_ptr<SkCodec> SkCodec::MakeFromStream(std::unique_ptr<SkStream> strea
         return nullptr;
     }
 
+    if (selectionPolicy != SelectionPolicy::kPreferStillImage
+            && selectionPolicy != SelectionPolicy::kPreferAnimation) {
+        *outResult = kInvalidParameters;
+        return nullptr;
+    }
+
     constexpr size_t bytesToRead = MinBufferedBytesNeeded();
 
     char buffer[bytesToRead];
@@ -121,6 +124,12 @@ std::unique_ptr<SkCodec> SkCodec::MakeFromStream(std::unique_ptr<SkStream> strea
             }
         }
 
+#ifdef SK_HAS_HEIF_LIBRARY
+        if (SkHeifCodec::IsHeif(buffer, bytesRead)) {
+            return SkHeifCodec::MakeFromStream(std::move(stream), selectionPolicy, outResult);
+        }
+#endif
+
 #ifdef SK_CODEC_DECODES_RAW
         // Try to treat the input as RAW if all the other checks failed.
         return SkRawCodec::MakeFromStream(std::move(stream), outResult);

src/codec/SkHeifCodec.cpp

@@ -118,49 +118,86 @@ private:
     std::unique_ptr<SkStream> fStream;
 };
 
-std::unique_ptr<SkCodec> SkHeifCodec::MakeFromStream(
+#ifndef SK_LEGACY_HEIF_API
-        std::unique_ptr<SkStream> stream, Result* result) {
+static void releaseProc(const void* ptr, void* context) {
+    delete reinterpret_cast<std::vector<uint8_t>*>(context);
+}
+#endif
+
+std::unique_ptr<SkCodec> SkHeifCodec::MakeFromStream(std::unique_ptr<SkStream> stream,
+        SkCodec::SelectionPolicy selectionPolicy, Result* result) {
     std::unique_ptr<HeifDecoder> heifDecoder(createHeifDecoder());
     if (heifDecoder.get() == nullptr) {
         *result = kInternalError;
         return nullptr;
     }
 
-    HeifFrameInfo frameInfo;
+    HeifFrameInfo heifInfo;
-    if (!heifDecoder->init(new SkHeifStreamWrapper(stream.release()),
+    if (!heifDecoder->init(new SkHeifStreamWrapper(stream.release()), &heifInfo)) {
-                           &frameInfo)) {
         *result = kInvalidInput;
         return nullptr;
     }
 
+#ifndef SK_LEGACY_HEIF_API
+    size_t frameCount = 1;
+    if (selectionPolicy == SkCodec::SelectionPolicy::kPreferAnimation) {
+        HeifFrameInfo sequenceInfo;
+        if (heifDecoder->getSequenceInfo(&sequenceInfo, &frameCount) &&
+                frameCount > 1) {
+            heifInfo = std::move(sequenceInfo);
+        }
+    }
+#endif
+
     std::unique_ptr<SkEncodedInfo::ICCProfile> profile = nullptr;
-    if ((frameInfo.mIccSize > 0) && (frameInfo.mIccData != nullptr)) {
+#ifdef SK_LEGACY_HEIF_API
+    if ((heifInfo.mIccSize > 0) && (heifInfo.mIccData != nullptr)) {
         // FIXME: Would it be possible to use MakeWithoutCopy?
-        auto icc = SkData::MakeWithCopy(frameInfo.mIccData.get(), frameInfo.mIccSize);
+        auto icc = SkData::MakeWithCopy(heifInfo.mIccData.get(), heifInfo.mIccSize);
         profile = SkEncodedInfo::ICCProfile::Make(std::move(icc));
     }
+#else
+    if (heifInfo.mIccData.size() > 0) {
+        auto iccData = new std::vector<uint8_t>(std::move(heifInfo.mIccData));
+        auto icc = SkData::MakeWithProc(iccData->data(), iccData->size(), releaseProc, iccData);
+        profile = SkEncodedInfo::ICCProfile::Make(std::move(icc));
+    }
+#endif
     if (profile && profile->profile()->data_color_space != skcms_Signature_RGB) {
         // This will result in sRGB.
         profile = nullptr;
     }
 
-    SkEncodedInfo info = SkEncodedInfo::Make(frameInfo.mWidth, frameInfo.mHeight,
+    SkEncodedInfo info = SkEncodedInfo::Make(heifInfo.mWidth, heifInfo.mHeight,
             SkEncodedInfo::kYUV_Color, SkEncodedInfo::kOpaque_Alpha, 8, std::move(profile));
-    SkEncodedOrigin orientation = get_orientation(frameInfo);
+    SkEncodedOrigin orientation = get_orientation(heifInfo);
 
     *result = kSuccess;
-    return std::unique_ptr<SkCodec>(new SkHeifCodec(std::move(info), heifDecoder.release(),
+    return std::unique_ptr<SkCodec>(new SkHeifCodec(
-                                                    orientation));
+            std::move(info), heifDecoder.release(), orientation
+#ifndef SK_LEGACY_HEIF_API
+            , frameCount > 1
+#endif
+            ));
 }
 
-SkHeifCodec::SkHeifCodec(SkEncodedInfo&& info, HeifDecoder* heifDecoder, SkEncodedOrigin origin)
+SkHeifCodec::SkHeifCodec(
+        SkEncodedInfo&& info,
+        HeifDecoder* heifDecoder,
+        SkEncodedOrigin origin
+#ifndef SK_LEGACY_HEIF_API
+        , bool useAnimation
+#endif
+        )
     : INHERITED(std::move(info), skcms_PixelFormat_RGBA_8888, nullptr, origin)
     , fHeifDecoder(heifDecoder)
     , fSwizzleSrcRow(nullptr)
     , fColorXformSrcRow(nullptr)
+#ifndef SK_LEGACY_HEIF_API
+    , fUseAnimation(useAnimation)
+#endif
 {}
 
-
 bool SkHeifCodec::conversionSupported(const SkImageInfo& dstInfo, bool srcIsOpaque,
                                       bool needsColorXform) {
     SkASSERT(srcIsOpaque);
@@ -249,6 +286,79 @@ int SkHeifCodec::readRows(const SkImageInfo& dstInfo, void* dst, size_t rowBytes
     return count;
 }
 
+#ifndef SK_LEGACY_HEIF_API
+int SkHeifCodec::onGetFrameCount() {
+    if (!fUseAnimation) {
+        return 1;
+    }
+
+    if (fFrameHolder.size() == 0) {
+        size_t frameCount;
+        HeifFrameInfo frameInfo;
+        if (!fHeifDecoder->getSequenceInfo(&frameInfo, &frameCount)
+                || frameCount <= 1) {
+            fUseAnimation = false;
+            return 1;
+        }
+        fFrameHolder.reserve(frameCount);
+        for (size_t i = 0; i < frameCount; i++) {
+            Frame* frame = fFrameHolder.appendNewFrame();
+            frame->setXYWH(0, 0, frameInfo.mWidth, frameInfo.mHeight);
+            frame->setDisposalMethod(SkCodecAnimation::DisposalMethod::kKeep);
+            // TODO: fill in per-frame durations
+            // Currently we don't know the duration until the frame is actually
+            // decoded (onGetFrameInfo is also called before frame is decoded).
+            // For now, fill it base on the value reported for the sequence.
+            frame->setDuration(frameInfo.mDurationUs / 1000);
+            frame->setRequiredFrame(SkCodec::kNoFrame);
+            frame->setHasAlpha(false);
+        }
+    }
+
+    return fFrameHolder.size();
+}
+
+const SkFrame* SkHeifCodec::FrameHolder::onGetFrame(int i) const {
+    return static_cast<const SkFrame*>(this->frame(i));
+}
+
+SkHeifCodec::Frame* SkHeifCodec::FrameHolder::appendNewFrame() {
+    const int i = this->size();
+    fFrames.emplace_back(i); // TODO: need to handle frame duration here
+    return &fFrames[i];
+}
+
+const SkHeifCodec::Frame* SkHeifCodec::FrameHolder::frame(int i) const {
+    SkASSERT(i >= 0 && i < this->size());
+    return &fFrames[i];
+}
+
+bool SkHeifCodec::onGetFrameInfo(int i, FrameInfo* frameInfo) const {
+    if (i >= fFrameHolder.size()) {
+        return false;
+    }
+
+    const Frame* frame = fFrameHolder.frame(i);
+    if (!frame) {
+        return false;
+    }
+
+    if (frameInfo) {
+        frameInfo->fRequiredFrame = SkCodec::kNoFrame;
+        frameInfo->fDuration = frame->getDuration();
+        frameInfo->fFullyReceived = true;
+        frameInfo->fAlphaType = kOpaque_SkAlphaType;
+        frameInfo->fDisposalMethod = SkCodecAnimation::DisposalMethod::kKeep;
+    }
+
+    return true;
+}
+
+int SkHeifCodec::onGetRepetitionCount() {
+    return kRepetitionCountInfinite;
+}
+#endif // SK_LEGACY_HEIF_API
+
 /*
  * Performs the heif decode
  */
@@ -263,9 +373,22 @@ SkCodec::Result SkHeifCodec::onGetPixels(const SkImageInfo& dstInfo,
         return kUnimplemented;
     }
 
+#ifdef SK_LEGACY_HEIF_API
     if (!fHeifDecoder->decode(&fFrameInfo)) {
         return kInvalidInput;
     }
+#else
+    bool success;
+    if (fUseAnimation) {
+        success = fHeifDecoder->decodeSequence(options.fFrameIndex, &fFrameInfo);
+    } else {
+        success = fHeifDecoder->decode(&fFrameInfo);
+    }
+
+    if (!success) {
+        return kInvalidInput;
+    }
+#endif // SK_LEGACY_HEIF_API
 
     fSwizzler.reset(nullptr);
     this->allocateStorage(dstInfo);

src/codec/SkHeifCodec.h

@@ -12,6 +12,7 @@
 #include "include/codec/SkEncodedOrigin.h"
 #include "include/core/SkImageInfo.h"
 #include "include/core/SkStream.h"
+#include "src/codec/SkFrameHolder.h"
 #include "src/codec/SkSwizzler.h"
 
 #if __has_include("HeifDecoderAPI.h")
@@ -27,7 +28,8 @@ public:
     /*
      * Assumes IsHeif was called and returned true.
      */
-    static std::unique_ptr<SkCodec> MakeFromStream(std::unique_ptr<SkStream>, Result*);
+    static std::unique_ptr<SkCodec> MakeFromStream(
+            std::unique_ptr<SkStream>, SkCodec::SelectionPolicy selectionPolicy, Result*);
 
 protected:
 
@@ -41,6 +43,15 @@ protected:
         return SkEncodedImageFormat::kHEIF;
     }
 
+#ifndef SK_LEGACY_HEIF_API
+    int onGetFrameCount() override;
+    bool onGetFrameInfo(int, FrameInfo*) const override;
+    int onGetRepetitionCount() override;
+    const SkFrameHolder* getFrameHolder() const override {
+        return &fFrameHolder;
+    }
+#endif
+
     bool conversionSupported(const SkImageInfo&, bool, bool) override;
 
     bool onRewind() override;
@@ -50,7 +61,11 @@ private:
      * Creates an instance of the decoder
      * Called only by NewFromStream
      */
-    SkHeifCodec(SkEncodedInfo&&, HeifDecoder*, SkEncodedOrigin);
+    SkHeifCodec(SkEncodedInfo&&, HeifDecoder*, SkEncodedOrigin
+#ifndef SK_LEGACY_HEIF_API
+                , bool animation
+#endif
+                );
 
     void initializeSwizzler(const SkImageInfo& dstInfo, const Options& options);
     void allocateStorage(const SkImageInfo& dstInfo);
@@ -73,7 +88,47 @@ private:
     uint32_t*                          fColorXformSrcRow;
 
     std::unique_ptr<SkSwizzler>        fSwizzler;
+#ifndef SK_LEGACY_HEIF_API
+    bool                               fUseAnimation;
 
+    class Frame : public SkFrame {
+    public:
+        Frame(int i) : INHERITED(i) {}
+
+    protected:
+        SkEncodedInfo::Alpha onReportedAlpha() const override {
+            return SkEncodedInfo::Alpha::kOpaque_Alpha;
+        }
+
+    private:
+        typedef SkFrame INHERITED;
+    };
+
+    class FrameHolder : public SkFrameHolder {
+    public:
+        ~FrameHolder() override {}
+        void setScreenSize(int w, int h) {
+            fScreenWidth = w;
+            fScreenHeight = h;
+        }
+        Frame* appendNewFrame();
+        const Frame* frame(int i) const;
+        int size() const {
+            return static_cast<int>(fFrames.size());
+        }
+        void reserve(int size) {
+            fFrames.reserve(size);
+        }
+
+    protected:
+        const SkFrame* onGetFrame(int i) const override;
+
+    private:
+        std::vector<Frame> fFrames;
+    };
+
+    FrameHolder fFrameHolder;
+#endif // SK_LEGACY_HEIF_API
     typedef SkCodec INHERITED;
 };
 

src/codec/SkStubHeifDecoderAPI.h

@@ -32,6 +32,7 @@ struct HeifStream {
 };
 
 struct HeifFrameInfo {
+#ifdef SK_LEGACY_HEIF_API
     int mRotationAngle;
     int mWidth;
     int mHeight;
@@ -39,6 +40,14 @@ struct HeifFrameInfo {
 
     size_t                  mIccSize;
     std::unique_ptr<char[]> mIccData;
+#else
+    uint32_t mWidth;
+    uint32_t mHeight;
+    int32_t  mRotationAngle;           // Rotation angle, clockwise, should be multiple of 90
+    uint32_t mBytesPerPixel;           // Number of bytes for one pixel
+    int64_t mDurationUs;               // Duration of the frame in us
+    std::vector<uint8_t> mIccData;     // ICC data array
+#endif
 };
 
 struct HeifDecoder {
@@ -47,10 +56,22 @@ struct HeifDecoder {
         return false;
     }
 
+#ifndef SK_LEGACY_HEIF_API
+    bool getSequenceInfo(HeifFrameInfo* frameInfo, size_t *frameCount) {
+        return false;
+    }
+#endif
+
     bool decode(HeifFrameInfo*) {
         return false;
     }
 
+#ifndef SK_LEGACY_HEIF_API
+    bool decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) {
+        return false;
+    }
+#endif
+
     bool setOutputColor(HeifColorFormat) {
         return false;
     }