[canvaskit] add audio asset support to skottie-bindings

Change-Id: If4c36f0261a18ed068cd745a4c454c127d0e96bd
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/360916
Reviewed-by: Florin Malita <fmalita@chromium.org>
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Commit-Queue: Jorge Betancourt <jmbetancourt@google.com>

modules/canvaskit/canvaskit/types/index.d.ts

@@ -385,9 +385,11 @@ export interface CanvasKit {
      * @param assets - a dictionary of named blobs: { key: ArrayBuffer, ... }
      * @param filterPrefix - an optional string acting as a name filter for selecting "interesting"
      *                       Lottie properties (surfaced in the embedded player controls)
+     * @param soundMap - an optional mapping of sound identifiers (strings) to AudioPlayers.
+     *                   Only needed if the animation supports sound.
      */
     MakeManagedAnimation(json: string, assets?: Record<string, ArrayBuffer>,
-                         filterPrefix?: string): ManagedSkottieAnimation;
+                         filterPrefix?: string, soundMap?: SoundMap): ManagedSkottieAnimation;
 
     /**
      * Returns a Particles effect built from the provided json string and assets.
@@ -677,6 +679,29 @@ export interface MallocObj {
     toTypedArray(): TypedArray;
 }
 
+/**
+ * This object maintains a single audio layer during skottie playback
+ */
+export interface AudioPlayer {
+    /**
+     * Playback control callback, emitted for each corresponding Animation::seek().
+     *
+     * Will seek to time t (seconds) relative to the layer's timeline origin.
+     * Negative t values are used to signal off state (stop playback outside layer span).
+     */
+    seek(t: number): void;
+}
+
+/**
+ * Mapping of sound names (strings) to AudioPlayers
+ */
+export interface SoundMap {
+    /**
+     * Returns AudioPlayer for a certain audio layer
+     * @param key string identifier, name of audio file the desired AudioPlayer manages
+     */
+    getPlayer(key: string): AudioPlayer;
+}
 export interface ManagedSkottieAnimation extends SkottieAnimation {
     setColor(key: string, color: InputColor): void;
     setOpacity(key: string, opacity: number): void;

modules/canvaskit/skottie.js

@@ -7,7 +7,8 @@
 
 // prop_filter_prefix is an optional string acting as a name filter for selecting
 // "interesting" Lottie properties (surfaced in the embedded player controls)
-CanvasKit.MakeManagedAnimation = function(json, assets, prop_filter_prefix) {
+
+CanvasKit.MakeManagedAnimation = function(json, assets, prop_filter_prefix, soundMap) {
   if (!CanvasKit._MakeManagedAnimation) {
     throw 'Not compiled with MakeManagedAnimation';
   }
@@ -15,7 +16,7 @@ CanvasKit.MakeManagedAnimation = function(json, assets, prop_filter_prefix) {
     prop_filter_prefix = '';
   }
   if (!assets) {
-    return CanvasKit._MakeManagedAnimation(json, 0, nullptr, nullptr, nullptr, prop_filter_prefix);
+    return CanvasKit._MakeManagedAnimation(json, 0, nullptr, nullptr, nullptr, prop_filter_prefix, soundMap);
   }
   var assetNamePtrs = [];
   var assetDataPtrs = [];
@@ -49,7 +50,7 @@ CanvasKit.MakeManagedAnimation = function(json, assets, prop_filter_prefix) {
   var assetSizesPtr = copy1dArray(assetSizes,    "HEAPU32");
 
   var anim = CanvasKit._MakeManagedAnimation(json, assetKeys.length, namesPtr,
-                                             assetsPtr, assetSizesPtr, prop_filter_prefix);
+                                             assetsPtr, assetSizesPtr, prop_filter_prefix, soundMap);
 
   // The C++ code has made copies of the asset and string data, so free our copies.
   CanvasKit._free(namesPtr);

modules/canvaskit/skottie_bindings.cpp

@@ -30,6 +30,20 @@ using namespace emscripten;
 #if SK_INCLUDE_MANAGED_SKOTTIE
 namespace {
 
+// WebTrack wraps a JS object that has a 'seek' method.
+// Playback logic is kept there.
+class WebTrack final : public skresources::ExternalTrackAsset {
+public:
+    explicit WebTrack(emscripten::val player) : fPlayer(std::move(player)) {}
+
+private:
+    void seek(float t) override {
+        fPlayer.call<void>("seek", val(t));
+    }
+
+    const emscripten::val fPlayer;
+};
+
 class SkottieAssetProvider : public skottie::ResourceProvider {
 public:
     ~SkottieAssetProvider() override = default;
@@ -40,12 +54,8 @@ public:
     // confusing enscripten.
     using AssetVec = std::vector<std::pair<SkString, sk_sp<SkData>>>;
 
-    static sk_sp<SkottieAssetProvider> Make(AssetVec assets) {
-        if (assets.empty()) {
-            return nullptr;
-        }
-
-        return sk_sp<SkottieAssetProvider>(new SkottieAssetProvider(std::move(assets)));
+    static sk_sp<SkottieAssetProvider> Make(AssetVec assets, emscripten::val soundMap) {
+        return sk_sp<SkottieAssetProvider>(new SkottieAssetProvider(std::move(assets), std::move(soundMap)));
     }
 
     sk_sp<skottie::ImageAsset> loadImageAsset(const char[] /* path */,
@@ -59,6 +69,17 @@ public:
         return nullptr;
     }
 
+    sk_sp<skresources::ExternalTrackAsset> loadAudioAsset(const char[] /* path */,
+                                                          const char name[],
+                                                          const char[] /*id*/) override {
+        emscripten::val player = this->findSoundAsset(name);
+        if (player.as<bool>()) {
+            return sk_make_sp<WebTrack>(std::move(player));
+        }
+
+        return nullptr;
+    }
+
     sk_sp<SkData> loadFont(const char name[], const char[] /* url */) const override {
         // Same as images paths, we ignore font URLs.
         return this->findAsset(name);
@@ -70,7 +91,9 @@ public:
     }
 
 private:
-    explicit SkottieAssetProvider(AssetVec assets) : fAssets(std::move(assets)) {}
+    explicit SkottieAssetProvider(AssetVec assets, emscripten::val soundMap)
+    : fAssets(std::move(assets))
+    , fSoundMap(std::move(soundMap)) {}
 
     sk_sp<SkData> findAsset(const char name[]) const {
         for (const auto& asset : fAssets) {
@@ -83,7 +106,18 @@ private:
         return nullptr;
     }
 
+    emscripten::val findSoundAsset(const char name[]) const {
+        if (fSoundMap.as<bool>() && fSoundMap.hasOwnProperty("getPlayer")) {
+            emscripten::val player = fSoundMap.call<emscripten::val>("getPlayer", val(name));
+            if (player.as<bool>() && player.hasOwnProperty("seek")) {
+                return player;
+            }
+        }
+        return emscripten::val::null();
+    }
+
     const AssetVec fAssets;
+    const emscripten::val fSoundMap;
 };
 
 class ManagedAnimation final : public SkRefCnt {
@@ -263,7 +297,8 @@ EMSCRIPTEN_BINDINGS(Skottie) {
                                                            uintptr_t /* char**     */ nptr,
                                                            uintptr_t /* uint8_t**  */ dptr,
                                                            uintptr_t /* size_t*    */ sptr,
-                                                           std::string prop_prefix)
+                                                           std::string prop_prefix,
+                                                           emscripten::val soundMap)
                                                         ->sk_sp<ManagedAnimation> {
         // See the comment in canvaskit_bindings.cpp about the use of uintptr_t
         const auto assetNames = reinterpret_cast<char**   >(nptr);
@@ -281,7 +316,7 @@ EMSCRIPTEN_BINDINGS(Skottie) {
 
         return ManagedAnimation::Make(json,
                                       skresources::DataURIResourceProviderProxy::Make(
-                                          SkottieAssetProvider::Make(std::move(assets))),
+                                          SkottieAssetProvider::Make(std::move(assets), std::move(soundMap))),
                                       prop_prefix);
     }));
     constant("managed_skottie", true);

modules/canvaskit/tests/skottie.spec.js

@@ -78,4 +78,42 @@ describe('Skottie behavior', () => {
         animation.render(canvas, bounds);
         animation.delete();
     }, washPromise);
+
+    it('can load audio assets', (done) => {
+        if (!CanvasKit.skottie || !CanvasKit.managed_skottie) {
+            console.warn('Skipping test because not compiled with skottie');
+            return;
+        }
+        const mockSoundMap = {
+            map : new Map(),
+            getPlayer : function(name) {return this.map.get(name)},
+            setPlayer : function(name, player) {this.map.set(name, player)},
+        };
+        function mockPlayer(name) {
+            this.name = name;
+            this.wasPlayed = false,
+            this.seek = function(t) {
+                this.wasPlayed = true;
+            }
+        }
+        for (let i = 0; i < 20; i++) {
+            var name = 'aud_' + i + '.mp3';
+            mockSoundMap.setPlayer(name, new mockPlayer(name));
+        }
+        fetch('/assets/audio_external.json')
+        .then((response) => response.text())
+        .then((lottie) => {
+            const animation = CanvasKit.MakeManagedAnimation(lottie, null, null, mockSoundMap);
+            expect(animation).toBeTruthy();
+            // 190 frames in sample lottie
+            for (let t = 0; t < 190; t++) {
+                animation.seekFrame(t);
+            }
+            animation.delete();
+            for(const player of mockSoundMap.map.values()) {
+                expect(player.wasPlayed).toBeTrue(player.name + " was not played");
+            }
+            done();
+        });
+    });
 });