[wasm-gc][turbofan] Speculative direct calls for call_ref

Behind the --wasm-inlining flag, we introduce speculative direct calls
as an alternative to invoking functions through references.
In pseudocode, call_ref(func_ref, args...) reduces to
  if (func_ref == function_reference_at(expected_index)) {
    call_direct(expected_index, args...)
  } else call_ref(func_ref, args...)
The introduced direct call can later get inlined in WasmInliningPhase.
Currently, we always speculate that the reference is the function at
index 0. Proper heuristics, based on liftoff runtime feedback, will come
later.

Bug: v8:12166, v8:7748
Change-Id: Icd1319d3091b436e71906717fd8a2662bfbb8481
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3162602
Commit-Queue: Manos Koukoutos <manoskouk@chromium.org>
Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Cr-Commit-Position: refs/heads/main@{#76884}

src/compiler/wasm-compiler.cc

@@ -3227,6 +3227,22 @@ Node* WasmGraphBuilder::BuildCallRef(uint32_t sig_index,
   return call;
 }
 
+void WasmGraphBuilder::CompareToExternalFunctionAtIndex(
+    Node* func_ref, uint32_t function_index, Node** success_control,
+    Node** failure_control) {
+  // Since we are comparing to a function reference, it is guaranteed that
+  // instance->wasm_external_functions() has been initialized.
+  Node* external_functions = gasm_->LoadFromObject(
+      MachineType::TaggedPointer(), GetInstance(),
+      wasm::ObjectAccess::ToTagged(
+          WasmInstanceObject::kWasmExternalFunctionsOffset));
+  Node* function_ref = gasm_->LoadFixedArrayElement(
+      external_functions, gasm_->IntPtrConstant(function_index),
+      MachineType::AnyTagged());
+  gasm_->Branch(gasm_->WordEqual(function_ref, func_ref), success_control,
+                failure_control, BranchHint::kTrue);
+}
+
 Node* WasmGraphBuilder::CallRef(uint32_t sig_index, base::Vector<Node*> args,
                                 base::Vector<Node*> rets,
                                 WasmGraphBuilder::CheckForNull null_check,

src/compiler/wasm-compiler.h

@@ -328,6 +328,9 @@ class WasmGraphBuilder {
   Node* CallRef(uint32_t sig_index, base::Vector<Node*> args,
                 base::Vector<Node*> rets, CheckForNull null_check,
                 wasm::WasmCodePosition position);
+  void CompareToExternalFunctionAtIndex(Node* func_ref, uint32_t function_index,
+                                        Node** success_control,
+                                        Node** failure_control);
 
   Node* ReturnCall(uint32_t index, base::Vector<Node*> args,
                    wasm::WasmCodePosition position);

src/wasm/graph-builder-interface.cc

@@ -660,13 +660,93 @@ class WasmGraphBuildingInterface {
   void CallRef(FullDecoder* decoder, const Value& func_ref,
                const FunctionSig* sig, uint32_t sig_index, const Value args[],
                Value returns[]) {
+    if (!FLAG_wasm_inlining) {
+      DoCall(decoder, kCallRef, 0, NullCheckFor(func_ref.type), func_ref.node,
+             sig, sig_index, args, returns);
+      return;
+    }
+
+    // Check for equality against a function at a specific index, and if
+    // successful, just emit a direct call.
+    // TODO(12166): For now, we check against function 0. Decide the index based
+    // on liftoff feedback.
+    const uint32_t expected_function_index = 0;
+
+    TFNode* success_control;
+    TFNode* failure_control;
+    builder_->CompareToExternalFunctionAtIndex(
+        func_ref.node, expected_function_index, &success_control,
+        &failure_control);
+    TFNode* initial_effect = effect();
+
+    builder_->SetControl(success_control);
+    ssa_env_->control = success_control;
+    Value* returns_direct =
+        decoder->zone()->NewArray<Value>(sig->return_count());
+    DoCall(decoder, kCallDirect, expected_function_index,
+           CheckForNull::kWithoutNullCheck, nullptr,
+           decoder->module_->signature(sig_index), sig_index, args,
+           returns_direct);
+    TFNode* control_direct = control();
+    TFNode* effect_direct = effect();
+
+    builder_->SetEffectControl(initial_effect, failure_control);
+    ssa_env_->effect = initial_effect;
+    ssa_env_->control = failure_control;
+    Value* returns_ref = decoder->zone()->NewArray<Value>(sig->return_count());
     DoCall(decoder, kCallRef, 0, NullCheckFor(func_ref.type), func_ref.node,
-           sig, sig_index, args, returns);
+           sig, sig_index, args, returns_ref);
+    TFNode* control_ref = control();
+    TFNode* effect_ref = effect();
+
+    TFNode* control_args[] = {control_direct, control_ref};
+    TFNode* control = builder_->Merge(2, control_args);
+
+    TFNode* effect_args[] = {effect_direct, effect_ref, control};
+    TFNode* effect = builder_->EffectPhi(2, effect_args);
+
+    ssa_env_->control = control;
+    ssa_env_->effect = effect;
+    builder_->SetEffectControl(effect, control);
+
+    for (uint32_t i = 0; i < sig->return_count(); i++) {
+      TFNode* phi_args[] = {returns_direct[i].node, returns_ref[i].node,
+                            control};
+      returns[i].node = builder_->Phi(sig->GetReturn(i), 2, phi_args);
+    }
   }
 
   void ReturnCallRef(FullDecoder* decoder, const Value& func_ref,
                      const FunctionSig* sig, uint32_t sig_index,
                      const Value args[]) {
+    if (!FLAG_wasm_inlining) {
+      DoReturnCall(decoder, kCallRef, 0, NullCheckFor(func_ref.type), func_ref,
+                   sig, sig_index, args);
+      return;
+    }
+
+    // Check for equality against a function at a specific index, and if
+    // successful, just emit a direct call.
+    // TODO(12166): For now, we check against function 0. Decide the index based
+    // on liftoff feedback.
+    const uint32_t expected_function_index = 0;
+
+    TFNode* success_control;
+    TFNode* failure_control;
+    builder_->CompareToExternalFunctionAtIndex(
+        func_ref.node, expected_function_index, &success_control,
+        &failure_control);
+    TFNode* initial_effect = effect();
+
+    builder_->SetControl(success_control);
+    ssa_env_->control = success_control;
+    DoReturnCall(decoder, kCallDirect, 0, CheckForNull::kWithoutNullCheck,
+                 Value{nullptr, kWasmBottom}, sig, expected_function_index,
+                 args);
+
+    builder_->SetEffectControl(initial_effect, failure_control);
+    ssa_env_->effect = initial_effect;
+    ssa_env_->control = failure_control;
     DoReturnCall(decoder, kCallRef, 0, NullCheckFor(func_ref.type), func_ref,
                  sig, sig_index, args);
   }

test/mjsunit/wasm/inlining.js

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 // Flags: --wasm-inlining --no-liftoff --experimental-wasm-return-call
+// Flags: --experimental-wasm-typed-funcref
 
 d8.file.execute("test/mjsunit/wasm/wasm-module-builder.js");
 
@@ -239,3 +240,91 @@ d8.file.execute("test/mjsunit/wasm/wasm-module-builder.js");
   let instance = builder.instantiate();
   assertEquals(20, instance.exports.main(10, 20));
 })();
+
+(function CallRefSpecSucceededTest() {
+  let builder = new WasmModuleBuilder();
+
+  // f(x) = x - 1
+  let callee = builder.addFunction("callee", kSig_i_i)
+    .addBody([kExprLocalGet, 0, kExprI32Const, 1, kExprI32Sub]);
+
+  let global = builder.addGlobal(wasmRefType(0), false,
+                                 WasmInitExpr.RefFunc(callee.index));
+
+  // g(x) = f(5) + x
+  builder.addFunction("main", kSig_i_i)
+    .addBody([kExprI32Const, 5, kExprGlobalGet, global.index, kExprCallRef,
+              kExprLocalGet, 0, kExprI32Add])
+    .exportAs("main");
+
+  let instance = builder.instantiate();
+  assertEquals(14, instance.exports.main(10));
+})();
+
+(function CallRefSpecFailedTest() {
+  let builder = new WasmModuleBuilder();
+
+  // h(x) = x - 1
+  builder.addFunction("callee", kSig_i_i)
+    .addBody([kExprLocalGet, 0, kExprI32Const, 1, kExprI32Sub]);
+
+  // f(x) = x - 2
+  let callee = builder.addFunction("callee", kSig_i_i)
+    .addBody([kExprLocalGet, 0, kExprI32Const, 2, kExprI32Sub]);
+
+  let global = builder.addGlobal(wasmRefType(1), false,
+                                 WasmInitExpr.RefFunc(callee.index));
+
+  // g(x) = f(5) + x
+  builder.addFunction("main", kSig_i_i)
+    .addBody([kExprI32Const, 5, kExprGlobalGet, global.index, kExprCallRef,
+              kExprLocalGet, 0, kExprI32Add])
+    .exportAs("main");
+
+  let instance = builder.instantiate();
+  assertEquals(13, instance.exports.main(10));
+})();
+
+(function CallReturnRefSpecSucceededTest() {
+  let builder = new WasmModuleBuilder();
+
+  // f(x) = x - 1
+  let callee = builder.addFunction("callee", kSig_i_i)
+    .addBody([kExprLocalGet, 0, kExprI32Const, 1, kExprI32Sub]);
+
+  let global = builder.addGlobal(wasmRefType(0), false,
+                                 WasmInitExpr.RefFunc(callee.index));
+
+  // g(x) = f(5 + x)
+  builder.addFunction("main", kSig_i_i)
+    .addBody([kExprI32Const, 5, kExprLocalGet, 0, kExprI32Add,
+              kExprGlobalGet, global.index, kExprReturnCallRef])
+    .exportAs("main");
+
+  let instance = builder.instantiate();
+  assertEquals(14, instance.exports.main(10));
+})();
+
+(function CallReturnRefSpecFailedTest() {
+  let builder = new WasmModuleBuilder();
+
+  // h(x) = x - 1
+  builder.addFunction("callee", kSig_i_i)
+    .addBody([kExprLocalGet, 0, kExprI32Const, 1, kExprI32Sub]);
+
+  // f(x) = x - 2
+  let callee = builder.addFunction("callee", kSig_i_i)
+    .addBody([kExprLocalGet, 0, kExprI32Const, 2, kExprI32Sub]);
+
+  let global = builder.addGlobal(wasmRefType(1), false,
+                                 WasmInitExpr.RefFunc(callee.index));
+
+  // g(x) = f(5 + x)
+  builder.addFunction("main", kSig_i_i)
+    .addBody([kExprI32Const, 5, kExprLocalGet, 0, kExprI32Add,
+              kExprGlobalGet, global.index, kExprReturnCallRef])
+    .exportAs("main");
+
+  let instance = builder.instantiate();
+  assertEquals(13, instance.exports.main(10));
+})();