[wasm] Avoid redundant code copy for import wrappers.

This avoids creating an on-heap copy for import wrappers by directly
adding the {WasmCode} into the native heap instead. It reduces
compilation time as well as useless GC pressure.

R=clemensh@chromium.org
BUG=v8:8423

Change-Id: Ia063523834c963591027c7d1ed78b795d24907bf
Reviewed-on: https://chromium-review.googlesource.com/c/1335566
Commit-Queue: Michael Starzinger <mstarzinger@chromium.org>
Reviewed-by: Andreas Haas <ahaas@chromium.org>
Reviewed-by: Clemens Hammacher <clemensh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#57511}

src/compiler/pipeline.cc

@@ -1111,7 +1111,8 @@ PipelineWasmCompilationJob::ExecuteJobImpl() {
       code_generator->GetSafepointTableOffset(),
       code_generator->GetHandlerTableOffset(),
       code_generator->GetProtectedInstructions(),
-      code_generator->GetSourcePositionTable(), wasm::WasmCode::kTurbofan);
+      code_generator->GetSourcePositionTable(), wasm::WasmCode::kFunction,
+      wasm::WasmCode::kTurbofan);
 
   if (data_.info()->trace_turbo_json_enabled()) {
     TurboJsonFile json_of(data_.info(), std::ios_base::app);
@@ -2233,7 +2234,99 @@ MaybeHandle<Code> Pipeline::GenerateCodeForCodeStub(
 }
 
 // static
-MaybeHandle<Code> Pipeline::GenerateCodeForWasmStub(
+wasm::WasmCode* Pipeline::GenerateCodeForWasmNativeStub(
+    wasm::WasmEngine* wasm_engine, CallDescriptor* call_descriptor,
+    MachineGraph* mcgraph, Code::Kind kind, const char* debug_name,
+    const AssemblerOptions& options, wasm::NativeModule* native_module,
+    SourcePositionTable* source_positions) {
+  Graph* graph = mcgraph->graph();
+  OptimizedCompilationInfo info(CStrVector(debug_name), graph->zone(), kind);
+  // Construct a pipeline for scheduling and code generation.
+  ZoneStats zone_stats(wasm_engine->allocator());
+  NodeOriginTable* node_positions = new (graph->zone()) NodeOriginTable(graph);
+  PipelineData data(&zone_stats, wasm_engine, &info, mcgraph, nullptr,
+                    source_positions, node_positions, -1, options);
+  std::unique_ptr<PipelineStatistics> pipeline_statistics;
+  if (FLAG_turbo_stats || FLAG_turbo_stats_nvp) {
+    pipeline_statistics.reset(new PipelineStatistics(
+        &info, wasm_engine->GetOrCreateTurboStatistics(), &zone_stats));
+    pipeline_statistics->BeginPhaseKind("wasm stub codegen");
+  }
+
+  PipelineImpl pipeline(&data);
+
+  if (info.trace_turbo_json_enabled() || info.trace_turbo_graph_enabled()) {
+    CodeTracer::Scope tracing_scope(data.GetCodeTracer());
+    OFStream os(tracing_scope.file());
+    os << "---------------------------------------------------\n"
+       << "Begin compiling method " << info.GetDebugName().get()
+       << " using Turbofan" << std::endl;
+  }
+
+  if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
+    StdoutStream{} << "-- wasm stub " << Code::Kind2String(kind) << " graph -- "
+                   << std::endl
+                   << AsRPO(*graph);
+  }
+
+  if (info.trace_turbo_json_enabled()) {
+    TurboJsonFile json_of(&info, std::ios_base::trunc);
+    json_of << "{\"function\":\"" << info.GetDebugName().get()
+            << "\", \"source\":\"\",\n\"phases\":[";
+  }
+  // TODO(rossberg): Should this really be untyped?
+  pipeline.RunPrintAndVerify("machine", true);
+  pipeline.ComputeScheduledGraph();
+
+  Linkage linkage(call_descriptor);
+  if (!pipeline.SelectInstructions(&linkage)) return nullptr;
+  pipeline.AssembleCode(&linkage);
+
+  CodeGenerator* code_generator = pipeline.code_generator();
+  CodeDesc code_desc;
+  code_generator->tasm()->GetCode(nullptr, &code_desc);
+
+  // TODO(mstarzinger): This is specific to Wasm-to-JS wrappers, fix this before
+  // using it for other wrappers (like the interpreter entry wrapper).
+  wasm::WasmCode* code = native_module->AddCode(
+      data.wasm_function_index(), code_desc,
+      code_generator->frame()->GetTotalFrameSlotCount(),
+      code_generator->GetSafepointTableOffset(),
+      code_generator->GetHandlerTableOffset(),
+      code_generator->GetProtectedInstructions(),
+      code_generator->GetSourcePositionTable(),
+      wasm::WasmCode::kWasmToJsWrapper, wasm::WasmCode::kOther);
+
+  if (info.trace_turbo_json_enabled()) {
+    TurboJsonFile json_of(&info, std::ios_base::app);
+    json_of << "{\"name\":\"disassembly\",\"type\":\"disassembly\",\"data\":\"";
+#ifdef ENABLE_DISASSEMBLER
+    std::stringstream disassembler_stream;
+    Disassembler::Decode(
+        nullptr, &disassembler_stream, code->instructions().start(),
+        code->instructions().start() + code->safepoint_table_offset(),
+        CodeReference(code));
+    for (auto const c : disassembler_stream.str()) {
+      json_of << AsEscapedUC16ForJSON(c);
+    }
+#endif  // ENABLE_DISASSEMBLER
+    json_of << "\"}\n]";
+    json_of << "\n}";
+  }
+
+  if (info.trace_turbo_json_enabled() || info.trace_turbo_graph_enabled()) {
+    CodeTracer::Scope tracing_scope(data.GetCodeTracer());
+    OFStream os(tracing_scope.file());
+    os << "---------------------------------------------------\n"
+       << "Finished compiling method " << info.GetDebugName().get()
+       << " using Turbofan" << std::endl;
+  }
+
+  return code;
+}
+
+// static
+MaybeHandle<Code> Pipeline::GenerateCodeForWasmHeapStub(
     Isolate* isolate, CallDescriptor* call_descriptor, Graph* graph,
     Code::Kind kind, const char* debug_name, const AssemblerOptions& options,
     SourcePositionTable* source_positions) {

src/compiler/pipeline.h

@@ -21,11 +21,10 @@ class RegisterConfiguration;
 class JumpOptimizationInfo;
 
 namespace wasm {
-enum ModuleOrigin : uint8_t;
 struct FunctionBody;
 class NativeModule;
+class WasmCode;
 class WasmEngine;
-struct WasmModule;
 }  // namespace wasm
 
 namespace compiler {
@@ -54,7 +53,15 @@ class Pipeline : public AllStatic {
       int function_index);
 
   // Run the pipeline on a machine graph and generate code.
-  static MaybeHandle<Code> GenerateCodeForWasmStub(
+  static wasm::WasmCode* GenerateCodeForWasmNativeStub(
+      wasm::WasmEngine* wasm_engine, CallDescriptor* call_descriptor,
+      MachineGraph* mcgraph, Code::Kind kind, const char* debug_name,
+      const AssemblerOptions& assembler_options,
+      wasm::NativeModule* native_module,
+      SourcePositionTable* source_positions = nullptr);
+
+  // Run the pipeline on a machine graph and generate code.
+  static MaybeHandle<Code> GenerateCodeForWasmHeapStub(
       Isolate* isolate, CallDescriptor* call_descriptor, Graph* graph,
       Code::Kind kind, const char* debug_name,
       const AssemblerOptions& assembler_options,

src/compiler/wasm-compiler.cc

@@ -4968,7 +4968,7 @@ MaybeHandle<Code> CompileJSToWasmWrapper(Isolate* isolate,
   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
       &zone, false, params + 1, CallDescriptor::kNoFlags);
 
-  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmStub(
+  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
       isolate, incoming, &graph, Code::JS_TO_WASM_FUNCTION, debug_name,
       WasmAssemblerOptions());
   Handle<Code> code;
@@ -5078,25 +5078,11 @@ wasm::WasmCode* CompileWasmImportCallWrapper(Isolate* isolate,
   if (machine.Is32()) {
     incoming = GetI32WasmCallDescriptor(&zone, incoming);
   }
-  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmStub(
-      isolate, incoming, &graph, Code::WASM_TO_JS_FUNCTION, func_name,
-      AssemblerOptions::Default(isolate), source_position_table);
-  Handle<Code> code = maybe_code.ToHandleChecked();
-#ifdef ENABLE_DISASSEMBLER
-  if (FLAG_print_opt_code) {
-    CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
-    OFStream os(tracing_scope.file());
-    code->Disassemble(func_name, os);
-  }
-#endif
-
-  if (must_record_function_compilation(isolate)) {
-    RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, "%s",
-                              func_name);
-  }
-
-  // TODO(wasm): No need to compile the code onto the heap and copy back.
-  wasm::WasmCode* wasm_code = native_module->AddImportCallWrapper(code);
+  wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
+      isolate->wasm_engine(), incoming, &jsgraph, Code::WASM_TO_JS_FUNCTION,
+      func_name, AssemblerOptions::Default(isolate), native_module,
+      source_position_table);
+  CHECK_NOT_NULL(wasm_code);
 
   return wasm_code;
 }
@@ -5136,7 +5122,7 @@ wasm::WasmCode* CompileWasmInterpreterEntry(Isolate* isolate,
   func_name.Truncate(
       SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
 
-  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmStub(
+  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
       isolate, incoming, &graph, Code::WASM_INTERPRETER_ENTRY,
       func_name.start(), AssemblerOptions::Default(isolate));
   Handle<Code> code = maybe_code.ToHandleChecked();
@@ -5189,7 +5175,7 @@ MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
   char debug_name[kMaxNameLen] = "c-wasm-entry:";
   AppendSignature(debug_name, kMaxNameLen, sig);
 
-  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmStub(
+  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
       isolate, incoming, &graph, Code::C_WASM_ENTRY, debug_name,
       AssemblerOptions::Default(isolate));
   Handle<Code> code;

src/wasm/baseline/liftoff-compiler.cc

@@ -1907,7 +1907,7 @@ bool LiftoffCompilationUnit::ExecuteCompilation(CompilationEnv* env,
   WasmCode* code = wasm_unit_->native_module_->AddCode(
       wasm_unit_->func_index_, desc, frame_slot_count, safepoint_table_offset,
       0, std::move(protected_instructions), std::move(source_positions),
-      WasmCode::kLiftoff);
+      WasmCode::kFunction, WasmCode::kLiftoff);
   wasm_unit_->SetResult(code, counters);
 
   return true;

src/wasm/wasm-code-manager.cc

@@ -440,11 +440,6 @@ WasmCode* NativeModule::AddInterpreterEntry(Handle<Code> code,
   return ret;
 }
 
-WasmCode* NativeModule::AddImportCallWrapper(Handle<Code> code) {
-  WasmCode* ret = AddAnonymousCode(code, WasmCode::kWasmToJsWrapper);
-  return ret;
-}
-
 WasmCode* NativeModule::AddCodeForTesting(Handle<Code> code) {
   WasmCode* ret = AddAnonymousCode(code, WasmCode::kFunction);
   return ret;
@@ -546,7 +541,8 @@ WasmCode* NativeModule::AddCode(
     uint32_t index, const CodeDesc& desc, uint32_t stack_slots,
     size_t safepoint_table_offset, size_t handler_table_offset,
     OwnedVector<trap_handler::ProtectedInstructionData> protected_instructions,
-    OwnedVector<const byte> source_pos_table, WasmCode::Tier tier) {
+    OwnedVector<const byte> source_pos_table, WasmCode::Kind kind,
+    WasmCode::Tier tier) {
   OwnedVector<byte> reloc_info = OwnedVector<byte>::New(desc.reloc_size);
   memcpy(reloc_info.start(), desc.buffer + desc.buffer_size - desc.reloc_size,
          desc.reloc_size);
@@ -555,7 +551,7 @@ WasmCode* NativeModule::AddCode(
                    stack_slots, safepoint_table_offset, handler_table_offset,
                    desc.instr_size - desc.constant_pool_size,
                    std::move(protected_instructions), std::move(reloc_info),
-                   std::move(source_pos_table), WasmCode::kFunction, tier);
+                   std::move(source_pos_table), kind, tier);
 
   // Apply the relocation delta by iterating over the RelocInfo.
   intptr_t delta = ret->instructions().start() - desc.buffer;

src/wasm/wasm-code-manager.h

@@ -221,7 +221,7 @@ class V8_EXPORT_PRIVATE NativeModule final {
                     OwnedVector<trap_handler::ProtectedInstructionData>
                         protected_instructions,
                     OwnedVector<const byte> source_position_table,
-                    WasmCode::Tier tier);
+                    WasmCode::Kind kind, WasmCode::Tier tier);
 
   WasmCode* AddDeserializedCode(
       uint32_t index, Vector<const byte> instructions, uint32_t stack_slots,
@@ -238,9 +238,6 @@ class V8_EXPORT_PRIVATE NativeModule final {
   // self-identify as the {index} function.
   WasmCode* AddInterpreterEntry(Handle<Code> code, uint32_t index);
 
-  // Add an import wrapper for a JS-to-Wasm call.
-  WasmCode* AddImportCallWrapper(Handle<Code> code);
-
   // Adds anonymous code for testing purposes.
   WasmCode* AddCodeForTesting(Handle<Code> code);
 

test/cctest/wasm/test-run-wasm.cc

@@ -3437,9 +3437,9 @@ TEST(Liftoff_tier_up) {
     memcpy(buffer.get(), sub_code->instructions().start(), sub_size);
     desc.buffer = buffer.get();
     desc.instr_size = static_cast<int>(sub_size);
-    WasmCode* code =
-        native_module->AddCode(add.function_index(), desc, 0, 0, 0, {},
-                               OwnedVector<byte>(), WasmCode::kOther);
+    WasmCode* code = native_module->AddCode(
+        add.function_index(), desc, 0, 0, 0, {}, OwnedVector<byte>(),
+        WasmCode::kFunction, WasmCode::kOther);
     native_module->PublishCode(code);
 
     // Second run should now execute {sub}.

test/unittests/wasm/wasm-code-manager-unittest.cc

@@ -176,7 +176,7 @@ class WasmCodeManagerTest : public TestWithContext,
     desc.buffer = exec_buff.get();
     desc.instr_size = static_cast<int>(size);
     return native_module->AddCode(index, desc, 0, 0, 0, {}, OwnedVector<byte>(),
-                                  WasmCode::kOther);
+                                  WasmCode::kFunction, WasmCode::kOther);
   }
 
   size_t page() const { return AllocatePageSize(); }