From 8b7bbcb0c98951d3580785eabe0c45b23c89f98a Mon Sep 17 00:00:00 2001 From: Liu Yu Date: Sun, 29 Jan 2023 15:51:53 +0800 Subject: [PATCH] [loong64][mips64][wasm][memory64] Fix atomics Port commit 76a817e03a83414a5eb727e5f1c73cfa3831cf30 Bug: v8:13636, v8:10949 Change-Id: Iabb31939b4d5aa582e67ce7f1076ac5946803a0a Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4196757 Auto-Submit: Liu Yu Reviewed-by: Zhao Jiazhong Commit-Queue: Zhao Jiazhong Cr-Commit-Position: refs/heads/main@{#85532} --- .../loong64/liftoff-assembler-loong64.h | 104 ++++++++++-------- .../mips64/liftoff-assembler-mips64.h | 98 +++++++++-------- 2 files changed, 110 insertions(+), 92 deletions(-) diff --git a/src/wasm/baseline/loong64/liftoff-assembler-loong64.h b/src/wasm/baseline/loong64/liftoff-assembler-loong64.h index 18d2ac981c..8adcd9b21d 100644 --- a/src/wasm/baseline/loong64/liftoff-assembler-loong64.h +++ b/src/wasm/baseline/loong64/liftoff-assembler-loong64.h @@ -537,9 +537,11 @@ void LiftoffAssembler::Store(Register dst_addr, Register offset_reg, void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr, Register offset_reg, uintptr_t offset_imm, - LoadType type, LiftoffRegList pinned) { + LoadType type, LiftoffRegList pinned, + bool i64_offset) { UseScratchRegisterScope temps(this); - MemOperand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm); + MemOperand src_op = + liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm, i64_offset); switch (type.value()) { case LoadType::kI32Load8U: case LoadType::kI64Load8U: { @@ -575,9 +577,11 @@ void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr, void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg, uintptr_t offset_imm, LiftoffRegister src, - StoreType type, LiftoffRegList pinned) { + StoreType type, LiftoffRegList pinned, + bool i64_offset) { UseScratchRegisterScope temps(this); - MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); + MemOperand dst_op = + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); switch (type.value()) { case StoreType::kI64Store8: case StoreType::kI32Store8: { @@ -625,43 +629,44 @@ void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg, dbar(0); \ } while (0) -#define ATOMIC_BINOP_CASE(name, inst32, inst64, opcode) \ - void LiftoffAssembler::Atomic##name( \ - Register dst_addr, Register offset_reg, uintptr_t offset_imm, \ - LiftoffRegister value, LiftoffRegister result, StoreType type) { \ - LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \ - Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - MemOperand dst_op = \ - liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); \ - Add_d(temp0, dst_op.base(), dst_op.offset()); \ - switch (type.value()) { \ - case StoreType::kI64Store8: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 8, inst64, 7); \ - break; \ - case StoreType::kI32Store8: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 8, inst32, 3); \ - break; \ - case StoreType::kI64Store16: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 16, inst64, 7); \ - break; \ - case StoreType::kI32Store16: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 16, inst32, 3); \ - break; \ - case StoreType::kI64Store32: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 32, inst64, 7); \ - break; \ - case StoreType::kI32Store: \ - am##opcode##_db_w(result.gp(), value.gp(), temp0); \ - break; \ - case StoreType::kI64Store: \ - am##opcode##_db_d(result.gp(), value.gp(), temp0); \ - break; \ - default: \ - UNREACHABLE(); \ - } \ +#define ATOMIC_BINOP_CASE(name, inst32, inst64, opcode) \ + void LiftoffAssembler::Atomic##name( \ + Register dst_addr, Register offset_reg, uintptr_t offset_imm, \ + LiftoffRegister value, LiftoffRegister result, StoreType type, \ + bool i64_offset) { \ + LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \ + Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + MemOperand dst_op = \ + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); \ + Add_d(temp0, dst_op.base(), dst_op.offset()); \ + switch (type.value()) { \ + case StoreType::kI64Store8: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 8, inst64, 7); \ + break; \ + case StoreType::kI32Store8: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 8, inst32, 3); \ + break; \ + case StoreType::kI64Store16: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 16, inst64, 7); \ + break; \ + case StoreType::kI32Store16: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 16, inst32, 3); \ + break; \ + case StoreType::kI64Store32: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 32, inst64, 7); \ + break; \ + case StoreType::kI32Store: \ + am##opcode##_db_w(result.gp(), value.gp(), temp0); \ + break; \ + case StoreType::kI64Store: \ + am##opcode##_db_d(result.gp(), value.gp(), temp0); \ + break; \ + default: \ + UNREACHABLE(); \ + } \ } ATOMIC_BINOP_CASE(Add, Add_w, Add_d, add) @@ -683,13 +688,15 @@ ATOMIC_BINOP_CASE(Xor, Xor, Xor, xor) void LiftoffAssembler::AtomicSub(Register dst_addr, Register offset_reg, uintptr_t offset_imm, LiftoffRegister value, - LiftoffRegister result, StoreType type) { + LiftoffRegister result, StoreType type, + bool i64_offset) { LiftoffRegList pinned{dst_addr, offset_reg, value, result}; Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); - MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); + MemOperand dst_op = + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); Add_d(temp0, dst_op.base(), dst_op.offset()); switch (type.value()) { case StoreType::kI64Store8: @@ -741,12 +748,14 @@ void LiftoffAssembler::AtomicSub(Register dst_addr, Register offset_reg, void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg, uintptr_t offset_imm, LiftoffRegister value, - LiftoffRegister result, StoreType type) { + LiftoffRegister result, StoreType type, + bool i64_offset) { LiftoffRegList pinned{dst_addr, offset_reg, value, result}; Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); - MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); + MemOperand dst_op = + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); Add_d(temp0, dst_op.base(), dst_op.offset()); switch (type.value()) { case StoreType::kI64Store8: @@ -816,12 +825,13 @@ void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg, void LiftoffAssembler::AtomicCompareExchange( Register dst_addr, Register offset_reg, uintptr_t offset_imm, LiftoffRegister expected, LiftoffRegister new_value, LiftoffRegister result, - StoreType type) { + StoreType type, bool i64_offset) { LiftoffRegList pinned{dst_addr, offset_reg, expected, new_value, result}; Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); - MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); + MemOperand dst_op = + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); Add_d(temp0, dst_op.base(), dst_op.offset()); switch (type.value()) { case StoreType::kI64Store8: diff --git a/src/wasm/baseline/mips64/liftoff-assembler-mips64.h b/src/wasm/baseline/mips64/liftoff-assembler-mips64.h index c301e9e2cd..6f5bf76c69 100644 --- a/src/wasm/baseline/mips64/liftoff-assembler-mips64.h +++ b/src/wasm/baseline/mips64/liftoff-assembler-mips64.h @@ -663,9 +663,11 @@ void LiftoffAssembler::Store(Register dst_addr, Register offset_reg, void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr, Register offset_reg, uintptr_t offset_imm, - LoadType type, LiftoffRegList pinned) { + LoadType type, LiftoffRegList pinned, + bool i64_offset) { UseScratchRegisterScope temps(this); - MemOperand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm); + MemOperand src_op = + liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm, i64_offset); switch (type.value()) { case LoadType::kI32Load8U: case LoadType::kI64Load8U: { @@ -701,9 +703,11 @@ void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr, void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg, uintptr_t offset_imm, LiftoffRegister src, - StoreType type, LiftoffRegList pinned) { + StoreType type, LiftoffRegList pinned, + bool i64_offset) { UseScratchRegisterScope temps(this); - MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); + MemOperand dst_op = + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); switch (type.value()) { case StoreType::kI64Store8: case StoreType::kI32Store8: { @@ -763,43 +767,44 @@ void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg, sync(); \ } while (0) -#define ATOMIC_BINOP_CASE(name, inst32, inst64) \ - void LiftoffAssembler::Atomic##name( \ - Register dst_addr, Register offset_reg, uintptr_t offset_imm, \ - LiftoffRegister value, LiftoffRegister result, StoreType type) { \ - LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \ - Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ - MemOperand dst_op = \ - liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); \ - Daddu(temp0, dst_op.rm(), dst_op.offset()); \ - switch (type.value()) { \ - case StoreType::kI64Store8: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 8, inst64, 7); \ - break; \ - case StoreType::kI32Store8: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 8, inst32, 3); \ - break; \ - case StoreType::kI64Store16: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 16, inst64, 7); \ - break; \ - case StoreType::kI32Store16: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 16, inst32, 3); \ - break; \ - case StoreType::kI64Store32: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 32, inst64, 7); \ - break; \ - case StoreType::kI32Store: \ - ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32); \ - break; \ - case StoreType::kI64Store: \ - ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64); \ - break; \ - default: \ - UNREACHABLE(); \ - } \ +#define ATOMIC_BINOP_CASE(name, inst32, inst64) \ + void LiftoffAssembler::Atomic##name( \ + Register dst_addr, Register offset_reg, uintptr_t offset_imm, \ + LiftoffRegister value, LiftoffRegister result, StoreType type, \ + bool i64_offset) { \ + LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \ + Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \ + MemOperand dst_op = \ + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); \ + Daddu(temp0, dst_op.rm(), dst_op.offset()); \ + switch (type.value()) { \ + case StoreType::kI64Store8: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 8, inst64, 7); \ + break; \ + case StoreType::kI32Store8: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 8, inst32, 3); \ + break; \ + case StoreType::kI64Store16: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 16, inst64, 7); \ + break; \ + case StoreType::kI32Store16: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 16, inst32, 3); \ + break; \ + case StoreType::kI64Store32: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 32, inst64, 7); \ + break; \ + case StoreType::kI32Store: \ + ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32); \ + break; \ + case StoreType::kI64Store: \ + ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64); \ + break; \ + default: \ + UNREACHABLE(); \ + } \ } ATOMIC_BINOP_CASE(Add, Addu, Daddu) @@ -843,12 +848,14 @@ ATOMIC_BINOP_CASE(Xor, Xor, Xor) void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg, uintptr_t offset_imm, LiftoffRegister value, - LiftoffRegister result, StoreType type) { + LiftoffRegister result, StoreType type, + bool i64_offset) { LiftoffRegList pinned{dst_addr, offset_reg, value, result}; Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); - MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); + MemOperand dst_op = + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); Daddu(temp0, dst_op.rm(), dst_op.offset()); switch (type.value()) { case StoreType::kI64Store8: @@ -919,12 +926,13 @@ void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg, void LiftoffAssembler::AtomicCompareExchange( Register dst_addr, Register offset_reg, uintptr_t offset_imm, LiftoffRegister expected, LiftoffRegister new_value, LiftoffRegister result, - StoreType type) { + StoreType type, bool i64_offset) { LiftoffRegList pinned{dst_addr, offset_reg, expected, new_value, result}; Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); - MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); + MemOperand dst_op = + liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); Daddu(temp0, dst_op.rm(), dst_op.offset()); switch (type.value()) { case StoreType::kI64Store8: