[loong64][mips64][wasm][memory64] Fix atomics
Port commit 76a817e03a
Bug: v8:13636, v8:10949
Change-Id: Iabb31939b4d5aa582e67ce7f1076ac5946803a0a
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4196757
Auto-Submit: Liu Yu <liuyu@loongson.cn>
Reviewed-by: Zhao Jiazhong <zhaojiazhong-hf@loongson.cn>
Commit-Queue: Zhao Jiazhong <zhaojiazhong-hf@loongson.cn>
Cr-Commit-Position: refs/heads/main@{#85532}
This commit is contained in:
Liu Yu
V8 LUCI CQ
parent
82cbc72c0e
commit
8b7bbcb0c9
@ -537,9 +537,11 @@ void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
|
||||
|
||||
void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr,
|
||||
Register offset_reg, uintptr_t offset_imm,
|
||||
LoadType type, LiftoffRegList pinned) {
|
||||
LoadType type, LiftoffRegList pinned,
|
||||
bool i64_offset) {
|
||||
UseScratchRegisterScope temps(this);
|
||||
MemOperand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
|
||||
MemOperand src_op =
|
||||
liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm, i64_offset);
|
||||
switch (type.value()) {
|
||||
case LoadType::kI32Load8U:
|
||||
case LoadType::kI64Load8U: {
|
||||
@ -575,9 +577,11 @@ void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr,
|
||||
|
||||
void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg,
|
||||
uintptr_t offset_imm, LiftoffRegister src,
|
||||
StoreType type, LiftoffRegList pinned) {
|
||||
StoreType type, LiftoffRegList pinned,
|
||||
bool i64_offset) {
|
||||
UseScratchRegisterScope temps(this);
|
||||
MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
|
||||
MemOperand dst_op =
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset);
|
||||
switch (type.value()) {
|
||||
case StoreType::kI64Store8:
|
||||
case StoreType::kI32Store8: {
|
||||
@ -625,43 +629,44 @@ void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg,
|
||||
dbar(0); \
|
||||
} while (0)
|
||||
|
||||
#define ATOMIC_BINOP_CASE(name, inst32, inst64, opcode) \
|
||||
void LiftoffAssembler::Atomic##name( \
|
||||
Register dst_addr, Register offset_reg, uintptr_t offset_imm, \
|
||||
LiftoffRegister value, LiftoffRegister result, StoreType type) { \
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
MemOperand dst_op = \
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); \
|
||||
Add_d(temp0, dst_op.base(), dst_op.offset()); \
|
||||
switch (type.value()) { \
|
||||
case StoreType::kI64Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 8, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 8, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 16, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 16, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store32: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 32, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store: \
|
||||
am##opcode##_db_w(result.gp(), value.gp(), temp0); \
|
||||
break; \
|
||||
case StoreType::kI64Store: \
|
||||
am##opcode##_db_d(result.gp(), value.gp(), temp0); \
|
||||
break; \
|
||||
default: \
|
||||
UNREACHABLE(); \
|
||||
} \
|
||||
#define ATOMIC_BINOP_CASE(name, inst32, inst64, opcode) \
|
||||
void LiftoffAssembler::Atomic##name( \
|
||||
Register dst_addr, Register offset_reg, uintptr_t offset_imm, \
|
||||
LiftoffRegister value, LiftoffRegister result, StoreType type, \
|
||||
bool i64_offset) { \
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
MemOperand dst_op = \
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); \
|
||||
Add_d(temp0, dst_op.base(), dst_op.offset()); \
|
||||
switch (type.value()) { \
|
||||
case StoreType::kI64Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 8, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 8, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 16, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_w, Sc_w, 16, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store32: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll_d, Sc_d, 32, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store: \
|
||||
am##opcode##_db_w(result.gp(), value.gp(), temp0); \
|
||||
break; \
|
||||
case StoreType::kI64Store: \
|
||||
am##opcode##_db_d(result.gp(), value.gp(), temp0); \
|
||||
break; \
|
||||
default: \
|
||||
UNREACHABLE(); \
|
||||
} \
|
||||
}
|
||||
|
||||
ATOMIC_BINOP_CASE(Add, Add_w, Add_d, add)
|
||||
@ -683,13 +688,15 @@ ATOMIC_BINOP_CASE(Xor, Xor, Xor, xor)
|
||||
|
||||
void LiftoffAssembler::AtomicSub(Register dst_addr, Register offset_reg,
|
||||
uintptr_t offset_imm, LiftoffRegister value,
|
||||
LiftoffRegister result, StoreType type) {
|
||||
LiftoffRegister result, StoreType type,
|
||||
bool i64_offset) {
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, value, result};
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
|
||||
MemOperand dst_op =
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset);
|
||||
Add_d(temp0, dst_op.base(), dst_op.offset());
|
||||
switch (type.value()) {
|
||||
case StoreType::kI64Store8:
|
||||
@ -741,12 +748,14 @@ void LiftoffAssembler::AtomicSub(Register dst_addr, Register offset_reg,
|
||||
void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg,
|
||||
uintptr_t offset_imm,
|
||||
LiftoffRegister value,
|
||||
LiftoffRegister result, StoreType type) {
|
||||
LiftoffRegister result, StoreType type,
|
||||
bool i64_offset) {
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, value, result};
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
|
||||
MemOperand dst_op =
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset);
|
||||
Add_d(temp0, dst_op.base(), dst_op.offset());
|
||||
switch (type.value()) {
|
||||
case StoreType::kI64Store8:
|
||||
@ -816,12 +825,13 @@ void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg,
|
||||
void LiftoffAssembler::AtomicCompareExchange(
|
||||
Register dst_addr, Register offset_reg, uintptr_t offset_imm,
|
||||
LiftoffRegister expected, LiftoffRegister new_value, LiftoffRegister result,
|
||||
StoreType type) {
|
||||
StoreType type, bool i64_offset) {
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, expected, new_value, result};
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
|
||||
MemOperand dst_op =
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset);
|
||||
Add_d(temp0, dst_op.base(), dst_op.offset());
|
||||
switch (type.value()) {
|
||||
case StoreType::kI64Store8:
|
||||
|
||||
@ -663,9 +663,11 @@ void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
|
||||
|
||||
void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr,
|
||||
Register offset_reg, uintptr_t offset_imm,
|
||||
LoadType type, LiftoffRegList pinned) {
|
||||
LoadType type, LiftoffRegList pinned,
|
||||
bool i64_offset) {
|
||||
UseScratchRegisterScope temps(this);
|
||||
MemOperand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
|
||||
MemOperand src_op =
|
||||
liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm, i64_offset);
|
||||
switch (type.value()) {
|
||||
case LoadType::kI32Load8U:
|
||||
case LoadType::kI64Load8U: {
|
||||
@ -701,9 +703,11 @@ void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr,
|
||||
|
||||
void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg,
|
||||
uintptr_t offset_imm, LiftoffRegister src,
|
||||
StoreType type, LiftoffRegList pinned) {
|
||||
StoreType type, LiftoffRegList pinned,
|
||||
bool i64_offset) {
|
||||
UseScratchRegisterScope temps(this);
|
||||
MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
|
||||
MemOperand dst_op =
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset);
|
||||
switch (type.value()) {
|
||||
case StoreType::kI64Store8:
|
||||
case StoreType::kI32Store8: {
|
||||
@ -763,43 +767,44 @@ void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg,
|
||||
sync(); \
|
||||
} while (0)
|
||||
|
||||
#define ATOMIC_BINOP_CASE(name, inst32, inst64) \
|
||||
void LiftoffAssembler::Atomic##name( \
|
||||
Register dst_addr, Register offset_reg, uintptr_t offset_imm, \
|
||||
LiftoffRegister value, LiftoffRegister result, StoreType type) { \
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
MemOperand dst_op = \
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); \
|
||||
Daddu(temp0, dst_op.rm(), dst_op.offset()); \
|
||||
switch (type.value()) { \
|
||||
case StoreType::kI64Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 8, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 8, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 16, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 16, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store32: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 32, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store: \
|
||||
ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32); \
|
||||
break; \
|
||||
case StoreType::kI64Store: \
|
||||
ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64); \
|
||||
break; \
|
||||
default: \
|
||||
UNREACHABLE(); \
|
||||
} \
|
||||
#define ATOMIC_BINOP_CASE(name, inst32, inst64) \
|
||||
void LiftoffAssembler::Atomic##name( \
|
||||
Register dst_addr, Register offset_reg, uintptr_t offset_imm, \
|
||||
LiftoffRegister value, LiftoffRegister result, StoreType type, \
|
||||
bool i64_offset) { \
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, value, result}; \
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
|
||||
MemOperand dst_op = \
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset); \
|
||||
Daddu(temp0, dst_op.rm(), dst_op.offset()); \
|
||||
switch (type.value()) { \
|
||||
case StoreType::kI64Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 8, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store8: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 8, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 16, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store16: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 16, inst32, 3); \
|
||||
break; \
|
||||
case StoreType::kI64Store32: \
|
||||
ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 32, inst64, 7); \
|
||||
break; \
|
||||
case StoreType::kI32Store: \
|
||||
ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32); \
|
||||
break; \
|
||||
case StoreType::kI64Store: \
|
||||
ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64); \
|
||||
break; \
|
||||
default: \
|
||||
UNREACHABLE(); \
|
||||
} \
|
||||
}
|
||||
|
||||
ATOMIC_BINOP_CASE(Add, Addu, Daddu)
|
||||
@ -843,12 +848,14 @@ ATOMIC_BINOP_CASE(Xor, Xor, Xor)
|
||||
void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg,
|
||||
uintptr_t offset_imm,
|
||||
LiftoffRegister value,
|
||||
LiftoffRegister result, StoreType type) {
|
||||
LiftoffRegister result, StoreType type,
|
||||
bool i64_offset) {
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, value, result};
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
|
||||
MemOperand dst_op =
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset);
|
||||
Daddu(temp0, dst_op.rm(), dst_op.offset());
|
||||
switch (type.value()) {
|
||||
case StoreType::kI64Store8:
|
||||
@ -919,12 +926,13 @@ void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg,
|
||||
void LiftoffAssembler::AtomicCompareExchange(
|
||||
Register dst_addr, Register offset_reg, uintptr_t offset_imm,
|
||||
LiftoffRegister expected, LiftoffRegister new_value, LiftoffRegister result,
|
||||
StoreType type) {
|
||||
StoreType type, bool i64_offset) {
|
||||
LiftoffRegList pinned{dst_addr, offset_reg, expected, new_value, result};
|
||||
Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
|
||||
MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
|
||||
MemOperand dst_op =
|
||||
liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm, i64_offset);
|
||||
Daddu(temp0, dst_op.rm(), dst_op.offset());
|
||||
switch (type.value()) {
|
||||
case StoreType::kI64Store8:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user