diff --git a/src/ia32/assembler-ia32.h b/src/ia32/assembler-ia32.h index 78bcec73a4..76594f7bd4 100644 --- a/src/ia32/assembler-ia32.h +++ b/src/ia32/assembler-ia32.h @@ -676,6 +676,7 @@ class Assembler : public AssemblerBase { // Arithmetics void adc(Register dst, int32_t imm32); + void adc(Register dst, Register src) { adc(dst, Operand(src)); } void adc(Register dst, Operand src); void add(Register dst, Register src) { add(dst, Operand(src)); } @@ -766,6 +767,7 @@ class Assembler : public AssemblerBase { void sar_cl(Register dst) { sar_cl(Operand(dst)); } void sar_cl(Operand dst); + void sbb(Register dst, Register src) { sbb(dst, Operand(src)); } void sbb(Register dst, Operand src); void shl(Register dst, uint8_t imm8) { shl(Operand(dst), imm8); } diff --git a/src/wasm/baseline/arm/liftoff-assembler-arm.h b/src/wasm/baseline/arm/liftoff-assembler-arm.h index 6ea4253cd3..36a2622639 100644 --- a/src/wasm/baseline/arm/liftoff-assembler-arm.h +++ b/src/wasm/baseline/arm/liftoff-assembler-arm.h @@ -103,6 +103,11 @@ void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) { Register rhs) { \ BAILOUT("gp binop: " #name); \ } +#define UNIMPLEMENTED_I64_BINOP(name) \ + void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister lhs, \ + LiftoffRegister rhs) { \ + BAILOUT("i64 binop" #name); \ + } #define UNIMPLEMENTED_GP_UNOP(name) \ bool LiftoffAssembler::emit_##name(Register dst, Register src) { \ BAILOUT("gp unop: " #name); \ @@ -134,6 +139,8 @@ UNIMPLEMENTED_GP_BINOP(i32_mul) UNIMPLEMENTED_GP_BINOP(i32_and) UNIMPLEMENTED_GP_BINOP(i32_or) UNIMPLEMENTED_GP_BINOP(i32_xor) +UNIMPLEMENTED_I64_BINOP(i64_add) +UNIMPLEMENTED_I64_BINOP(i64_sub) UNIMPLEMENTED_I32_SHIFTOP(i32_shl) UNIMPLEMENTED_I32_SHIFTOP(i32_sar) UNIMPLEMENTED_I32_SHIFTOP(i32_shr) @@ -143,7 +150,6 @@ UNIMPLEMENTED_I64_SHIFTOP(i64_shr) UNIMPLEMENTED_GP_UNOP(i32_clz) UNIMPLEMENTED_GP_UNOP(i32_ctz) UNIMPLEMENTED_GP_UNOP(i32_popcnt) -UNIMPLEMENTED_GP_BINOP(ptrsize_add) UNIMPLEMENTED_FP_BINOP(f32_add) UNIMPLEMENTED_FP_BINOP(f32_sub) UNIMPLEMENTED_FP_BINOP(f32_mul) @@ -160,6 +166,7 @@ UNIMPLEMENTED_FP_UNOP(f64_neg) UNIMPLEMENTED_FP_UNOP(f64_sqrt) #undef UNIMPLEMENTED_GP_BINOP +#undef UNIMPLEMENTED_I64_BINOP #undef UNIMPLEMENTED_GP_UNOP #undef UNIMPLEMENTED_FP_BINOP #undef UNIMPLEMENTED_FP_UNOP diff --git a/src/wasm/baseline/arm64/liftoff-assembler-arm64.h b/src/wasm/baseline/arm64/liftoff-assembler-arm64.h index 9a479f7a63..bc041f97eb 100644 --- a/src/wasm/baseline/arm64/liftoff-assembler-arm64.h +++ b/src/wasm/baseline/arm64/liftoff-assembler-arm64.h @@ -103,6 +103,11 @@ void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) { Register rhs) { \ BAILOUT("gp binop: " #name); \ } +#define UNIMPLEMENTED_I64_BINOP(name) \ + void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister lhs, \ + LiftoffRegister rhs) { \ + BAILOUT("i64 binop" #name); \ + } #define UNIMPLEMENTED_GP_UNOP(name) \ bool LiftoffAssembler::emit_##name(Register dst, Register src) { \ BAILOUT("gp unop: " #name); \ @@ -134,6 +139,8 @@ UNIMPLEMENTED_GP_BINOP(i32_mul) UNIMPLEMENTED_GP_BINOP(i32_and) UNIMPLEMENTED_GP_BINOP(i32_or) UNIMPLEMENTED_GP_BINOP(i32_xor) +UNIMPLEMENTED_I64_BINOP(i64_add) +UNIMPLEMENTED_I64_BINOP(i64_sub) UNIMPLEMENTED_I32_SHIFTOP(i32_shl) UNIMPLEMENTED_I32_SHIFTOP(i32_sar) UNIMPLEMENTED_I32_SHIFTOP(i32_shr) @@ -143,7 +150,6 @@ UNIMPLEMENTED_I64_SHIFTOP(i64_shr) UNIMPLEMENTED_GP_UNOP(i32_clz) UNIMPLEMENTED_GP_UNOP(i32_ctz) UNIMPLEMENTED_GP_UNOP(i32_popcnt) -UNIMPLEMENTED_GP_BINOP(ptrsize_add) UNIMPLEMENTED_FP_BINOP(f32_add) UNIMPLEMENTED_FP_BINOP(f32_sub) UNIMPLEMENTED_FP_BINOP(f32_mul) @@ -160,6 +166,7 @@ UNIMPLEMENTED_FP_UNOP(f64_neg) UNIMPLEMENTED_FP_UNOP(f64_sqrt) #undef UNIMPLEMENTED_GP_BINOP +#undef UNIMPLEMENTED_I64_BINOP #undef UNIMPLEMENTED_GP_UNOP #undef UNIMPLEMENTED_FP_BINOP #undef UNIMPLEMENTED_FP_UNOP diff --git a/src/wasm/baseline/ia32/liftoff-assembler-ia32.h b/src/wasm/baseline/ia32/liftoff-assembler-ia32.h index 3630bf0c55..1791c90c12 100644 --- a/src/wasm/baseline/ia32/liftoff-assembler-ia32.h +++ b/src/wasm/baseline/ia32/liftoff-assembler-ia32.h @@ -547,6 +547,47 @@ bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) { return true; } +namespace liftoff { +template +inline void OpWithCarry(LiftoffAssembler* assm, LiftoffRegister dst, + LiftoffRegister lhs, LiftoffRegister rhs) { + // First, compute the low half of the result, potentially into a temporary dst + // register if {dst.low_gp()} equals {rhs.low_gp()} or any register we need to + // keep alive for computing the upper half. + LiftoffRegList keep_alive = LiftoffRegList::ForRegs(lhs.high_gp(), rhs); + Register dst_low = keep_alive.has(dst.low_gp()) + ? assm->GetUnusedRegister(kGpReg, keep_alive).gp() + : dst.low_gp(); + + if (dst_low != lhs.low_gp()) assm->mov(dst_low, lhs.low_gp()); + (assm->*op)(dst_low, rhs.low_gp()); + + // Now compute the upper half, while keeping alive the previous result. + keep_alive = LiftoffRegList::ForRegs(dst_low, rhs.high_gp()); + Register dst_high = keep_alive.has(dst.high_gp()) + ? assm->GetUnusedRegister(kGpReg, keep_alive).gp() + : dst.high_gp(); + + if (dst_high != lhs.high_gp()) assm->mov(dst_high, lhs.high_gp()); + (assm->*op_with_carry)(dst_high, rhs.high_gp()); + + // If necessary, move result into the right registers. + LiftoffRegister tmp_result = LiftoffRegister::ForPair(dst_low, dst_high); + if (tmp_result != dst) assm->Move(dst, tmp_result, kWasmI64); +} +} // namespace liftoff + +void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs) { + liftoff::OpWithCarry<&Assembler::add, &Assembler::adc>(this, dst, lhs, rhs); +} + +void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs) { + liftoff::OpWithCarry<&Assembler::sub, &Assembler::sbb>(this, dst, lhs, rhs); +} + namespace liftoff { inline bool PairContains(LiftoffRegister pair, Register reg) { return pair.low_gp() == reg || pair.high_gp() == reg; @@ -615,11 +656,6 @@ void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src, &TurboAssembler::ShrPair_cl, pinned); } -void LiftoffAssembler::emit_ptrsize_add(Register dst, Register lhs, - Register rhs) { - emit_i32_add(dst, lhs, rhs); -} - void LiftoffAssembler::emit_f32_add(DoubleRegister dst, DoubleRegister lhs, DoubleRegister rhs) { if (CpuFeatures::IsSupported(AVX)) { diff --git a/src/wasm/baseline/liftoff-assembler.h b/src/wasm/baseline/liftoff-assembler.h index 1a5c0ff946..22b5575209 100644 --- a/src/wasm/baseline/liftoff-assembler.h +++ b/src/wasm/baseline/liftoff-assembler.h @@ -397,6 +397,10 @@ class LiftoffAssembler : public TurboAssembler { inline bool emit_i32_popcnt(Register dst, Register src); // i64 binops. + inline void emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); + inline void emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); inline void emit_i64_shl(LiftoffRegister dst, LiftoffRegister src, Register amount, LiftoffRegList pinned = {}); inline void emit_i64_sar(LiftoffRegister dst, LiftoffRegister src, @@ -404,7 +408,14 @@ class LiftoffAssembler : public TurboAssembler { inline void emit_i64_shr(LiftoffRegister dst, LiftoffRegister src, Register amount, LiftoffRegList pinned = {}); - inline void emit_ptrsize_add(Register dst, Register lhs, Register rhs); + inline void emit_ptrsize_add(Register dst, Register lhs, Register rhs) { + if (kPointerSize == 4) { + emit_i64_add(LiftoffRegister(dst), LiftoffRegister(lhs), + LiftoffRegister(rhs)); + } else { + emit_i32_add(dst, lhs, rhs); + } + } // f32 binops. inline void emit_f32_add(DoubleRegister dst, DoubleRegister lhs, diff --git a/src/wasm/baseline/liftoff-compiler.cc b/src/wasm/baseline/liftoff-compiler.cc index 52f30ff6e8..3927fc39f8 100644 --- a/src/wasm/baseline/liftoff-compiler.cc +++ b/src/wasm/baseline/liftoff-compiler.cc @@ -687,6 +687,12 @@ class LiftoffCompiler { [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp()); \ }); +#define CASE_I64_BINOP(opcode, fn) \ + case WasmOpcode::kExpr##opcode: \ + return EmitBinOp( \ + [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ + __ emit_##fn(dst, lhs, rhs); \ + }); #define CASE_FLOAT_BINOP(opcode, type, fn) \ case WasmOpcode::kExpr##opcode: \ return EmitBinOp( \ @@ -752,6 +758,8 @@ class LiftoffCompiler { CASE_I32_CMPOP(I32LeU, kUnsignedLessEqual) CASE_I32_CMPOP(I32GeS, kSignedGreaterEqual) CASE_I32_CMPOP(I32GeU, kUnsignedGreaterEqual) + CASE_I64_BINOP(I64Add, i64_add) + CASE_I64_BINOP(I64Sub, i64_sub) CASE_I64_CMPOP(I64Eq, kEqual) CASE_I64_CMPOP(I64Ne, kUnequal) CASE_I64_CMPOP(I64LtS, kSignedLessThan) @@ -788,6 +796,7 @@ class LiftoffCompiler { return unsupported(decoder, WasmOpcodes::OpcodeName(opcode)); } #undef CASE_I32_BINOP +#undef CASE_I64_BINOP #undef CASE_FLOAT_BINOP #undef CASE_I32_CMPOP #undef CASE_I64_CMPOP diff --git a/src/wasm/baseline/liftoff-register.h b/src/wasm/baseline/liftoff-register.h index 94a6c19e81..3db7bdc1a4 100644 --- a/src/wasm/baseline/liftoff-register.h +++ b/src/wasm/baseline/liftoff-register.h @@ -233,6 +233,8 @@ class LiftoffRegList { } return (regs_ & (storage_t{1} << reg.liftoff_code())) != 0; } + bool has(Register reg) const { return has(LiftoffRegister(reg)); } + bool has(DoubleRegister reg) const { return has(LiftoffRegister(reg)); } constexpr bool is_empty() const { return regs_ == 0; } diff --git a/src/wasm/baseline/mips/liftoff-assembler-mips.h b/src/wasm/baseline/mips/liftoff-assembler-mips.h index 0b61e649ca..57920073b1 100644 --- a/src/wasm/baseline/mips/liftoff-assembler-mips.h +++ b/src/wasm/baseline/mips/liftoff-assembler-mips.h @@ -408,11 +408,6 @@ I32_BINOP(xor, xor_) #undef I32_BINOP -void LiftoffAssembler::emit_ptrsize_add(Register dst, Register lhs, - Register rhs) { - emit_i32_add(dst, lhs, rhs); -} - bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) { TurboAssembler::Clz(dst, src); return true; @@ -440,6 +435,17 @@ I32_SHIFTOP(shr, srlv) #undef I32_SHIFTOP +#define UNIMPLEMENTED_I64_BINOP(name) \ + void LiftoffAssembler::emit_i64_##name( \ + LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ + BAILOUT("i64 binop: " #name); \ + } + +UNIMPLEMENTED_I64_BINOP(add) +UNIMPLEMENTED_I64_BINOP(sub) + +#undef UNIMPLEMENTED_I64_BINOP + namespace liftoff { inline bool IsRegInRegPair(LiftoffRegister pair, Register reg) { diff --git a/src/wasm/baseline/mips64/liftoff-assembler-mips64.h b/src/wasm/baseline/mips64/liftoff-assembler-mips64.h index beac6b3750..669ed42db9 100644 --- a/src/wasm/baseline/mips64/liftoff-assembler-mips64.h +++ b/src/wasm/baseline/mips64/liftoff-assembler-mips64.h @@ -355,11 +355,6 @@ I32_BINOP(xor, xor_) #undef I32_BINOP -void LiftoffAssembler::emit_ptrsize_add(Register dst, Register lhs, - Register rhs) { - TurboAssembler::Daddu(dst, lhs, rhs); -} - bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) { TurboAssembler::Clz(dst, src); return true; @@ -387,6 +382,16 @@ I32_SHIFTOP(shr, srlv) #undef I32_SHIFTOP +void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs) { + TurboAssembler::Daddu(dst.gp(), lhs.gp(), rhs.gp()); +} + +void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs) { + BAILOUT("i64_sub"); +} + #define I64_SHIFTOP(name, instruction) \ void LiftoffAssembler::emit_i64_##name(LiftoffRegister dst, \ LiftoffRegister src, Register amount, \ diff --git a/src/wasm/baseline/ppc/liftoff-assembler-ppc.h b/src/wasm/baseline/ppc/liftoff-assembler-ppc.h index df54fd0e62..1657753afa 100644 --- a/src/wasm/baseline/ppc/liftoff-assembler-ppc.h +++ b/src/wasm/baseline/ppc/liftoff-assembler-ppc.h @@ -103,6 +103,11 @@ void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) { Register rhs) { \ BAILOUT("gp binop: " #name); \ } +#define UNIMPLEMENTED_I64_BINOP(name) \ + void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister lhs, \ + LiftoffRegister rhs) { \ + BAILOUT("i64 binop" #name); \ + } #define UNIMPLEMENTED_GP_UNOP(name) \ bool LiftoffAssembler::emit_##name(Register dst, Register src) { \ BAILOUT("gp unop: " #name); \ @@ -134,6 +139,8 @@ UNIMPLEMENTED_GP_BINOP(i32_mul) UNIMPLEMENTED_GP_BINOP(i32_and) UNIMPLEMENTED_GP_BINOP(i32_or) UNIMPLEMENTED_GP_BINOP(i32_xor) +UNIMPLEMENTED_I64_BINOP(i64_add) +UNIMPLEMENTED_I64_BINOP(i64_sub) UNIMPLEMENTED_I32_SHIFTOP(i32_shl) UNIMPLEMENTED_I32_SHIFTOP(i32_sar) UNIMPLEMENTED_I32_SHIFTOP(i32_shr) @@ -143,7 +150,6 @@ UNIMPLEMENTED_I64_SHIFTOP(i64_shr) UNIMPLEMENTED_GP_UNOP(i32_clz) UNIMPLEMENTED_GP_UNOP(i32_ctz) UNIMPLEMENTED_GP_UNOP(i32_popcnt) -UNIMPLEMENTED_GP_BINOP(ptrsize_add) UNIMPLEMENTED_FP_BINOP(f32_add) UNIMPLEMENTED_FP_BINOP(f32_sub) UNIMPLEMENTED_FP_BINOP(f32_mul) @@ -160,6 +166,7 @@ UNIMPLEMENTED_FP_UNOP(f64_neg) UNIMPLEMENTED_FP_UNOP(f64_sqrt) #undef UNIMPLEMENTED_GP_BINOP +#undef UNIMPLEMENTED_I64_BINOP #undef UNIMPLEMENTED_GP_UNOP #undef UNIMPLEMENTED_FP_BINOP #undef UNIMPLEMENTED_FP_UNOP diff --git a/src/wasm/baseline/s390/liftoff-assembler-s390.h b/src/wasm/baseline/s390/liftoff-assembler-s390.h index 53ffe128cb..56f78cb4f2 100644 --- a/src/wasm/baseline/s390/liftoff-assembler-s390.h +++ b/src/wasm/baseline/s390/liftoff-assembler-s390.h @@ -103,6 +103,11 @@ void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) { Register rhs) { \ BAILOUT("gp binop: " #name); \ } +#define UNIMPLEMENTED_I64_BINOP(name) \ + void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister lhs, \ + LiftoffRegister rhs) { \ + BAILOUT("i64 binop" #name); \ + } #define UNIMPLEMENTED_GP_UNOP(name) \ bool LiftoffAssembler::emit_##name(Register dst, Register src) { \ BAILOUT("gp unop: " #name); \ @@ -134,6 +139,8 @@ UNIMPLEMENTED_GP_BINOP(i32_mul) UNIMPLEMENTED_GP_BINOP(i32_and) UNIMPLEMENTED_GP_BINOP(i32_or) UNIMPLEMENTED_GP_BINOP(i32_xor) +UNIMPLEMENTED_I64_BINOP(i64_add) +UNIMPLEMENTED_I64_BINOP(i64_sub) UNIMPLEMENTED_I32_SHIFTOP(i32_shl) UNIMPLEMENTED_I32_SHIFTOP(i32_sar) UNIMPLEMENTED_I32_SHIFTOP(i32_shr) @@ -143,7 +150,6 @@ UNIMPLEMENTED_I64_SHIFTOP(i64_shr) UNIMPLEMENTED_GP_UNOP(i32_clz) UNIMPLEMENTED_GP_UNOP(i32_ctz) UNIMPLEMENTED_GP_UNOP(i32_popcnt) -UNIMPLEMENTED_GP_BINOP(ptrsize_add) UNIMPLEMENTED_FP_BINOP(f32_add) UNIMPLEMENTED_FP_BINOP(f32_sub) UNIMPLEMENTED_FP_BINOP(f32_mul) @@ -160,6 +166,7 @@ UNIMPLEMENTED_FP_UNOP(f64_neg) UNIMPLEMENTED_FP_UNOP(f64_sqrt) #undef UNIMPLEMENTED_GP_BINOP +#undef UNIMPLEMENTED_I64_BINOP #undef UNIMPLEMENTED_GP_UNOP #undef UNIMPLEMENTED_FP_BINOP #undef UNIMPLEMENTED_FP_UNOP diff --git a/src/wasm/baseline/x64/liftoff-assembler-x64.h b/src/wasm/baseline/x64/liftoff-assembler-x64.h index 0ba0ba5f86..8d6f1370be 100644 --- a/src/wasm/baseline/x64/liftoff-assembler-x64.h +++ b/src/wasm/baseline/x64/liftoff-assembler-x64.h @@ -497,6 +497,26 @@ bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) { return true; } +void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs) { + if (lhs.gp() != dst.gp()) { + leap(dst.gp(), Operand(lhs.gp(), rhs.gp(), times_1, 0)); + } else { + addp(dst.gp(), rhs.gp()); + } +} + +void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs) { + if (dst.gp() == rhs.gp()) { + negq(dst.gp()); + addq(dst.gp(), lhs.gp()); + } else { + if (dst.gp() != lhs.gp()) movq(dst.gp(), lhs.gp()); + subq(dst.gp(), rhs.gp()); + } +} + void LiftoffAssembler::emit_i64_shl(LiftoffRegister dst, LiftoffRegister src, Register amount, LiftoffRegList pinned) { liftoff::EmitShiftOperation(this, dst.gp(), src.gp(), amount, @@ -515,15 +535,6 @@ void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src, &Assembler::shrq_cl, pinned); } -void LiftoffAssembler::emit_ptrsize_add(Register dst, Register lhs, - Register rhs) { - if (lhs != dst) { - leap(dst, Operand(lhs, rhs, times_1, 0)); - } else { - addp(dst, rhs); - } -} - void LiftoffAssembler::emit_f32_add(DoubleRegister dst, DoubleRegister lhs, DoubleRegister rhs) { if (CpuFeatures::IsSupported(AVX)) {