From 5641559d44023e82fc820a0a8b0c1142fcfccbf3 Mon Sep 17 00:00:00 2001 From: "Miran.Karic" Date: Wed, 5 Jul 2017 17:29:13 +0200 Subject: [PATCH] MIPS[64]: Add optimizations to memory load/store helper. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CL replaces several helper functions for memory load/store using base register and offset with one helper function that contains several optimizations. BUG= Change-Id: I187e7e882131552abd9a0b3a0070d78adefd25b6 Reviewed-on: https://chromium-review.googlesource.com/552119 Commit-Queue: Miran Karić Reviewed-by: Ivica Bogosavljevic Cr-Commit-Position: refs/heads/master@{#46420} --- src/mips/assembler-mips.cc | 262 +++++++++++---------- src/mips/assembler-mips.h | 16 +- src/mips/macro-assembler-mips.cc | 166 ++++++-------- src/mips64/assembler-mips64.cc | 156 ++++++++++--- src/mips64/assembler-mips64.h | 14 +- src/mips64/macro-assembler-mips64.cc | 328 +++++++++------------------ src/mips64/macro-assembler-mips64.h | 4 - 7 files changed, 476 insertions(+), 470 deletions(-) diff --git a/src/mips/assembler-mips.cc b/src/mips/assembler-mips.cc index 1a52e3d3cc..80bc895289 100644 --- a/src/mips/assembler-mips.cc +++ b/src/mips/assembler-mips.cc @@ -1930,107 +1930,151 @@ void Assembler::lsa(Register rd, Register rt, Register rs, uint8_t sa) { // ------------Memory-instructions------------- -// Helper for base-reg + offset, when offset is larger than int16. -void Assembler::LoadRegPlusOffsetToAt(const MemOperand& src) { - DCHECK(!src.rm().is(at)); - if (IsMipsArchVariant(kMips32r6)) { - int32_t hi = (src.offset_ >> kLuiShift) & kImm16Mask; - if (src.offset_ & kNegOffset) { - hi += 1; +void Assembler::AdjustBaseAndOffset(MemOperand& src, + OffsetAccessType access_type, + int second_access_add_to_offset) { + // This method is used to adjust the base register and offset pair + // for a load/store when the offset doesn't fit into int16_t. + // It is assumed that 'base + offset' is sufficiently aligned for memory + // operands that are machine word in size or smaller. For doubleword-sized + // operands it's assumed that 'base' is a multiple of 8, while 'offset' + // may be a multiple of 4 (e.g. 4-byte-aligned long and double arguments + // and spilled variables on the stack accessed relative to the stack + // pointer register). + // We preserve the "alignment" of 'offset' by adjusting it by a multiple of 8. + + bool doubleword_aligned = (src.offset() & (kDoubleSize - 1)) == 0; + bool two_accesses = static_cast(access_type) || !doubleword_aligned; + DCHECK(second_access_add_to_offset <= 7); // Must be <= 7. + + // is_int16 must be passed a signed value, hence the static cast below. + if (is_int16(src.offset()) && + (!two_accesses || is_int16(static_cast( + src.offset() + second_access_add_to_offset)))) { + // Nothing to do: 'offset' (and, if needed, 'offset + 4', or other specified + // value) fits into int16_t. + return; + } + + DCHECK(!src.rm().is( + at)); // Must not overwrite the register 'base' while loading 'offset'. + +#ifdef DEBUG + // Remember the "(mis)alignment" of 'offset', it will be checked at the end. + uint32_t misalignment = src.offset() & (kDoubleSize - 1); +#endif + + // Do not load the whole 32-bit 'offset' if it can be represented as + // a sum of two 16-bit signed offsets. This can save an instruction or two. + // To simplify matters, only do this for a symmetric range of offsets from + // about -64KB to about +64KB, allowing further addition of 4 when accessing + // 64-bit variables with two 32-bit accesses. + constexpr int32_t kMinOffsetForSimpleAdjustment = + 0x7ff8; // Max int16_t that's a multiple of 8. + constexpr int32_t kMaxOffsetForSimpleAdjustment = + 2 * kMinOffsetForSimpleAdjustment; + if (0 <= src.offset() && src.offset() <= kMaxOffsetForSimpleAdjustment) { + addiu(at, src.rm(), kMinOffsetForSimpleAdjustment); + src.offset_ -= kMinOffsetForSimpleAdjustment; + } else if (-kMaxOffsetForSimpleAdjustment <= src.offset() && + src.offset() < 0) { + addiu(at, src.rm(), -kMinOffsetForSimpleAdjustment); + src.offset_ += kMinOffsetForSimpleAdjustment; + } else if (IsMipsArchVariant(kMips32r6)) { + // On r6 take advantage of the aui instruction, e.g.: + // aui at, base, offset_high + // lw reg_lo, offset_low(at) + // lw reg_hi, (offset_low+4)(at) + // or when offset_low+4 overflows int16_t: + // aui at, base, offset_high + // addiu at, at, 8 + // lw reg_lo, (offset_low-8)(at) + // lw reg_hi, (offset_low-4)(at) + int16_t offset_high = static_cast(src.offset() >> 16); + int16_t offset_low = static_cast(src.offset()); + offset_high += (offset_low < 0) + ? 1 + : 0; // Account for offset sign extension in load/store. + aui(at, src.rm(), offset_high); + if (two_accesses && !is_int16(static_cast( + offset_low + second_access_add_to_offset))) { + // Avoid overflow in the 16-bit offset of the load/store instruction when + // adding 4. + addiu(at, at, kDoubleSize); + offset_low -= kDoubleSize; } - aui(at, src.rm(), hi); - addiu(at, at, src.offset_ & kImm16Mask); + src.offset_ = offset_low; } else { - lui(at, (src.offset_ >> kLuiShift) & kImm16Mask); - ori(at, at, src.offset_ & kImm16Mask); // Load 32-bit offset. - addu(at, at, src.rm()); // Add base register. + // Do not load the whole 32-bit 'offset' if it can be represented as + // a sum of three 16-bit signed offsets. This can save an instruction. + // To simplify matters, only do this for a symmetric range of offsets from + // about -96KB to about +96KB, allowing further addition of 4 when accessing + // 64-bit variables with two 32-bit accesses. + constexpr int32_t kMinOffsetForMediumAdjustment = + 2 * kMinOffsetForSimpleAdjustment; + constexpr int32_t kMaxOffsetForMediumAdjustment = + 3 * kMinOffsetForSimpleAdjustment; + if (0 <= src.offset() && src.offset() <= kMaxOffsetForMediumAdjustment) { + addiu(at, src.rm(), kMinOffsetForMediumAdjustment / 2); + addiu(at, at, kMinOffsetForMediumAdjustment / 2); + src.offset_ -= kMinOffsetForMediumAdjustment; + } else if (-kMaxOffsetForMediumAdjustment <= src.offset() && + src.offset() < 0) { + addiu(at, src.rm(), -kMinOffsetForMediumAdjustment / 2); + addiu(at, at, -kMinOffsetForMediumAdjustment / 2); + src.offset_ += kMinOffsetForMediumAdjustment; + } else { + // Now that all shorter options have been exhausted, load the full 32-bit + // offset. + int32_t loaded_offset = RoundDown(src.offset(), kDoubleSize); + lui(at, (loaded_offset >> kLuiShift) & kImm16Mask); + ori(at, at, loaded_offset & kImm16Mask); // Load 32-bit offset. + addu(at, at, src.rm()); + src.offset_ -= loaded_offset; + } } -} + src.rm_ = at; -// Helper for base-reg + upper part of offset, when offset is larger than int16. -// Loads higher part of the offset to AT register. -// Returns lower part of the offset to be used as offset -// in Load/Store instructions -int32_t Assembler::LoadRegPlusUpperOffsetPartToAt(const MemOperand& src) { - DCHECK(!src.rm().is(at)); - int32_t hi = (src.offset_ >> kLuiShift) & kImm16Mask; - // If the highest bit of the lower part of the offset is 1, this would make - // the offset in the load/store instruction negative. We need to compensate - // for this by adding 1 to the upper part of the offset. - if (src.offset_ & kNegOffset) { - hi += 1; + DCHECK(is_int16(src.offset())); + if (two_accesses) { + DCHECK(is_int16( + static_cast(src.offset() + second_access_add_to_offset))); } - - if (IsMipsArchVariant(kMips32r6)) { - aui(at, src.rm(), hi); - } else { - lui(at, hi); - addu(at, at, src.rm()); - } - return (src.offset_ & kImm16Mask); -} - -// Helper for loading base-reg + upper offset's part to AT reg when we are using -// two 32-bit loads/stores instead of one 64-bit -int32_t Assembler::LoadUpperOffsetForTwoMemoryAccesses(const MemOperand& src) { - DCHECK(!src.rm().is(at)); - if (is_int16((src.offset_ & kImm16Mask) + kIntSize)) { - // Only if lower part of offset + kIntSize fits in 16bits - return LoadRegPlusUpperOffsetPartToAt(src); - } - // In case offset's lower part + kIntSize doesn't fit in 16bits, - // load reg + hole offset to AT - LoadRegPlusOffsetToAt(src); - return 0; + DCHECK(misalignment == (src.offset() & (kDoubleSize - 1))); } void Assembler::lb(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(LB, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(LB, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(LB, source.rm(), rd, source.offset()); } void Assembler::lbu(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(LBU, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(LBU, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(LBU, source.rm(), rd, source.offset()); } void Assembler::lh(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(LH, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(LH, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(LH, source.rm(), rd, source.offset()); } void Assembler::lhu(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(LHU, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(LHU, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(LHU, source.rm(), rd, source.offset()); } void Assembler::lw(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(LW, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(LW, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(LW, source.rm(), rd, source.offset()); } @@ -2051,32 +2095,23 @@ void Assembler::lwr(Register rd, const MemOperand& rs) { void Assembler::sb(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(SB, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to store. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(SB, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(SB, source.rm(), rd, source.offset()); } void Assembler::sh(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(SH, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to store. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(SH, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(SH, source.rm(), rd, source.offset()); } void Assembler::sw(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset_)) { - GenInstrImmediate(SW, rs.rm(), rd, rs.offset_); - } else { // Offset > 16 bits, use multiple instructions to store. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - GenInstrImmediate(SW, at, rd, off16); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + GenInstrImmediate(SW, source.rm(), rd, source.offset()); } @@ -2361,22 +2396,16 @@ void Assembler::seb(Register rd, Register rt) { // Load, store, move. void Assembler::lwc1(FPURegister fd, const MemOperand& src) { - if (is_int16(src.offset_)) { - GenInstrImmediate(LWC1, src.rm(), fd, src.offset_); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src); - GenInstrImmediate(LWC1, at, fd, off16); - } + MemOperand tmp = src; + AdjustBaseAndOffset(tmp); + GenInstrImmediate(LWC1, tmp.rm(), fd, tmp.offset()); } void Assembler::swc1(FPURegister fd, const MemOperand& src) { - if (is_int16(src.offset_)) { - GenInstrImmediate(SWC1, src.rm(), fd, src.offset_); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src); - GenInstrImmediate(SWC1, at, fd, off16); - } + MemOperand tmp = src; + AdjustBaseAndOffset(tmp); + GenInstrImmediate(SWC1, tmp.rm(), fd, tmp.offset()); } @@ -3052,14 +3081,17 @@ MSA_BRANCH_LIST(MSA_BRANCH) V(st_w, ST_W) \ V(st_d, ST_D) -#define MSA_LD_ST(name, opcode) \ - void Assembler::name(MSARegister wd, const MemOperand& rs) { \ - if (is_int10(rs.offset())) { \ - GenInstrMsaMI10(opcode, rs.offset(), rs.rm(), wd); \ - } else { \ - LoadRegPlusOffsetToAt(rs); \ - GenInstrMsaMI10(opcode, 0, at, wd); \ - } \ +#define MSA_LD_ST(name, opcode) \ + void Assembler::name(MSARegister wd, const MemOperand& rs) { \ + MemOperand source = rs; \ + AdjustBaseAndOffset(source); \ + if (is_int10(source.offset())) { \ + GenInstrMsaMI10(opcode, source.offset(), source.rm(), wd); \ + } else { \ + DCHECK(!rs.rm().is(at)); \ + addiu(at, source.rm(), source.offset()); \ + GenInstrMsaMI10(opcode, 0, at, wd); \ + } \ } MSA_LD_ST_LIST(MSA_LD_ST) diff --git a/src/mips/assembler-mips.h b/src/mips/assembler-mips.h index f1421d0c1d..33c8ada1f0 100644 --- a/src/mips/assembler-mips.h +++ b/src/mips/assembler-mips.h @@ -1866,10 +1866,18 @@ class Assembler : public AssemblerBase { // Load Scaled Address instruction. void lsa(Register rd, Register rt, Register rs, uint8_t sa); - // Helpers. - void LoadRegPlusOffsetToAt(const MemOperand& src); - int32_t LoadRegPlusUpperOffsetPartToAt(const MemOperand& src); - int32_t LoadUpperOffsetForTwoMemoryAccesses(const MemOperand& src); + // Readable constants for base and offset adjustment helper, these indicate if + // aside from offset, another value like offset + 4 should fit into int16. + enum class OffsetAccessType : bool { + SINGLE_ACCESS = false, + TWO_ACCESSES = true + }; + + // Helper function for memory load/store using base register and offset. + void AdjustBaseAndOffset( + MemOperand& src, + OffsetAccessType access_type = OffsetAccessType::SINGLE_ACCESS, + int second_access_add_to_offset = 4); int32_t buffer_space() const { return reloc_info_writer.pos() - pc_; } diff --git a/src/mips/macro-assembler-mips.cc b/src/mips/macro-assembler-mips.cc index f3c5669d54..c9ac4a9cec 100644 --- a/src/mips/macro-assembler-mips.cc +++ b/src/mips/macro-assembler-mips.cc @@ -1133,20 +1133,17 @@ void MacroAssembler::Ulw(Register rd, const MemOperand& rs) { } else { DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson)); - if (is_int16(rs.offset() + kMipsLwrOffset) && - is_int16(rs.offset() + kMipsLwlOffset)) { - if (!rd.is(rs.rm())) { - lwr(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset)); - lwl(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset)); - } else { - lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset)); - lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset)); - mov(rd, at); - } - } else { // Offset > 16 bits, use multiple instructions to load. - LoadRegPlusOffsetToAt(rs); - lwr(rd, MemOperand(at, kMipsLwrOffset)); - lwl(rd, MemOperand(at, kMipsLwlOffset)); + DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3); + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 3 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3); + if (!rd.is(source.rm())) { + lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset)); + lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset)); + } else { + lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset)); + lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset)); + mov(rd, at); } } } @@ -1155,20 +1152,18 @@ void MacroAssembler::Ulw(Register rd, const MemOperand& rs) { void MacroAssembler::Usw(Register rd, const MemOperand& rs) { DCHECK(!rd.is(at)); DCHECK(!rs.rm().is(at)); + DCHECK(!rd.is(rs.rm())); if (IsMipsArchVariant(kMips32r6)) { sw(rd, rs); } else { DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson)); - if (is_int16(rs.offset() + kMipsSwrOffset) && - is_int16(rs.offset() + kMipsSwlOffset)) { - swr(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwrOffset)); - swl(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwlOffset)); - } else { - LoadRegPlusOffsetToAt(rs); - swr(rd, MemOperand(at, kMipsSwrOffset)); - swl(rd, MemOperand(at, kMipsSwlOffset)); - } + DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3); + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 3 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3); + swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset)); + swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset)); } } @@ -1180,22 +1175,24 @@ void MacroAssembler::Ulh(Register rd, const MemOperand& rs) { } else { DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson)); - if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) { + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 1 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1); + if (source.rm().is(at)) { #if defined(V8_TARGET_LITTLE_ENDIAN) - lbu(at, rs); - lb(rd, MemOperand(rs.rm(), rs.offset() + 1)); + lb(rd, MemOperand(source.rm(), source.offset() + 1)); + lbu(at, source); #elif defined(V8_TARGET_BIG_ENDIAN) - lbu(at, MemOperand(rs.rm(), rs.offset() + 1)); - lb(rd, rs); + lb(rd, source); + lbu(at, MemOperand(source.rm(), source.offset() + 1)); #endif - } else { // Offset > 16 bits, use multiple instructions to load. - LoadRegPlusOffsetToAt(rs); + } else { #if defined(V8_TARGET_LITTLE_ENDIAN) - lb(rd, MemOperand(at, 1)); - lbu(at, MemOperand(at, 0)); + lbu(at, source); + lb(rd, MemOperand(source.rm(), source.offset() + 1)); #elif defined(V8_TARGET_BIG_ENDIAN) - lb(rd, MemOperand(at, 0)); - lbu(at, MemOperand(at, 1)); + lbu(at, MemOperand(source.rm(), source.offset() + 1)); + lb(rd, source); #endif } sll(rd, rd, 8); @@ -1211,22 +1208,24 @@ void MacroAssembler::Ulhu(Register rd, const MemOperand& rs) { } else { DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson)); - if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) { + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 1 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1); + if (source.rm().is(at)) { #if defined(V8_TARGET_LITTLE_ENDIAN) - lbu(at, rs); - lbu(rd, MemOperand(rs.rm(), rs.offset() + 1)); + lbu(rd, MemOperand(source.rm(), source.offset() + 1)); + lbu(at, source); #elif defined(V8_TARGET_BIG_ENDIAN) - lbu(at, MemOperand(rs.rm(), rs.offset() + 1)); - lbu(rd, rs); + lbu(rd, source); + lbu(at, MemOperand(source.rm(), source.offset() + 1)); #endif - } else { // Offset > 16 bits, use multiple instructions to load. - LoadRegPlusOffsetToAt(rs); + } else { #if defined(V8_TARGET_LITTLE_ENDIAN) - lbu(rd, MemOperand(at, 1)); - lbu(at, MemOperand(at, 0)); + lbu(at, source); + lbu(rd, MemOperand(source.rm(), source.offset() + 1)); #elif defined(V8_TARGET_BIG_ENDIAN) - lbu(rd, MemOperand(at, 0)); - lbu(at, MemOperand(at, 1)); + lbu(at, MemOperand(source.rm(), source.offset() + 1)); + lbu(rd, source); #endif } sll(rd, rd, 8); @@ -1245,11 +1244,8 @@ void MacroAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) { DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson)); MemOperand source = rs; - // If offset > 16 bits, load address to at with offset 0. - if (!is_int16(rs.offset()) || !is_int16(rs.offset() + 1)) { - LoadRegPlusOffsetToAt(rs); - source = MemOperand(at, 0); - } + // Adjust offset for two accesses and check if offset + 1 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1); if (!scratch.is(rd)) { mov(scratch, rd); @@ -1324,70 +1320,44 @@ void MacroAssembler::Usdc1(FPURegister fd, const MemOperand& rs, void MacroAssembler::Ldc1(FPURegister fd, const MemOperand& src) { // Workaround for non-8-byte alignment of HeapNumber, convert 64-bit // load to two 32-bit loads. + DCHECK(Register::kMantissaOffset <= 4 && Register::kExponentOffset <= 4); + MemOperand tmp = src; + AdjustBaseAndOffset(tmp, OffsetAccessType::TWO_ACCESSES); + lwc1(fd, MemOperand(tmp.rm(), tmp.offset() + Register::kMantissaOffset)); if (IsFp32Mode()) { // fp32 mode. - if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) { - lwc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset)); - FPURegister nextfpreg; - nextfpreg.setcode(fd.code() + 1); - lwc1(nextfpreg, - MemOperand(src.rm(), src.offset() + Register::kExponentOffset)); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src); - lwc1(fd, MemOperand(at, off16 + Register::kMantissaOffset)); - FPURegister nextfpreg; - nextfpreg.setcode(fd.code() + 1); - lwc1(nextfpreg, MemOperand(at, off16 + Register::kExponentOffset)); - } + FPURegister nextfpreg; + nextfpreg.setcode(fd.code() + 1); + lwc1(nextfpreg, + MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset)); } else { DCHECK(IsFp64Mode() || IsFpxxMode()); // Currently we support FPXX and FP64 on Mips32r2 and Mips32r6 DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)); - if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) { - lwc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset)); - lw(at, MemOperand(src.rm(), src.offset() + Register::kExponentOffset)); - mthc1(at, fd); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src); - lwc1(fd, MemOperand(at, off16 + Register::kMantissaOffset)); - lw(at, MemOperand(at, off16 + Register::kExponentOffset)); - mthc1(at, fd); - } + DCHECK(!src.rm().is(at)); + lw(at, MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset)); + mthc1(at, fd); } } void MacroAssembler::Sdc1(FPURegister fd, const MemOperand& src) { // Workaround for non-8-byte alignment of HeapNumber, convert 64-bit // store to two 32-bit stores. - DCHECK(!src.rm().is(at)); - DCHECK(!src.rm().is(t8)); + DCHECK(Register::kMantissaOffset <= 4 && Register::kExponentOffset <= 4); + MemOperand tmp = src; + AdjustBaseAndOffset(tmp, OffsetAccessType::TWO_ACCESSES); + swc1(fd, MemOperand(tmp.rm(), tmp.offset() + Register::kMantissaOffset)); if (IsFp32Mode()) { // fp32 mode. - if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) { - swc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset)); - FPURegister nextfpreg; - nextfpreg.setcode(fd.code() + 1); - swc1(nextfpreg, - MemOperand(src.rm(), src.offset() + Register::kExponentOffset)); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src); - swc1(fd, MemOperand(at, off16 + Register::kMantissaOffset)); - FPURegister nextfpreg; - nextfpreg.setcode(fd.code() + 1); - swc1(nextfpreg, MemOperand(at, off16 + Register::kExponentOffset)); - } + FPURegister nextfpreg; + nextfpreg.setcode(fd.code() + 1); + swc1(nextfpreg, + MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset)); } else { DCHECK(IsFp64Mode() || IsFpxxMode()); // Currently we support FPXX and FP64 on Mips32r2 and Mips32r6 DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)); - if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) { - swc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset)); - mfhc1(at, fd); - sw(at, MemOperand(src.rm(), src.offset() + Register::kExponentOffset)); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src); - swc1(fd, MemOperand(at, off16 + Register::kMantissaOffset)); - mfhc1(t8, fd); - sw(t8, MemOperand(at, off16 + Register::kExponentOffset)); - } + DCHECK(!src.rm().is(at)); + mfhc1(at, fd); + sw(at, MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset)); } } diff --git a/src/mips64/assembler-mips64.cc b/src/mips64/assembler-mips64.cc index 9b41e79145..56cdd30fa3 100644 --- a/src/mips64/assembler-mips64.cc +++ b/src/mips64/assembler-mips64.cc @@ -2090,31 +2090,130 @@ void Assembler::dlsa(Register rd, Register rt, Register rs, uint8_t sa) { // ------------Memory-instructions------------- -// Helper for base-reg + offset, when offset is larger than int16. -void Assembler::LoadRegPlusOffsetToAt(const MemOperand& src) { - DCHECK(!src.rm().is(at)); - DCHECK(is_int32(src.offset_)); +void Assembler::AdjustBaseAndOffset(MemOperand& src, + OffsetAccessType access_type, + int second_access_add_to_offset) { + // This method is used to adjust the base register and offset pair + // for a load/store when the offset doesn't fit into int16_t. + // It is assumed that 'base + offset' is sufficiently aligned for memory + // operands that are machine word in size or smaller. For doubleword-sized + // operands it's assumed that 'base' is a multiple of 8, while 'offset' + // may be a multiple of 4 (e.g. 4-byte-aligned long and double arguments + // and spilled variables on the stack accessed relative to the stack + // pointer register). + // We preserve the "alignment" of 'offset' by adjusting it by a multiple of 8. - if (kArchVariant == kMips64r6) { - int32_t hi = (src.offset_ >> kLuiShift) & kImm16Mask; - if (src.offset_ & kNegOffset) { - if ((hi & kNegOffset) != ((hi + 1) & kNegOffset)) { - lui(at, (src.offset_ >> kLuiShift) & kImm16Mask); - ori(at, at, src.offset_ & kImm16Mask); // Load 32-bit offset. - daddu(at, at, src.rm()); // Add base register. - return; - } + bool doubleword_aligned = (src.offset() & (kDoubleSize - 1)) == 0; + bool two_accesses = static_cast(access_type) || !doubleword_aligned; + DCHECK(second_access_add_to_offset <= 7); // Must be <= 7. - hi += 1; + // is_int16 must be passed a signed value, hence the static cast below. + if (is_int16(src.offset()) && + (!two_accesses || is_int16(static_cast( + src.offset() + second_access_add_to_offset)))) { + // Nothing to do: 'offset' (and, if needed, 'offset + 4', or other specified + // value) fits into int16_t. + return; + } + + DCHECK(!src.rm().is( + at)); // Must not overwrite the register 'base' while loading 'offset'. + +#ifdef DEBUG + // Remember the "(mis)alignment" of 'offset', it will be checked at the end. + uint32_t misalignment = src.offset() & (kDoubleSize - 1); +#endif + + // Do not load the whole 32-bit 'offset' if it can be represented as + // a sum of two 16-bit signed offsets. This can save an instruction or two. + // To simplify matters, only do this for a symmetric range of offsets from + // about -64KB to about +64KB, allowing further addition of 4 when accessing + // 64-bit variables with two 32-bit accesses. + constexpr int32_t kMinOffsetForSimpleAdjustment = + 0x7ff8; // Max int16_t that's a multiple of 8. + constexpr int32_t kMaxOffsetForSimpleAdjustment = + 2 * kMinOffsetForSimpleAdjustment; + + if (0 <= src.offset() && src.offset() <= kMaxOffsetForSimpleAdjustment) { + daddiu(at, src.rm(), kMinOffsetForSimpleAdjustment); + src.offset_ -= kMinOffsetForSimpleAdjustment; + } else if (-kMaxOffsetForSimpleAdjustment <= src.offset() && + src.offset() < 0) { + daddiu(at, src.rm(), -kMinOffsetForSimpleAdjustment); + src.offset_ += kMinOffsetForSimpleAdjustment; + } else if (kArchVariant == kMips64r6) { + // On r6 take advantage of the daui instruction, e.g.: + // daui AT, base, offset_high + // [dahi AT, 1] // When `offset` is close to +2GB. + // lw reg_lo, offset_low(AT) + // [lw reg_hi, (offset_low+4)(AT)] // If misaligned 64-bit load. + // or when offset_low+4 overflows int16_t: + // daui AT, base, offset_high + // daddiu AT, AT, 8 + // lw reg_lo, (offset_low-8)(AT) + // lw reg_hi, (offset_low-4)(AT) + int16_t offset_low = static_cast(src.offset()); + int32_t offset_low32 = offset_low; + int16_t offset_high = static_cast(src.offset() >> 16); + bool increment_hi16 = offset_low < 0; + bool overflow_hi16 = false; + + if (increment_hi16) { + offset_high++; + overflow_hi16 = (offset_high == -32768); + } + daui(at, src.rm(), offset_high); + + if (overflow_hi16) { + dahi(at, 1); } - daui(at, src.rm(), hi); - daddiu(at, at, src.offset_ & kImm16Mask); + if (two_accesses && !is_int16(static_cast( + offset_low32 + second_access_add_to_offset))) { + // Avoid overflow in the 16-bit offset of the load/store instruction when + // adding 4. + daddiu(at, at, kDoubleSize); + offset_low32 -= kDoubleSize; + } + + src.offset_ = offset_low32; } else { - lui(at, (src.offset_ >> kLuiShift) & kImm16Mask); - ori(at, at, src.offset_ & kImm16Mask); // Load 32-bit offset. - daddu(at, at, src.rm()); // Add base register. + // Do not load the whole 32-bit 'offset' if it can be represented as + // a sum of three 16-bit signed offsets. This can save an instruction. + // To simplify matters, only do this for a symmetric range of offsets from + // about -96KB to about +96KB, allowing further addition of 4 when accessing + // 64-bit variables with two 32-bit accesses. + constexpr int32_t kMinOffsetForMediumAdjustment = + 2 * kMinOffsetForSimpleAdjustment; + constexpr int32_t kMaxOffsetForMediumAdjustment = + 3 * kMinOffsetForSimpleAdjustment; + if (0 <= src.offset() && src.offset() <= kMaxOffsetForMediumAdjustment) { + daddiu(at, src.rm(), kMinOffsetForMediumAdjustment / 2); + daddiu(at, at, kMinOffsetForMediumAdjustment / 2); + src.offset_ -= kMinOffsetForMediumAdjustment; + } else if (-kMaxOffsetForMediumAdjustment <= src.offset() && + src.offset() < 0) { + daddiu(at, src.rm(), -kMinOffsetForMediumAdjustment / 2); + daddiu(at, at, -kMinOffsetForMediumAdjustment / 2); + src.offset_ += kMinOffsetForMediumAdjustment; + } else { + // Now that all shorter options have been exhausted, load the full 32-bit + // offset. + int32_t loaded_offset = RoundDown(src.offset(), kDoubleSize); + lui(at, (loaded_offset >> kLuiShift) & kImm16Mask); + ori(at, at, loaded_offset & kImm16Mask); // Load 32-bit offset. + daddu(at, at, src.rm()); + src.offset_ -= loaded_offset; + } } + src.rm_ = at; + + DCHECK(is_int16(src.offset())); + if (two_accesses) { + DCHECK(is_int16( + static_cast(src.offset() + second_access_add_to_offset))); + } + DCHECK(misalignment == (src.offset() & (kDoubleSize - 1))); } void Assembler::lb(Register rd, const MemOperand& rs) { @@ -3285,14 +3384,17 @@ MSA_BRANCH_LIST(MSA_BRANCH) V(st_w, ST_W) \ V(st_d, ST_D) -#define MSA_LD_ST(name, opcode) \ - void Assembler::name(MSARegister wd, const MemOperand& rs) { \ - if (is_int10(rs.offset())) { \ - GenInstrMsaMI10(opcode, rs.offset(), rs.rm(), wd); \ - } else { \ - LoadRegPlusOffsetToAt(rs); \ - GenInstrMsaMI10(opcode, 0, at, wd); \ - } \ +#define MSA_LD_ST(name, opcode) \ + void Assembler::name(MSARegister wd, const MemOperand& rs) { \ + MemOperand source = rs; \ + AdjustBaseAndOffset(source); \ + if (is_int10(source.offset())) { \ + GenInstrMsaMI10(opcode, source.offset(), source.rm(), wd); \ + } else { \ + DCHECK(!rs.rm().is(at)); \ + daddiu(at, source.rm(), source.offset()); \ + GenInstrMsaMI10(opcode, 0, at, wd); \ + } \ } MSA_LD_ST_LIST(MSA_LD_ST) diff --git a/src/mips64/assembler-mips64.h b/src/mips64/assembler-mips64.h index 37ba828289..f85bb8740e 100644 --- a/src/mips64/assembler-mips64.h +++ b/src/mips64/assembler-mips64.h @@ -1918,8 +1918,18 @@ class Assembler : public AssemblerBase { void lsa(Register rd, Register rt, Register rs, uint8_t sa); void dlsa(Register rd, Register rt, Register rs, uint8_t sa); - // Helpers. - void LoadRegPlusOffsetToAt(const MemOperand& src); + // Readable constants for base and offset adjustment helper, these indicate if + // aside from offset, another value like offset + 4 should fit into int16. + enum class OffsetAccessType : bool { + SINGLE_ACCESS = false, + TWO_ACCESSES = true + }; + + // Helper function for memory load/store using base register and offset. + void AdjustBaseAndOffset( + MemOperand& src, + OffsetAccessType access_type = OffsetAccessType::SINGLE_ACCESS, + int second_access_add_to_offset = 4); inline static void set_target_internal_reference_encoded_at(Address pc, Address target); diff --git a/src/mips64/macro-assembler-mips64.cc b/src/mips64/macro-assembler-mips64.cc index 8c20dfb019..b3fd37865e 100644 --- a/src/mips64/macro-assembler-mips64.cc +++ b/src/mips64/macro-assembler-mips64.cc @@ -194,62 +194,6 @@ MemOperand MacroAssembler::SafepointRegistersAndDoublesSlot(Register reg) { return MemOperand(sp, doubles_size + register_offset); } -// Helper for base-reg + offset, when offset is larger than int16. -void MacroAssembler::LoadRegPlusOffsetToAt(const MemOperand& src) { - DCHECK(!src.rm().is(at)); - DCHECK(is_int32(src.offset())); - - if (kArchVariant == kMips64r6) { - int32_t hi = (src.offset() >> kLuiShift) & kImm16Mask; - if (src.offset() & kNegOffset) { - if ((hi & kNegOffset) != ((hi + 1) & kNegOffset)) { - lui(at, (src.offset() >> kLuiShift) & kImm16Mask); - ori(at, at, src.offset() & kImm16Mask); // Load 32-bit offset. - daddu(at, at, src.rm()); // Add base register. - return; - } - - hi += 1; - } - - daui(at, src.rm(), hi); - daddiu(at, at, src.offset() & kImm16Mask); - } else { - lui(at, (src.offset() >> kLuiShift) & kImm16Mask); - ori(at, at, src.offset() & kImm16Mask); // Load 32-bit offset. - daddu(at, at, src.rm()); // Add base register. - } -} - -// Helper for base-reg + upper part of offset, when offset is larger than int16. -// Loads higher part of the offset to AT register. -// Returns lower part of the offset to be used as offset -// in Load/Store instructions -int32_t MacroAssembler::LoadRegPlusUpperOffsetPartToAt(const MemOperand& src) { - DCHECK(!src.rm().is(at)); - DCHECK(is_int32(src.offset())); - int32_t hi = (src.offset() >> kLuiShift) & kImm16Mask; - // If the highest bit of the lower part of the offset is 1, this would make - // the offset in the load/store instruction negative. We need to compensate - // for this by adding 1 to the upper part of the offset. - if (src.offset() & kNegOffset) { - if ((hi & kNegOffset) != ((hi + 1) & kNegOffset)) { - LoadRegPlusOffsetToAt(src); - return 0; - } - - hi += 1; - } - - if (kArchVariant == kMips64r6) { - daui(at, src.rm(), hi); - } else { - lui(at, hi); - daddu(at, at, src.rm()); - } - return (src.offset() & kImm16Mask); -} - void MacroAssembler::InNewSpace(Register object, Register scratch, Condition cc, @@ -1321,20 +1265,17 @@ void MacroAssembler::Ulw(Register rd, const MemOperand& rs) { Lw(rd, rs); } else { DCHECK(kArchVariant == kMips64r2); - if (is_int16(rs.offset() + kMipsLwrOffset) && - is_int16(rs.offset() + kMipsLwlOffset)) { - if (!rd.is(rs.rm())) { - lwr(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset)); - lwl(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset)); - } else { - lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset)); - lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset)); - mov(rd, at); - } - } else { // Offset > 16 bits, use multiple instructions to load. - LoadRegPlusOffsetToAt(rs); - lwr(rd, MemOperand(at, kMipsLwrOffset)); - lwl(rd, MemOperand(at, kMipsLwlOffset)); + DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3); + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 3 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3); + if (!rd.is(source.rm())) { + lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset)); + lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset)); + } else { + lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset)); + lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset)); + mov(rd, at); } } } @@ -1353,19 +1294,17 @@ void MacroAssembler::Ulwu(Register rd, const MemOperand& rs) { void MacroAssembler::Usw(Register rd, const MemOperand& rs) { DCHECK(!rd.is(at)); DCHECK(!rs.rm().is(at)); + DCHECK(!rd.is(rs.rm())); if (kArchVariant == kMips64r6) { Sw(rd, rs); } else { DCHECK(kArchVariant == kMips64r2); - if (is_int16(rs.offset() + kMipsSwrOffset) && - is_int16(rs.offset() + kMipsSwlOffset)) { - swr(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwrOffset)); - swl(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwlOffset)); - } else { - LoadRegPlusOffsetToAt(rs); - swr(rd, MemOperand(at, kMipsSwrOffset)); - swl(rd, MemOperand(at, kMipsSwlOffset)); - } + DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3); + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 3 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3); + swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset)); + swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset)); } } @@ -1376,22 +1315,24 @@ void MacroAssembler::Ulh(Register rd, const MemOperand& rs) { Lh(rd, rs); } else { DCHECK(kArchVariant == kMips64r2); - if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) { + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 1 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1); + if (source.rm().is(at)) { #if defined(V8_TARGET_LITTLE_ENDIAN) - Lbu(at, rs); - Lb(rd, MemOperand(rs.rm(), rs.offset() + 1)); + Lb(rd, MemOperand(source.rm(), source.offset() + 1)); + Lbu(at, source); #elif defined(V8_TARGET_BIG_ENDIAN) - Lbu(at, MemOperand(rs.rm(), rs.offset() + 1)); - Lb(rd, rs); + Lb(rd, source); + Lbu(at, MemOperand(source.rm(), source.offset() + 1)); #endif - } else { // Offset > 16 bits, use multiple instructions to load. - LoadRegPlusOffsetToAt(rs); + } else { #if defined(V8_TARGET_LITTLE_ENDIAN) - Lb(rd, MemOperand(at, 1)); - Lbu(at, MemOperand(at, 0)); + Lbu(at, source); + Lb(rd, MemOperand(source.rm(), source.offset() + 1)); #elif defined(V8_TARGET_BIG_ENDIAN) - Lb(rd, MemOperand(at, 0)); - Lbu(at, MemOperand(at, 1)); + Lbu(at, MemOperand(source.rm(), source.offset() + 1)); + Lb(rd, source); #endif } dsll(rd, rd, 8); @@ -1406,22 +1347,24 @@ void MacroAssembler::Ulhu(Register rd, const MemOperand& rs) { Lhu(rd, rs); } else { DCHECK(kArchVariant == kMips64r2); - if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) { + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 1 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1); + if (source.rm().is(at)) { #if defined(V8_TARGET_LITTLE_ENDIAN) - Lbu(at, rs); - Lbu(rd, MemOperand(rs.rm(), rs.offset() + 1)); + Lbu(rd, MemOperand(source.rm(), source.offset() + 1)); + Lbu(at, source); #elif defined(V8_TARGET_BIG_ENDIAN) - Lbu(at, MemOperand(rs.rm(), rs.offset() + 1)); - Lbu(rd, rs); + Lbu(rd, source); + Lbu(at, MemOperand(source.rm(), source.offset() + 1)); #endif - } else { // Offset > 16 bits, use multiple instructions to load. - LoadRegPlusOffsetToAt(rs); + } else { #if defined(V8_TARGET_LITTLE_ENDIAN) - Lbu(rd, MemOperand(at, 1)); - Lbu(at, MemOperand(at, 0)); + Lbu(at, source); + Lbu(rd, MemOperand(source.rm(), source.offset() + 1)); #elif defined(V8_TARGET_BIG_ENDIAN) - Lbu(rd, MemOperand(at, 0)); - Lbu(at, MemOperand(at, 1)); + Lbu(at, MemOperand(source.rm(), source.offset() + 1)); + Lbu(rd, source); #endif } dsll(rd, rd, 8); @@ -1439,11 +1382,8 @@ void MacroAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) { } else { DCHECK(kArchVariant == kMips64r2); MemOperand source = rs; - // If offset > 16 bits, load address to at with offset 0. - if (!is_int16(rs.offset()) || !is_int16(rs.offset() + 1)) { - LoadRegPlusOffsetToAt(rs); - source = MemOperand(at, 0); - } + // Adjust offset for two accesses and check if offset + 1 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1); if (!scratch.is(rd)) { mov(scratch, rd); @@ -1468,20 +1408,17 @@ void MacroAssembler::Uld(Register rd, const MemOperand& rs) { Ld(rd, rs); } else { DCHECK(kArchVariant == kMips64r2); - if (is_int16(rs.offset() + kMipsLdrOffset) && - is_int16(rs.offset() + kMipsLdlOffset)) { - if (!rd.is(rs.rm())) { - ldr(rd, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset)); - ldl(rd, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset)); - } else { - ldr(at, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset)); - ldl(at, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset)); - mov(rd, at); - } - } else { // Offset > 16 bits, use multiple instructions to load. - LoadRegPlusOffsetToAt(rs); - ldr(rd, MemOperand(at, kMipsLdrOffset)); - ldl(rd, MemOperand(at, kMipsLdlOffset)); + DCHECK(kMipsLdrOffset <= 7 && kMipsLdlOffset <= 7); + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 7 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 7); + if (!rd.is(source.rm())) { + ldr(rd, MemOperand(source.rm(), source.offset() + kMipsLdrOffset)); + ldl(rd, MemOperand(source.rm(), source.offset() + kMipsLdlOffset)); + } else { + ldr(at, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset)); + ldl(at, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset)); + mov(rd, at); } } } @@ -1505,15 +1442,12 @@ void MacroAssembler::Usd(Register rd, const MemOperand& rs) { Sd(rd, rs); } else { DCHECK(kArchVariant == kMips64r2); - if (is_int16(rs.offset() + kMipsSdrOffset) && - is_int16(rs.offset() + kMipsSdlOffset)) { - sdr(rd, MemOperand(rs.rm(), rs.offset() + kMipsSdrOffset)); - sdl(rd, MemOperand(rs.rm(), rs.offset() + kMipsSdlOffset)); - } else { - LoadRegPlusOffsetToAt(rs); - sdr(rd, MemOperand(at, kMipsSdrOffset)); - sdl(rd, MemOperand(at, kMipsSdlOffset)); - } + DCHECK(kMipsSdrOffset <= 7 && kMipsSdlOffset <= 7); + MemOperand source = rs; + // Adjust offset for two accesses and check if offset + 7 fits into int16_t. + AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 7); + sdr(rd, MemOperand(source.rm(), source.offset() + kMipsSdrOffset)); + sdl(rd, MemOperand(source.rm(), source.offset() + kMipsSdlOffset)); } } @@ -1573,139 +1507,93 @@ void MacroAssembler::Usdc1(FPURegister fd, const MemOperand& rs, } void MacroAssembler::Lb(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - lb(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - lb(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + lb(rd, source); } void MacroAssembler::Lbu(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - lbu(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - lbu(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + lbu(rd, source); } void MacroAssembler::Sb(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - sb(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to store. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - sb(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + sb(rd, source); } void MacroAssembler::Lh(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - lh(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - lh(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + lh(rd, source); } void MacroAssembler::Lhu(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - lhu(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - lhu(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + lhu(rd, source); } void MacroAssembler::Sh(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - sh(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to store. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - sh(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + sh(rd, source); } void MacroAssembler::Lw(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - lw(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - lw(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + lw(rd, source); } void MacroAssembler::Lwu(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - lwu(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - lwu(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + lwu(rd, source); } void MacroAssembler::Sw(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - sw(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to store. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - sw(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + sw(rd, source); } void MacroAssembler::Ld(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - ld(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - ld(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + ld(rd, source); } void MacroAssembler::Sd(Register rd, const MemOperand& rs) { - if (is_int16(rs.offset())) { - sd(rd, rs); - } else { // Offset > 16 bits, use multiple instructions to store. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs); - sd(rd, MemOperand(at, off16)); - } + MemOperand source = rs; + AdjustBaseAndOffset(source); + sd(rd, source); } void MacroAssembler::Lwc1(FPURegister fd, const MemOperand& src) { - if (is_int16(src.offset())) { - lwc1(fd, src); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src); - lwc1(fd, MemOperand(at, off16)); - } + MemOperand tmp = src; + AdjustBaseAndOffset(tmp); + lwc1(fd, tmp); } void MacroAssembler::Swc1(FPURegister fs, const MemOperand& src) { - if (is_int16(src.offset())) { - swc1(fs, src); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src); - swc1(fs, MemOperand(at, off16)); - } + MemOperand tmp = src; + AdjustBaseAndOffset(tmp); + swc1(fs, tmp); } void MacroAssembler::Ldc1(FPURegister fd, const MemOperand& src) { - if (is_int16(src.offset())) { - ldc1(fd, src); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src); - ldc1(fd, MemOperand(at, off16)); - } + MemOperand tmp = src; + AdjustBaseAndOffset(tmp); + ldc1(fd, tmp); } void MacroAssembler::Sdc1(FPURegister fs, const MemOperand& src) { - DCHECK(!src.rm().is(at)); - if (is_int16(src.offset())) { - sdc1(fs, src); - } else { // Offset > 16 bits, use multiple instructions to load. - int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src); - sdc1(fs, MemOperand(at, off16)); - } + MemOperand tmp = src; + AdjustBaseAndOffset(tmp); + sdc1(fs, tmp); } void MacroAssembler::li(Register dst, Handle value, LiFlags mode) { diff --git a/src/mips64/macro-assembler-mips64.h b/src/mips64/macro-assembler-mips64.h index 6d69bfe7c5..9c51682069 100644 --- a/src/mips64/macro-assembler-mips64.h +++ b/src/mips64/macro-assembler-mips64.h @@ -1873,10 +1873,6 @@ const Operand& rt = Operand(zero_reg), BranchDelaySlot bd = PROTECT MemOperand SafepointRegisterSlot(Register reg); MemOperand SafepointRegistersAndDoublesSlot(Register reg); - // Helpers. - void LoadRegPlusOffsetToAt(const MemOperand& src); - int32_t LoadRegPlusUpperOffsetPartToAt(const MemOperand& src); - bool has_frame_; bool has_double_zero_reg_set_; Isolate* isolate_;