[arm64] Merge some stack operations and add padding

Merge some stack operations to work on an even number of registers, adding
a padding register where necessary. Some lightly-used macro assembler
functions are inlined, to make pairing registers easier. Not all merges
create an even number of register arguments yet.

This is a step towards aligning the stack pointer to 16-bytes.

Bug: v8:6644
Change-Id: I995510cf91fa1f7af659a8d9b83acf7857787100
Reviewed-on: https://chromium-review.googlesource.com/654607
Commit-Queue: Martyn Capewell <martyn.capewell@arm.com>
Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#47975}
This commit is contained in:
Martyn Capewell 2017-09-08 15:23:13 +01:00 committed by Commit Bot
parent faea50313f
commit 53add533de
6 changed files with 162 additions and 154 deletions

View File

@ -84,7 +84,7 @@ const int kNumSafepointRegisters = 32;
// space, i.e. kNumSafepointSavedRegisters <= kNumSafepointRegisters.
#define kSafepointSavedRegisters CPURegList::GetSafepointSavedRegisters().list()
#define kNumSafepointSavedRegisters \
CPURegList::GetSafepointSavedRegisters().Count();
CPURegList::GetSafepointSavedRegisters().Count()
// Some CPURegister methods can return Register and VRegister types, so we
// need to declare them in advance.
@ -456,6 +456,9 @@ ALIAS_REGISTER(Register, lr, x30);
ALIAS_REGISTER(Register, xzr, x31);
ALIAS_REGISTER(Register, wzr, w31);
// Register used for padding stack slots.
ALIAS_REGISTER(Register, padreg, x31);
// Keeps the 0 double value.
ALIAS_REGISTER(VRegister, fp_zero, d15);
// MacroAssembler fixed V Registers.

View File

@ -33,8 +33,7 @@ namespace internal {
void ArrayNArgumentsConstructorStub::Generate(MacroAssembler* masm) {
__ Mov(x5, Operand(x0, LSL, kPointerSizeLog2));
__ Str(x1, MemOperand(jssp, x5));
__ Push(x1);
__ Push(x2);
__ Push(x1, x2);
__ Add(x0, x0, Operand(3));
__ TailCallRuntime(Runtime::kNewArray);
}
@ -620,25 +619,26 @@ void JSEntryStub::Generate(MacroAssembler* masm) {
ExternalReference js_entry_sp(IsolateAddressId::kJSEntrySPAddress, isolate());
__ Mov(x10, ExternalReference(js_entry_sp));
__ Ldr(x11, MemOperand(x10));
__ Cbnz(x11, &non_outermost_js);
__ Str(fp, MemOperand(x10));
__ Mov(x12, StackFrame::OUTERMOST_JSENTRY_FRAME);
__ Push(x12);
__ B(&done);
__ Bind(&non_outermost_js);
// We spare one instruction by pushing xzr since the marker is 0.
// Select between the inner and outermost frame marker, based on the JS entry
// sp. We assert that the inner marker is zero, so we can use xzr to save a
// move instruction.
DCHECK(StackFrame::INNER_JSENTRY_FRAME == 0);
__ Push(xzr);
__ Cmp(x11, 0); // If x11 is zero, this is the outermost frame.
__ Csel(x12, xzr, StackFrame::OUTERMOST_JSENTRY_FRAME, ne);
__ B(ne, &done);
__ Str(fp, MemOperand(x10));
__ Bind(&done);
__ Push(x12);
// The frame set up looks like this:
// jssp[0] : JS entry frame marker.
// jssp[1] : C entry FP.
// jssp[2] : stack frame marker.
// jssp[3] : stack frmae marker.
// jssp[3] : stack frame marker.
// jssp[4] : bad frame pointer 0xfff...ff <- fp points here.
// Jump to a faked try block that does the invoke, with a faked catch
// block that sets the pending exception.
__ B(&invoke);
@ -665,7 +665,22 @@ void JSEntryStub::Generate(MacroAssembler* masm) {
// Invoke: Link this frame into the handler chain.
__ Bind(&invoke);
__ PushStackHandler();
// Push new stack handler.
DCHECK(jssp.Is(__ StackPointer()));
static_assert(StackHandlerConstants::kSize == 1 * kPointerSize,
"Unexpected offset for StackHandlerConstants::kSize");
static_assert(StackHandlerConstants::kNextOffset == 0 * kPointerSize,
"Unexpected offset for StackHandlerConstants::kNextOffset");
// Link the current handler as the next handler.
__ Mov(x11, ExternalReference(IsolateAddressId::kHandlerAddress, isolate()));
__ Ldr(x10, MemOperand(x11));
__ Push(x10);
// Set this new handler as the current one.
__ Str(jssp, MemOperand(x11));
// If an exception not caught by another handler occurs, this handler
// returns control to the code after the B(&invoke) above, which
// restores all callee-saved registers (including cp and fp) to their
@ -689,9 +704,13 @@ void JSEntryStub::Generate(MacroAssembler* masm) {
__ Call(BUILTIN_CODE(isolate(), JSEntryTrampoline), RelocInfo::CODE_TARGET);
}
// Unlink this frame from the handler chain.
__ PopStackHandler();
// Pop the stack handler and unlink this frame from the handler chain.
static_assert(StackHandlerConstants::kNextOffset == 0 * kPointerSize,
"Unexpected offset for StackHandlerConstants::kNextOffset");
__ Pop(x10);
__ Mov(x11, ExternalReference(IsolateAddressId::kHandlerAddress, isolate()));
__ Drop(StackHandlerConstants::kSize - kXRegSize, kByteSizeInBytes);
__ Str(x10, MemOperand(x11));
__ Bind(&exit);
// x0 holds the result.
@ -705,17 +724,20 @@ void JSEntryStub::Generate(MacroAssembler* masm) {
// Check if the current stack frame is marked as the outermost JS frame.
Label non_outermost_js_2;
__ Pop(x10);
__ Cmp(x10, StackFrame::OUTERMOST_JSENTRY_FRAME);
__ B(ne, &non_outermost_js_2);
__ Mov(x11, ExternalReference(js_entry_sp));
__ Str(xzr, MemOperand(x11));
__ Bind(&non_outermost_js_2);
{
Register c_entry_fp = x11;
__ Pop(x10, c_entry_fp);
__ Cmp(x10, StackFrame::OUTERMOST_JSENTRY_FRAME);
__ B(ne, &non_outermost_js_2);
__ Mov(x12, ExternalReference(js_entry_sp));
__ Str(xzr, MemOperand(x12));
__ Bind(&non_outermost_js_2);
// Restore the top frame descriptors from the stack.
__ Pop(x10);
__ Mov(x11, ExternalReference(IsolateAddressId::kCEntryFPAddress, isolate()));
__ Str(x10, MemOperand(x11));
// Restore the top frame descriptors from the stack.
__ Mov(x12,
ExternalReference(IsolateAddressId::kCEntryFPAddress, isolate()));
__ Str(c_entry_fp, MemOperand(x12));
}
// Reset the stack to the callee saved registers.
__ Drop(-EntryFrameConstants::kCallerFPOffset, kByteSizeInBytes);
@ -1195,8 +1217,10 @@ void NameDictionaryLookupStub::GenerateNegativeLookup(MacroAssembler* masm,
}
CPURegList spill_list(CPURegister::kRegister, kXRegSizeInBits, 0, 6);
spill_list.Combine(lr);
spill_list.Remove(scratch0); // Scratch registers don't need to be preserved.
spill_list.Combine(lr);
spill_list.Combine(padreg); // Add padreg to make the list of even length.
DCHECK_EQ(spill_list.Count() % 2, 0);
__ PushCPURegList(spill_list);
@ -1811,22 +1835,20 @@ void CallApiCallbackStub::Generate(MacroAssembler* masm) {
STATIC_ASSERT(FCA::kIsolateIndex == 1);
STATIC_ASSERT(FCA::kHolderIndex == 0);
// new target
__ PushRoot(Heap::kUndefinedValueRootIndex);
Register undef = x7;
__ LoadRoot(undef, Heap::kUndefinedValueRootIndex);
// context, callee and call data.
__ Push(context, callee, call_data);
// Push new target, context, callee and call data.
__ Push(undef, context, callee, call_data);
Register scratch = call_data;
__ LoadRoot(scratch, Heap::kUndefinedValueRootIndex);
Register isolate_reg = x5;
__ Mov(isolate_reg, ExternalReference::isolate_address(masm->isolate()));
// FunctionCallbackArguments:
// return value, return value default, isolate, holder.
__ Push(scratch, scratch, isolate_reg, holder);
__ Push(undef, undef, isolate_reg, holder);
// Enter a new context
// Enter a new context.
if (is_lazy()) {
// ----------- S t a t e -------------------------------------
// -- sp[0] : holder
@ -1839,8 +1861,9 @@ void CallApiCallbackStub::Generate(MacroAssembler* masm) {
// -- sp[(FCA::kArgsLength + argc + 1) * 8] : accessor_holder
// -----------------------------------------------------------
// Load context from accessor_holder
// Load context from accessor_holder.
Register accessor_holder = context;
Register scratch = undef;
Register scratch2 = callee;
__ Ldr(accessor_holder,
MemOperand(__ StackPointer(),
@ -1852,10 +1875,10 @@ void CallApiCallbackStub::Generate(MacroAssembler* masm) {
__ Tst(scratch2, Operand(1 << Map::kIsConstructor));
__ B(ne, &skip_looking_for_constructor);
__ GetMapConstructor(context, scratch, scratch, scratch2);
__ bind(&skip_looking_for_constructor);
__ Bind(&skip_looking_for_constructor);
__ Ldr(context, FieldMemOperand(context, JSFunction::kContextOffset));
} else {
// Load context from callee
// Load context from callee.
__ Ldr(context, FieldMemOperand(callee, JSFunction::kContextOffset));
}

View File

@ -1355,6 +1355,9 @@ void TurboAssembler::Drop(const Register& count, uint64_t unit_size) {
}
}
void TurboAssembler::DropArguments(const Register& count, uint64_t unit_size) {
Drop(count, unit_size);
}
void MacroAssembler::DropBySMI(const Register& count_smi, uint64_t unit_size) {
DCHECK(unit_size == 0 || base::bits::IsPowerOfTwo(unit_size));

View File

@ -2440,10 +2440,8 @@ void TurboAssembler::EnterFrame(StackFrame::Type type) {
if (type == StackFrame::INTERNAL) {
DCHECK(jssp.Is(StackPointer()));
Mov(type_reg, StackFrame::TypeToMarker(type));
Push(lr, fp);
Push(type_reg);
Mov(code_reg, Operand(CodeObject()));
Push(code_reg);
Push(lr, fp, type_reg, code_reg);
Add(fp, jssp, InternalFrameConstants::kFixedFrameSizeFromFp);
// jssp[4] : lr
// jssp[3] : fp
@ -2454,7 +2452,7 @@ void TurboAssembler::EnterFrame(StackFrame::Type type) {
Mov(type_reg, StackFrame::TypeToMarker(type));
Push(lr, fp);
Mov(fp, csp);
Push(type_reg, xzr);
Push(type_reg, padreg);
// csp[3] : lr
// csp[2] : fp
// csp[1] : type
@ -2462,8 +2460,7 @@ void TurboAssembler::EnterFrame(StackFrame::Type type) {
} else {
DCHECK(jssp.Is(StackPointer()));
Mov(type_reg, StackFrame::TypeToMarker(type));
Push(lr, fp);
Push(type_reg);
Push(lr, fp, type_reg);
Add(fp, jssp, TypedFrameConstants::kFixedFrameSizeFromFp);
// jssp[2] : lr
// jssp[1] : fp
@ -2663,34 +2660,6 @@ void MacroAssembler::MaybeDropFrames() {
ne);
}
void MacroAssembler::PushStackHandler() {
DCHECK(jssp.Is(StackPointer()));
// Adjust this code if the asserts don't hold.
STATIC_ASSERT(StackHandlerConstants::kSize == 1 * kPointerSize);
STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0 * kPointerSize);
// For the JSEntry handler, we must preserve the live registers x0-x4.
// (See JSEntryStub::GenerateBody().)
// Link the current handler as the next handler.
Mov(x11, ExternalReference(IsolateAddressId::kHandlerAddress, isolate()));
Ldr(x10, MemOperand(x11));
Push(x10);
// Set this new handler as the current one.
Str(jssp, MemOperand(x11));
}
void MacroAssembler::PopStackHandler() {
STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
Pop(x10);
Mov(x11, ExternalReference(IsolateAddressId::kHandlerAddress, isolate()));
Drop(StackHandlerConstants::kSize - kXRegSize, kByteSizeInBytes);
Str(x10, MemOperand(x11));
}
void MacroAssembler::Allocate(int object_size,
Register result,
Register scratch1,
@ -2922,14 +2891,6 @@ void MacroAssembler::GetMapConstructor(Register result, Register map,
Bind(&done);
}
void MacroAssembler::PushRoot(Heap::RootListIndex index) {
UseScratchRegisterScope temps(this);
Register temp = temps.AcquireX();
LoadRoot(temp, index);
Push(temp);
}
void MacroAssembler::CompareRoot(const Register& obj,
Heap::RootListIndex index) {
UseScratchRegisterScope temps(this);
@ -3049,6 +3010,9 @@ void MacroAssembler::RememberedSetHelper(Register object, // For debug tests.
void MacroAssembler::PopSafepointRegisters() {
const int num_unsaved = kNumSafepointRegisters - kNumSafepointSavedRegisters;
DCHECK_GE(num_unsaved, 0);
DCHECK_EQ(num_unsaved % 2, 0);
DCHECK_EQ(kSafepointSavedRegisters % 2, 0);
PopXRegList(kSafepointSavedRegisters);
Drop(num_unsaved);
}
@ -3058,7 +3022,9 @@ void MacroAssembler::PushSafepointRegisters() {
// Safepoints expect a block of kNumSafepointRegisters values on the stack, so
// adjust the stack for unsaved registers.
const int num_unsaved = kNumSafepointRegisters - kNumSafepointSavedRegisters;
DCHECK(num_unsaved >= 0);
DCHECK_GE(num_unsaved, 0);
DCHECK_EQ(num_unsaved % 2, 0);
DCHECK_EQ(kSafepointSavedRegisters % 2, 0);
Claim(num_unsaved);
PushXRegList(kSafepointSavedRegisters);
}
@ -3219,14 +3185,14 @@ void MacroAssembler::RecordWriteForMap(Register object,
// Record the actual write.
if (lr_status == kLRHasNotBeenSaved) {
Push(lr);
Push(padreg, lr);
}
Add(dst, object, HeapObject::kMapOffset - kHeapObjectTag);
RecordWriteStub stub(isolate(), object, map, dst, OMIT_REMEMBERED_SET,
fp_mode);
CallStub(&stub);
if (lr_status == kLRHasNotBeenSaved) {
Pop(lr);
Pop(lr, padreg);
}
Bind(&done);
@ -3293,13 +3259,13 @@ void MacroAssembler::RecordWrite(
// Record the actual write.
if (lr_status == kLRHasNotBeenSaved) {
Push(lr);
Push(padreg, lr);
}
RecordWriteStub stub(isolate(), object, value, address, remembered_set_action,
fp_mode);
CallStub(&stub);
if (lr_status == kLRHasNotBeenSaved) {
Pop(lr);
Pop(lr, padreg);
}
Bind(&done);

View File

@ -702,6 +702,14 @@ class TurboAssembler : public Assembler {
inline void Drop(int64_t count, uint64_t unit_size = kXRegSize);
inline void Drop(const Register& count, uint64_t unit_size = kXRegSize);
// Drop arguments from stack without actually accessing memory.
// This will currently drop 'count' arguments of the given size from the
// stack.
// TODO(arm64): Update this to round up the number of bytes dropped to
// a multiple of 16, so that we can remove jssp.
inline void DropArguments(const Register& count,
uint64_t unit_size = kXRegSize);
// Re-synchronizes the system stack pointer (csp) with the current stack
// pointer (according to StackPointer()).
//
@ -1874,16 +1882,6 @@ class MacroAssembler : public TurboAssembler {
// Frame restart support
void MaybeDropFrames();
// Exception handling
// Push a new stack handler and link into stack handler chain.
void PushStackHandler();
// Unlink the stack handler on top of the stack from the stack handler chain.
// Must preserve the result register.
void PopStackHandler();
// ---------------------------------------------------------------------------
// Allocation support
@ -2001,9 +1999,6 @@ class MacroAssembler : public TurboAssembler {
// register.
void LoadElementsKindFromMap(Register result, Register map);
// Load the value from the root list and push it onto the stack.
void PushRoot(Heap::RootListIndex index);
// Compare the object in a register to a value from the root list.
void CompareRoot(const Register& obj, Heap::RootListIndex index);
@ -2292,11 +2287,6 @@ class MacroAssembler : public TurboAssembler {
const CPURegister& arg2 = NoCPUReg,
const CPURegister& arg3 = NoCPUReg);
// Return true if the sequence is a young sequence geneated by
// EmitFrameSetupForCodeAgePatching. Otherwise, this method asserts that the
// sequence is a code age sequence (emitted by EmitCodeAgeSequence).
static bool IsYoungSequence(Isolate* isolate, byte* sequence);
private:
// Helper for implementing JumpIfNotInNewSpace and JumpIfInNewSpace.
void InNewSpace(Register object,

View File

@ -203,12 +203,10 @@ void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
{
FrameScope scope(masm, StackFrame::CONSTRUCT);
__ LoadRoot(x10, Heap::kTheHoleValueRootIndex);
// Preserve the incoming parameters on the stack.
__ SmiTag(x0);
__ Push(cp, x0);
__ SmiUntag(x0);
__ PushRoot(Heap::kTheHoleValueRootIndex);
__ SmiTag(x11, x0);
__ Push(cp, x11, x10);
// Set up pointer to last argument.
__ Add(x2, fp, StandardFrameConstants::kCallerSPOffset);
@ -1111,14 +1109,15 @@ void Builtins::Generate_InterpreterPushArgsThenCallImpl(
Label stack_overflow;
// Add one for the receiver.
__ add(x3, x0, Operand(1));
__ Add(x3, x0, 1);
// Add a stack check before pushing arguments.
Generate_StackOverflowCheck(masm, x3, x6, &stack_overflow);
// Push "undefined" as the receiver arg if we need to.
if (receiver_mode == ConvertReceiverMode::kNullOrUndefined) {
__ PushRoot(Heap::kUndefinedValueRootIndex);
__ LoadRoot(x10, Heap::kUndefinedValueRootIndex);
__ Push(x10);
__ Mov(x3, x0); // Argument count is correct.
}
@ -1451,62 +1450,88 @@ void Builtins::Generate_InstantiateAsmJs(MacroAssembler* masm) {
// -- x1 : new target (preserved for callee)
// -- x3 : target function (preserved for callee)
// -----------------------------------
Register argc = x0;
Register new_target = x1;
Register target = x3;
Label failed;
{
FrameScope scope(masm, StackFrame::INTERNAL);
// Preserve argument count for later compare.
__ Move(x4, x0);
// Push a copy of the target function and the new target.
__ SmiTag(x0);
// Push another copy as a parameter to the runtime call.
__ Push(x0, x1, x3, x1);
// Copy arguments from caller (stdlib, foreign, heap).
// Push argument count, a copy of the target function and the new target,
// together with some padding to maintain 16-byte alignment.
__ SmiTag(argc);
__ Push(argc, new_target, target, padreg);
// Push another copy of new target as a parameter to the runtime call and
// copy the rest of the arguments from caller (stdlib, foreign, heap).
Label args_done;
for (int j = 0; j < 4; ++j) {
Label over;
if (j < 3) {
__ cmp(x4, Operand(j));
__ B(ne, &over);
}
for (int i = j - 1; i >= 0; --i) {
__ ldr(x4, MemOperand(fp, StandardFrameConstants::kCallerSPOffset +
i * kPointerSize));
__ push(x4);
}
for (int i = 0; i < 3 - j; ++i) {
__ PushRoot(Heap::kUndefinedValueRootIndex);
}
if (j < 3) {
__ jmp(&args_done);
__ bind(&over);
}
}
__ bind(&args_done);
Register undef = x10;
Register scratch1 = x12;
Register scratch2 = x13;
Register scratch3 = x14;
__ LoadRoot(undef, Heap::kUndefinedValueRootIndex);
Label at_least_one_arg;
Label three_args;
DCHECK(Smi::kZero == 0);
__ Cbnz(argc, &at_least_one_arg);
// No arguments.
__ Push(new_target, undef, undef, undef);
__ B(&args_done);
__ Bind(&at_least_one_arg);
// Load two arguments, though we may only use one (for the one arg case).
__ Ldp(scratch2, scratch1,
MemOperand(fp, StandardFrameConstants::kCallerSPOffset));
// Set flags for determining the value of smi-tagged argc.
// lt => 1, eq => 2, gt => 3.
__ Cmp(argc, Smi::FromInt(2));
__ B(gt, &three_args);
// One or two arguments.
// If there is one argument (flags are lt), scratch2 contains that argument,
// and scratch1 must be undefined.
__ CmovX(scratch1, scratch2, lt);
__ CmovX(scratch2, undef, lt);
__ Push(new_target, scratch1, scratch2, undef);
__ B(&args_done);
// Three arguments.
__ Bind(&three_args);
__ Ldr(scratch3, MemOperand(fp, StandardFrameConstants::kCallerSPOffset +
2 * kPointerSize));
__ Push(new_target, scratch3, scratch1, scratch2);
__ Bind(&args_done);
// Call runtime, on success unwind frame, and parent frame.
__ CallRuntime(Runtime::kInstantiateAsmJs, 4);
// A smi 0 is returned on failure, an object on success.
__ JumpIfSmi(x0, &failed);
__ Drop(2);
__ pop(x4);
__ SmiUntag(x4);
// Peek the argument count from the stack, untagging at the same time.
__ Ldr(w4, UntagSmiMemOperand(__ StackPointer(), 3 * kPointerSize));
__ Drop(4);
scope.GenerateLeaveFrame();
__ add(x4, x4, Operand(1));
__ Drop(x4);
// Drop arguments and receiver.
__ Add(x4, x4, 1);
__ DropArguments(x4);
__ Ret();
__ bind(&failed);
__ Bind(&failed);
// Restore target function and new target.
__ Pop(x3, x1, x0);
__ SmiUntag(x0);
__ Pop(padreg, target, new_target, argc);
__ SmiUntag(argc);
}
// On failure, tail call back to regular js by re-calling the function
// which has be reset to the compile lazy builtin.
__ Ldr(x4, FieldMemOperand(x1, JSFunction::kCodeOffset));
__ Add(x4, x4, Operand(Code::kHeaderSize - kHeapObjectTag));
__ Ldr(x4, FieldMemOperand(new_target, JSFunction::kCodeOffset));
__ Add(x4, x4, Code::kHeaderSize - kHeapObjectTag);
__ Jump(x4);
}
@ -2164,14 +2189,12 @@ void Builtins::Generate_CallFunction(MacroAssembler* masm,
// in the fast case? (fall back to AllocateInNewSpace?)
FrameScope scope(masm, StackFrame::INTERNAL);
__ SmiTag(x0);
__ Push(x0, x1);
__ Push(padreg, x0, x1, cp);
__ Mov(x0, x3);
__ Push(cp);
__ Call(BUILTIN_CODE(masm->isolate(), ToObject),
RelocInfo::CODE_TARGET);
__ Pop(cp);
__ Mov(x3, x0);
__ Pop(x1, x0);
__ Pop(cp, x1, x0, padreg);
__ SmiUntag(x0);
}
__ Ldr(x2, FieldMemOperand(x1, JSFunction::kSharedFunctionInfoOffset));
@ -2195,10 +2218,10 @@ void Builtins::Generate_CallFunction(MacroAssembler* masm,
__ InvokeFunctionCode(x1, no_reg, expected, actual, JUMP_FUNCTION);
// The function is a "classConstructor", need to raise an exception.
__ bind(&class_constructor);
__ Bind(&class_constructor);
{
FrameScope frame(masm, StackFrame::INTERNAL);
__ Push(x1);
__ Push(padreg, x1);
__ CallRuntime(Runtime::kThrowConstructorNonCallableError);
}
}