diff --git a/src/arm/code-stubs-arm.cc b/src/arm/code-stubs-arm.cc index 7a4923d0d9..8cc87ef071 100644 --- a/src/arm/code-stubs-arm.cc +++ b/src/arm/code-stubs-arm.cc @@ -12,6 +12,7 @@ #include "src/codegen.h" #include "src/ic/handler-compiler.h" #include "src/ic/ic.h" +#include "src/ic/stub-cache.h" #include "src/isolate.h" #include "src/jsregexp.h" #include "src/regexp-macro-assembler.h" @@ -4323,15 +4324,15 @@ void StubFailureTrampolineStub::Generate(MacroAssembler* masm) { void LoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorLoadStub stub(isolate(), state()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawLoadStub stub(isolate(), state()); + stub.GenerateForTrampoline(masm); } void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorKeyedLoadStub stub(isolate()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawKeyedLoadStub stub(isolate()); + stub.GenerateForTrampoline(masm); } @@ -4349,6 +4350,239 @@ void CallIC_ArrayTrampolineStub::Generate(MacroAssembler* masm) { } +void VectorRawLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +static void HandleArrayCases(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register feedback, Register scratch1, + Register scratch2, Register scratch3, + bool is_polymorphic, Label* miss) { + // feedback initially contains the feedback array + Label next_loop, prepare_next; + Label load_smi_map, compare_map; + Label start_polymorphic; + + Register receiver_map = scratch1; + Register cached_map = scratch2; + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &load_smi_map); + __ ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ bind(&compare_map); + __ ldr(cached_map, + FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); + __ ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ cmp(receiver_map, cached_map); + __ b(ne, &start_polymorphic); + // found, now call handler. + Register handler = feedback; + __ ldr(handler, FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(1))); + __ add(pc, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + + + Register length = scratch3; + __ bind(&start_polymorphic); + __ ldr(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); + if (!is_polymorphic) { + // If the IC could be monomorphic we have to make sure we don't go past the + // end of the feedback array. + __ cmp(length, Operand(Smi::FromInt(2))); + __ b(eq, miss); + } + + Register too_far = length; + Register pointer_reg = feedback; + + // +-----+------+------+-----+-----+ ... ----+ + // | map | len | wm0 | h0 | wm1 | hN | + // +-----+------+------+-----+-----+ ... ----+ + // 0 1 2 len-1 + // ^ ^ + // | | + // pointer_reg too_far + // aka feedback scratch3 + // also need receiver_map (aka scratch1) + // use cached_map (scratch2) to look in the weak map values. + __ add(too_far, feedback, Operand::PointerOffsetFromSmiKey(length)); + __ add(too_far, too_far, Operand(FixedArray::kHeaderSize - kHeapObjectTag)); + __ add(pointer_reg, feedback, + Operand(FixedArray::OffsetOfElementAt(2) - kHeapObjectTag)); + + __ bind(&next_loop); + __ ldr(cached_map, MemOperand(pointer_reg)); + __ ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ cmp(receiver_map, cached_map); + __ b(ne, &prepare_next); + __ ldr(handler, MemOperand(pointer_reg, kPointerSize)); + __ add(pc, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + + __ bind(&prepare_next); + __ add(pointer_reg, pointer_reg, Operand(kPointerSize * 2)); + __ cmp(pointer_reg, too_far); + __ b(lt, &next_loop); + + // We exhausted our array of map handler pairs. + __ jmp(miss); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ jmp(&compare_map); +} + + +static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register weak_cell, Register scratch, + Label* miss) { + // feedback initially contains the feedback array + Label compare_smi_map; + Register receiver_map = scratch; + Register cached_map = weak_cell; + + // Move the weak map into the weak_cell register. + __ ldr(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset)); + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &compare_smi_map); + __ ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ cmp(cached_map, receiver_map); + __ b(ne, miss); + + Register handler = weak_cell; + __ add(handler, vector, Operand::PointerOffsetFromSmiKey(slot)); + __ ldr(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ add(pc, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + + // In microbenchmarks, it made sense to unroll this code so that the call to + // the handler is duplicated for a HeapObject receiver and a Smi receiver. + __ bind(&compare_smi_map); + __ CompareRoot(weak_cell, Heap::kHeapNumberMapRootIndex); + __ b(ne, miss); + __ add(handler, vector, Operand::PointerOffsetFromSmiKey(slot)); + __ ldr(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ add(pc, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); +} + + +void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // r1 + Register name = VectorLoadICDescriptor::NameRegister(); // r2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // r3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // r0 + Register feedback = r4; + Register scratch1 = r5; + + __ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot)); + __ ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ CompareRoot(scratch1, Heap::kWeakCellMapRootIndex); + __ b(ne, &try_array); + HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, + &miss); + + // Is it a fixed array? + __ bind(&try_array); + __ CompareRoot(scratch1, Heap::kFixedArrayMapRootIndex); + __ b(ne, ¬_array); + HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, r8, + r9, true, &miss); + + __ bind(¬_array); + __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex); + __ b(ne, &miss); + Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags( + Code::ComputeHandlerFlags(Code::LOAD_IC)); + masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, + false, receiver, name, feedback, + scratch1, r8, r9); + + __ bind(&miss); + LoadIC::GenerateMiss(masm); +} + + +void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // r1 + Register key = VectorLoadICDescriptor::NameRegister(); // r2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // r3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // r0 + Register feedback = r4; + Register scratch1 = r5; + + __ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot)); + __ ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ CompareRoot(scratch1, Heap::kWeakCellMapRootIndex); + __ b(ne, &try_array); + __ JumpIfNotSmi(key, &miss); + HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, + &miss); + + __ bind(&try_array); + // Is it a fixed array? + __ CompareRoot(scratch1, Heap::kFixedArrayMapRootIndex); + __ b(ne, ¬_array); + // We have a polymorphic element handler. + __ JumpIfNotSmi(key, &miss); + + Label polymorphic, try_poly_name; + __ bind(&polymorphic); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, r8, + r9, true, &miss); + + __ bind(¬_array); + // Is it generic? + __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex); + __ b(ne, &try_poly_name); + Handle megamorphic_stub = + KeyedLoadIC::ChooseMegamorphicStub(masm->isolate()); + __ Jump(megamorphic_stub, RelocInfo::CODE_TARGET); + + __ bind(&try_poly_name); + // We might have a name in feedback, and a fixed array in the next slot. + __ cmp(key, feedback); + __ b(ne, &miss); + // If the name comparison succeeded, we know we have a fixed array with + // at least one map/handler pair. + __ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot)); + __ ldr(feedback, + FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, r8, + r9, false, &miss); + + __ bind(&miss); + KeyedLoadIC::GenerateMiss(masm); +} + + void ProfileEntryHookStub::MaybeCallEntryHook(MacroAssembler* masm) { if (masm->isolate()->function_entry_hook() != NULL) { ProfileEntryHookStub stub(masm->isolate()); diff --git a/src/arm64/code-stubs-arm64.cc b/src/arm64/code-stubs-arm64.cc index a92e143188..cc499d1844 100644 --- a/src/arm64/code-stubs-arm64.cc +++ b/src/arm64/code-stubs-arm64.cc @@ -11,6 +11,7 @@ #include "src/codegen.h" #include "src/ic/handler-compiler.h" #include "src/ic/ic.h" +#include "src/ic/stub-cache.h" #include "src/isolate.h" #include "src/jsregexp.h" #include "src/regexp-macro-assembler.h" @@ -4451,15 +4452,15 @@ void StubFailureTrampolineStub::Generate(MacroAssembler* masm) { void LoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorLoadStub stub(isolate(), state()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawLoadStub stub(isolate(), state()); + stub.GenerateForTrampoline(masm); } void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorKeyedLoadStub stub(isolate()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawKeyedLoadStub stub(isolate()); + stub.GenerateForTrampoline(masm); } @@ -4477,6 +4478,236 @@ void CallIC_ArrayTrampolineStub::Generate(MacroAssembler* masm) { } +void VectorRawLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +static void HandleArrayCases(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register feedback, Register scratch1, + Register scratch2, Register scratch3, + bool is_polymorphic, Label* miss) { + // feedback initially contains the feedback array + Label next_loop, prepare_next; + Label load_smi_map, compare_map; + Label start_polymorphic; + + Register receiver_map = scratch1; + Register cached_map = scratch2; + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &load_smi_map); + __ Ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ Bind(&compare_map); + __ Ldr(cached_map, + FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); + __ Ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ Cmp(receiver_map, cached_map); + __ B(ne, &start_polymorphic); + // found, now call handler. + Register handler = feedback; + __ Ldr(handler, FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(1))); + __ Add(handler, handler, Code::kHeaderSize - kHeapObjectTag); + __ Jump(feedback); + + Register length = scratch3; + __ Bind(&start_polymorphic); + __ Ldr(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); + if (!is_polymorphic) { + __ Cmp(length, Operand(Smi::FromInt(2))); + __ B(eq, miss); + } + + Register too_far = length; + Register pointer_reg = feedback; + + // +-----+------+------+-----+-----+ ... ----+ + // | map | len | wm0 | h0 | wm1 | hN | + // +-----+------+------+-----+-----+ ... ----+ + // 0 1 2 len-1 + // ^ ^ + // | | + // pointer_reg too_far + // aka feedback scratch3 + // also need receiver_map (aka scratch1) + // use cached_map (scratch2) to look in the weak map values. + __ Add(too_far, feedback, + Operand::UntagSmiAndScale(length, kPointerSizeLog2)); + __ Add(too_far, too_far, FixedArray::kHeaderSize - kHeapObjectTag); + __ Add(pointer_reg, feedback, + FixedArray::OffsetOfElementAt(2) - kHeapObjectTag); + + __ Bind(&next_loop); + __ Ldr(cached_map, MemOperand(pointer_reg)); + __ Ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ Cmp(receiver_map, cached_map); + __ B(ne, &prepare_next); + __ Ldr(handler, MemOperand(pointer_reg, kPointerSize)); + __ Add(handler, handler, Code::kHeaderSize - kHeapObjectTag); + __ Jump(handler); + + __ Bind(&prepare_next); + __ Add(pointer_reg, pointer_reg, kPointerSize * 2); + __ Cmp(pointer_reg, too_far); + __ B(lt, &next_loop); + + // We exhausted our array of map handler pairs. + __ jmp(miss); + + __ Bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ jmp(&compare_map); +} + + +static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register weak_cell, Register scratch, + Label* miss) { + // feedback initially contains the feedback array + Label compare_smi_map; + Register receiver_map = scratch; + Register cached_map = weak_cell; + + // Move the weak map into the weak_cell register. + __ Ldr(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset)); + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &compare_smi_map); + __ Ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ Cmp(cached_map, receiver_map); + __ B(ne, miss); + + Register handler = weak_cell; + __ Add(handler, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); + __ Ldr(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ Add(handler, handler, Code::kHeaderSize - kHeapObjectTag); + __ Jump(weak_cell); + + // In microbenchmarks, it made sense to unroll this code so that the call to + // the handler is duplicated for a HeapObject receiver and a Smi receiver. + // TODO(mvstanton): does this hold on ARM? + __ Bind(&compare_smi_map); + __ JumpIfNotRoot(weak_cell, Heap::kHeapNumberMapRootIndex, miss); + __ Add(handler, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); + __ Ldr(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ Add(handler, handler, Code::kHeaderSize - kHeapObjectTag); + __ Jump(handler); +} + + +void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // x1 + Register name = VectorLoadICDescriptor::NameRegister(); // x2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // x3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // x0 + Register feedback = x4; + Register scratch1 = x5; + + __ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); + __ Ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ Ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ JumpIfNotRoot(scratch1, Heap::kWeakCellMapRootIndex, &try_array); + HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, + &miss); + + // Is it a fixed array? + __ Bind(&try_array); + __ JumpIfNotRoot(scratch1, Heap::kFixedArrayMapRootIndex, ¬_array); + HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, x6, + x7, true, &miss); + + __ Bind(¬_array); + __ JumpIfNotRoot(feedback, Heap::kmegamorphic_symbolRootIndex, &miss); + Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags( + Code::ComputeHandlerFlags(Code::LOAD_IC)); + masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, + false, receiver, name, feedback, + scratch1, x6, x7); + + __ Bind(&miss); + LoadIC::GenerateMiss(masm); +} + + +void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // x1 + Register key = VectorLoadICDescriptor::NameRegister(); // x2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // x3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // x0 + Register feedback = x4; + Register scratch1 = x5; + + __ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); + __ Ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ Ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ JumpIfNotRoot(scratch1, Heap::kWeakCellMapRootIndex, &try_array); + __ JumpIfNotSmi(key, &miss); + HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, + &miss); + + __ Bind(&try_array); + // Is it a fixed array? + __ JumpIfNotRoot(scratch1, Heap::kFixedArrayMapRootIndex, ¬_array); + // We have a polymorphic element handler. + __ JumpIfNotSmi(key, &miss); + + Label polymorphic, try_poly_name; + __ Bind(&polymorphic); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, x6, + x7, true, &miss); + + __ Bind(¬_array); + // Is it generic? + __ JumpIfNotRoot(feedback, Heap::kmegamorphic_symbolRootIndex, + &try_poly_name); + Handle megamorphic_stub = + KeyedLoadIC::ChooseMegamorphicStub(masm->isolate()); + __ Jump(megamorphic_stub, RelocInfo::CODE_TARGET); + + __ Bind(&try_poly_name); + // We might have a name in feedback, and a fixed array in the next slot. + __ Cmp(key, feedback); + __ B(ne, &miss); + // If the name comparison succeeded, we know we have a fixed array with + // at least one map/handler pair. + __ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); + __ Ldr(feedback, + FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, x6, + x7, false, &miss); + + __ Bind(&miss); + KeyedLoadIC::GenerateMiss(masm); +} + + // The entry hook is a "BumpSystemStackPointer" instruction (sub), followed by // a "Push lr" instruction, followed by a call. static const unsigned int kProfileEntryHookCallSize = diff --git a/src/code-stubs-hydrogen.cc b/src/code-stubs-hydrogen.cc index f776abc043..95fa2a2e15 100644 --- a/src/code-stubs-hydrogen.cc +++ b/src/code-stubs-hydrogen.cc @@ -100,21 +100,6 @@ class CodeStubGraphBuilderBase : public HGraphBuilder { HValue* shared_info, HValue* native_context); - // Tail calls handler found at array[map_index + 1]. - void TailCallHandler(HValue* receiver, HValue* name, HValue* array, - HValue* map_index, HValue* slot, HValue* vector); - - // Tail calls handler_code. - void TailCallHandler(HValue* receiver, HValue* name, HValue* slot, - HValue* vector, HValue* handler_code); - - void TailCallMiss(HValue* receiver, HValue* name, HValue* slot, - HValue* vector, bool keyed_load); - - // Handle MONOMORPHIC and POLYMORPHIC LoadIC and KeyedLoadIC cases. - void HandleArrayCases(HValue* array, HValue* receiver, HValue* name, - HValue* slot, HValue* vector, bool keyed_load); - private: HValue* BuildArraySingleArgumentConstructor(JSArrayBuilder* builder); HValue* BuildArrayNArgumentsConstructor(JSArrayBuilder* builder, @@ -2029,211 +2014,6 @@ Handle KeyedLoadGenericStub::GenerateCode() { } -void CodeStubGraphBuilderBase::TailCallHandler(HValue* receiver, HValue* name, - HValue* array, HValue* map_index, - HValue* slot, HValue* vector) { - // The handler is at array[map_index + 1]. Compute this with a custom offset - // to HLoadKeyed. - int offset = - GetDefaultHeaderSizeForElementsKind(FAST_ELEMENTS) + kPointerSize; - HValue* handler_code = Add( - array, map_index, nullptr, FAST_ELEMENTS, NEVER_RETURN_HOLE, offset); - TailCallHandler(receiver, name, slot, vector, handler_code); -} - - -void CodeStubGraphBuilderBase::TailCallHandler(HValue* receiver, HValue* name, - HValue* slot, HValue* vector, - HValue* handler_code) { - VectorLoadICDescriptor descriptor(isolate()); - HValue* op_vals[] = {context(), receiver, name, slot, vector}; - Add(handler_code, 0, descriptor, - Vector(op_vals, 5), TAIL_CALL); - // We never return here, it is a tail call. -} - - -void CodeStubGraphBuilderBase::TailCallMiss(HValue* receiver, HValue* name, - HValue* slot, HValue* vector, - bool keyed_load) { - DCHECK(FLAG_vector_ics); - Add( - receiver, name, slot, vector, - HTailCallThroughMegamorphicCache::ComputeFlags(keyed_load, true)); - // We never return here, it is a tail call. -} - - -void CodeStubGraphBuilderBase::HandleArrayCases(HValue* array, HValue* receiver, - HValue* name, HValue* slot, - HValue* vector, - bool keyed_load) { - HConstant* constant_two = Add(2); - HConstant* constant_three = Add(3); - - IfBuilder if_receiver_heap_object(this); - if_receiver_heap_object.IfNot(receiver); - if_receiver_heap_object.Then(); - Push(AddLoadMap(receiver, nullptr)); - if_receiver_heap_object.Else(); - HConstant* heap_number_map = - Add(isolate()->factory()->heap_number_map()); - Push(heap_number_map); - if_receiver_heap_object.End(); - HValue* receiver_map = Pop(); - - HValue* start = - keyed_load ? graph()->GetConstant1() : graph()->GetConstant0(); - HValue* weak_cell = - Add(array, start, nullptr, FAST_ELEMENTS, ALLOW_RETURN_HOLE); - // Load the weak cell value. It may be Smi(0), or a map. Compare nonetheless - // against the receiver_map. - HValue* array_map = Add(weak_cell, nullptr, - HObjectAccess::ForWeakCellValue()); - - IfBuilder if_correct_map(this); - if_correct_map.If(receiver_map, array_map); - if_correct_map.Then(); - { TailCallHandler(receiver, name, array, start, slot, vector); } - if_correct_map.Else(); - { - // If our array has more elements, the ic is polymorphic. Look for the - // receiver map in the rest of the array. - HValue* length = AddLoadFixedArrayLength(array, nullptr); - LoopBuilder builder(this, context(), LoopBuilder::kPostIncrement, - constant_two); - start = keyed_load ? constant_three : constant_two; - HValue* key = builder.BeginBody(start, length, Token::LT); - { - HValue* weak_cell = Add(array, key, nullptr, FAST_ELEMENTS, - ALLOW_RETURN_HOLE); - HValue* array_map = Add( - weak_cell, nullptr, HObjectAccess::ForWeakCellValue()); - IfBuilder if_correct_poly_map(this); - if_correct_poly_map.If(receiver_map, - array_map); - if_correct_poly_map.Then(); - { TailCallHandler(receiver, name, array, key, slot, vector); } - } - builder.EndBody(); - } - if_correct_map.End(); -} - - -template <> -HValue* CodeStubGraphBuilder::BuildCodeStub() { - HValue* receiver = GetParameter(VectorLoadICDescriptor::kReceiverIndex); - HValue* name = GetParameter(VectorLoadICDescriptor::kNameIndex); - HValue* slot = GetParameter(VectorLoadICDescriptor::kSlotIndex); - HValue* vector = GetParameter(VectorLoadICDescriptor::kVectorIndex); - - // If the feedback is an array, then the IC is in the monomorphic or - // polymorphic state. - HValue* feedback = - Add(vector, slot, nullptr, FAST_ELEMENTS, ALLOW_RETURN_HOLE); - IfBuilder array_checker(this); - array_checker.If(feedback, - isolate()->factory()->fixed_array_map()); - array_checker.Then(); - { HandleArrayCases(feedback, receiver, name, slot, vector, false); } - array_checker.Else(); - { - // Is the IC megamorphic? - IfBuilder mega_checker(this); - HConstant* megamorphic_symbol = - Add(isolate()->factory()->megamorphic_symbol()); - mega_checker.If(feedback, megamorphic_symbol); - mega_checker.Then(); - { - // Probe the stub cache. - Add( - receiver, name, slot, vector, - HTailCallThroughMegamorphicCache::ComputeFlags(false, false)); - } - mega_checker.End(); - } - array_checker.End(); - - TailCallMiss(receiver, name, slot, vector, false); - return graph()->GetConstant0(); -} - - -Handle VectorLoadStub::GenerateCode() { return DoGenerateCode(this); } - - -template <> -HValue* CodeStubGraphBuilder::BuildCodeStub() { - HValue* receiver = GetParameter(VectorLoadICDescriptor::kReceiverIndex); - HValue* name = GetParameter(VectorLoadICDescriptor::kNameIndex); - HValue* slot = GetParameter(VectorLoadICDescriptor::kSlotIndex); - HValue* vector = GetParameter(VectorLoadICDescriptor::kVectorIndex); - HConstant* zero = graph()->GetConstant0(); - - // If the feedback is an array, then the IC is in the monomorphic or - // polymorphic state. - HValue* feedback = - Add(vector, slot, nullptr, FAST_ELEMENTS, ALLOW_RETURN_HOLE); - IfBuilder array_checker(this); - array_checker.If(feedback, - isolate()->factory()->fixed_array_map()); - array_checker.Then(); - { - // If feedback[0] is 0, then the IC has element handlers and name should be - // a smi. If feedback[0] is a string, verify that it matches name. - HValue* recorded_name = Add(feedback, zero, nullptr, - FAST_ELEMENTS, ALLOW_RETURN_HOLE); - - IfBuilder recorded_name_is_zero(this); - recorded_name_is_zero.If(recorded_name, zero); - recorded_name_is_zero.Then(); - { Add(name); } - recorded_name_is_zero.Else(); - { - IfBuilder strings_match(this); - strings_match.IfNot(name, recorded_name); - strings_match.Then(); - TailCallMiss(receiver, name, slot, vector, true); - strings_match.End(); - } - recorded_name_is_zero.End(); - - HandleArrayCases(feedback, receiver, name, slot, vector, true); - } - array_checker.Else(); - { - // Check if the IC is in megamorphic state. - IfBuilder megamorphic_checker(this); - HConstant* megamorphic_symbol = - Add(isolate()->factory()->megamorphic_symbol()); - megamorphic_checker.If(feedback, - megamorphic_symbol); - megamorphic_checker.Then(); - { - // Tail-call to the megamorphic KeyedLoadIC, treating it like a handler. - Handle stub = KeyedLoadIC::ChooseMegamorphicStub(isolate()); - HValue* constant_stub = Add(stub); - LoadDescriptor descriptor(isolate()); - HValue* op_vals[] = {context(), receiver, name}; - Add(constant_stub, 0, descriptor, - Vector(op_vals, 3), TAIL_CALL); - // We never return here, it is a tail call. - } - megamorphic_checker.End(); - } - array_checker.End(); - - TailCallMiss(receiver, name, slot, vector, true); - return zero; -} - - -Handle VectorKeyedLoadStub::GenerateCode() { - return DoGenerateCode(this); -} - - Handle MegamorphicLoadStub::GenerateCode() { return DoGenerateCode(this); } diff --git a/src/code-stubs.cc b/src/code-stubs.cc index 0d32028aa7..f600cf316d 100644 --- a/src/code-stubs.cc +++ b/src/code-stubs.cc @@ -620,26 +620,6 @@ CallInterfaceDescriptor StoreTransitionStub::GetCallInterfaceDescriptor() { } -static void InitializeVectorLoadStub(Isolate* isolate, - CodeStubDescriptor* descriptor, - Address deoptimization_handler) { - DCHECK(FLAG_vector_ics); - descriptor->Initialize(deoptimization_handler); -} - - -void VectorLoadStub::InitializeDescriptor(CodeStubDescriptor* descriptor) { - InitializeVectorLoadStub(isolate(), descriptor, - FUNCTION_ADDR(LoadIC_MissFromStubFailure)); -} - - -void VectorKeyedLoadStub::InitializeDescriptor(CodeStubDescriptor* descriptor) { - InitializeVectorLoadStub(isolate(), descriptor, - FUNCTION_ADDR(KeyedLoadIC_MissFromStubFailure)); -} - - void MegamorphicLoadStub::InitializeDescriptor(CodeStubDescriptor* d) {} diff --git a/src/code-stubs.h b/src/code-stubs.h index 5a2d2691a0..f5eeb09492 100644 --- a/src/code-stubs.h +++ b/src/code-stubs.h @@ -85,8 +85,8 @@ namespace internal { V(StringAdd) \ V(ToBoolean) \ V(TransitionElementsKind) \ - V(VectorKeyedLoad) \ - V(VectorLoad) \ + V(VectorRawKeyedLoad) \ + V(VectorRawLoad) \ /* IC Handler stubs */ \ V(LoadConstant) \ V(LoadField) \ @@ -2062,38 +2062,49 @@ class MegamorphicLoadStub : public HydrogenCodeStub { }; -class VectorLoadStub : public HydrogenCodeStub { +class VectorRawLoadStub : public PlatformCodeStub { public: - explicit VectorLoadStub(Isolate* isolate, const LoadICState& state) - : HydrogenCodeStub(isolate) { - set_sub_minor_key(state.GetExtraICState()); + explicit VectorRawLoadStub(Isolate* isolate, const LoadICState& state) + : PlatformCodeStub(isolate) { + minor_key_ = state.GetExtraICState(); } - Code::Kind GetCodeKind() const OVERRIDE { return Code::LOAD_IC; } + void GenerateForTrampoline(MacroAssembler* masm); - InlineCacheState GetICState() const FINAL { return DEFAULT; } + virtual Code::Kind GetCodeKind() const OVERRIDE { return Code::LOAD_IC; } - ExtraICState GetExtraICState() const FINAL { - return static_cast(sub_minor_key()); + virtual InlineCacheState GetICState() const FINAL OVERRIDE { return DEFAULT; } + + virtual ExtraICState GetExtraICState() const FINAL OVERRIDE { + return static_cast(minor_key_); } - private: - LoadICState state() const { return LoadICState(GetExtraICState()); } - DEFINE_CALL_INTERFACE_DESCRIPTOR(VectorLoadIC); - DEFINE_HYDROGEN_CODE_STUB(VectorLoad, HydrogenCodeStub); + DEFINE_PLATFORM_CODE_STUB(VectorRawLoad, PlatformCodeStub); + + protected: + void GenerateImpl(MacroAssembler* masm, bool in_frame); }; -class VectorKeyedLoadStub : public VectorLoadStub { +class VectorRawKeyedLoadStub : public PlatformCodeStub { public: - explicit VectorKeyedLoadStub(Isolate* isolate) - : VectorLoadStub(isolate, LoadICState(0)) {} + explicit VectorRawKeyedLoadStub(Isolate* isolate) + : PlatformCodeStub(isolate) {} - Code::Kind GetCodeKind() const OVERRIDE { return Code::KEYED_LOAD_IC; } + void GenerateForTrampoline(MacroAssembler* masm); + + virtual Code::Kind GetCodeKind() const OVERRIDE { + return Code::KEYED_LOAD_IC; + } + + virtual InlineCacheState GetICState() const FINAL OVERRIDE { return DEFAULT; } DEFINE_CALL_INTERFACE_DESCRIPTOR(VectorLoadIC); - DEFINE_HYDROGEN_CODE_STUB(VectorKeyedLoad, VectorLoadStub); + DEFINE_PLATFORM_CODE_STUB(VectorRawKeyedLoad, PlatformCodeStub); + + protected: + void GenerateImpl(MacroAssembler* masm, bool in_frame); }; diff --git a/src/ia32/code-stubs-ia32.cc b/src/ia32/code-stubs-ia32.cc index 74715d9a83..4f8de8ae38 100644 --- a/src/ia32/code-stubs-ia32.cc +++ b/src/ia32/code-stubs-ia32.cc @@ -12,6 +12,7 @@ #include "src/codegen.h" #include "src/ic/handler-compiler.h" #include "src/ic/ic.h" +#include "src/ic/stub-cache.h" #include "src/isolate.h" #include "src/jsregexp.h" #include "src/regexp-macro-assembler.h" @@ -4399,15 +4400,236 @@ void StubFailureTrampolineStub::Generate(MacroAssembler* masm) { void LoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorLoadStub stub(isolate(), state()); - __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawLoadStub stub(isolate(), state()); + stub.GenerateForTrampoline(masm); } void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorKeyedLoadStub stub(isolate()); - __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawKeyedLoadStub stub(isolate()); + stub.GenerateForTrampoline(masm); +} + + +static void HandleArrayCases(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register feedback, bool is_polymorphic, + Label* miss) { + // feedback initially contains the feedback array + Label next, next_loop, prepare_next; + Label load_smi_map, compare_map; + Label start_polymorphic; + + __ push(receiver); + __ push(vector); + + Register receiver_map = receiver; + Register cached_map = vector; + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &load_smi_map); + __ mov(receiver_map, FieldOperand(receiver, 0)); + __ bind(&compare_map); + __ mov(cached_map, FieldOperand(feedback, FixedArray::OffsetOfElementAt(0))); + + // A named keyed load might have a 2 element array, all other cases can count + // on an array with at least 2 {map, handler} pairs, so they can go right + // into polymorphic array handling. + __ cmp(receiver_map, FieldOperand(cached_map, WeakCell::kValueOffset)); + __ j(not_equal, is_polymorphic ? &start_polymorphic : &next); + + // found, now call handler. + Register handler = feedback; + __ mov(handler, FieldOperand(feedback, FixedArray::OffsetOfElementAt(1))); + __ pop(vector); + __ pop(receiver); + __ lea(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); + + if (!is_polymorphic) { + __ bind(&next); + __ cmp(FieldOperand(feedback, FixedArray::kLengthOffset), + Immediate(Smi::FromInt(2))); + __ j(not_equal, &start_polymorphic); + __ pop(vector); + __ pop(receiver); + __ jmp(miss); + } + + // Polymorphic, we have to loop from 2 to N + __ bind(&start_polymorphic); + __ push(key); + Register counter = key; + __ mov(counter, Immediate(Smi::FromInt(2))); + __ bind(&next_loop); + __ mov(cached_map, FieldOperand(feedback, counter, times_half_pointer_size, + FixedArray::kHeaderSize)); + __ cmp(receiver_map, FieldOperand(cached_map, WeakCell::kValueOffset)); + __ j(not_equal, &prepare_next); + __ mov(handler, FieldOperand(feedback, counter, times_half_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + __ pop(key); + __ pop(vector); + __ pop(receiver); + __ lea(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); + + __ bind(&prepare_next); + __ add(counter, Immediate(Smi::FromInt(2))); + __ cmp(counter, FieldOperand(feedback, FixedArray::kLengthOffset)); + __ j(less, &next_loop); + + // We exhausted our array of map handler pairs. + __ pop(key); + __ pop(vector); + __ pop(receiver); + __ jmp(miss); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ jmp(&compare_map); +} + + +static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register weak_cell, Label* miss) { + // feedback initially contains the feedback array + Label compare_smi_map; + + // Move the weak map into the weak_cell register. + Register ic_map = weak_cell; + __ mov(ic_map, FieldOperand(weak_cell, WeakCell::kValueOffset)); + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &compare_smi_map); + __ cmp(ic_map, FieldOperand(receiver, 0)); + __ j(not_equal, miss); + Register handler = weak_cell; + __ mov(handler, FieldOperand(vector, slot, times_half_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + __ lea(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); + + // In microbenchmarks, it made sense to unroll this code so that the call to + // the handler is duplicated for a HeapObject receiver and a Smi receiver. + __ bind(&compare_smi_map); + __ CompareRoot(ic_map, Heap::kHeapNumberMapRootIndex); + __ j(not_equal, miss); + __ mov(handler, FieldOperand(vector, slot, times_half_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + __ lea(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); +} + + +void VectorRawLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // edx + Register name = VectorLoadICDescriptor::NameRegister(); // ecx + Register vector = VectorLoadICDescriptor::VectorRegister(); // ebx + Register slot = VectorLoadICDescriptor::SlotRegister(); // eax + Register scratch = edi; + __ mov(scratch, FieldOperand(vector, slot, times_half_pointer_size, + FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ CompareRoot(FieldOperand(scratch, 0), Heap::kWeakCellMapRootIndex); + __ j(not_equal, &try_array); + HandleMonomorphicCase(masm, receiver, name, vector, slot, scratch, &miss); + + // Is it a fixed array? + __ bind(&try_array); + __ CompareRoot(FieldOperand(scratch, 0), Heap::kFixedArrayMapRootIndex); + __ j(not_equal, ¬_array); + HandleArrayCases(masm, receiver, name, vector, slot, scratch, true, &miss); + + __ bind(¬_array); + __ CompareRoot(scratch, Heap::kmegamorphic_symbolRootIndex); + __ j(not_equal, &miss); + __ push(slot); + __ push(vector); + Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags( + Code::ComputeHandlerFlags(Code::LOAD_IC)); + masm->isolate()->stub_cache()->GenerateProbe( + masm, Code::LOAD_IC, code_flags, false, receiver, name, vector, scratch); + __ pop(vector); + __ pop(slot); + + __ bind(&miss); + LoadIC::GenerateMiss(masm); +} + + +void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // edx + Register key = VectorLoadICDescriptor::NameRegister(); // ecx + Register vector = VectorLoadICDescriptor::VectorRegister(); // ebx + Register slot = VectorLoadICDescriptor::SlotRegister(); // eax + Register feedback = edi; + __ mov(feedback, FieldOperand(vector, slot, times_half_pointer_size, + FixedArray::kHeaderSize)); + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ CompareRoot(FieldOperand(feedback, 0), Heap::kWeakCellMapRootIndex); + __ j(not_equal, &try_array); + __ JumpIfNotSmi(key, &miss); + HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, &miss); + + __ bind(&try_array); + // Is it a fixed array? + __ CompareRoot(FieldOperand(feedback, 0), Heap::kFixedArrayMapRootIndex); + __ j(not_equal, ¬_array); + // We have a polymorphic element handler. + __ JumpIfNotSmi(key, &miss); + + Label polymorphic, try_poly_name; + __ bind(&polymorphic); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, true, &miss); + + __ bind(¬_array); + // Is it generic? + __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex); + __ j(not_equal, &try_poly_name); + Handle megamorphic_stub = + KeyedLoadIC::ChooseMegamorphicStub(masm->isolate()); + __ jmp(megamorphic_stub, RelocInfo::CODE_TARGET); + + __ bind(&try_poly_name); + // We might have a name in feedback, and a fixed array in the next slot. + __ cmp(key, feedback); + __ j(not_equal, &miss); + // If the name comparison succeeded, we know we have a fixed array with + // at least one map/handler pair. + __ mov(feedback, FieldOperand(vector, slot, times_half_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, false, &miss); + + __ bind(&miss); + KeyedLoadIC::GenerateMiss(masm); } diff --git a/src/ic/ic.cc b/src/ic/ic.cc index 92a6e29109..6c31429f16 100644 --- a/src/ic/ic.cc +++ b/src/ic/ic.cc @@ -988,7 +988,7 @@ Handle LoadIC::load_global(Isolate* isolate, Handle global, Handle LoadIC::initialize_stub_in_optimized_code( Isolate* isolate, ExtraICState extra_state, State initialization_state) { if (FLAG_vector_ics) { - return VectorLoadStub(isolate, LoadICState(extra_state)).GetCode(); + return VectorRawLoadStub(isolate, LoadICState(extra_state)).GetCode(); } return PropertyICCompiler::ComputeLoad(isolate, initialization_state, extra_state); @@ -1007,7 +1007,7 @@ Handle KeyedLoadIC::initialize_stub(Isolate* isolate) { Handle KeyedLoadIC::initialize_stub_in_optimized_code( Isolate* isolate, State initialization_state) { if (FLAG_vector_ics) { - return VectorKeyedLoadStub(isolate).GetCode(); + return VectorRawKeyedLoadStub(isolate).GetCode(); } switch (initialization_state) { case UNINITIALIZED: diff --git a/src/mips/code-stubs-mips.cc b/src/mips/code-stubs-mips.cc index dd608e6325..60ba09f063 100644 --- a/src/mips/code-stubs-mips.cc +++ b/src/mips/code-stubs-mips.cc @@ -12,6 +12,7 @@ #include "src/codegen.h" #include "src/ic/handler-compiler.h" #include "src/ic/ic.h" +#include "src/ic/stub-cache.h" #include "src/isolate.h" #include "src/jsregexp.h" #include "src/regexp-macro-assembler.h" @@ -4554,15 +4555,15 @@ void StubFailureTrampolineStub::Generate(MacroAssembler* masm) { void LoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorLoadStub stub(isolate(), state()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawLoadStub stub(isolate(), state()); + stub.GenerateForTrampoline(masm); } void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorKeyedLoadStub stub(isolate()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawKeyedLoadStub stub(isolate()); + stub.GenerateForTrampoline(masm); } @@ -4580,6 +4581,243 @@ void CallIC_ArrayTrampolineStub::Generate(MacroAssembler* masm) { } +void VectorRawLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +static void HandleArrayCases(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register feedback, Register scratch1, + Register scratch2, Register scratch3, + bool is_polymorphic, Label* miss) { + // feedback initially contains the feedback array + Label next_loop, prepare_next; + Label load_smi_map, compare_map; + Label start_polymorphic; + + Register receiver_map = scratch1; + Register cached_map = scratch2; + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &load_smi_map); + __ lw(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ bind(&compare_map); + __ lw(cached_map, + FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); + __ lw(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ Branch(&start_polymorphic, ne, receiver_map, Operand(cached_map)); + // found, now call handler. + Register handler = feedback; + __ lw(handler, FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(1))); + __ Addu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + __ Jump(t9); + + + Register length = scratch3; + __ bind(&start_polymorphic); + __ lw(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); + if (!is_polymorphic) { + // If the IC could be monomorphic we have to make sure we don't go past the + // end of the feedback array. + __ Branch(miss, eq, length, Operand(Smi::FromInt(2))); + } + + Register too_far = length; + Register pointer_reg = feedback; + + // +-----+------+------+-----+-----+ ... ----+ + // | map | len | wm0 | h0 | wm1 | hN | + // +-----+------+------+-----+-----+ ... ----+ + // 0 1 2 len-1 + // ^ ^ + // | | + // pointer_reg too_far + // aka feedback scratch3 + // also need receiver_map (aka scratch1) + // use cached_map (scratch2) to look in the weak map values. + __ sll(at, length, kPointerSizeLog2 - kSmiTagSize); + __ Addu(too_far, feedback, Operand(at)); + __ Addu(too_far, too_far, Operand(FixedArray::kHeaderSize - kHeapObjectTag)); + __ Addu(pointer_reg, feedback, + Operand(FixedArray::OffsetOfElementAt(2) - kHeapObjectTag)); + + __ bind(&next_loop); + __ lw(cached_map, MemOperand(pointer_reg)); + __ lw(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ Branch(&prepare_next, ne, receiver_map, Operand(cached_map)); + __ lw(handler, MemOperand(pointer_reg, kPointerSize)); + __ Addu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + __ Jump(t9); + + __ bind(&prepare_next); + __ Addu(pointer_reg, pointer_reg, Operand(kPointerSize * 2)); + __ Branch(&next_loop, lt, pointer_reg, Operand(too_far)); + + // We exhausted our array of map handler pairs. + __ jmp(miss); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ jmp(&compare_map); +} + + +static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register weak_cell, Register scratch, + Label* miss) { + // feedback initially contains the feedback array + Label compare_smi_map; + Register receiver_map = scratch; + Register cached_map = weak_cell; + + // Move the weak map into the weak_cell register. + __ lw(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset)); + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &compare_smi_map); + __ lw(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ Branch(miss, ne, cached_map, Operand(receiver_map)); + + Register handler = weak_cell; + __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); + __ Addu(handler, vector, Operand(at)); + __ lw(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ Addu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + __ Jump(t9); + + // In microbenchmarks, it made sense to unroll this code so that the call to + // the handler is duplicated for a HeapObject receiver and a Smi receiver. + __ bind(&compare_smi_map); + __ LoadRoot(at, Heap::kHeapNumberMapRootIndex); + __ Branch(miss, ne, at, Operand(weak_cell)); + __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); + __ Addu(handler, vector, Operand(at)); + __ lw(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ Addu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + __ Jump(t9); +} + + +void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // a1 + Register name = VectorLoadICDescriptor::NameRegister(); // a2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 + Register feedback = t0; + Register scratch1 = t1; + + __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); + __ Addu(feedback, vector, Operand(at)); + __ lw(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ lw(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ LoadRoot(at, Heap::kWeakCellMapRootIndex); + __ Branch(&try_array, ne, at, Operand(scratch1)); + HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, + &miss); + + // Is it a fixed array? + __ bind(&try_array); + __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); + __ Branch(¬_array, ne, at, Operand(scratch1)); + HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, t4, + t5, true, &miss); + + __ bind(¬_array); + __ LoadRoot(at, Heap::kmegamorphic_symbolRootIndex); + __ Branch(&miss, ne, at, Operand(feedback)); + Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags( + Code::ComputeHandlerFlags(Code::LOAD_IC)); + masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, + false, receiver, name, feedback, + scratch1, t4, t5); + + __ bind(&miss); + LoadIC::GenerateMiss(masm); +} + + +void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // a1 + Register key = VectorLoadICDescriptor::NameRegister(); // a2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 + Register feedback = t0; + Register scratch1 = t1; + + __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); + __ Addu(feedback, vector, Operand(at)); + __ lw(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ lw(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ LoadRoot(at, Heap::kWeakCellMapRootIndex); + __ Branch(&try_array, ne, at, Operand(scratch1)); + __ JumpIfNotSmi(key, &miss); + HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, + &miss); + + __ bind(&try_array); + // Is it a fixed array? + __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); + __ Branch(¬_array, ne, at, Operand(scratch1)); + // We have a polymorphic element handler. + __ JumpIfNotSmi(key, &miss); + + Label polymorphic, try_poly_name; + __ bind(&polymorphic); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, t4, + t5, true, &miss); + + __ bind(¬_array); + // Is it generic? + __ LoadRoot(at, Heap::kmegamorphic_symbolRootIndex); + __ Branch(&try_poly_name, ne, at, Operand(feedback)); + Handle megamorphic_stub = + KeyedLoadIC::ChooseMegamorphicStub(masm->isolate()); + __ Jump(megamorphic_stub, RelocInfo::CODE_TARGET); + + __ bind(&try_poly_name); + // We might have a name in feedback, and a fixed array in the next slot. + __ Branch(&miss, ne, key, Operand(feedback)); + // If the name comparison succeeded, we know we have a fixed array with + // at least one map/handler pair. + __ sll(at, slot, kPointerSizeLog2 - kSmiTagSize); + __ Addu(feedback, vector, Operand(at)); + __ lw(feedback, + FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, t4, + t5, false, &miss); + + __ bind(&miss); + KeyedLoadIC::GenerateMiss(masm); +} + + void ProfileEntryHookStub::MaybeCallEntryHook(MacroAssembler* masm) { if (masm->isolate()->function_entry_hook() != NULL) { ProfileEntryHookStub stub(masm->isolate()); diff --git a/src/mips64/code-stubs-mips64.cc b/src/mips64/code-stubs-mips64.cc index 7f367fe9a8..458a5da0c4 100644 --- a/src/mips64/code-stubs-mips64.cc +++ b/src/mips64/code-stubs-mips64.cc @@ -11,6 +11,7 @@ #include "src/codegen.h" #include "src/ic/handler-compiler.h" #include "src/ic/ic.h" +#include "src/ic/stub-cache.h" #include "src/isolate.h" #include "src/jsregexp.h" #include "src/regexp-macro-assembler.h" @@ -4597,15 +4598,15 @@ void StubFailureTrampolineStub::Generate(MacroAssembler* masm) { void LoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorLoadStub stub(isolate(), state()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawLoadStub stub(isolate(), state()); + stub.GenerateForTrampoline(masm); } void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorKeyedLoadStub stub(isolate()); - __ Jump(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawKeyedLoadStub stub(isolate()); + stub.GenerateForTrampoline(masm); } @@ -4623,6 +4624,243 @@ void CallIC_ArrayTrampolineStub::Generate(MacroAssembler* masm) { } +void VectorRawLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +static void HandleArrayCases(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register feedback, Register scratch1, + Register scratch2, Register scratch3, + bool is_polymorphic, Label* miss) { + // feedback initially contains the feedback array + Label next_loop, prepare_next; + Label load_smi_map, compare_map; + Label start_polymorphic; + + Register receiver_map = scratch1; + Register cached_map = scratch2; + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &load_smi_map); + __ ld(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ bind(&compare_map); + __ ld(cached_map, + FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); + __ ld(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ Branch(&start_polymorphic, ne, receiver_map, Operand(cached_map)); + // found, now call handler. + Register handler = feedback; + __ ld(handler, FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(1))); + __ Daddu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + __ Jump(t9); + + Register length = scratch3; + __ bind(&start_polymorphic); + __ ld(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); + if (!is_polymorphic) { + // If the IC could be monomorphic we have to make sure we don't go past the + // end of the feedback array. + __ Branch(miss, eq, length, Operand(Smi::FromInt(2))); + } + + Register too_far = length; + Register pointer_reg = feedback; + + // +-----+------+------+-----+-----+ ... ----+ + // | map | len | wm0 | h0 | wm1 | hN | + // +-----+------+------+-----+-----+ ... ----+ + // 0 1 2 len-1 + // ^ ^ + // | | + // pointer_reg too_far + // aka feedback scratch3 + // also need receiver_map (aka scratch1) + // use cached_map (scratch2) to look in the weak map values. + __ SmiScale(too_far, length, kPointerSizeLog2); + __ Daddu(too_far, feedback, Operand(too_far)); + __ Daddu(too_far, too_far, Operand(FixedArray::kHeaderSize - kHeapObjectTag)); + __ Daddu(pointer_reg, feedback, + Operand(FixedArray::OffsetOfElementAt(2) - kHeapObjectTag)); + + __ bind(&next_loop); + __ ld(cached_map, MemOperand(pointer_reg)); + __ ld(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); + __ Branch(&prepare_next, ne, receiver_map, Operand(cached_map)); + __ ld(handler, MemOperand(pointer_reg, kPointerSize)); + __ Daddu(t9, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); + __ Jump(t9); + + __ bind(&prepare_next); + __ Daddu(pointer_reg, pointer_reg, Operand(kPointerSize * 2)); + __ Branch(&next_loop, lt, pointer_reg, Operand(too_far)); + + // We exhausted our array of map handler pairs. + __ Branch(miss); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ Branch(&compare_map); +} + + +static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register weak_cell, Register scratch, + Label* miss) { + // feedback initially contains the feedback array + Label compare_smi_map; + Register receiver_map = scratch; + Register cached_map = weak_cell; + + // Move the weak map into the weak_cell register. + __ ld(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset)); + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &compare_smi_map); + __ ld(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); + __ Branch(miss, ne, cached_map, Operand(receiver_map)); + + Register handler = weak_cell; + __ SmiScale(handler, slot, kPointerSizeLog2); + __ Daddu(handler, vector, Operand(handler)); + __ ld(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ Daddu(t9, handler, Code::kHeaderSize - kHeapObjectTag); + __ Jump(t9); + + // In microbenchmarks, it made sense to unroll this code so that the call to + // the handler is duplicated for a HeapObject receiver and a Smi receiver. + // TODO(mvstanton): does this hold on ARM? + __ bind(&compare_smi_map); + __ LoadRoot(at, Heap::kHeapNumberMapRootIndex); + __ Branch(miss, ne, weak_cell, Operand(at)); + __ SmiScale(handler, slot, kPointerSizeLog2); + __ Daddu(handler, vector, Operand(handler)); + __ ld(handler, + FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); + __ Daddu(t9, handler, Code::kHeaderSize - kHeapObjectTag); + __ Jump(t9); +} + + +void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // a1 + Register name = VectorLoadICDescriptor::NameRegister(); // a2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 + Register feedback = a4; + Register scratch1 = a5; + + __ SmiScale(feedback, slot, kPointerSizeLog2); + __ Daddu(feedback, vector, Operand(feedback)); + __ ld(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ ld(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ LoadRoot(at, Heap::kWeakCellMapRootIndex); + __ Branch(&try_array, ne, scratch1, Operand(at)); + HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, + &miss); + + // Is it a fixed array? + __ bind(&try_array); + __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); + __ Branch(¬_array, ne, scratch1, Operand(at)); + HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, a6, + a7, true, &miss); + + __ bind(¬_array); + __ LoadRoot(at, Heap::kmegamorphic_symbolRootIndex); + __ Branch(&miss, ne, feedback, Operand(at)); + Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags( + Code::ComputeHandlerFlags(Code::LOAD_IC)); + masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, + false, receiver, name, feedback, + scratch1, a6, a7); + + __ bind(&miss); + LoadIC::GenerateMiss(masm); +} + + +void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // a1 + Register key = VectorLoadICDescriptor::NameRegister(); // a2 + Register vector = VectorLoadICDescriptor::VectorRegister(); // a3 + Register slot = VectorLoadICDescriptor::SlotRegister(); // a0 + Register feedback = a4; + Register scratch1 = a5; + + __ SmiScale(feedback, slot, kPointerSizeLog2); + __ Daddu(feedback, vector, Operand(feedback)); + __ ld(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ ld(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); + __ LoadRoot(at, Heap::kWeakCellMapRootIndex); + __ Branch(&try_array, ne, scratch1, Operand(at)); + __ JumpIfNotSmi(key, &miss); + HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, + &miss); + + __ bind(&try_array); + // Is it a fixed array? + __ LoadRoot(at, Heap::kFixedArrayMapRootIndex); + __ Branch(¬_array, ne, scratch1, Operand(at)); + // We have a polymorphic element handler. + __ JumpIfNotSmi(key, &miss); + + Label polymorphic, try_poly_name; + __ bind(&polymorphic); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, a6, + a7, true, &miss); + + __ bind(¬_array); + // Is it generic? + __ LoadRoot(at, Heap::kmegamorphic_symbolRootIndex); + __ Branch(&try_poly_name, ne, feedback, Operand(at)); + Handle megamorphic_stub = + KeyedLoadIC::ChooseMegamorphicStub(masm->isolate()); + __ Jump(megamorphic_stub, RelocInfo::CODE_TARGET); + + __ bind(&try_poly_name); + // We might have a name in feedback, and a fixed array in the next slot. + __ Branch(&miss, ne, key, Operand(feedback)); + // If the name comparison succeeded, we know we have a fixed array with + // at least one map/handler pair. + __ SmiScale(feedback, slot, kPointerSizeLog2); + __ Daddu(feedback, vector, Operand(feedback)); + __ ld(feedback, + FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, a6, + a7, false, &miss); + + __ bind(&miss); + KeyedLoadIC::GenerateMiss(masm); +} + + void ProfileEntryHookStub::MaybeCallEntryHook(MacroAssembler* masm) { if (masm->isolate()->function_entry_hook() != NULL) { ProfileEntryHookStub stub(masm->isolate()); diff --git a/src/type-feedback-vector.cc b/src/type-feedback-vector.cc index b0be315b2b..371a309ea7 100644 --- a/src/type-feedback-vector.cc +++ b/src/type-feedback-vector.cc @@ -82,8 +82,8 @@ Handle TypeFeedbackVector::Allocate( const int ic_slot_count = spec.ic_slots(); const int index_count = FLAG_vector_ics ? VectorICComputer::word_count(ic_slot_count) : 0; - const int length = - slot_count + ic_slot_count + index_count + kReservedIndexCount; + const int length = slot_count + (ic_slot_count * elements_per_ic_slot()) + + index_count + kReservedIndexCount; if (length == kReservedIndexCount) { return Handle::cast( isolate->factory()->empty_fixed_array()); @@ -207,16 +207,28 @@ Handle FeedbackNexus::EnsureArrayOfSize(int length) { } -void FeedbackNexus::InstallHandlers(int start_index, MapHandleList* maps, - CodeHandleList* handlers) { +Handle FeedbackNexus::EnsureExtraArrayOfSize(int length) { Isolate* isolate = GetIsolate(); - Handle array = handle(FixedArray::cast(GetFeedback()), isolate); + Handle feedback_extra = handle(GetFeedbackExtra(), isolate); + if (!feedback_extra->IsFixedArray() || + FixedArray::cast(*feedback_extra)->length() != length) { + Handle array = isolate->factory()->NewFixedArray(length); + SetFeedbackExtra(*array); + return array; + } + return Handle::cast(feedback_extra); +} + + +void FeedbackNexus::InstallHandlers(Handle array, + MapHandleList* maps, + CodeHandleList* handlers) { int receiver_count = maps->length(); for (int current = 0; current < receiver_count; ++current) { Handle map = maps->at(current); Handle cell = Map::WeakCellForMap(map); - array->set(start_index + (current * 2), *cell); - array->set(start_index + (current * 2 + 1), *handlers->at(current)); + array->set(current * 2, *cell); + array->set(current * 2 + 1, *handlers->at(current)); } } @@ -224,6 +236,7 @@ void FeedbackNexus::InstallHandlers(int start_index, MapHandleList* maps, InlineCacheState LoadICNexus::StateFromFeedback() const { Isolate* isolate = GetIsolate(); Object* feedback = GetFeedback(); + if (feedback == *vector()->UninitializedSentinel(isolate)) { return UNINITIALIZED; } else if (feedback == *vector()->MegamorphicSentinel(isolate)) { @@ -233,10 +246,10 @@ InlineCacheState LoadICNexus::StateFromFeedback() const { } else if (feedback->IsFixedArray()) { // Determine state purely by our structure, don't check if the maps are // cleared. - FixedArray* array = FixedArray::cast(feedback); - int length = array->length(); - DCHECK(length >= 2); - return length == 2 ? MONOMORPHIC : POLYMORPHIC; + return POLYMORPHIC; + } else if (feedback->IsWeakCell()) { + // Don't check if the map is cleared. + return MONOMORPHIC; } return UNINITIALIZED; @@ -246,6 +259,7 @@ InlineCacheState LoadICNexus::StateFromFeedback() const { InlineCacheState KeyedLoadICNexus::StateFromFeedback() const { Isolate* isolate = GetIsolate(); Object* feedback = GetFeedback(); + if (feedback == *vector()->UninitializedSentinel(isolate)) { return UNINITIALIZED; } else if (feedback == *vector()->PremonomorphicSentinel(isolate)) { @@ -255,10 +269,14 @@ InlineCacheState KeyedLoadICNexus::StateFromFeedback() const { } else if (feedback->IsFixedArray()) { // Determine state purely by our structure, don't check if the maps are // cleared. - FixedArray* array = FixedArray::cast(feedback); - int length = array->length(); - DCHECK(length >= 3); - return length == 3 ? MONOMORPHIC : POLYMORPHIC; + return POLYMORPHIC; + } else if (feedback->IsWeakCell()) { + // Don't check if the map is cleared. + return MONOMORPHIC; + } else if (feedback->IsName()) { + Object* extra = GetFeedbackExtra(); + FixedArray* extra_array = FixedArray::cast(extra); + return extra_array->length() > 2 ? POLYMORPHIC : MONOMORPHIC; } return UNINITIALIZED; @@ -268,6 +286,8 @@ InlineCacheState KeyedLoadICNexus::StateFromFeedback() const { InlineCacheState CallICNexus::StateFromFeedback() const { Isolate* isolate = GetIsolate(); Object* feedback = GetFeedback(); + DCHECK(!FLAG_vector_ics || + GetFeedbackExtra() == *vector()->UninitializedSentinel(isolate)); if (feedback == *vector()->MegamorphicSentinel(isolate)) { return GENERIC; @@ -311,56 +331,68 @@ void CallICNexus::ConfigureMonomorphic(Handle function) { void KeyedLoadICNexus::ConfigureMegamorphic() { - SetFeedback(*vector()->MegamorphicSentinel(GetIsolate()), SKIP_WRITE_BARRIER); + Isolate* isolate = GetIsolate(); + SetFeedback(*vector()->MegamorphicSentinel(isolate), SKIP_WRITE_BARRIER); + SetFeedbackExtra(*vector()->UninitializedSentinel(isolate), + SKIP_WRITE_BARRIER); } void LoadICNexus::ConfigureMegamorphic() { SetFeedback(*vector()->MegamorphicSentinel(GetIsolate()), SKIP_WRITE_BARRIER); + SetFeedbackExtra(*vector()->UninitializedSentinel(GetIsolate()), + SKIP_WRITE_BARRIER); } void LoadICNexus::ConfigurePremonomorphic() { SetFeedback(*vector()->PremonomorphicSentinel(GetIsolate()), SKIP_WRITE_BARRIER); + SetFeedbackExtra(*vector()->UninitializedSentinel(GetIsolate()), + SKIP_WRITE_BARRIER); } void KeyedLoadICNexus::ConfigurePremonomorphic() { - SetFeedback(*vector()->PremonomorphicSentinel(GetIsolate()), - SKIP_WRITE_BARRIER); + Isolate* isolate = GetIsolate(); + SetFeedback(*vector()->PremonomorphicSentinel(isolate), SKIP_WRITE_BARRIER); + SetFeedbackExtra(*vector()->UninitializedSentinel(isolate), + SKIP_WRITE_BARRIER); } void LoadICNexus::ConfigureMonomorphic(Handle receiver_map, Handle handler) { - Handle array = EnsureArrayOfSize(2); Handle cell = Map::WeakCellForMap(receiver_map); - array->set(0, *cell); - array->set(1, *handler); + SetFeedback(*cell); + SetFeedbackExtra(*handler); } void KeyedLoadICNexus::ConfigureMonomorphic(Handle name, Handle receiver_map, Handle handler) { - Handle array = EnsureArrayOfSize(3); - if (name.is_null()) { - array->set(0, Smi::FromInt(0)); - } else { - array->set(0, *name); - } Handle cell = Map::WeakCellForMap(receiver_map); - array->set(1, *cell); - array->set(2, *handler); + if (name.is_null()) { + SetFeedback(*cell); + SetFeedbackExtra(*handler); + } else { + SetFeedback(*name); + Handle array = EnsureExtraArrayOfSize(2); + array->set(0, *cell); + array->set(1, *handler); + } } void LoadICNexus::ConfigurePolymorphic(MapHandleList* maps, CodeHandleList* handlers) { + Isolate* isolate = GetIsolate(); int receiver_count = maps->length(); - EnsureArrayOfSize(receiver_count * 2); - InstallHandlers(0, maps, handlers); + Handle array = EnsureArrayOfSize(receiver_count * 2); + InstallHandlers(array, maps, handlers); + SetFeedbackExtra(*vector()->UninitializedSentinel(isolate), + SKIP_WRITE_BARRIER); } @@ -368,26 +400,34 @@ void KeyedLoadICNexus::ConfigurePolymorphic(Handle name, MapHandleList* maps, CodeHandleList* handlers) { int receiver_count = maps->length(); - Handle array = EnsureArrayOfSize(1 + receiver_count * 2); + DCHECK(receiver_count > 1); + Handle array; if (name.is_null()) { - array->set(0, Smi::FromInt(0)); + array = EnsureArrayOfSize(receiver_count * 2); + SetFeedbackExtra(*vector()->UninitializedSentinel(GetIsolate()), + SKIP_WRITE_BARRIER); } else { - array->set(0, *name); + SetFeedback(*name); + array = EnsureExtraArrayOfSize(receiver_count * 2); } - InstallHandlers(1, maps, handlers); + + InstallHandlers(array, maps, handlers); } -int FeedbackNexus::ExtractMaps(int start_index, MapHandleList* maps) const { +int FeedbackNexus::ExtractMaps(MapHandleList* maps) const { Isolate* isolate = GetIsolate(); Object* feedback = GetFeedback(); - if (feedback->IsFixedArray()) { + if (feedback->IsFixedArray() || feedback->IsString()) { int found = 0; + if (feedback->IsString()) { + feedback = GetFeedbackExtra(); + } FixedArray* array = FixedArray::cast(feedback); // The array should be of the form [], then // [map, handler, map, handler, ... ] - DCHECK(array->length() >= (2 + start_index)); - for (int i = start_index; i < array->length(); i += 2) { + DCHECK(array->length() >= 2); + for (int i = 0; i < array->length(); i += 2) { WeakCell* cell = WeakCell::cast(array->get(i)); if (!cell->cleared()) { Map* map = Map::cast(cell->value()); @@ -396,18 +436,27 @@ int FeedbackNexus::ExtractMaps(int start_index, MapHandleList* maps) const { } } return found; + } else if (feedback->IsWeakCell()) { + WeakCell* cell = WeakCell::cast(feedback); + if (!cell->cleared()) { + Map* map = Map::cast(cell->value()); + maps->Add(handle(map, isolate)); + return 1; + } } return 0; } -MaybeHandle FeedbackNexus::FindHandlerForMap(int start_index, - Handle map) const { +MaybeHandle FeedbackNexus::FindHandlerForMap(Handle map) const { Object* feedback = GetFeedback(); - if (feedback->IsFixedArray()) { + if (feedback->IsFixedArray() || feedback->IsString()) { + if (feedback->IsString()) { + feedback = GetFeedbackExtra(); + } FixedArray* array = FixedArray::cast(feedback); - for (int i = start_index; i < array->length(); i += 2) { + for (int i = 0; i < array->length(); i += 2) { WeakCell* cell = WeakCell::cast(array->get(i)); if (!cell->cleared()) { Map* array_map = Map::cast(cell->value()); @@ -418,23 +467,34 @@ MaybeHandle FeedbackNexus::FindHandlerForMap(int start_index, } } } + } else if (feedback->IsWeakCell()) { + WeakCell* cell = WeakCell::cast(feedback); + if (!cell->cleared()) { + Map* cell_map = Map::cast(cell->value()); + if (cell_map == *map) { + Code* code = Code::cast(GetFeedbackExtra()); + DCHECK(code->kind() == Code::HANDLER); + return handle(code); + } + } } return MaybeHandle(); } -bool FeedbackNexus::FindHandlers(int start_index, CodeHandleList* code_list, - int length) const { +bool FeedbackNexus::FindHandlers(CodeHandleList* code_list, int length) const { Object* feedback = GetFeedback(); int count = 0; - if (feedback->IsFixedArray()) { + if (feedback->IsFixedArray() || feedback->IsString()) { + if (feedback->IsString()) { + feedback = GetFeedbackExtra(); + } FixedArray* array = FixedArray::cast(feedback); - // The array should be of the form [], then - // [map, handler, map, handler, ... ]. Be sure to skip handlers whose maps - // have been cleared. - DCHECK(array->length() >= (2 + start_index)); - for (int i = start_index; i < array->length(); i += 2) { + // The array should be of the form [map, handler, map, handler, ... ]. + // Be sure to skip handlers whose maps have been cleared. + DCHECK(array->length() >= 2); + for (int i = 0; i < array->length(); i += 2) { WeakCell* cell = WeakCell::cast(array->get(i)); if (!cell->cleared()) { Code* code = Code::cast(array->get(i + 1)); @@ -443,16 +503,19 @@ bool FeedbackNexus::FindHandlers(int start_index, CodeHandleList* code_list, count++; } } + } else if (feedback->IsWeakCell()) { + WeakCell* cell = WeakCell::cast(feedback); + if (!cell->cleared()) { + Code* code = Code::cast(GetFeedbackExtra()); + DCHECK(code->kind() == Code::HANDLER); + code_list->Add(handle(code)); + count++; + } } return count == length; } -int LoadICNexus::ExtractMaps(MapHandleList* maps) const { - return FeedbackNexus::ExtractMaps(0, maps); -} - - void LoadICNexus::Clear(Code* host) { LoadIC::Clear(GetIsolate(), host, this); } @@ -461,39 +524,10 @@ void KeyedLoadICNexus::Clear(Code* host) { } -int KeyedLoadICNexus::ExtractMaps(MapHandleList* maps) const { - return FeedbackNexus::ExtractMaps(1, maps); -} - - -MaybeHandle LoadICNexus::FindHandlerForMap(Handle map) const { - return FeedbackNexus::FindHandlerForMap(0, map); -} - - -MaybeHandle KeyedLoadICNexus::FindHandlerForMap(Handle map) const { - return FeedbackNexus::FindHandlerForMap(1, map); -} - - -bool LoadICNexus::FindHandlers(CodeHandleList* code_list, int length) const { - return FeedbackNexus::FindHandlers(0, code_list, length); -} - - -bool KeyedLoadICNexus::FindHandlers(CodeHandleList* code_list, - int length) const { - return FeedbackNexus::FindHandlers(1, code_list, length); -} - - Name* KeyedLoadICNexus::FindFirstName() const { Object* feedback = GetFeedback(); - if (feedback->IsFixedArray()) { - FixedArray* array = FixedArray::cast(feedback); - DCHECK(array->length() >= 3); - Object* name = array->get(0); - if (name->IsName()) return Name::cast(name); + if (feedback->IsString()) { + return Name::cast(feedback); } return NULL; } diff --git a/src/type-feedback-vector.h b/src/type-feedback-vector.h index b7abad51f1..aba42a6c91 100644 --- a/src/type-feedback-vector.h +++ b/src/type-feedback-vector.h @@ -74,6 +74,8 @@ class TypeFeedbackVector : public FixedArray { static const int kWithTypesIndex = 1; static const int kGenericCountIndex = 2; + static int elements_per_ic_slot() { return FLAG_vector_ics ? 2 : 1; } + int first_ic_slot_index() const { DCHECK(length() >= kReservedIndexCount); return Smi::cast(get(kFirstICSlotIndex))->value(); @@ -114,7 +116,7 @@ class TypeFeedbackVector : public FixedArray { int ICSlots() const { if (length() == 0) return 0; - return length() - first_ic_slot_index(); + return (length() - first_ic_slot_index()) / elements_per_ic_slot(); } // Conversion from a slot or ic slot to an integer index to the underlying @@ -127,7 +129,7 @@ class TypeFeedbackVector : public FixedArray { int GetIndex(FeedbackVectorICSlot slot) const { int first_ic_slot = first_ic_slot_index(); DCHECK(slot.ToInt() < ICSlots()); - return first_ic_slot + slot.ToInt(); + return first_ic_slot + slot.ToInt() * elements_per_ic_slot(); } // Conversion from an integer index to either a slot or an ic slot. The caller @@ -140,7 +142,8 @@ class TypeFeedbackVector : public FixedArray { FeedbackVectorICSlot ToICSlot(int index) const { DCHECK(index >= first_ic_slot_index() && index < length()); - return FeedbackVectorICSlot(index - first_ic_slot_index()); + int ic_slot = (index - first_ic_slot_index()) / elements_per_ic_slot(); + return FeedbackVectorICSlot(ic_slot); } Object* Get(FeedbackVectorSlot slot) const { return get(GetIndex(slot)); } @@ -244,14 +247,17 @@ class FeedbackNexus { void FindAllMaps(MapHandleList* maps) const { ExtractMaps(maps); } virtual InlineCacheState StateFromFeedback() const = 0; - virtual int ExtractMaps(MapHandleList* maps) const = 0; - virtual MaybeHandle FindHandlerForMap(Handle map) const = 0; - virtual bool FindHandlers(CodeHandleList* code_list, int length = -1) const { - return length == 0; - } + virtual int ExtractMaps(MapHandleList* maps) const; + virtual MaybeHandle FindHandlerForMap(Handle map) const; + virtual bool FindHandlers(CodeHandleList* code_list, int length = -1) const; virtual Name* FindFirstName() const { return NULL; } Object* GetFeedback() const { return vector()->Get(slot()); } + Object* GetFeedbackExtra() const { + DCHECK(TypeFeedbackVector::elements_per_ic_slot() > 1); + int extra_index = vector()->GetIndex(slot()) + 1; + return vector()->get(extra_index); + } protected: Isolate* GetIsolate() const { return vector()->GetIsolate(); } @@ -261,13 +267,17 @@ class FeedbackNexus { vector()->Set(slot(), feedback, mode); } + void SetFeedbackExtra(Object* feedback_extra, + WriteBarrierMode mode = UPDATE_WRITE_BARRIER) { + DCHECK(TypeFeedbackVector::elements_per_ic_slot() > 1); + int index = vector()->GetIndex(slot()) + 1; + vector()->set(index, feedback_extra, mode); + } + Handle EnsureArrayOfSize(int length); - void InstallHandlers(int start_index, MapHandleList* maps, + Handle EnsureExtraArrayOfSize(int length); + void InstallHandlers(Handle array, MapHandleList* maps, CodeHandleList* handlers); - int ExtractMaps(int start_index, MapHandleList* maps) const; - MaybeHandle FindHandlerForMap(int start_index, Handle map) const; - bool FindHandlers(int start_index, CodeHandleList* code_list, - int length) const; private: // The reason for having a vector handle and a raw pointer is that we can and @@ -334,10 +344,6 @@ class LoadICNexus : public FeedbackNexus { void ConfigurePolymorphic(MapHandleList* maps, CodeHandleList* handlers); InlineCacheState StateFromFeedback() const OVERRIDE; - int ExtractMaps(MapHandleList* maps) const OVERRIDE; - MaybeHandle FindHandlerForMap(Handle map) const OVERRIDE; - virtual bool FindHandlers(CodeHandleList* code_list, - int length = -1) const OVERRIDE; }; @@ -364,10 +370,6 @@ class KeyedLoadICNexus : public FeedbackNexus { CodeHandleList* handlers); InlineCacheState StateFromFeedback() const OVERRIDE; - int ExtractMaps(MapHandleList* maps) const OVERRIDE; - MaybeHandle FindHandlerForMap(Handle map) const OVERRIDE; - virtual bool FindHandlers(CodeHandleList* code_list, - int length = -1) const OVERRIDE; Name* FindFirstName() const OVERRIDE; }; } diff --git a/src/x64/code-stubs-x64.cc b/src/x64/code-stubs-x64.cc index e5836fd33e..1d1f9b4424 100644 --- a/src/x64/code-stubs-x64.cc +++ b/src/x64/code-stubs-x64.cc @@ -11,6 +11,7 @@ #include "src/codegen.h" #include "src/ic/handler-compiler.h" #include "src/ic/ic.h" +#include "src/ic/stub-cache.h" #include "src/isolate.h" #include "src/jsregexp.h" #include "src/regexp-macro-assembler.h" @@ -4337,15 +4338,230 @@ void StubFailureTrampolineStub::Generate(MacroAssembler* masm) { void LoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorLoadStub stub(isolate(), state()); - __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawLoadStub stub(isolate(), state()); + stub.GenerateForTrampoline(masm); } void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) { EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister()); - VectorKeyedLoadStub stub(isolate()); - __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET); + VectorRawKeyedLoadStub stub(isolate()); + stub.GenerateForTrampoline(masm); +} + + +static void HandleArrayCases(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register feedback, Register scratch1, + Register scratch2, Register scratch3, + Register scratch4, bool is_polymorphic, + Label* miss) { + // feedback initially contains the feedback array + Label next_loop, prepare_next; + Label load_smi_map, compare_map; + Label start_polymorphic; + + Register receiver_map = scratch1; + Register counter = scratch2; + Register length = scratch3; + Register cached_map = scratch4; + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &load_smi_map); + __ movp(receiver_map, FieldOperand(receiver, 0)); + __ bind(&compare_map); + __ movp(cached_map, FieldOperand(feedback, FixedArray::OffsetOfElementAt(0))); + __ cmpp(receiver_map, FieldOperand(cached_map, WeakCell::kValueOffset)); + __ j(not_equal, &start_polymorphic); + + // found, now call handler. + Register handler = feedback; + __ movp(handler, FieldOperand(feedback, FixedArray::OffsetOfElementAt(1))); + __ leap(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); + + // Polymorphic, we have to loop from 2 to N + __ bind(&start_polymorphic); + __ SmiToInteger32(length, FieldOperand(feedback, FixedArray::kLengthOffset)); + if (!is_polymorphic) { + // If the IC could be monomorphic we have to make sure we don't go past the + // end of the feedback array. + __ cmpl(length, Immediate(2)); + __ j(equal, miss); + } + __ movl(counter, Immediate(2)); + + __ bind(&next_loop); + __ movp(cached_map, FieldOperand(feedback, counter, times_pointer_size, + FixedArray::kHeaderSize)); + __ cmpp(receiver_map, FieldOperand(cached_map, WeakCell::kValueOffset)); + __ j(not_equal, &prepare_next); + __ movp(handler, FieldOperand(feedback, counter, times_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + __ leap(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); + + __ bind(&prepare_next); + __ addl(counter, Immediate(2)); + __ cmpl(counter, length); + __ j(less, &next_loop); + + // We exhausted our array of map handler pairs. + __ jmp(miss); + + __ bind(&load_smi_map); + __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex); + __ jmp(&compare_map); +} + + +static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, + Register key, Register vector, Register slot, + Register weak_cell, Register integer_slot, + Label* miss) { + // feedback initially contains the feedback array + Label compare_smi_map; + + // Move the weak map into the weak_cell register. + Register ic_map = weak_cell; + __ movp(ic_map, FieldOperand(weak_cell, WeakCell::kValueOffset)); + + // Receiver might not be a heap object. + __ JumpIfSmi(receiver, &compare_smi_map); + __ cmpp(ic_map, FieldOperand(receiver, 0)); + __ j(not_equal, miss); + Register handler = weak_cell; + __ movp(handler, FieldOperand(vector, integer_slot, times_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + __ leap(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); + + // In microbenchmarks, it made sense to unroll this code so that the call to + // the handler is duplicated for a HeapObject receiver and a Smi receiver. + __ bind(&compare_smi_map); + __ CompareRoot(ic_map, Heap::kHeapNumberMapRootIndex); + __ j(not_equal, miss); + __ movp(handler, FieldOperand(vector, integer_slot, times_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + __ leap(handler, FieldOperand(handler, Code::kHeaderSize)); + __ jmp(handler); +} + + +void VectorRawLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // rdx + Register name = VectorLoadICDescriptor::NameRegister(); // rcx + Register vector = VectorLoadICDescriptor::VectorRegister(); // rbx + Register slot = VectorLoadICDescriptor::SlotRegister(); // rax + Register feedback = rdi; + Register integer_slot = r8; + + __ SmiToInteger32(integer_slot, slot); + __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size, + FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ CompareRoot(FieldOperand(feedback, 0), Heap::kWeakCellMapRootIndex); + __ j(not_equal, &try_array); + HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, + integer_slot, &miss); + + // Is it a fixed array? + __ bind(&try_array); + __ CompareRoot(FieldOperand(feedback, 0), Heap::kFixedArrayMapRootIndex); + __ j(not_equal, ¬_array); + HandleArrayCases(masm, receiver, name, vector, slot, feedback, integer_slot, + r9, r11, r15, true, &miss); + + __ bind(¬_array); + __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex); + __ j(not_equal, &miss); + Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags( + Code::ComputeHandlerFlags(Code::LOAD_IC)); + masm->isolate()->stub_cache()->GenerateProbe( + masm, Code::LOAD_IC, code_flags, false, receiver, name, feedback, no_reg); + + __ bind(&miss); + LoadIC::GenerateMiss(masm); +} + + +void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) { + GenerateImpl(masm, false); +} + + +void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) { + GenerateImpl(masm, true); +} + + +void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { + Register receiver = VectorLoadICDescriptor::ReceiverRegister(); // rdx + Register key = VectorLoadICDescriptor::NameRegister(); // rcx + Register vector = VectorLoadICDescriptor::VectorRegister(); // rbx + Register slot = VectorLoadICDescriptor::SlotRegister(); // rax + Register feedback = rdi; + Register integer_slot = r8; + + __ SmiToInteger32(integer_slot, slot); + __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size, + FixedArray::kHeaderSize)); + + // Is it a weak cell? + Label try_array; + Label not_array, smi_key, key_okay, miss; + __ CompareRoot(FieldOperand(feedback, 0), Heap::kWeakCellMapRootIndex); + __ j(not_equal, &try_array); + __ JumpIfNotSmi(key, &miss); + HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, + integer_slot, &miss); + + __ bind(&try_array); + // Is it a fixed array? + __ CompareRoot(FieldOperand(feedback, 0), Heap::kFixedArrayMapRootIndex); + __ j(not_equal, ¬_array); + // We have a polymorphic element handler. + __ JumpIfNotSmi(key, &miss); + + Label polymorphic, try_poly_name; + __ bind(&polymorphic); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, integer_slot, + r9, r11, r15, true, &miss); + + __ bind(¬_array); + // Is it generic? + __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex); + __ j(not_equal, &try_poly_name); + Handle megamorphic_stub = + KeyedLoadIC::ChooseMegamorphicStub(masm->isolate()); + __ jmp(megamorphic_stub, RelocInfo::CODE_TARGET); + + __ bind(&try_poly_name); + // We might have a name in feedback, and a fixed array in the next slot. + __ cmpp(key, feedback); + __ j(not_equal, &miss); + // If the name comparison succeeded, we know we have a fixed array with + // at least one map/handler pair. + __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size, + FixedArray::kHeaderSize + kPointerSize)); + HandleArrayCases(masm, receiver, key, vector, slot, feedback, integer_slot, + r9, r11, r15, false, &miss); + + __ bind(&miss); + KeyedLoadIC::GenerateMiss(masm); } diff --git a/test/cctest/test-feedback-vector.cc b/test/cctest/test-feedback-vector.cc index fe7198cded..d375ba36a0 100644 --- a/test/cctest/test-feedback-vector.cc +++ b/test/cctest/test-feedback-vector.cc @@ -71,8 +71,8 @@ TEST(VectorStructure) { CHECK_EQ(index, TypeFeedbackVector::kReservedIndexCount + metadata_length + 3); CHECK(FeedbackVectorICSlot(0) == vector->ToICSlot(index)); - - CHECK_EQ(TypeFeedbackVector::kReservedIndexCount + metadata_length + 3 + 5, + CHECK_EQ(TypeFeedbackVector::kReservedIndexCount + metadata_length + 3 + + 5 * TypeFeedbackVector::elements_per_ic_slot(), vector->length()); }