diff --git a/src/compiler/effect-control-linearizer.cc b/src/compiler/effect-control-linearizer.cc index cd697e0a9d..959712f075 100644 --- a/src/compiler/effect-control-linearizer.cc +++ b/src/compiler/effect-control-linearizer.cc @@ -706,6 +706,9 @@ bool EffectControlLinearizer::TryWireInStateEffect(Node* node, case IrOpcode::kPlainPrimitiveToFloat64: state = LowerPlainPrimitiveToFloat64(node, *effect, *control); break; + case IrOpcode::kTransitionElementsKind: + state = LowerTransitionElementsKind(node, *effect, *control); + break; default: return false; } @@ -2121,6 +2124,65 @@ EffectControlLinearizer::LowerPlainPrimitiveToFloat64(Node* node, Node* effect, return ValueEffectControl(value, effect, control); } +EffectControlLinearizer::ValueEffectControl +EffectControlLinearizer::LowerTransitionElementsKind(Node* node, Node* effect, + Node* control) { + ElementsTransition const transition = ElementsTransitionOf(node->op()); + Node* object = node->InputAt(0); + Node* source_map = node->InputAt(1); + Node* target_map = node->InputAt(2); + + // Load the current map of {object}. + Node* object_map = effect = + graph()->NewNode(simplified()->LoadField(AccessBuilder::ForMap()), object, + effect, control); + + // Check if {object_map} is the same as {source_map}. + Node* check = + graph()->NewNode(machine()->WordEqual(), object_map, source_map); + Node* branch = + graph()->NewNode(common()->Branch(BranchHint::kFalse), check, control); + + // Migrate the {object} from {source_map} to {target_map}. + Node* if_true = graph()->NewNode(common()->IfTrue(), branch); + Node* etrue = effect; + { + switch (transition) { + case ElementsTransition::kFastTransition: { + // In-place migration of {object}, just store the {target_map}. + etrue = + graph()->NewNode(simplified()->StoreField(AccessBuilder::ForMap()), + object, target_map, etrue, if_true); + break; + } + case ElementsTransition::kSlowTransition: { + // Instance migration, call out to the runtime for {object}. + Operator::Properties properties = + Operator::kNoDeopt | Operator::kNoThrow; + Runtime::FunctionId id = Runtime::kTransitionElementsKind; + CallDescriptor const* desc = Linkage::GetRuntimeCallDescriptor( + graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags); + etrue = graph()->NewNode( + common()->Call(desc), jsgraph()->CEntryStubConstant(1), object, + target_map, + jsgraph()->ExternalConstant(ExternalReference(id, isolate())), + jsgraph()->Int32Constant(2), jsgraph()->NoContextConstant(), etrue, + if_true); + break; + } + } + } + + // Nothing to do if the {object} doesn't have the {source_map}. + Node* if_false = graph()->NewNode(common()->IfFalse(), branch); + Node* efalse = effect; + + control = graph()->NewNode(common()->Merge(2), if_true, if_false); + effect = graph()->NewNode(common()->EffectPhi(2), etrue, efalse, control); + + return ValueEffectControl(nullptr, effect, control); +} + Factory* EffectControlLinearizer::factory() const { return isolate()->factory(); } diff --git a/src/compiler/effect-control-linearizer.h b/src/compiler/effect-control-linearizer.h index 5b59bd71aa..a38bc0de92 100644 --- a/src/compiler/effect-control-linearizer.h +++ b/src/compiler/effect-control-linearizer.h @@ -122,6 +122,8 @@ class EffectControlLinearizer { Node* control); ValueEffectControl LowerPlainPrimitiveToFloat64(Node* node, Node* effect, Node* control); + ValueEffectControl LowerTransitionElementsKind(Node* node, Node* effect, + Node* control); ValueEffectControl AllocateHeapNumberWithValue(Node* node, Node* effect, Node* control); diff --git a/src/compiler/js-native-context-specialization.cc b/src/compiler/js-native-context-specialization.cc index 3492af1d7f..72086fa2a3 100644 --- a/src/compiler/js-native-context-specialization.cc +++ b/src/compiler/js-native-context-specialization.cc @@ -488,7 +488,6 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess( DCHECK(node->opcode() == IrOpcode::kJSLoadProperty || node->opcode() == IrOpcode::kJSStoreProperty); Node* receiver = NodeProperties::GetValueInput(node, 0); - Node* context = NodeProperties::GetContextInput(node); Node* effect = NodeProperties::GetEffectInput(node); Node* control = NodeProperties::GetControlInput(node); Node* frame_state = NodeProperties::FindFrameStateBefore(node); @@ -525,12 +524,6 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess( receiver = effect = graph()->NewNode(simplified()->CheckTaggedPointer(), receiver, effect, control); - // Load the {receiver} map. The resulting effect is the dominating effect for - // all (polymorphic) branches. - Node* receiver_map = effect = - graph()->NewNode(simplified()->LoadField(AccessBuilder::ForMap()), - receiver, effect, control); - // Generate code for the various different element access patterns. Node* fallthrough_control = control; for (size_t j = 0; j < access_infos.size(); ++j) { @@ -538,8 +531,28 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess( Node* this_receiver = receiver; Node* this_value = value; Node* this_index = index; - Node* this_effect; - Node* this_control; + Node* this_effect = effect; + Node* this_control = fallthrough_control; + + // Perform possible elements kind transitions. + for (auto transition : access_info.transitions()) { + Handle const transition_source = transition.first; + Handle const transition_target = transition.second; + this_effect = graph()->NewNode( + simplified()->TransitionElementsKind( + IsSimpleMapChangeTransition(transition_source->elements_kind(), + transition_target->elements_kind()) + ? ElementsTransition::kFastTransition + : ElementsTransition::kSlowTransition), + receiver, jsgraph()->HeapConstant(transition_source), + jsgraph()->HeapConstant(transition_target), this_effect, + this_control); + } + + // Load the {receiver} map. + Node* receiver_map = this_effect = + graph()->NewNode(simplified()->LoadField(AccessBuilder::ForMap()), + receiver, this_effect, this_control); // Perform map check on {receiver}. Type* receiver_type = access_info.receiver_type(); @@ -547,7 +560,6 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess( { ZoneVector this_controls(zone()); ZoneVector this_effects(zone()); - size_t num_transitions = access_info.transitions().size(); int num_classes = access_info.receiver_type()->NumClasses(); for (auto i = access_info.receiver_type()->Classes(); !i.Done(); i.Advance()) { @@ -556,16 +568,15 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess( Node* check = graph()->NewNode(simplified()->ReferenceEqual(Type::Any()), receiver_map, jsgraph()->Constant(map)); - if (--num_classes == 0 && num_transitions == 0 && - j == access_infos.size() - 1) { + if (--num_classes == 0 && j == access_infos.size() - 1) { // Last map check on the fallthrough control path, do a conditional // eager deoptimization exit here. // TODO(turbofan): This is ugly as hell! We should probably introduce // macro-ish operators for property access that encapsulate this whole // mess. - check = graph()->NewNode(simplified()->CheckIf(), check, effect, - fallthrough_control); - this_controls.push_back(fallthrough_control); + check = graph()->NewNode(simplified()->CheckIf(), check, this_effect, + this_control); + this_controls.push_back(this_control); this_effects.push_back(check); fallthrough_control = nullptr; } else { @@ -578,57 +589,6 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess( if (!map->IsJSArrayMap()) receiver_is_jsarray = false; } - // Generate possible elements kind transitions. - for (auto transition : access_info.transitions()) { - DCHECK_LT(0u, num_transitions); - Handle transition_source = transition.first; - Handle transition_target = transition.second; - Node* transition_control; - Node* transition_effect = effect; - - // Check if {receiver} has the specified {transition_source} map. - Node* check = graph()->NewNode( - simplified()->ReferenceEqual(Type::Any()), receiver_map, - jsgraph()->HeapConstant(transition_source)); - if (--num_transitions == 0 && j == access_infos.size() - 1) { - transition_effect = - graph()->NewNode(simplified()->CheckIf(), check, - transition_effect, fallthrough_control); - transition_control = fallthrough_control; - fallthrough_control = nullptr; - } else { - Node* branch = - graph()->NewNode(common()->Branch(), check, fallthrough_control); - fallthrough_control = graph()->NewNode(common()->IfFalse(), branch); - transition_control = graph()->NewNode(common()->IfTrue(), branch); - } - - // Migrate {receiver} from {transition_source} to {transition_target}. - if (IsSimpleMapChangeTransition(transition_source->elements_kind(), - transition_target->elements_kind())) { - // In-place migration, just store the {transition_target} map. - transition_effect = graph()->NewNode( - simplified()->StoreField(AccessBuilder::ForMap()), receiver, - jsgraph()->HeapConstant(transition_target), transition_effect, - transition_control); - } else { - // Instance migration, let the stub deal with the {receiver}. - TransitionElementsKindStub stub(isolate(), - transition_source->elements_kind(), - transition_target->elements_kind()); - CallDescriptor const* const desc = Linkage::GetStubCallDescriptor( - isolate(), graph()->zone(), stub.GetCallInterfaceDescriptor(), 0, - CallDescriptor::kNeedsFrameState, node->op()->properties()); - transition_effect = graph()->NewNode( - common()->Call(desc), jsgraph()->HeapConstant(stub.GetCode()), - receiver, jsgraph()->HeapConstant(transition_target), context, - frame_state, transition_effect, transition_control); - } - - this_controls.push_back(transition_control); - this_effects.push_back(transition_effect); - } - // Create single chokepoint for the control. int const this_control_count = static_cast(this_controls.size()); if (this_control_count == 1) { @@ -642,15 +602,15 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess( this_effect = graph()->NewNode(common()->EffectPhi(this_control_count), this_control_count + 1, &this_effects.front()); - } - // TODO(turbofan): The effect/control linearization will not find a - // FrameState after the StoreField or Call that is generated for the - // elements kind transition above. This is because those operators - // don't have the kNoWrite flag on it, even though they are not - // observable by JavaScript. - this_effect = graph()->NewNode(common()->Checkpoint(), frame_state, - this_effect, this_control); + // TODO(turbofan): The effect/control linearization will not find a + // FrameState after the StoreField or Call that is generated for the + // elements kind transition above. This is because those operators + // don't have the kNoWrite flag on it, even though they are not + // observable by JavaScript. + this_effect = graph()->NewNode(common()->Checkpoint(), frame_state, + this_effect, this_control); + } } // Certain stores need a prototype chain check because shape changes diff --git a/src/compiler/opcodes.h b/src/compiler/opcodes.h index 0d0f0428ea..e2964dd7c1 100644 --- a/src/compiler/opcodes.h +++ b/src/compiler/opcodes.h @@ -281,7 +281,8 @@ V(ObjectIsReceiver) \ V(ObjectIsSmi) \ V(ObjectIsString) \ - V(ObjectIsUndetectable) + V(ObjectIsUndetectable) \ + V(TransitionElementsKind) #define SIMPLIFIED_OP_LIST(V) \ SIMPLIFIED_CHANGE_OP_LIST(V) \ diff --git a/src/compiler/simplified-lowering.cc b/src/compiler/simplified-lowering.cc index 5a9376e2d5..449c882e79 100644 --- a/src/compiler/simplified-lowering.cc +++ b/src/compiler/simplified-lowering.cc @@ -2131,6 +2131,10 @@ class RepresentationSelector { } return; } + case IrOpcode::kTransitionElementsKind: { + VisitInputs(node); + return SetOutput(node, MachineRepresentation::kNone); + } //------------------------------------------------------------------ // Machine-level operators. diff --git a/src/compiler/simplified-operator.cc b/src/compiler/simplified-operator.cc index ac5134d758..193f655144 100644 --- a/src/compiler/simplified-operator.cc +++ b/src/compiler/simplified-operator.cc @@ -241,6 +241,26 @@ CheckTaggedHoleMode CheckTaggedHoleModeOf(const Operator* op) { return OpParameter(op); } +size_t hash_value(ElementsTransition transition) { + return static_cast(transition); +} + +std::ostream& operator<<(std::ostream& os, ElementsTransition transition) { + switch (transition) { + case ElementsTransition::kFastTransition: + return os << "fast-transition"; + case ElementsTransition::kSlowTransition: + return os << "slow-transition"; + } + UNREACHABLE(); + return os; +} + +ElementsTransition ElementsTransitionOf(const Operator* op) { + DCHECK_EQ(IrOpcode::kTransitionElementsKind, op->opcode()); + return OpParameter(op); +} + BinaryOperationHints::Hint BinaryOperationHintOf(const Operator* op) { DCHECK(op->opcode() == IrOpcode::kSpeculativeNumberAdd || op->opcode() == IrOpcode::kSpeculativeNumberSubtract || @@ -516,6 +536,16 @@ const Operator* SimplifiedOperatorBuilder::ReferenceEqual(Type* type) { "ReferenceEqual", 2, 0, 0, 1, 0, 0); } +const Operator* SimplifiedOperatorBuilder::TransitionElementsKind( + ElementsTransition transition) { + return new (zone()) Operator1( // -- + IrOpcode::kTransitionElementsKind, // opcode + Operator::kNoDeopt | Operator::kNoThrow, // flags + "TransitionElementsKind", // name + 3, 1, 1, 0, 1, 0, // counts + transition); // parameter +} + const Operator* SimplifiedOperatorBuilder::Allocate(PretenureFlag pretenure) { switch (pretenure) { case NOT_TENURED: diff --git a/src/compiler/simplified-operator.h b/src/compiler/simplified-operator.h index 53a7007760..9cd721f229 100644 --- a/src/compiler/simplified-operator.h +++ b/src/compiler/simplified-operator.h @@ -140,7 +140,15 @@ std::ostream& operator<<(std::ostream&, CheckForMinusZeroMode); CheckForMinusZeroMode CheckMinusZeroModeOf(const Operator*) WARN_UNUSED_RESULT; -Type* TypeOf(const Operator* op) WARN_UNUSED_RESULT; +// A descriptor for elements kind transitions. +enum class ElementsTransition : uint8_t { + kFastTransition, // simple transition, just updating the map. + kSlowTransition // full transition, round-trip to the runtime. +}; + +std::ostream& operator<<(std::ostream&, ElementsTransition); + +ElementsTransition ElementsTransitionOf(const Operator* op) WARN_UNUSED_RESULT; BinaryOperationHints::Hint BinaryOperationHintOf(const Operator* op); @@ -288,6 +296,9 @@ class SimplifiedOperatorBuilder final : public ZoneObject { const Operator* ObjectIsString(); const Operator* ObjectIsUndetectable(); + // transition-elements-kind object, from-map, to-map + const Operator* TransitionElementsKind(ElementsTransition transition); + const Operator* Allocate(PretenureFlag pretenure = NOT_TENURED); const Operator* LoadField(FieldAccess const&); diff --git a/src/compiler/typer.cc b/src/compiler/typer.cc index 04e0b5af62..1e8d64af0e 100644 --- a/src/compiler/typer.cc +++ b/src/compiler/typer.cc @@ -725,6 +725,11 @@ Type* Typer::Visitor::TypeLoopExitEffect(Node* node) { return nullptr; } +Type* Typer::Visitor::TypeTransitionElementsKind(Node* node) { + UNREACHABLE(); + return nullptr; +} + Type* Typer::Visitor::TypeCheckpoint(Node* node) { UNREACHABLE(); return nullptr; diff --git a/src/compiler/verifier.cc b/src/compiler/verifier.cc index b59d1015ca..46cd825ff9 100644 --- a/src/compiler/verifier.cc +++ b/src/compiler/verifier.cc @@ -830,6 +830,12 @@ void Verifier::Visitor::Check(Node* node) { CheckValueInputIs(node, 0, Type::PlainNumber()); CheckUpperIs(node, Type::TaggedPointer()); break; + case IrOpcode::kTransitionElementsKind: + CheckValueInputIs(node, 0, Type::Any()); + CheckValueInputIs(node, 1, Type::Internal()); + CheckValueInputIs(node, 2, Type::Internal()); + CheckNotTyped(node); + break; case IrOpcode::kChangeTaggedSignedToInt32: { // Signed32 /\ Tagged -> Signed32 /\ UntaggedInt32 diff --git a/src/elements.cc b/src/elements.cc index 3cf8378162..031d516929 100644 --- a/src/elements.cc +++ b/src/elements.cc @@ -783,6 +783,45 @@ class ElementsAccessorBase : public ElementsAccessor { return new_elements; } + static void TransitionElementsKindImpl(Handle object, + Handle to_map) { + Handle from_map = handle(object->map()); + ElementsKind from_kind = from_map->elements_kind(); + ElementsKind to_kind = to_map->elements_kind(); + if (IsFastHoleyElementsKind(from_kind)) { + to_kind = GetHoleyElementsKind(to_kind); + } + if (from_kind != to_kind) { + // This method should never be called for any other case. + DCHECK(IsFastElementsKind(from_kind)); + DCHECK(IsFastElementsKind(to_kind)); + DCHECK_NE(TERMINAL_FAST_ELEMENTS_KIND, from_kind); + + Handle from_elements(object->elements()); + if (object->elements() == object->GetHeap()->empty_fixed_array() || + IsFastDoubleElementsKind(from_kind) == + IsFastDoubleElementsKind(to_kind)) { + // No change is needed to the elements() buffer, the transition + // only requires a map change. + JSObject::MigrateToMap(object, to_map); + } else { + DCHECK((IsFastSmiElementsKind(from_kind) && + IsFastDoubleElementsKind(to_kind)) || + (IsFastDoubleElementsKind(from_kind) && + IsFastObjectElementsKind(to_kind))); + uint32_t capacity = static_cast(object->elements()->length()); + Handle elements = ConvertElementsWithCapacity( + object, from_elements, from_kind, capacity); + JSObject::SetMapAndElements(object, to_map, elements); + } + if (FLAG_trace_elements_transitions) { + JSObject::PrintElementsTransition(stdout, object, from_kind, + from_elements, to_kind, + handle(object->elements())); + } + } + } + static void GrowCapacityAndConvertImpl(Handle object, uint32_t capacity) { ElementsKind from_kind = object->GetElementsKind(); @@ -822,6 +861,10 @@ class ElementsAccessorBase : public ElementsAccessor { } } + void TransitionElementsKind(Handle object, Handle map) final { + Subclass::TransitionElementsKindImpl(object, map); + } + void GrowCapacityAndConvert(Handle object, uint32_t capacity) final { Subclass::GrowCapacityAndConvertImpl(object, capacity); @@ -2255,6 +2298,11 @@ class SloppyArgumentsElementsAccessor } } + static void TransitionElementsKindImpl(Handle object, + Handle map) { + UNREACHABLE(); + } + static void GrowCapacityAndConvertImpl(Handle object, uint32_t capacity) { UNREACHABLE(); diff --git a/src/elements.h b/src/elements.h index 9f70f2a0c7..ae48d4ef07 100644 --- a/src/elements.h +++ b/src/elements.h @@ -110,6 +110,8 @@ class ElementsAccessor { KeyAccumulator* accumulator, AddKeyConversion convert) = 0; + virtual void TransitionElementsKind(Handle object, + Handle map) = 0; virtual void GrowCapacityAndConvert(Handle object, uint32_t capacity) = 0; diff --git a/src/runtime/runtime-array.cc b/src/runtime/runtime-array.cc index a92215c85d..68543d9a26 100644 --- a/src/runtime/runtime-array.cc +++ b/src/runtime/runtime-array.cc @@ -86,11 +86,12 @@ RUNTIME_FUNCTION(Runtime_FixedArraySet) { RUNTIME_FUNCTION(Runtime_TransitionElementsKind) { HandleScope scope(isolate); - DCHECK(args.length() == 2); - CONVERT_ARG_HANDLE_CHECKED(JSArray, array, 0); - CONVERT_ARG_HANDLE_CHECKED(Map, map, 1); - JSObject::TransitionElementsKind(array, map->elements_kind()); - return *array; + DCHECK_EQ(2, args.length()); + CONVERT_ARG_HANDLE_CHECKED(JSArray, object, 0); + CONVERT_ARG_HANDLE_CHECKED(Map, to_map, 1); + ElementsKind to_kind = to_map->elements_kind(); + ElementsAccessor::ForKind(to_kind)->TransitionElementsKind(object, to_map); + return *object; }