[stringrefs] Optimize get_codeunit
This patch doubles the performance of iterating over a stringview_wtf16. It does this by: - changing string.as_wtf16 to flatten any Cons strings; in TF this is represented by a new operator - introducing a Turbofan operator PrepareStringForGetCodeunit that inspects the given string's internal representation and retrieves the pointer to the actual characters - adapting the code emitted for `get_codeunit` to consume the output of this operator - improving WasmLoadElimination to deduplicate both new operators for peeled loops, so that as much work as possible only needs to be done once. This patch was authored about half-and-half by manoskouk@chromium.org and jkummerow@chromium.org. Bug: v8:12868 Change-Id: If9cf4c3ffeb5e1ca08b864cbc0bf868656ca2dec Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4198142 Commit-Queue: Jakob Kummerow <jkummerow@chromium.org> Reviewed-by: Tobias Tebbi <tebbi@chromium.org> Reviewed-by: Manos Koukoutos <manoskouk@chromium.org> Cr-Commit-Position: refs/heads/main@{#85628}
This commit is contained in:
Jakob Kummerow
V8 LUCI CQ
parent
994c2a575d
commit
c07d8535f8
@ -699,6 +699,10 @@ builtin ThrowWasmTrapArrayTooLarge(): JSAny {
|
||||
tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapArrayTooLarge));
|
||||
}
|
||||
|
||||
builtin ThrowWasmTrapStringOffsetOutOfBounds(): JSAny {
|
||||
tail WasmTrap(SmiConstant(MessageTemplate::kWasmTrapStringOffsetOutOfBounds));
|
||||
}
|
||||
|
||||
macro TryNumberToIntptr(value: JSAny): intptr labels Failure {
|
||||
typeswitch (value) {
|
||||
case (s: Smi): {
|
||||
@ -939,6 +943,13 @@ builtin WasmStringNewWtf16Array(
|
||||
}
|
||||
}
|
||||
|
||||
// Contract: input is any string, output is a string that the TF operator
|
||||
// "StringPrepareForGetCodeunit" can handle.
|
||||
builtin WasmStringAsWtf16(str: String): String {
|
||||
const cons = Cast<ConsString>(str) otherwise return str;
|
||||
return Flatten(cons);
|
||||
}
|
||||
|
||||
builtin WasmStringConst(index: uint32): String {
|
||||
const instance = LoadInstanceFromFrame();
|
||||
tail runtime::WasmStringConst(
|
||||
|
||||
@ -2035,7 +2035,8 @@ enum IsolateAddressId {
|
||||
V(TrapNullDereference) \
|
||||
V(TrapIllegalCast) \
|
||||
V(TrapArrayOutOfBounds) \
|
||||
V(TrapArrayTooLarge)
|
||||
V(TrapArrayTooLarge) \
|
||||
V(TrapStringOffsetOutOfBounds)
|
||||
|
||||
enum KeyedAccessLoadMode {
|
||||
STANDARD_LOAD,
|
||||
|
||||
@ -4152,11 +4152,10 @@ Node* EffectControlLinearizer::StringCharCodeAt(Node* receiver,
|
||||
|
||||
__ Bind(&if_seqstring);
|
||||
{
|
||||
Node* receiver_is_onebyte = __ Word32Equal(
|
||||
Node* receiver_is_onebyte =
|
||||
__ Word32Equal(__ Word32And(receiver_instance_type,
|
||||
__ Int32Constant(kStringEncodingMask)),
|
||||
__ Int32Constant(kTwoByteStringTag)),
|
||||
__ Int32Constant(0));
|
||||
__ Int32Constant(kOneByteStringTag));
|
||||
Node* result = LoadFromSeqString(receiver, position, receiver_is_onebyte);
|
||||
__ Goto(&loop_done, result);
|
||||
}
|
||||
|
||||
@ -565,7 +565,9 @@
|
||||
V(WasmArrayGet) \
|
||||
V(WasmArraySet) \
|
||||
V(WasmArrayLength) \
|
||||
V(WasmArrayInitializeLength)
|
||||
V(WasmArrayInitializeLength) \
|
||||
V(StringAsWtf16) \
|
||||
V(StringPrepareForGetCodeunit)
|
||||
|
||||
#define SIMPLIFIED_OP_LIST(V) \
|
||||
SIMPLIFIED_CHANGE_OP_LIST(V) \
|
||||
|
||||
@ -2224,6 +2224,8 @@ struct WasmGCOptimizationPhase {
|
||||
temp_zone);
|
||||
WasmGCOperatorReducer wasm_gc(&graph_reducer, temp_zone, data->mcgraph(),
|
||||
module);
|
||||
// Note: if we want to add DeadCodeElimination here, we'll have to update
|
||||
// the existing reducers to handle kDead and kDeadValue nodes everywhere.
|
||||
AddReducer(data, &graph_reducer, &load_elimination);
|
||||
AddReducer(data, &graph_reducer, &wasm_gc);
|
||||
graph_reducer.ReduceGraph();
|
||||
|
||||
@ -1286,6 +1286,22 @@ struct SimplifiedOperatorGlobalCache final {
|
||||
"WasmArrayInitializeLength", 2, 1, 1, 0, 1, 0) {}
|
||||
};
|
||||
WasmArrayInitializeLengthOperator kWasmArrayInitializeLength;
|
||||
|
||||
struct StringAsWtf16Operator final : public Operator {
|
||||
StringAsWtf16Operator()
|
||||
: Operator(IrOpcode::kStringAsWtf16, Operator::kEliminatable,
|
||||
"StringAsWtf16", 1, 1, 1, 1, 1, 1) {}
|
||||
};
|
||||
StringAsWtf16Operator kStringAsWtf16;
|
||||
|
||||
struct StringPrepareForGetCodeunitOperator final : public Operator {
|
||||
StringPrepareForGetCodeunitOperator()
|
||||
: Operator(IrOpcode::kStringPrepareForGetCodeunit,
|
||||
Operator::kEliminatable, "StringPrepareForGetCodeunit", 1, 1,
|
||||
1, 3, 1, 1) {}
|
||||
};
|
||||
StringPrepareForGetCodeunitOperator kStringPrepareForGetCodeunit;
|
||||
|
||||
#endif
|
||||
|
||||
#define SPECULATIVE_NUMBER_BINOP(Name) \
|
||||
@ -1519,6 +1535,14 @@ const Operator* SimplifiedOperatorBuilder::IsNotNull() {
|
||||
return &cache_.kIsNotNull;
|
||||
}
|
||||
|
||||
const Operator* SimplifiedOperatorBuilder::StringAsWtf16() {
|
||||
return &cache_.kStringAsWtf16;
|
||||
}
|
||||
|
||||
const Operator* SimplifiedOperatorBuilder::StringPrepareForGetCodeunit() {
|
||||
return &cache_.kStringPrepareForGetCodeunit;
|
||||
}
|
||||
|
||||
const Operator* SimplifiedOperatorBuilder::WasmExternInternalize() {
|
||||
return zone()->New<Operator>(IrOpcode::kWasmExternInternalize,
|
||||
Operator::kEliminatable, "WasmExternInternalize",
|
||||
|
||||
@ -1162,6 +1162,8 @@ class V8_EXPORT_PRIVATE SimplifiedOperatorBuilder final
|
||||
const Operator* WasmArraySet(const wasm::ArrayType* type);
|
||||
const Operator* WasmArrayLength();
|
||||
const Operator* WasmArrayInitializeLength();
|
||||
const Operator* StringAsWtf16();
|
||||
const Operator* StringPrepareForGetCodeunit();
|
||||
#endif
|
||||
|
||||
const Operator* DateNow();
|
||||
|
||||
@ -1731,6 +1731,8 @@ void Verifier::Visitor::Check(Node* node, const AllNodes& all) {
|
||||
case IrOpcode::kWasmArraySet:
|
||||
case IrOpcode::kWasmArrayLength:
|
||||
case IrOpcode::kWasmArrayInitializeLength:
|
||||
case IrOpcode::kStringAsWtf16:
|
||||
case IrOpcode::kStringPrepareForGetCodeunit:
|
||||
// TODO(manoskouk): What are the constraints here?
|
||||
break;
|
||||
#endif // V8_ENABLE_WEBASSEMBLY
|
||||
|
||||
@ -42,6 +42,8 @@ V8_INLINE bool operator==(const WasmTypeCheckConfig& p1,
|
||||
return p1.from == p2.from && p1.to == p2.to;
|
||||
}
|
||||
|
||||
static constexpr int kCharWidthBailoutSentinel = 3;
|
||||
|
||||
} // namespace compiler
|
||||
} // namespace internal
|
||||
} // namespace v8
|
||||
|
||||
@ -39,6 +39,7 @@
|
||||
#include "src/logging/counters.h"
|
||||
#include "src/objects/heap-number.h"
|
||||
#include "src/objects/instance-type.h"
|
||||
#include "src/objects/string.h"
|
||||
#include "src/roots/roots.h"
|
||||
#include "src/tracing/trace-event.h"
|
||||
#include "src/trap-handler/trap-handler.h"
|
||||
@ -2965,38 +2966,14 @@ Node* WasmGraphBuilder::BuildIndirectCall(uint32_t table_index,
|
||||
}
|
||||
}
|
||||
|
||||
Node* WasmGraphBuilder::BuildLoadExternalPointerFromObject(
|
||||
Node* object, int offset, ExternalPointerTag tag) {
|
||||
#ifdef V8_ENABLE_SANDBOX
|
||||
DCHECK_NE(tag, kExternalPointerNullTag);
|
||||
DCHECK(!IsSharedExternalPointerType(tag));
|
||||
Node* external_pointer = gasm_->LoadFromObject(
|
||||
MachineType::Uint32(), object, wasm::ObjectAccess::ToTagged(offset));
|
||||
static_assert(kExternalPointerIndexShift > kSystemPointerSizeLog2);
|
||||
Node* shift_amount =
|
||||
gasm_->Int32Constant(kExternalPointerIndexShift - kSystemPointerSizeLog2);
|
||||
Node* scaled_index = gasm_->Word32Shr(external_pointer, shift_amount);
|
||||
Node* isolate_root = BuildLoadIsolateRoot();
|
||||
Node* table =
|
||||
gasm_->LoadFromObject(MachineType::Pointer(), isolate_root,
|
||||
IsolateData::external_pointer_table_offset() +
|
||||
Internals::kExternalPointerTableBufferOffset);
|
||||
Node* decoded_ptr = gasm_->Load(MachineType::Pointer(), table, scaled_index);
|
||||
return gasm_->WordAnd(decoded_ptr, gasm_->IntPtrConstant(~tag));
|
||||
#else
|
||||
return gasm_->LoadFromObject(MachineType::Pointer(), object,
|
||||
wasm::ObjectAccess::ToTagged(offset));
|
||||
#endif // V8_ENABLE_SANDBOX
|
||||
}
|
||||
|
||||
Node* WasmGraphBuilder::BuildLoadCallTargetFromExportedFunctionData(
|
||||
Node* function) {
|
||||
Node* internal = gasm_->LoadFromObject(
|
||||
MachineType::TaggedPointer(), function,
|
||||
wasm::ObjectAccess::ToTagged(WasmExportedFunctionData::kInternalOffset));
|
||||
return BuildLoadExternalPointerFromObject(
|
||||
return gasm_->BuildLoadExternalPointerFromObject(
|
||||
internal, WasmInternalFunction::kCallTargetOffset,
|
||||
kWasmInternalFunctionCallTargetTag);
|
||||
kWasmInternalFunctionCallTargetTag, BuildLoadIsolateRoot());
|
||||
}
|
||||
|
||||
// TODO(9495): Support CAPI function refs.
|
||||
@ -3019,9 +2996,9 @@ Node* WasmGraphBuilder::BuildCallRef(const wasm::FunctionSig* sig,
|
||||
MachineType::TaggedPointer(), function,
|
||||
wasm::ObjectAccess::ToTagged(WasmInternalFunction::kRefOffset));
|
||||
|
||||
Node* target = BuildLoadExternalPointerFromObject(
|
||||
Node* target = gasm_->BuildLoadExternalPointerFromObject(
|
||||
function, WasmInternalFunction::kCallTargetOffset,
|
||||
kWasmInternalFunctionCallTargetTag);
|
||||
kWasmInternalFunctionCallTargetTag, BuildLoadIsolateRoot());
|
||||
Node* is_null_target = gasm_->WordEqual(target, gasm_->IntPtrConstant(0));
|
||||
gasm_->GotoIfNot(is_null_target, &end_label, target);
|
||||
{
|
||||
@ -6065,6 +6042,14 @@ Node* WasmGraphBuilder::StringEncodeWtf16(uint32_t memory, Node* string,
|
||||
offset, gasm_->SmiConstant(memory));
|
||||
}
|
||||
|
||||
Node* WasmGraphBuilder::StringAsWtf16(Node* string, CheckForNull null_check,
|
||||
wasm::WasmCodePosition position) {
|
||||
if (null_check == kWithNullCheck) {
|
||||
string = AssertNotNull(string, position);
|
||||
}
|
||||
return gasm_->StringAsWtf16(string);
|
||||
}
|
||||
|
||||
Node* WasmGraphBuilder::StringEncodeWtf16Array(
|
||||
Node* string, CheckForNull string_null_check, Node* array,
|
||||
CheckForNull array_null_check, Node* start,
|
||||
@ -6167,9 +6152,56 @@ Node* WasmGraphBuilder::StringViewWtf16GetCodeUnit(
|
||||
if (null_check == kWithNullCheck) {
|
||||
string = AssertNotNull(string, position);
|
||||
}
|
||||
return gasm_->CallBuiltin(Builtin::kWasmStringViewWtf16GetCodeUnit,
|
||||
Operator::kNoDeopt | Operator::kNoThrow, string,
|
||||
offset);
|
||||
Node* prepare = gasm_->StringPrepareForGetCodeunit(string);
|
||||
Node* base = gasm_->Projection(0, prepare);
|
||||
Node* base_offset = gasm_->Projection(1, prepare);
|
||||
Node* charwidth_shift = gasm_->Projection(2, prepare);
|
||||
|
||||
// Bounds check.
|
||||
Node* length = gasm_->LoadImmutableFromObject(
|
||||
MachineType::Int32(), string,
|
||||
wasm::ObjectAccess::ToTagged(String::kLengthOffset));
|
||||
TrapIfFalse(wasm::kTrapStringOffsetOutOfBounds,
|
||||
gasm_->Uint32LessThan(offset, length), position);
|
||||
|
||||
auto onebyte = gasm_->MakeLabel();
|
||||
auto bailout = gasm_->MakeDeferredLabel();
|
||||
auto done = gasm_->MakeLabel(MachineRepresentation::kWord32);
|
||||
gasm_->GotoIf(
|
||||
gasm_->Word32Equal(charwidth_shift,
|
||||
gasm_->Int32Constant(kCharWidthBailoutSentinel)),
|
||||
&bailout);
|
||||
gasm_->GotoIf(gasm_->Word32Equal(charwidth_shift, gasm_->Int32Constant(0)),
|
||||
&onebyte);
|
||||
|
||||
// Two-byte.
|
||||
Node* object_offset =
|
||||
gasm_->IntAdd(gasm_->IntMul(gasm_->BuildChangeInt32ToIntPtr(offset),
|
||||
gasm_->IntPtrConstant(2)),
|
||||
base_offset);
|
||||
Node* result = gasm_->LoadImmutableFromObject(MachineType::Uint16(), base,
|
||||
object_offset);
|
||||
gasm_->Goto(&done, result);
|
||||
|
||||
// One-byte.
|
||||
gasm_->Bind(&onebyte);
|
||||
object_offset =
|
||||
gasm_->IntAdd(gasm_->BuildChangeInt32ToIntPtr(offset), base_offset);
|
||||
result =
|
||||
gasm_->LoadImmutableFromObject(MachineType::Uint8(), base, object_offset);
|
||||
gasm_->Goto(&done, result);
|
||||
|
||||
gasm_->Bind(&bailout);
|
||||
gasm_->Goto(&done,
|
||||
gasm_->CallBuiltin(Builtin::kWasmStringViewWtf16GetCodeUnit,
|
||||
Operator::kPure, string, offset));
|
||||
|
||||
gasm_->Bind(&done);
|
||||
// Make sure the original string is kept alive as long as we're operating
|
||||
// on pointers extracted from it (otherwise e.g. external strings' resources
|
||||
// might get freed prematurely).
|
||||
gasm_->Retain(string);
|
||||
return done.PhiAt(0);
|
||||
}
|
||||
|
||||
Node* WasmGraphBuilder::StringViewWtf16Encode(uint32_t memory, Node* string,
|
||||
@ -6870,9 +6902,9 @@ class WasmWrapperGraphBuilder : public WasmGraphBuilder {
|
||||
Node* internal = gasm_->LoadFromObject(
|
||||
MachineType::TaggedPointer(), function_data,
|
||||
wasm::ObjectAccess::ToTagged(WasmFunctionData::kInternalOffset));
|
||||
args[0] = BuildLoadExternalPointerFromObject(
|
||||
args[0] = gasm_->BuildLoadExternalPointerFromObject(
|
||||
internal, WasmInternalFunction::kCallTargetOffset,
|
||||
kWasmInternalFunctionCallTargetTag);
|
||||
kWasmInternalFunctionCallTargetTag, BuildLoadIsolateRoot());
|
||||
Node* instance_node = gasm_->LoadFromObject(
|
||||
MachineType::TaggedPointer(), internal,
|
||||
wasm::ObjectAccess::ToTagged(WasmInternalFunction::kRefOffset));
|
||||
|
||||
@ -534,6 +534,8 @@ class WasmGraphBuilder {
|
||||
Node* start, Node* end);
|
||||
Node* StringNewWtf16(uint32_t memory, Node* offset, Node* size);
|
||||
Node* StringNewWtf16Array(Node* array, Node* start, Node* end);
|
||||
Node* StringAsWtf16(Node* string, CheckForNull null_check,
|
||||
wasm::WasmCodePosition position);
|
||||
Node* StringConst(uint32_t index);
|
||||
Node* StringMeasureUtf8(Node* string, CheckForNull null_check,
|
||||
wasm::WasmCodePosition position);
|
||||
@ -821,10 +823,6 @@ class WasmGraphBuilder {
|
||||
Node* BuildMultiReturnFixedArrayFromIterable(const wasm::FunctionSig* sig,
|
||||
Node* iterable, Node* context);
|
||||
|
||||
Node* BuildLoadExternalPointerFromObject(
|
||||
Node* object, int offset,
|
||||
ExternalPointerTag tag = kForeignForeignAddressTag);
|
||||
|
||||
Node* BuildLoadCallTargetFromExportedFunctionData(Node* function_data);
|
||||
|
||||
//-----------------------------------------------------------------------
|
||||
|
||||
@ -74,16 +74,24 @@ Reduction WasmGCLowering::Reduce(Node* node) {
|
||||
return ReduceWasmArrayLength(node);
|
||||
case IrOpcode::kWasmArrayInitializeLength:
|
||||
return ReduceWasmArrayInitializeLength(node);
|
||||
case IrOpcode::kStringAsWtf16:
|
||||
return ReduceStringAsWtf16(node);
|
||||
case IrOpcode::kStringPrepareForGetCodeunit:
|
||||
return ReduceStringPrepareForGetCodeunit(node);
|
||||
default:
|
||||
return NoChange();
|
||||
}
|
||||
}
|
||||
|
||||
Node* WasmGCLowering::RootNode(RootIndex index) {
|
||||
Node* WasmGCLowering::IsolateRoot() {
|
||||
// TODO(13449): Use root register instead of isolate.
|
||||
Node* isolate_root = gasm_.LoadImmutable(
|
||||
return gasm_.LoadImmutable(
|
||||
MachineType::Pointer(), instance_node_,
|
||||
WasmInstanceObject::kIsolateRootOffset - kHeapObjectTag);
|
||||
}
|
||||
|
||||
Node* WasmGCLowering::RootNode(RootIndex index) {
|
||||
Node* isolate_root = IsolateRoot();
|
||||
return gasm_.LoadImmutable(MachineType::Pointer(), isolate_root,
|
||||
IsolateData::root_slot_offset(index));
|
||||
}
|
||||
@ -445,6 +453,204 @@ Reduction WasmGCLowering::ReduceWasmArrayInitializeLength(Node* node) {
|
||||
return Replace(set_length);
|
||||
}
|
||||
|
||||
Reduction WasmGCLowering::ReduceStringAsWtf16(Node* node) {
|
||||
DCHECK_EQ(node->opcode(), IrOpcode::kStringAsWtf16);
|
||||
Node* effect = NodeProperties::GetEffectInput(node);
|
||||
Node* control = NodeProperties::GetControlInput(node);
|
||||
Node* str = NodeProperties::GetValueInput(node, 0);
|
||||
|
||||
gasm_.InitializeEffectControl(effect, control);
|
||||
|
||||
auto done = gasm_.MakeLabel(MachineRepresentation::kTaggedPointer);
|
||||
Node* instance_type = gasm_.LoadInstanceType(gasm_.LoadMap(str));
|
||||
Node* string_representation = gasm_.Word32And(
|
||||
instance_type, gasm_.Int32Constant(kStringRepresentationMask));
|
||||
gasm_.GotoIf(gasm_.Word32Equal(string_representation,
|
||||
gasm_.Int32Constant(kSeqStringTag)),
|
||||
&done, str);
|
||||
gasm_.Goto(&done, gasm_.CallBuiltin(Builtin::kWasmStringAsWtf16,
|
||||
Operator::kPure, str));
|
||||
gasm_.Bind(&done);
|
||||
ReplaceWithValue(node, done.PhiAt(0), gasm_.effect(), gasm_.control());
|
||||
node->Kill();
|
||||
return Replace(done.PhiAt(0));
|
||||
}
|
||||
|
||||
Reduction WasmGCLowering::ReduceStringPrepareForGetCodeunit(Node* node) {
|
||||
DCHECK_EQ(node->opcode(), IrOpcode::kStringPrepareForGetCodeunit);
|
||||
Node* effect = NodeProperties::GetEffectInput(node);
|
||||
Node* control = NodeProperties::GetControlInput(node);
|
||||
Node* original_string = NodeProperties::GetValueInput(node, 0);
|
||||
|
||||
gasm_.InitializeEffectControl(effect, control);
|
||||
|
||||
auto dispatch =
|
||||
gasm_.MakeLoopLabel(MachineRepresentation::kTaggedPointer, // String.
|
||||
MachineRepresentation::kWord32, // Instance type.
|
||||
MachineRepresentation::kWord32); // Offset.
|
||||
auto next = gasm_.MakeLabel(MachineRepresentation::kTaggedPointer, // String.
|
||||
MachineRepresentation::kWord32, // Instance type.
|
||||
MachineRepresentation::kWord32); // Offset.
|
||||
auto direct_string =
|
||||
gasm_.MakeLabel(MachineRepresentation::kTaggedPointer, // String.
|
||||
MachineRepresentation::kWord32, // Instance type.
|
||||
MachineRepresentation::kWord32); // Offset.
|
||||
|
||||
// These values will be used to replace the original node's projections.
|
||||
// The first, "string", is either a SeqString or Smi(0) (in case of external
|
||||
// string). Notably this makes it GC-safe: if that string moves, this pointer
|
||||
// will be updated accordingly.
|
||||
// The second, "offset", has full register width so that it can be used to
|
||||
// store external pointers: for external strings, we add up the character
|
||||
// backing store's base address and any slice offset.
|
||||
// The third, "character width", is a shift width, i.e. it is 0 for one-byte
|
||||
// strings, 1 for two-byte strings, kCharWidthBailoutSentinel for uncached
|
||||
// external strings (for which "string"/"offset" are invalid and unusable).
|
||||
auto done =
|
||||
gasm_.MakeLabel(MachineRepresentation::kTagged, // String.
|
||||
MachineType::PointerRepresentation(), // Offset.
|
||||
MachineRepresentation::kWord32); // Character width.
|
||||
|
||||
Node* original_type = gasm_.LoadInstanceType(gasm_.LoadMap(original_string));
|
||||
gasm_.Goto(&dispatch, original_string, original_type, gasm_.Int32Constant(0));
|
||||
|
||||
gasm_.Bind(&dispatch);
|
||||
{
|
||||
auto thin_string = gasm_.MakeLabel(MachineRepresentation::kTaggedPointer);
|
||||
auto cons_string = gasm_.MakeLabel(MachineRepresentation::kTaggedPointer);
|
||||
|
||||
Node* string = dispatch.PhiAt(0);
|
||||
Node* instance_type = dispatch.PhiAt(1);
|
||||
Node* offset = dispatch.PhiAt(2);
|
||||
static_assert(kIsIndirectStringTag == 1);
|
||||
static constexpr int kIsDirectStringTag = 0;
|
||||
gasm_.GotoIf(gasm_.Word32Equal(
|
||||
gasm_.Word32And(instance_type, gasm_.Int32Constant(
|
||||
kIsIndirectStringMask)),
|
||||
gasm_.Int32Constant(kIsDirectStringTag)),
|
||||
&direct_string, string, instance_type, offset);
|
||||
|
||||
// Handle indirect strings.
|
||||
Node* string_representation = gasm_.Word32And(
|
||||
instance_type, gasm_.Int32Constant(kStringRepresentationMask));
|
||||
gasm_.GotoIf(gasm_.Word32Equal(string_representation,
|
||||
gasm_.Int32Constant(kThinStringTag)),
|
||||
&thin_string, string);
|
||||
gasm_.GotoIf(gasm_.Word32Equal(string_representation,
|
||||
gasm_.Int32Constant(kConsStringTag)),
|
||||
&cons_string, string);
|
||||
|
||||
// Sliced string.
|
||||
Node* new_offset = gasm_.Int32Add(
|
||||
offset,
|
||||
gasm_.BuildChangeSmiToInt32(gasm_.LoadImmutableFromObject(
|
||||
MachineType::TaggedSigned(), string,
|
||||
wasm::ObjectAccess::ToTagged(SlicedString::kOffsetOffset))));
|
||||
Node* parent = gasm_.LoadImmutableFromObject(
|
||||
MachineType::TaggedPointer(), string,
|
||||
wasm::ObjectAccess::ToTagged(SlicedString::kParentOffset));
|
||||
Node* parent_type = gasm_.LoadInstanceType(gasm_.LoadMap(parent));
|
||||
gasm_.Goto(&next, parent, parent_type, new_offset);
|
||||
|
||||
// Thin string.
|
||||
gasm_.Bind(&thin_string);
|
||||
Node* actual = gasm_.LoadImmutableFromObject(
|
||||
MachineType::TaggedPointer(), string,
|
||||
wasm::ObjectAccess::ToTagged(ThinString::kActualOffset));
|
||||
Node* actual_type = gasm_.LoadInstanceType(gasm_.LoadMap(actual));
|
||||
// ThinStrings always reference (internalized) direct strings.
|
||||
gasm_.Goto(&direct_string, actual, actual_type, offset);
|
||||
|
||||
// Flat cons string. (Non-flat cons strings are ruled out by
|
||||
// string.as_wtf16.)
|
||||
gasm_.Bind(&cons_string);
|
||||
Node* first = gasm_.LoadImmutableFromObject(
|
||||
MachineType::TaggedPointer(), string,
|
||||
wasm::ObjectAccess::ToTagged(ConsString::kFirstOffset));
|
||||
Node* first_type = gasm_.LoadInstanceType(gasm_.LoadMap(first));
|
||||
gasm_.Goto(&next, first, first_type, offset);
|
||||
|
||||
gasm_.Bind(&next);
|
||||
gasm_.Goto(&dispatch, next.PhiAt(0), next.PhiAt(1), next.PhiAt(2));
|
||||
}
|
||||
|
||||
gasm_.Bind(&direct_string);
|
||||
{
|
||||
Node* string = direct_string.PhiAt(0);
|
||||
Node* instance_type = direct_string.PhiAt(1);
|
||||
Node* offset = direct_string.PhiAt(2);
|
||||
|
||||
Node* is_onebyte = gasm_.Word32And(
|
||||
instance_type, gasm_.Int32Constant(kStringEncodingMask));
|
||||
// Char width shift is 1 - (is_onebyte).
|
||||
static_assert(kStringEncodingMask == 1 << 3);
|
||||
Node* charwidth_shift =
|
||||
gasm_.Int32Sub(gasm_.Int32Constant(1),
|
||||
gasm_.Word32Shr(is_onebyte, gasm_.Int32Constant(3)));
|
||||
|
||||
auto external = gasm_.MakeLabel();
|
||||
Node* string_representation = gasm_.Word32And(
|
||||
instance_type, gasm_.Int32Constant(kStringRepresentationMask));
|
||||
gasm_.GotoIf(gasm_.Word32Equal(string_representation,
|
||||
gasm_.Int32Constant(kExternalStringTag)),
|
||||
&external);
|
||||
|
||||
// Sequential string.
|
||||
static_assert(SeqOneByteString::kCharsOffset ==
|
||||
SeqTwoByteString::kCharsOffset);
|
||||
Node* final_offset = gasm_.Int32Add(
|
||||
gasm_.Int32Constant(
|
||||
wasm::ObjectAccess::ToTagged(SeqOneByteString::kCharsOffset)),
|
||||
gasm_.Word32Shl(offset, charwidth_shift));
|
||||
gasm_.Goto(&done, string, gasm_.BuildChangeInt32ToIntPtr(final_offset),
|
||||
charwidth_shift);
|
||||
|
||||
// External string.
|
||||
gasm_.Bind(&external);
|
||||
gasm_.GotoIf(
|
||||
gasm_.Word32And(instance_type,
|
||||
gasm_.Int32Constant(kUncachedExternalStringMask)),
|
||||
&done, string, gasm_.IntPtrConstant(0),
|
||||
gasm_.Int32Constant(kCharWidthBailoutSentinel));
|
||||
Node* resource = gasm_.BuildLoadExternalPointerFromObject(
|
||||
string, ExternalString::kResourceDataOffset,
|
||||
kExternalStringResourceDataTag, IsolateRoot());
|
||||
Node* shifted_offset = gasm_.Word32Shl(offset, charwidth_shift);
|
||||
final_offset = gasm_.IntPtrAdd(
|
||||
resource, gasm_.BuildChangeInt32ToIntPtr(shifted_offset));
|
||||
gasm_.Goto(&done, gasm_.SmiConstant(0), final_offset, charwidth_shift);
|
||||
}
|
||||
|
||||
gasm_.Bind(&done);
|
||||
Node* base = done.PhiAt(0);
|
||||
Node* final_offset = done.PhiAt(1);
|
||||
Node* charwidth_shift = done.PhiAt(2);
|
||||
|
||||
Node* base_proj = NodeProperties::FindProjection(node, 0);
|
||||
Node* offset_proj = NodeProperties::FindProjection(node, 1);
|
||||
Node* charwidth_proj = NodeProperties::FindProjection(node, 2);
|
||||
if (base_proj) {
|
||||
ReplaceWithValue(base_proj, base, gasm_.effect(), gasm_.control());
|
||||
base_proj->Kill();
|
||||
}
|
||||
if (offset_proj) {
|
||||
ReplaceWithValue(offset_proj, final_offset, gasm_.effect(),
|
||||
gasm_.control());
|
||||
offset_proj->Kill();
|
||||
}
|
||||
if (charwidth_proj) {
|
||||
ReplaceWithValue(charwidth_proj, charwidth_shift, gasm_.effect(),
|
||||
gasm_.control());
|
||||
charwidth_proj->Kill();
|
||||
}
|
||||
|
||||
// Wire up the dangling end of the new effect chain.
|
||||
ReplaceWithValue(node, node, gasm_.effect(), gasm_.control());
|
||||
|
||||
node->Kill();
|
||||
return Replace(base);
|
||||
}
|
||||
|
||||
} // namespace compiler
|
||||
} // namespace internal
|
||||
} // namespace v8
|
||||
|
||||
@ -45,9 +45,14 @@ class WasmGCLowering final : public AdvancedReducer {
|
||||
Reduction ReduceWasmArraySet(Node* node);
|
||||
Reduction ReduceWasmArrayLength(Node* node);
|
||||
Reduction ReduceWasmArrayInitializeLength(Node* node);
|
||||
Reduction ReduceStringAsWtf16(Node* node);
|
||||
Reduction ReduceStringPrepareForGetCodeunit(Node* node);
|
||||
Node* IsolateRoot();
|
||||
Node* RootNode(RootIndex index);
|
||||
Node* Null();
|
||||
Node* IsNull(Node* object);
|
||||
Node* BuildLoadExternalPointerFromObject(Node* object, int offset,
|
||||
ExternalPointerTag tag);
|
||||
WasmGraphAssembler gasm_;
|
||||
const wasm::WasmModule* module_;
|
||||
Node* dead_;
|
||||
|
||||
@ -40,6 +40,8 @@ Reduction WasmGCOperatorReducer::Reduce(Node* node) {
|
||||
return ReduceIf(node, true);
|
||||
case IrOpcode::kIfFalse:
|
||||
return ReduceIf(node, false);
|
||||
case IrOpcode::kDead:
|
||||
return NoChange();
|
||||
case IrOpcode::kLoop:
|
||||
return TakeStatesFromFirstControl(node);
|
||||
default:
|
||||
|
||||
@ -173,6 +173,36 @@ Node* WasmGraphAssembler::InitializeImmutableInObject(ObjectAccess access,
|
||||
offset, value, effect(), control()));
|
||||
}
|
||||
|
||||
Node* WasmGraphAssembler::BuildLoadExternalPointerFromObject(
|
||||
Node* object, int offset, ExternalPointerTag tag, Node* isolate_root) {
|
||||
#ifdef V8_ENABLE_SANDBOX
|
||||
DCHECK_NE(tag, kExternalPointerNullTag);
|
||||
Node* external_pointer = LoadFromObject(MachineType::Uint32(), object,
|
||||
wasm::ObjectAccess::ToTagged(offset));
|
||||
static_assert(kExternalPointerIndexShift > kSystemPointerSizeLog2);
|
||||
Node* shift_amount =
|
||||
Int32Constant(kExternalPointerIndexShift - kSystemPointerSizeLog2);
|
||||
Node* scaled_index = Word32Shr(external_pointer, shift_amount);
|
||||
Node* table;
|
||||
if (IsSharedExternalPointerType(tag)) {
|
||||
Node* table_address =
|
||||
LoadFromObject(MachineType::Pointer(), isolate_root,
|
||||
IsolateData::shared_external_pointer_table_offset());
|
||||
table = LoadFromObject(MachineType::Pointer(), table_address,
|
||||
Internals::kExternalPointerTableBufferOffset);
|
||||
} else {
|
||||
table = LoadFromObject(MachineType::Pointer(), isolate_root,
|
||||
IsolateData::external_pointer_table_offset() +
|
||||
Internals::kExternalPointerTableBufferOffset);
|
||||
}
|
||||
Node* decoded_ptr = Load(MachineType::Pointer(), table, scaled_index);
|
||||
return WordAnd(decoded_ptr, IntPtrConstant(~tag));
|
||||
#else
|
||||
return LoadFromObject(MachineType::Pointer(), object,
|
||||
wasm::ObjectAccess::ToTagged(offset));
|
||||
#endif // V8_ENABLE_SANDBOX
|
||||
}
|
||||
|
||||
Node* WasmGraphAssembler::IsI31(Node* object) {
|
||||
if (COMPRESS_POINTERS_BOOL) {
|
||||
return Word32Equal(Word32And(object, Int32Constant(kSmiTagMask)),
|
||||
@ -409,6 +439,16 @@ void WasmGraphAssembler::ArrayInitializeLength(Node* array, Node* length) {
|
||||
length, effect(), control()));
|
||||
}
|
||||
|
||||
Node* WasmGraphAssembler::StringAsWtf16(Node* string) {
|
||||
return AddNode(graph()->NewNode(simplified_.StringAsWtf16(), string, effect(),
|
||||
control()));
|
||||
}
|
||||
|
||||
Node* WasmGraphAssembler::StringPrepareForGetCodeunit(Node* string) {
|
||||
return AddNode(graph()->NewNode(simplified_.StringPrepareForGetCodeunit(),
|
||||
string, effect(), control()));
|
||||
}
|
||||
|
||||
// Generic HeapObject helpers.
|
||||
|
||||
Node* WasmGraphAssembler::HasInstanceType(Node* heap_object,
|
||||
|
||||
@ -159,6 +159,10 @@ class WasmGraphAssembler : public GraphAssembler {
|
||||
value);
|
||||
}
|
||||
|
||||
Node* BuildLoadExternalPointerFromObject(Node* object, int offset,
|
||||
ExternalPointerTag tag,
|
||||
Node* isolate_root);
|
||||
|
||||
Node* IsI31(Node* object);
|
||||
|
||||
// Maps and their contents.
|
||||
@ -270,6 +274,10 @@ class WasmGraphAssembler : public GraphAssembler {
|
||||
|
||||
void ArrayInitializeLength(Node* array, Node* length);
|
||||
|
||||
Node* StringAsWtf16(Node* string);
|
||||
|
||||
Node* StringPrepareForGetCodeunit(Node* string);
|
||||
|
||||
// Generic helpers.
|
||||
|
||||
Node* HasInstanceType(Node* heap_object, InstanceType type);
|
||||
|
||||
@ -53,8 +53,11 @@ Node* ResolveAliases(Node* node) {
|
||||
return node;
|
||||
}
|
||||
|
||||
// We model array length as a field at index kArrayLengthFieldIndex.
|
||||
// We model array length and string canonicalization as fields at negative
|
||||
// indices.
|
||||
constexpr int kArrayLengthFieldIndex = -1;
|
||||
constexpr int kStringPrepareForGetCodeunitIndex = -2;
|
||||
constexpr int kStringAsWtf16Index = -3;
|
||||
} // namespace
|
||||
|
||||
Reduction WasmLoadElimination::UpdateState(Node* node,
|
||||
@ -121,6 +124,10 @@ Reduction WasmLoadElimination::Reduce(Node* node) {
|
||||
return ReduceWasmArrayLength(node);
|
||||
case IrOpcode::kWasmArrayInitializeLength:
|
||||
return ReduceWasmArrayInitializeLength(node);
|
||||
case IrOpcode::kStringPrepareForGetCodeunit:
|
||||
return ReduceStringPrepareForGetCodeunit(node);
|
||||
case IrOpcode::kStringAsWtf16:
|
||||
return ReduceStringAsWtf16(node);
|
||||
case IrOpcode::kEffectPhi:
|
||||
return ReduceEffectPhi(node);
|
||||
case IrOpcode::kDead:
|
||||
@ -284,6 +291,70 @@ Reduction WasmLoadElimination::ReduceWasmArrayInitializeLength(Node* node) {
|
||||
return UpdateState(node, new_state);
|
||||
}
|
||||
|
||||
Reduction WasmLoadElimination::ReduceStringPrepareForGetCodeunit(Node* node) {
|
||||
DCHECK_EQ(node->opcode(), IrOpcode::kStringPrepareForGetCodeunit);
|
||||
Node* object = ResolveAliases(NodeProperties::GetValueInput(node, 0));
|
||||
Node* effect = NodeProperties::GetEffectInput(node);
|
||||
Node* control = NodeProperties::GetControlInput(node);
|
||||
|
||||
AbstractState const* state = node_states_.Get(effect);
|
||||
if (state == nullptr) return NoChange();
|
||||
|
||||
HalfState const* mutable_state = &state->mutable_state;
|
||||
|
||||
FieldOrElementValue lookup_result =
|
||||
mutable_state->LookupField(kStringPrepareForGetCodeunitIndex, object);
|
||||
|
||||
if (!lookup_result.IsEmpty() && !lookup_result.value->IsDead()) {
|
||||
for (size_t i : {0, 1, 2}) {
|
||||
Node* proj_to_replace = NodeProperties::FindProjection(node, i);
|
||||
ReplaceWithValue(proj_to_replace,
|
||||
NodeProperties::FindProjection(lookup_result.value, i));
|
||||
proj_to_replace->Kill();
|
||||
}
|
||||
ReplaceWithValue(node, lookup_result.value, effect, control);
|
||||
node->Kill();
|
||||
return Replace(lookup_result.value);
|
||||
}
|
||||
|
||||
mutable_state =
|
||||
mutable_state->AddField(kStringPrepareForGetCodeunitIndex, object, node);
|
||||
|
||||
AbstractState const* new_state =
|
||||
zone()->New<AbstractState>(*mutable_state, state->immutable_state);
|
||||
|
||||
return UpdateState(node, new_state);
|
||||
}
|
||||
|
||||
Reduction WasmLoadElimination::ReduceStringAsWtf16(Node* node) {
|
||||
DCHECK_EQ(node->opcode(), IrOpcode::kStringAsWtf16);
|
||||
Node* object = ResolveAliases(NodeProperties::GetValueInput(node, 0));
|
||||
Node* effect = NodeProperties::GetEffectInput(node);
|
||||
Node* control = NodeProperties::GetControlInput(node);
|
||||
|
||||
AbstractState const* state = node_states_.Get(effect);
|
||||
if (state == nullptr) return NoChange();
|
||||
|
||||
HalfState const* immutable_state = &state->immutable_state;
|
||||
|
||||
FieldOrElementValue lookup_result =
|
||||
immutable_state->LookupField(kStringAsWtf16Index, object);
|
||||
|
||||
if (!lookup_result.IsEmpty() && !lookup_result.value->IsDead()) {
|
||||
ReplaceWithValue(node, lookup_result.value, effect, control);
|
||||
node->Kill();
|
||||
return Replace(lookup_result.value);
|
||||
}
|
||||
|
||||
immutable_state =
|
||||
immutable_state->AddField(kStringAsWtf16Index, object, node);
|
||||
|
||||
AbstractState const* new_state =
|
||||
zone()->New<AbstractState>(state->mutable_state, *immutable_state);
|
||||
|
||||
return UpdateState(node, new_state);
|
||||
}
|
||||
|
||||
Reduction WasmLoadElimination::ReduceOtherNode(Node* node) {
|
||||
if (node->op()->EffectOutputCount() == 0) return NoChange();
|
||||
DCHECK_EQ(node->op()->EffectInputCount(), 1);
|
||||
@ -296,6 +367,10 @@ Reduction WasmLoadElimination::ReduceOtherNode(Node* node) {
|
||||
// If this {node} has some uncontrolled side effects (i.e. it is a call
|
||||
// without {kNoWrite}), set its state to the immutable half-state of its
|
||||
// input state, otherwise to its input state.
|
||||
// Any cached StringPrepareForGetCodeUnit nodes must be killed at any point
|
||||
// that can cause internalization of strings (i.e. that can turn sequential
|
||||
// strings into thin strings). Currently, that can only happen in JS, so
|
||||
// from Wasm's point of view only in calls.
|
||||
return UpdateState(node, node->opcode() == IrOpcode::kCall &&
|
||||
!node->op()->HasProperty(Operator::kNoWrite)
|
||||
? zone()->New<AbstractState>(
|
||||
@ -308,6 +383,7 @@ Reduction WasmLoadElimination::ReduceStart(Node* node) {
|
||||
}
|
||||
|
||||
Reduction WasmLoadElimination::ReduceEffectPhi(Node* node) {
|
||||
DCHECK_EQ(node->opcode(), IrOpcode::kEffectPhi);
|
||||
Node* const effect0 = NodeProperties::GetEffectInput(node, 0);
|
||||
Node* const control = NodeProperties::GetControlInput(node);
|
||||
AbstractState const* state0 = node_states_.Get(effect0);
|
||||
|
||||
@ -84,8 +84,6 @@ class V8_EXPORT_PRIVATE WasmLoadElimination final
|
||||
map.Set(outer_key, map_copy);
|
||||
}
|
||||
|
||||
static void KillField(int field_index, Node* object,
|
||||
MachineRepresentation repr, Zone* zone);
|
||||
static void Print(const FieldInfos& infos);
|
||||
static void Print(const ElementInfos& infos);
|
||||
|
||||
@ -120,6 +118,8 @@ class V8_EXPORT_PRIVATE WasmLoadElimination final
|
||||
Reduction ReduceWasmStructSet(Node* node);
|
||||
Reduction ReduceWasmArrayLength(Node* node);
|
||||
Reduction ReduceWasmArrayInitializeLength(Node* node);
|
||||
Reduction ReduceStringPrepareForGetCodeunit(Node* node);
|
||||
Reduction ReduceStringAsWtf16(Node* node);
|
||||
Reduction ReduceEffectPhi(Node* node);
|
||||
Reduction ReduceStart(Node* node);
|
||||
Reduction ReduceOtherNode(Node* node);
|
||||
|
||||
@ -7041,7 +7041,22 @@ class LiftoffCompiler {
|
||||
}
|
||||
|
||||
void StringAsWtf16(FullDecoder* decoder, const Value& str, Value* result) {
|
||||
RefAsNonNull(decoder, str, result);
|
||||
LiftoffRegList pinned;
|
||||
|
||||
LiftoffRegister str_reg = pinned.set(__ PopToRegister(pinned));
|
||||
MaybeEmitNullCheck(decoder, str_reg.gp(), pinned, str.type);
|
||||
LiftoffAssembler::VarState str_var(kRef, str_reg, 0);
|
||||
|
||||
CallRuntimeStub(WasmCode::kWasmStringAsWtf16,
|
||||
MakeSig::Returns(kRef).Params(kRef),
|
||||
{
|
||||
str_var,
|
||||
},
|
||||
decoder->position());
|
||||
RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
|
||||
|
||||
LiftoffRegister result_reg(kReturnRegister0);
|
||||
__ PushRegister(kRef, result_reg);
|
||||
}
|
||||
|
||||
void StringViewWtf16GetCodeUnit(FullDecoder* decoder, const Value& view,
|
||||
|
||||
@ -1723,12 +1723,9 @@ class WasmGraphBuildingInterface {
|
||||
}
|
||||
|
||||
void StringAsWtf16(FullDecoder* decoder, const Value& str, Value* result) {
|
||||
// Since we implement stringview_wtf16 as string, that's the type we'll
|
||||
// use for the Node. (The decoder's Value type must be stringview_wtf16
|
||||
// because static type validation relies on it.)
|
||||
result->node = builder_->SetType(
|
||||
builder_->AssertNotNull(str.node, decoder->position()),
|
||||
ValueType::Ref(HeapType::kString));
|
||||
SetAndTypeNode(result,
|
||||
builder_->StringAsWtf16(str.node, NullCheckFor(str.type),
|
||||
decoder->position()));
|
||||
}
|
||||
|
||||
void StringViewWtf16GetCodeUnit(FullDecoder* decoder, const Value& view,
|
||||
|
||||
@ -132,6 +132,7 @@ struct WasmModule;
|
||||
V(WasmStringConcat) \
|
||||
V(WasmStringEqual) \
|
||||
V(WasmStringIsUSVSequence) \
|
||||
V(WasmStringAsWtf16) \
|
||||
V(WasmStringViewWtf16GetCodeUnit) \
|
||||
V(WasmStringViewWtf16Encode) \
|
||||
V(WasmStringViewWtf16Slice) \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user