[scanner] Prepare CharacterStreams for specializing scanner and parser by character type
This templatizes CharacterStream by char type, and makes them subclass ScannerStream. Methods that are widely used by tests are marked virtual on ScannerStream and final on CharacterStream<T> so the specialized scanner will know what to call. ParseInfo passes around ScannerStream, but the scanner requires the explicit CharacterStream<T>. Since AdvanceUntil is templatized by FunctionType, I couldn't mark that virtual; so instead I adjusted those tests to operate directly on ucs2 (not utf8 since we'll drop that in the future). In the end no functionality was changed. Some calls became virtual in tests. This is mainly just preparation. Change-Id: I0b4def65d3eb8fa5c806027c7e9123a590ebbdb5 Reviewed-on: https://chromium-review.googlesource.com/1156690 Commit-Queue: Toon Verwaest <verwaest@chromium.org> Reviewed-by: Michael Starzinger <mstarzinger@chromium.org> Reviewed-by: Marja Hölttä <marja@chromium.org> Cr-Commit-Position: refs/heads/master@{#54848}
This commit is contained in:
Toon Verwaest
Commit Bot
parent
65e3cea37f
commit
2d40e2f445
@ -234,13 +234,13 @@ UnoptimizedCompilationJob::Status AsmJsCompilationJob::ExecuteJobImpl() {
|
||||
Zone* compile_zone = compilation_info()->zone();
|
||||
Zone translate_zone(allocator_, ZONE_NAME);
|
||||
|
||||
Utf16CharacterStream* stream = parse_info()->character_stream();
|
||||
ScannerStream* stream = parse_info()->character_stream();
|
||||
base::Optional<AllowHandleDereference> allow_deref;
|
||||
if (stream->can_access_heap()) {
|
||||
allow_deref.emplace();
|
||||
}
|
||||
stream->Seek(compilation_info()->literal()->start_position());
|
||||
wasm::AsmJsParser parser(&translate_zone, stack_limit(), stream);
|
||||
wasm::AsmJsParser parser(&translate_zone, stack_limit(), stream,
|
||||
compilation_info()->literal()->start_position());
|
||||
if (!parser.Run()) {
|
||||
if (!FLAG_suppress_asm_messages) {
|
||||
ReportCompilationFailure(parse_info(), parser.failure_location(),
|
||||
|
||||
@ -69,9 +69,9 @@ namespace wasm {
|
||||
#define TOK(name) AsmJsScanner::kToken_##name
|
||||
|
||||
AsmJsParser::AsmJsParser(Zone* zone, uintptr_t stack_limit,
|
||||
Utf16CharacterStream* stream)
|
||||
ScannerStream* stream, int start)
|
||||
: zone_(zone),
|
||||
scanner_(stream),
|
||||
scanner_(static_cast<CharacterStream<uint16_t>*>(stream), start),
|
||||
module_builder_(new (zone) WasmModuleBuilder(zone)),
|
||||
return_type_(nullptr),
|
||||
stack_limit_(stack_limit),
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
namespace v8 {
|
||||
namespace internal {
|
||||
|
||||
class Utf16CharacterStream;
|
||||
class ScannerStream;
|
||||
|
||||
namespace wasm {
|
||||
|
||||
@ -49,8 +49,8 @@ class AsmJsParser {
|
||||
|
||||
typedef EnumSet<StandardMember, uint64_t> StdlibSet;
|
||||
|
||||
explicit AsmJsParser(Zone* zone, uintptr_t stack_limit,
|
||||
Utf16CharacterStream* stream);
|
||||
explicit AsmJsParser(Zone* zone, uintptr_t stack_limit, ScannerStream* stream,
|
||||
int start);
|
||||
bool Run();
|
||||
const char* failure_message() const { return failure_message_; }
|
||||
int failure_location() const { return failure_location_; }
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include "src/char-predicates-inl.h"
|
||||
#include "src/conversions.h"
|
||||
#include "src/flags.h"
|
||||
#include "src/parsing/scanner-character-streams.h"
|
||||
#include "src/parsing/scanner.h"
|
||||
#include "src/unicode-cache.h"
|
||||
|
||||
@ -19,7 +20,7 @@ namespace {
|
||||
static const int kMaxIdentifierCount = 0xF000000;
|
||||
};
|
||||
|
||||
AsmJsScanner::AsmJsScanner(Utf16CharacterStream* stream)
|
||||
AsmJsScanner::AsmJsScanner(CharacterStream<uint16_t>* stream, int start)
|
||||
: stream_(stream),
|
||||
token_(kUninitialized),
|
||||
preceding_token_(kUninitialized),
|
||||
@ -33,6 +34,7 @@ AsmJsScanner::AsmJsScanner(Utf16CharacterStream* stream)
|
||||
double_value_(0.0),
|
||||
unsigned_value_(0),
|
||||
preceded_by_newline_(false) {
|
||||
stream->Seek(start);
|
||||
#define V(name, _junk1, _junk2, _junk3) property_names_[#name] = kToken_##name;
|
||||
STDLIB_MATH_FUNCTION_LIST(V)
|
||||
STDLIB_ARRAY_TYPE_LIST(V)
|
||||
|
||||
@ -16,7 +16,8 @@
|
||||
namespace v8 {
|
||||
namespace internal {
|
||||
|
||||
class Utf16CharacterStream;
|
||||
template <typename Char>
|
||||
class CharacterStream;
|
||||
|
||||
// A custom scanner to extract the token stream needed to parse valid
|
||||
// asm.js: http://asmjs.org/spec/latest/
|
||||
@ -31,7 +32,7 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
|
||||
public:
|
||||
typedef int32_t token_t;
|
||||
|
||||
explicit AsmJsScanner(Utf16CharacterStream* stream);
|
||||
AsmJsScanner(CharacterStream<uint16_t>* stream, int start);
|
||||
|
||||
// Get current token.
|
||||
token_t Token() const { return token_; }
|
||||
@ -136,7 +137,7 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
|
||||
// clang-format on
|
||||
|
||||
private:
|
||||
Utf16CharacterStream* stream_;
|
||||
CharacterStream<uint16_t>* stream_;
|
||||
token_t token_;
|
||||
token_t preceding_token_;
|
||||
token_t next_token_; // Only set when in {rewind} state.
|
||||
|
||||
@ -133,7 +133,7 @@ void UnoptimizedCompileJob::PrepareOnMainThread(Isolate* isolate) {
|
||||
DCHECK(script->type() != Script::TYPE_NATIVE);
|
||||
Handle<String> source(String::cast(script->source()), isolate);
|
||||
if (source->IsExternalTwoByteString() || source->IsExternalOneByteString()) {
|
||||
std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
|
||||
std::unique_ptr<ScannerStream> stream(ScannerStream::For(
|
||||
isolate, source, shared_->StartPosition(), shared_->EndPosition()));
|
||||
parse_info_->set_character_stream(std::move(stream));
|
||||
} else {
|
||||
@ -191,7 +191,7 @@ void UnoptimizedCompileJob::PrepareOnMainThread(Isolate* isolate) {
|
||||
.ToHandleChecked();
|
||||
}
|
||||
wrapper_ = isolate->global_handles()->Create(*wrapper);
|
||||
std::unique_ptr<Utf16CharacterStream> stream(
|
||||
std::unique_ptr<ScannerStream> stream(
|
||||
ScannerStream::For(isolate, wrapper_, shared_->StartPosition() - offset,
|
||||
shared_->EndPosition() - offset));
|
||||
parse_info_->set_character_stream(std::move(stream));
|
||||
|
||||
@ -966,7 +966,7 @@ BackgroundCompileTask::BackgroundCompileTask(ScriptStreamingData* source,
|
||||
info->set_runtime_call_stats(nullptr);
|
||||
}
|
||||
info->set_toplevel();
|
||||
std::unique_ptr<Utf16CharacterStream> stream(
|
||||
std::unique_ptr<ScannerStream> stream(
|
||||
ScannerStream::For(source->source_stream.get(), source->encoding,
|
||||
info->runtime_call_stats()));
|
||||
info->set_character_stream(std::move(stream));
|
||||
|
||||
@ -198,7 +198,7 @@ void ParseInfo::AllocateSourceRangeMap() {
|
||||
void ParseInfo::ResetCharacterStream() { character_stream_.reset(); }
|
||||
|
||||
void ParseInfo::set_character_stream(
|
||||
std::unique_ptr<Utf16CharacterStream> character_stream) {
|
||||
std::unique_ptr<ScannerStream> character_stream) {
|
||||
DCHECK_NULL(character_stream_);
|
||||
character_stream_.swap(character_stream);
|
||||
}
|
||||
|
||||
@ -31,7 +31,7 @@ class RuntimeCallStats;
|
||||
class Logger;
|
||||
class SourceRangeMap;
|
||||
class UnicodeCache;
|
||||
class Utf16CharacterStream;
|
||||
class ScannerStream;
|
||||
class Zone;
|
||||
|
||||
// A container for the inputs, configuration options, and outputs of parsing.
|
||||
@ -97,11 +97,8 @@ class V8_EXPORT_PRIVATE ParseInfo {
|
||||
: NO_PARSE_RESTRICTION;
|
||||
}
|
||||
|
||||
Utf16CharacterStream* character_stream() const {
|
||||
return character_stream_.get();
|
||||
}
|
||||
void set_character_stream(
|
||||
std::unique_ptr<Utf16CharacterStream> character_stream);
|
||||
ScannerStream* character_stream() const { return character_stream_.get(); }
|
||||
void set_character_stream(std::unique_ptr<ScannerStream> character_stream);
|
||||
void ResetCharacterStream();
|
||||
|
||||
v8::Extension* extension() const { return extension_; }
|
||||
@ -274,7 +271,7 @@ class V8_EXPORT_PRIVATE ParseInfo {
|
||||
MaybeHandle<ScopeInfo> maybe_outer_scope_info_;
|
||||
|
||||
//----------- Inputs+Outputs of parsing and scope analysis -----------------
|
||||
std::unique_ptr<Utf16CharacterStream> character_stream_;
|
||||
std::unique_ptr<ScannerStream> character_stream_;
|
||||
ConsumedPreParsedScopeData consumed_preparsed_scope_data_;
|
||||
std::shared_ptr<AstValueFactory> ast_value_factory_;
|
||||
const class AstStringConstants* ast_string_constants_;
|
||||
|
||||
@ -507,7 +507,9 @@ FunctionLiteral* Parser::ParseProgram(Isolate* isolate, ParseInfo* info) {
|
||||
// Initialize parser state.
|
||||
DeserializeScopeChain(isolate, info, info->maybe_outer_scope_info());
|
||||
|
||||
scanner_.Initialize(info->character_stream(), info->is_module());
|
||||
auto stream =
|
||||
static_cast<CharacterStream<uint16_t>*>(info->character_stream());
|
||||
scanner_.Initialize(stream, info->is_module());
|
||||
FunctionLiteral* result = DoParseProgram(isolate, info);
|
||||
MaybeResetCharacterStream(info, result);
|
||||
|
||||
@ -701,7 +703,9 @@ FunctionLiteral* Parser::ParseFunction(Isolate* isolate, ParseInfo* info,
|
||||
// Initialize parser state.
|
||||
Handle<String> name(shared_info->Name(), isolate);
|
||||
info->set_function_name(ast_value_factory()->GetString(name));
|
||||
scanner_.Initialize(info->character_stream(), info->is_module());
|
||||
auto stream =
|
||||
static_cast<CharacterStream<uint16_t>*>(info->character_stream());
|
||||
scanner_.Initialize(stream, info->is_module());
|
||||
|
||||
FunctionLiteral* result =
|
||||
DoParseFunction(isolate, info, info->function_name());
|
||||
@ -3435,7 +3439,9 @@ void Parser::ParseOnBackground(ParseInfo* info) {
|
||||
DCHECK_NULL(info->literal());
|
||||
FunctionLiteral* result = nullptr;
|
||||
|
||||
scanner_.Initialize(info->character_stream(), info->is_module());
|
||||
auto stream =
|
||||
static_cast<CharacterStream<uint16_t>*>(info->character_stream());
|
||||
scanner_.Initialize(stream, info->is_module());
|
||||
DCHECK(info->maybe_outer_scope_info().is_null());
|
||||
|
||||
DCHECK(original_scope_);
|
||||
|
||||
@ -26,8 +26,7 @@ bool ParseProgram(ParseInfo* info, Isolate* isolate) {
|
||||
// Create a character stream for the parser.
|
||||
Handle<String> source(String::cast(info->script()->source()), isolate);
|
||||
isolate->counters()->total_parse_size()->Increment(source->length());
|
||||
std::unique_ptr<Utf16CharacterStream> stream(
|
||||
ScannerStream::For(isolate, source));
|
||||
std::unique_ptr<ScannerStream> stream(ScannerStream::For(isolate, source));
|
||||
info->set_character_stream(std::move(stream));
|
||||
|
||||
Parser parser(info);
|
||||
@ -61,7 +60,7 @@ bool ParseFunction(ParseInfo* info, Handle<SharedFunctionInfo> shared_info,
|
||||
// Create a character stream for the parser.
|
||||
Handle<String> source(String::cast(info->script()->source()), isolate);
|
||||
isolate->counters()->total_parse_size()->Increment(source->length());
|
||||
std::unique_ptr<Utf16CharacterStream> stream(
|
||||
std::unique_ptr<ScannerStream> stream(
|
||||
ScannerStream::For(isolate, source, shared_info->StartPosition(),
|
||||
shared_info->EndPosition()));
|
||||
info->set_character_stream(std::move(stream));
|
||||
|
||||
@ -157,7 +157,7 @@ class ChunkedStream {
|
||||
// Chars are buffered if either the underlying stream isn't utf-16 or the
|
||||
// underlying utf-16 stream might move (is on-heap).
|
||||
template <typename Char, template <typename T> class ByteStream>
|
||||
class BufferedCharacterStream : public Utf16CharacterStream {
|
||||
class BufferedCharacterStream : public CharacterStream<uint16_t> {
|
||||
public:
|
||||
template <class... TArgs>
|
||||
BufferedCharacterStream(size_t pos, TArgs... args) : byte_stream_(args...) {
|
||||
@ -194,7 +194,7 @@ class BufferedCharacterStream : public Utf16CharacterStream {
|
||||
// Provides a unbuffered utf-16 view on the bytes from the underlying
|
||||
// ByteStream.
|
||||
template <template <typename T> class ByteStream>
|
||||
class UnbufferedCharacterStream : public Utf16CharacterStream {
|
||||
class UnbufferedCharacterStream : public CharacterStream<uint16_t> {
|
||||
public:
|
||||
template <class... TArgs>
|
||||
UnbufferedCharacterStream(size_t pos, TArgs... args) : byte_stream_(args...) {
|
||||
@ -268,7 +268,7 @@ class RelocatingCharacterStream
|
||||
// even positions before the current).
|
||||
//
|
||||
// TODO(verwaest): Remove together with Utf8 external streaming streams.
|
||||
class BufferedUtf16CharacterStream : public Utf16CharacterStream {
|
||||
class BufferedUtf16CharacterStream : public CharacterStream<uint16_t> {
|
||||
public:
|
||||
BufferedUtf16CharacterStream();
|
||||
|
||||
@ -287,7 +287,7 @@ class BufferedUtf16CharacterStream : public Utf16CharacterStream {
|
||||
};
|
||||
|
||||
BufferedUtf16CharacterStream::BufferedUtf16CharacterStream()
|
||||
: Utf16CharacterStream(buffer_, buffer_, buffer_, 0) {}
|
||||
: CharacterStream(buffer_, buffer_, buffer_, 0) {}
|
||||
|
||||
bool BufferedUtf16CharacterStream::ReadBlock() {
|
||||
DCHECK_EQ(buffer_start_, buffer_);
|
||||
@ -585,13 +585,12 @@ size_t Utf8ExternalStreamingStream::FillBuffer(size_t position) {
|
||||
// ----------------------------------------------------------------------------
|
||||
// ScannerStream: Create stream instances.
|
||||
|
||||
Utf16CharacterStream* ScannerStream::For(Isolate* isolate,
|
||||
Handle<String> data) {
|
||||
ScannerStream* ScannerStream::For(Isolate* isolate, Handle<String> data) {
|
||||
return ScannerStream::For(isolate, data, 0, data->length());
|
||||
}
|
||||
|
||||
Utf16CharacterStream* ScannerStream::For(Isolate* isolate, Handle<String> data,
|
||||
int start_pos, int end_pos) {
|
||||
ScannerStream* ScannerStream::For(Isolate* isolate, Handle<String> data,
|
||||
int start_pos, int end_pos) {
|
||||
DCHECK_GE(start_pos, 0);
|
||||
DCHECK_LE(start_pos, end_pos);
|
||||
DCHECK_LE(end_pos, data->length());
|
||||
@ -629,20 +628,20 @@ Utf16CharacterStream* ScannerStream::For(Isolate* isolate, Handle<String> data,
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Utf16CharacterStream> ScannerStream::ForTesting(
|
||||
std::unique_ptr<CharacterStream<uint16_t>> ScannerStream::ForTesting(
|
||||
const char* data) {
|
||||
return ScannerStream::ForTesting(data, strlen(data));
|
||||
}
|
||||
|
||||
std::unique_ptr<Utf16CharacterStream> ScannerStream::ForTesting(
|
||||
std::unique_ptr<CharacterStream<uint16_t>> ScannerStream::ForTesting(
|
||||
const char* data, size_t length) {
|
||||
return std::unique_ptr<Utf16CharacterStream>(
|
||||
return std::unique_ptr<CharacterStream<uint16_t>>(
|
||||
new BufferedCharacterStream<uint8_t, ExternalStringStream>(
|
||||
static_cast<size_t>(0), reinterpret_cast<const uint8_t*>(data),
|
||||
static_cast<size_t>(length)));
|
||||
}
|
||||
|
||||
Utf16CharacterStream* ScannerStream::For(
|
||||
ScannerStream* ScannerStream::For(
|
||||
ScriptCompiler::ExternalSourceStream* source_stream,
|
||||
v8::ScriptCompiler::StreamedSource::Encoding encoding,
|
||||
RuntimeCallStats* stats) {
|
||||
|
||||
@ -5,6 +5,8 @@
|
||||
#ifndef V8_PARSING_SCANNER_CHARACTER_STREAMS_H_
|
||||
#define V8_PARSING_SCANNER_CHARACTER_STREAMS_H_
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "include/v8.h" // for v8::ScriptCompiler
|
||||
#include "src/globals.h"
|
||||
|
||||
@ -13,24 +15,182 @@ namespace internal {
|
||||
|
||||
template <typename T>
|
||||
class Handle;
|
||||
class Utf16CharacterStream;
|
||||
template <typename Char>
|
||||
class CharacterStream;
|
||||
class RuntimeCallStats;
|
||||
class String;
|
||||
|
||||
class V8_EXPORT_PRIVATE ScannerStream {
|
||||
public:
|
||||
static Utf16CharacterStream* For(Isolate* isolate, Handle<String> data);
|
||||
static Utf16CharacterStream* For(Isolate* isolate, Handle<String> data,
|
||||
int start_pos, int end_pos);
|
||||
static Utf16CharacterStream* For(
|
||||
ScriptCompiler::ExternalSourceStream* source_stream,
|
||||
ScriptCompiler::StreamedSource::Encoding encoding,
|
||||
RuntimeCallStats* stats);
|
||||
static const uc32 kEndOfInput = -1;
|
||||
|
||||
static ScannerStream* For(Isolate* isolate, Handle<String> data);
|
||||
static ScannerStream* For(Isolate* isolate, Handle<String> data,
|
||||
int start_pos, int end_pos);
|
||||
static ScannerStream* For(ScriptCompiler::ExternalSourceStream* source_stream,
|
||||
ScriptCompiler::StreamedSource::Encoding encoding,
|
||||
RuntimeCallStats* stats);
|
||||
|
||||
// For testing:
|
||||
static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data);
|
||||
static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data,
|
||||
size_t length);
|
||||
static std::unique_ptr<CharacterStream<uint16_t>> ForTesting(
|
||||
const char* data);
|
||||
static std::unique_ptr<CharacterStream<uint16_t>> ForTesting(const char* data,
|
||||
size_t length);
|
||||
|
||||
// Returns true if the stream could access the V8 heap after construction.
|
||||
virtual bool can_access_heap() = 0;
|
||||
virtual uc32 Advance() = 0;
|
||||
virtual void Seek(size_t pos) = 0;
|
||||
virtual size_t pos() const = 0;
|
||||
virtual void Back() = 0;
|
||||
|
||||
virtual ~ScannerStream() {}
|
||||
};
|
||||
|
||||
template <typename Char>
|
||||
class CharacterStream : public ScannerStream {
|
||||
public:
|
||||
// Returns and advances past the next UTF-16 code unit in the input
|
||||
// stream. If there are no more code units it returns kEndOfInput.
|
||||
inline uc32 Advance() final {
|
||||
if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
|
||||
return static_cast<uc32>(*(buffer_cursor_++));
|
||||
} else if (ReadBlockChecked()) {
|
||||
return static_cast<uc32>(*(buffer_cursor_++));
|
||||
} else {
|
||||
// Note: currently the following increment is necessary to avoid a
|
||||
// parser problem! The scanner treats the final kEndOfInput as
|
||||
// a code unit with a position, and does math relative to that
|
||||
// position.
|
||||
buffer_cursor_++;
|
||||
return kEndOfInput;
|
||||
}
|
||||
}
|
||||
|
||||
// Returns and advances past the next UTF-16 code unit in the input stream
|
||||
// that meets the checks requirement. If there are no more code units it
|
||||
// returns kEndOfInput.
|
||||
template <typename FunctionType>
|
||||
V8_INLINE uc32 AdvanceUntil(FunctionType check) {
|
||||
while (true) {
|
||||
auto next_cursor_pos =
|
||||
std::find_if(buffer_cursor_, buffer_end_, [&check](Char raw_c0) {
|
||||
uc32 c0 = static_cast<uc32>(raw_c0);
|
||||
return check(c0);
|
||||
});
|
||||
|
||||
if (next_cursor_pos == buffer_end_) {
|
||||
buffer_cursor_ = buffer_end_;
|
||||
if (!ReadBlockChecked()) {
|
||||
buffer_cursor_++;
|
||||
return kEndOfInput;
|
||||
}
|
||||
} else {
|
||||
buffer_cursor_ = next_cursor_pos + 1;
|
||||
return static_cast<uc32>(*next_cursor_pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Go back one by one character in the input stream.
|
||||
// This undoes the most recent Advance().
|
||||
inline void Back() final {
|
||||
// The common case - if the previous character is within
|
||||
// buffer_start_ .. buffer_end_ will be handles locally.
|
||||
// Otherwise, a new block is requested.
|
||||
if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
|
||||
buffer_cursor_--;
|
||||
} else {
|
||||
ReadBlockAt(pos() - 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Go back one by two characters in the input stream. (This is the same as
|
||||
// calling Back() twice. But Back() may - in some instances - do substantial
|
||||
// work. Back2() guarantees this work will be done only once.)
|
||||
inline void Back2() {
|
||||
if (V8_LIKELY(buffer_cursor_ - 2 >= buffer_start_)) {
|
||||
buffer_cursor_ -= 2;
|
||||
} else {
|
||||
ReadBlockAt(pos() - 2);
|
||||
}
|
||||
}
|
||||
|
||||
inline size_t pos() const final {
|
||||
return buffer_pos_ + (buffer_cursor_ - buffer_start_);
|
||||
}
|
||||
|
||||
inline void Seek(size_t pos) final {
|
||||
if (V8_LIKELY(pos >= buffer_pos_ &&
|
||||
pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
|
||||
buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
|
||||
} else {
|
||||
ReadBlockAt(pos);
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if the stream could access the V8 heap after construction.
|
||||
virtual bool can_access_heap() = 0;
|
||||
|
||||
protected:
|
||||
CharacterStream(const uint16_t* buffer_start, const uint16_t* buffer_cursor,
|
||||
const uint16_t* buffer_end, size_t buffer_pos)
|
||||
: buffer_start_(buffer_start),
|
||||
buffer_cursor_(buffer_cursor),
|
||||
buffer_end_(buffer_end),
|
||||
buffer_pos_(buffer_pos) {}
|
||||
CharacterStream() : CharacterStream(nullptr, nullptr, nullptr, 0) {}
|
||||
|
||||
bool ReadBlockChecked() {
|
||||
size_t position = pos();
|
||||
USE(position);
|
||||
bool success = ReadBlock();
|
||||
|
||||
// Post-conditions: 1, We should always be at the right position.
|
||||
// 2, Cursor should be inside the buffer.
|
||||
// 3, We should have more characters available iff success.
|
||||
DCHECK_EQ(pos(), position);
|
||||
DCHECK_LE(buffer_cursor_, buffer_end_);
|
||||
DCHECK_LE(buffer_start_, buffer_cursor_);
|
||||
DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
|
||||
return success;
|
||||
}
|
||||
|
||||
void ReadBlockAt(size_t new_pos) {
|
||||
// The callers of this method (Back/Back2/Seek) should handle the easy
|
||||
// case (seeking within the current buffer), and we should only get here
|
||||
// if we actually require new data.
|
||||
// (This is really an efficiency check, not a correctness invariant.)
|
||||
DCHECK(new_pos < buffer_pos_ ||
|
||||
new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_));
|
||||
|
||||
// Change pos() to point to new_pos.
|
||||
buffer_pos_ = new_pos;
|
||||
buffer_cursor_ = buffer_start_;
|
||||
DCHECK_EQ(pos(), new_pos);
|
||||
ReadBlockChecked();
|
||||
}
|
||||
|
||||
// Read more data, and update buffer_*_ to point to it.
|
||||
// Returns true if more data was available.
|
||||
//
|
||||
// ReadBlock() may modify any of the buffer_*_ members, but must sure that
|
||||
// the result of pos() remains unaffected.
|
||||
//
|
||||
// Examples:
|
||||
// - a stream could either fill a separate buffer. Then buffer_start_ and
|
||||
// buffer_cursor_ would point to the beginning of the buffer, and
|
||||
// buffer_pos would be the old pos().
|
||||
// - a stream with existing buffer chunks would set buffer_start_ and
|
||||
// buffer_end_ to cover the full chunk, and then buffer_cursor_ would
|
||||
// point into the middle of the buffer, while buffer_pos_ would describe
|
||||
// the start of the buffer.
|
||||
virtual bool ReadBlock() = 0;
|
||||
|
||||
const Char* buffer_start_;
|
||||
const Char* buffer_cursor_;
|
||||
const Char* buffer_end_;
|
||||
size_t buffer_pos_;
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
@ -188,7 +188,7 @@ Scanner::Scanner(UnicodeCache* unicode_cache)
|
||||
allow_harmony_bigint_(false),
|
||||
allow_harmony_numeric_separator_(false) {}
|
||||
|
||||
void Scanner::Initialize(Utf16CharacterStream* source, bool is_module) {
|
||||
void Scanner::Initialize(CharacterStream<uint16_t>* source, bool is_module) {
|
||||
DCHECK_NOT_NULL(source);
|
||||
source_ = source;
|
||||
is_module_ = is_module;
|
||||
|
||||
@ -7,13 +7,12 @@
|
||||
#ifndef V8_PARSING_SCANNER_H_
|
||||
#define V8_PARSING_SCANNER_H_
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "src/allocation.h"
|
||||
#include "src/base/logging.h"
|
||||
#include "src/char-predicates.h"
|
||||
#include "src/globals.h"
|
||||
#include "src/messages.h"
|
||||
#include "src/parsing/scanner-character-streams.h"
|
||||
#include "src/parsing/token.h"
|
||||
#include "src/unicode-decoder.h"
|
||||
#include "src/unicode.h"
|
||||
@ -30,161 +29,6 @@ class ExternalTwoByteString;
|
||||
class ParserRecorder;
|
||||
class UnicodeCache;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Buffered stream of UTF-16 code units, using an internal UTF-16 buffer.
|
||||
// A code unit is a 16 bit value representing either a 16 bit code point
|
||||
// or one part of a surrogate pair that make a single 21 bit code point.
|
||||
class Utf16CharacterStream {
|
||||
public:
|
||||
static const uc32 kEndOfInput = -1;
|
||||
|
||||
virtual ~Utf16CharacterStream() { }
|
||||
|
||||
// Returns and advances past the next UTF-16 code unit in the input
|
||||
// stream. If there are no more code units it returns kEndOfInput.
|
||||
inline uc32 Advance() {
|
||||
if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
|
||||
return static_cast<uc32>(*(buffer_cursor_++));
|
||||
} else if (ReadBlockChecked()) {
|
||||
return static_cast<uc32>(*(buffer_cursor_++));
|
||||
} else {
|
||||
// Note: currently the following increment is necessary to avoid a
|
||||
// parser problem! The scanner treats the final kEndOfInput as
|
||||
// a code unit with a position, and does math relative to that
|
||||
// position.
|
||||
buffer_cursor_++;
|
||||
return kEndOfInput;
|
||||
}
|
||||
}
|
||||
|
||||
// Returns and advances past the next UTF-16 code unit in the input stream
|
||||
// that meets the checks requirement. If there are no more code units it
|
||||
// returns kEndOfInput.
|
||||
template <typename FunctionType>
|
||||
V8_INLINE uc32 AdvanceUntil(FunctionType check) {
|
||||
while (true) {
|
||||
auto next_cursor_pos =
|
||||
std::find_if(buffer_cursor_, buffer_end_, [&check](uint16_t raw_c0_) {
|
||||
uc32 c0_ = static_cast<uc32>(raw_c0_);
|
||||
return check(c0_);
|
||||
});
|
||||
|
||||
if (next_cursor_pos == buffer_end_) {
|
||||
buffer_cursor_ = buffer_end_;
|
||||
if (!ReadBlockChecked()) {
|
||||
buffer_cursor_++;
|
||||
return kEndOfInput;
|
||||
}
|
||||
} else {
|
||||
buffer_cursor_ = next_cursor_pos + 1;
|
||||
return static_cast<uc32>(*next_cursor_pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Go back one by one character in the input stream.
|
||||
// This undoes the most recent Advance().
|
||||
inline void Back() {
|
||||
// The common case - if the previous character is within
|
||||
// buffer_start_ .. buffer_end_ will be handles locally.
|
||||
// Otherwise, a new block is requested.
|
||||
if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
|
||||
buffer_cursor_--;
|
||||
} else {
|
||||
ReadBlockAt(pos() - 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Go back one by two characters in the input stream. (This is the same as
|
||||
// calling Back() twice. But Back() may - in some instances - do substantial
|
||||
// work. Back2() guarantees this work will be done only once.)
|
||||
inline void Back2() {
|
||||
if (V8_LIKELY(buffer_cursor_ - 2 >= buffer_start_)) {
|
||||
buffer_cursor_ -= 2;
|
||||
} else {
|
||||
ReadBlockAt(pos() - 2);
|
||||
}
|
||||
}
|
||||
|
||||
inline size_t pos() const {
|
||||
return buffer_pos_ + (buffer_cursor_ - buffer_start_);
|
||||
}
|
||||
|
||||
inline void Seek(size_t pos) {
|
||||
if (V8_LIKELY(pos >= buffer_pos_ &&
|
||||
pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
|
||||
buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
|
||||
} else {
|
||||
ReadBlockAt(pos);
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if the stream could access the V8 heap after construction.
|
||||
virtual bool can_access_heap() = 0;
|
||||
|
||||
protected:
|
||||
Utf16CharacterStream(const uint16_t* buffer_start,
|
||||
const uint16_t* buffer_cursor,
|
||||
const uint16_t* buffer_end, size_t buffer_pos)
|
||||
: buffer_start_(buffer_start),
|
||||
buffer_cursor_(buffer_cursor),
|
||||
buffer_end_(buffer_end),
|
||||
buffer_pos_(buffer_pos) {}
|
||||
Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {}
|
||||
|
||||
bool ReadBlockChecked() {
|
||||
size_t position = pos();
|
||||
USE(position);
|
||||
bool success = ReadBlock();
|
||||
|
||||
// Post-conditions: 1, We should always be at the right position.
|
||||
// 2, Cursor should be inside the buffer.
|
||||
// 3, We should have more characters available iff success.
|
||||
DCHECK_EQ(pos(), position);
|
||||
DCHECK_LE(buffer_cursor_, buffer_end_);
|
||||
DCHECK_LE(buffer_start_, buffer_cursor_);
|
||||
DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
|
||||
return success;
|
||||
}
|
||||
|
||||
void ReadBlockAt(size_t new_pos) {
|
||||
// The callers of this method (Back/Back2/Seek) should handle the easy
|
||||
// case (seeking within the current buffer), and we should only get here
|
||||
// if we actually require new data.
|
||||
// (This is really an efficiency check, not a correctness invariant.)
|
||||
DCHECK(new_pos < buffer_pos_ ||
|
||||
new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_));
|
||||
|
||||
// Change pos() to point to new_pos.
|
||||
buffer_pos_ = new_pos;
|
||||
buffer_cursor_ = buffer_start_;
|
||||
DCHECK_EQ(pos(), new_pos);
|
||||
ReadBlockChecked();
|
||||
}
|
||||
|
||||
// Read more data, and update buffer_*_ to point to it.
|
||||
// Returns true if more data was available.
|
||||
//
|
||||
// ReadBlock() may modify any of the buffer_*_ members, but must sure that
|
||||
// the result of pos() remains unaffected.
|
||||
//
|
||||
// Examples:
|
||||
// - a stream could either fill a separate buffer. Then buffer_start_ and
|
||||
// buffer_cursor_ would point to the beginning of the buffer, and
|
||||
// buffer_pos would be the old pos().
|
||||
// - a stream with existing buffer chunks would set buffer_start_ and
|
||||
// buffer_end_ to cover the full chunk, and then buffer_cursor_ would
|
||||
// point into the middle of the buffer, while buffer_pos_ would describe
|
||||
// the start of the buffer.
|
||||
virtual bool ReadBlock() = 0;
|
||||
|
||||
const uint16_t* buffer_start_;
|
||||
const uint16_t* buffer_cursor_;
|
||||
const uint16_t* buffer_end_;
|
||||
size_t buffer_pos_;
|
||||
};
|
||||
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// JavaScript Scanner.
|
||||
|
||||
@ -232,11 +76,11 @@ class Scanner {
|
||||
|
||||
// -1 is outside of the range of any real source code.
|
||||
static const int kNoOctalLocation = -1;
|
||||
static const uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput;
|
||||
static const uc32 kEndOfInput = ScannerStream::kEndOfInput;
|
||||
|
||||
explicit Scanner(UnicodeCache* scanner_contants);
|
||||
|
||||
void Initialize(Utf16CharacterStream* source, bool is_module);
|
||||
void Initialize(CharacterStream<uint16_t>* source, bool is_module);
|
||||
|
||||
// Returns the next token and advances input.
|
||||
Token::Value Next();
|
||||
@ -847,8 +691,8 @@ class Scanner {
|
||||
TokenDesc next_; // desc for next token (one token look-ahead)
|
||||
TokenDesc next_next_; // desc for the token after next (after PeakAhead())
|
||||
|
||||
// Input stream. Must be initialized to an Utf16CharacterStream.
|
||||
Utf16CharacterStream* source_;
|
||||
// Input stream. Must be initialized to a CharacterStream.
|
||||
CharacterStream<uint16_t>* source_;
|
||||
|
||||
// Last-seen positions of potentially problematic tokens.
|
||||
Location octal_pos_;
|
||||
|
||||
@ -12,30 +12,29 @@ namespace {
|
||||
|
||||
// Implement ExternalSourceStream based on const char**.
|
||||
// This will take each string as one chunk. The last chunk must be empty.
|
||||
|
||||
template <typename Char = char>
|
||||
class ChunkSource : public v8::ScriptCompiler::ExternalSourceStream {
|
||||
public:
|
||||
explicit ChunkSource(const char** chunks) : current_(0) {
|
||||
explicit ChunkSource(const Char** chunks) : current_(0) {
|
||||
do {
|
||||
chunks_.push_back(
|
||||
{reinterpret_cast<const uint8_t*>(*chunks), strlen(*chunks)});
|
||||
chunks_.push_back({*chunks, string_length(*chunks)});
|
||||
chunks++;
|
||||
} while (chunks_.back().len > 0);
|
||||
}
|
||||
explicit ChunkSource(const char* chunks) : current_(0) {
|
||||
explicit ChunkSource(const Char* chunks) : current_(0) {
|
||||
do {
|
||||
chunks_.push_back(
|
||||
{reinterpret_cast<const uint8_t*>(chunks), strlen(chunks)});
|
||||
chunks += strlen(chunks) + 1;
|
||||
size_t length = string_length(chunks);
|
||||
chunks_.push_back({chunks, length});
|
||||
chunks += length + 1;
|
||||
} while (chunks_.back().len > 0);
|
||||
}
|
||||
ChunkSource(const uint8_t* data, size_t char_size, size_t len,
|
||||
bool extra_chunky)
|
||||
: current_(0) {
|
||||
ChunkSource(const Char* data, size_t len, bool extra_chunky) : current_(0) {
|
||||
// If extra_chunky, we'll use increasingly large chunk sizes. If not, we'll
|
||||
// have a single chunk of full length. Make sure that chunks are always
|
||||
// aligned to char-size though.
|
||||
size_t chunk_size = extra_chunky ? char_size : len;
|
||||
for (size_t i = 0; i < len; i += chunk_size, chunk_size += char_size) {
|
||||
size_t chunk_size = extra_chunky ? 1 : len;
|
||||
for (size_t i = 0; i < len; i += chunk_size, chunk_size += 1) {
|
||||
chunks_.push_back({data + i, i::Min(chunk_size, len - i)});
|
||||
}
|
||||
chunks_.push_back({nullptr, 0});
|
||||
@ -46,15 +45,23 @@ class ChunkSource : public v8::ScriptCompiler::ExternalSourceStream {
|
||||
size_t GetMoreData(const uint8_t** src) override {
|
||||
DCHECK_LT(current_, chunks_.size());
|
||||
Chunk& next = chunks_[current_++];
|
||||
uint8_t* chunk = new uint8_t[next.len];
|
||||
i::MemMove(chunk, next.ptr, next.len);
|
||||
*src = chunk;
|
||||
return next.len;
|
||||
const uint8_t* chunk = reinterpret_cast<const uint8_t*>(next.ptr);
|
||||
size_t bytelength = next.len * sizeof(Char);
|
||||
uint8_t* copy = new uint8_t[bytelength];
|
||||
i::MemMove(copy, chunk, bytelength);
|
||||
*src = copy;
|
||||
return bytelength;
|
||||
}
|
||||
|
||||
private:
|
||||
static size_t string_length(const Char* string) {
|
||||
size_t length = 0;
|
||||
while (*(string++) != 0) length++;
|
||||
return length;
|
||||
}
|
||||
|
||||
struct Chunk {
|
||||
const uint8_t* ptr;
|
||||
const Char* ptr;
|
||||
size_t len;
|
||||
};
|
||||
std::vector<Chunk> chunks_;
|
||||
@ -105,16 +112,15 @@ const uint16_t unicode_ucs2[] = {97, 98, 99, 228, 10784, 55357,
|
||||
|
||||
TEST(Utf8StreamAsciiOnly) {
|
||||
const char* chunks[] = {"abc", "def", "ghi", ""};
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
ChunkSource<char> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(i::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
|
||||
// Read the data without dying.
|
||||
v8::internal::uc32 c;
|
||||
do {
|
||||
c = stream->Advance();
|
||||
} while (c != v8::internal::Utf16CharacterStream::kEndOfInput);
|
||||
} while (c != i::ScannerStream::kEndOfInput);
|
||||
}
|
||||
|
||||
TEST(Utf8StreamBOM) {
|
||||
@ -123,16 +129,15 @@ TEST(Utf8StreamBOM) {
|
||||
strncpy(data + 3, unicode_utf8, arraysize(unicode_utf8));
|
||||
|
||||
const char* chunks[] = {data, "\0"};
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
ChunkSource<char> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
|
||||
// Read the data without tripping over the BOM.
|
||||
for (size_t i = 0; unicode_ucs2[i]; i++) {
|
||||
CHECK_EQ(unicode_ucs2[i], stream->Advance());
|
||||
}
|
||||
CHECK_EQ(v8::internal::Utf16CharacterStream::kEndOfInput, stream->Advance());
|
||||
CHECK_EQ(i::ScannerStream::kEndOfInput, stream->Advance());
|
||||
|
||||
// Make sure seek works.
|
||||
stream->Seek(0);
|
||||
@ -142,7 +147,7 @@ TEST(Utf8StreamBOM) {
|
||||
CHECK_EQ(unicode_ucs2[5], stream->Advance());
|
||||
|
||||
// Try again, but make sure we have to seek 'backwards'.
|
||||
while (v8::internal::Utf16CharacterStream::kEndOfInput != stream->Advance()) {
|
||||
while (i::ScannerStream::kEndOfInput != stream->Advance()) {
|
||||
// Do nothing. We merely advance the stream to the end of its input.
|
||||
}
|
||||
stream->Seek(5);
|
||||
@ -157,10 +162,9 @@ TEST(Utf8SplitBOM) {
|
||||
|
||||
{
|
||||
const char* chunks[] = {partial_bom, data, "\0"};
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
ChunkSource<char> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
|
||||
// Read the data without tripping over the BOM.
|
||||
for (size_t i = 0; unicode_ucs2[i]; i++) {
|
||||
@ -173,10 +177,9 @@ TEST(Utf8SplitBOM) {
|
||||
char bom_byte_2[] = "\xbb";
|
||||
{
|
||||
const char* chunks[] = {bom_byte_1, bom_byte_2, data, "\0"};
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
ChunkSource<char> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
|
||||
// Read the data without tripping over the BOM.
|
||||
for (size_t i = 0; unicode_ucs2[i]; i++) {
|
||||
@ -185,25 +188,27 @@ TEST(Utf8SplitBOM) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Utf8AdvanceUntil) {
|
||||
TEST(Ucs2AdvanceUntil) {
|
||||
// Test utf-8 advancing until a certain char.
|
||||
|
||||
const char line_term = '\n';
|
||||
const size_t kLen = arraysize(unicode_utf8);
|
||||
char data[kLen + 1];
|
||||
strncpy(data, unicode_utf8, kLen);
|
||||
const size_t kLen = arraysize(unicode_ucs2) - 1;
|
||||
uint16_t data[kLen + 1];
|
||||
memcpy(data, unicode_ucs2, kLen * 2);
|
||||
data[kLen - 1] = line_term;
|
||||
data[kLen] = '\0';
|
||||
|
||||
{
|
||||
const char* chunks[] = {data, "\0"};
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
uint16_t end[] = {0};
|
||||
const uint16_t* chunks[] = {data, end};
|
||||
ChunkSource<uint16_t> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
|
||||
|
||||
int32_t res = stream->AdvanceUntil(
|
||||
[](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
|
||||
int32_t res = static_cast<i::CharacterStream<uint16_t>*>(stream.get())
|
||||
->AdvanceUntil([](int32_t c0) {
|
||||
return unibrow::IsLineTerminator(c0);
|
||||
});
|
||||
CHECK_EQ(line_term, res);
|
||||
}
|
||||
}
|
||||
@ -215,68 +220,71 @@ TEST(AdvanceMatchAdvanceUntil) {
|
||||
|
||||
{
|
||||
const char* chunks[] = {data, "\0"};
|
||||
ChunkSource chunk_source_a(chunks);
|
||||
ChunkSource<char> chunk_source_a(chunks);
|
||||
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream_advance(
|
||||
std::unique_ptr<v8::internal::ScannerStream> stream_advance(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source_a, v8::ScriptCompiler::StreamedSource::UTF8,
|
||||
&chunk_source_a, v8::ScriptCompiler::StreamedSource::ONE_BYTE,
|
||||
nullptr));
|
||||
|
||||
ChunkSource chunk_source_au(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream_advance_until(
|
||||
ChunkSource<char> chunk_source_au(chunks);
|
||||
std::unique_ptr<v8::internal::ScannerStream> stream_advance_until(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source_au, v8::ScriptCompiler::StreamedSource::UTF8,
|
||||
&chunk_source_au, v8::ScriptCompiler::StreamedSource::ONE_BYTE,
|
||||
nullptr));
|
||||
|
||||
int32_t au_c0_ = stream_advance_until->AdvanceUntil(
|
||||
[](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
|
||||
int32_t au_c0 =
|
||||
static_cast<i::CharacterStream<uint16_t>*>(stream_advance_until.get())
|
||||
->AdvanceUntil(
|
||||
[](int32_t c0) { return unibrow::IsLineTerminator(c0); });
|
||||
|
||||
int32_t a_c0_ = '0';
|
||||
while (!unibrow::IsLineTerminator(a_c0_)) {
|
||||
a_c0_ = stream_advance->Advance();
|
||||
int32_t a_c0 = '0';
|
||||
while (!unibrow::IsLineTerminator(a_c0)) {
|
||||
a_c0 = stream_advance->Advance();
|
||||
}
|
||||
|
||||
// Check both advances methods have the same output
|
||||
CHECK_EQ(a_c0_, au_c0_);
|
||||
CHECK_EQ(a_c0, au_c0);
|
||||
|
||||
// Check if both set the cursor to the correct position by advancing both
|
||||
// streams by one character.
|
||||
a_c0_ = stream_advance->Advance();
|
||||
au_c0_ = stream_advance_until->Advance();
|
||||
CHECK_EQ(a_c0_, au_c0_);
|
||||
a_c0 = stream_advance->Advance();
|
||||
au_c0 = stream_advance_until->Advance();
|
||||
CHECK_EQ(a_c0, au_c0);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Utf8AdvanceUntilOverChunkBoundaries) {
|
||||
TEST(Ucs2AdvanceUntilOverChunkBoundaries) {
|
||||
// Test utf-8 advancing until a certain char, crossing chunk boundaries.
|
||||
|
||||
// Split the test string at each byte and pass it to the stream. This way,
|
||||
// we'll have a split at each possible boundary.
|
||||
size_t len = strlen(unicode_utf8);
|
||||
char buffer[arraysize(unicode_utf8) + 4];
|
||||
for (size_t i = 1; i < len; i++) {
|
||||
const size_t kLen = arraysize(unicode_ucs2) - 1;
|
||||
uint16_t buffer[kLen + 4];
|
||||
for (size_t i = 1; i < kLen; i++) {
|
||||
// Copy source string into buffer, splitting it at i.
|
||||
// Then add three chunks, 0..i-1, i..strlen-1, empty.
|
||||
strncpy(buffer, unicode_utf8, i);
|
||||
strncpy(buffer + i + 1, unicode_utf8 + i, len - i);
|
||||
memcpy(buffer, unicode_ucs2, i * 2);
|
||||
memcpy(buffer + i + 1, unicode_ucs2 + i, (kLen - i) * 2);
|
||||
buffer[i] = '\0';
|
||||
buffer[len + 1] = '\n';
|
||||
buffer[len + 2] = '\0';
|
||||
buffer[len + 3] = '\0';
|
||||
const char* chunks[] = {buffer, buffer + i + 1, buffer + len + 2};
|
||||
buffer[kLen + 1] = '\n';
|
||||
buffer[kLen + 2] = '\0';
|
||||
buffer[kLen + 3] = '\0';
|
||||
const uint16_t* chunks[] = {buffer, buffer + i + 1, buffer + kLen + 2};
|
||||
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
ChunkSource<uint16_t> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(i::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
|
||||
|
||||
int32_t res = stream->AdvanceUntil(
|
||||
[](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
|
||||
CHECK_EQ(buffer[len + 1], res);
|
||||
int32_t res = static_cast<i::CharacterStream<uint16_t>*>(stream.get())
|
||||
->AdvanceUntil([](int32_t c0) {
|
||||
return unibrow::IsLineTerminator(c0);
|
||||
});
|
||||
CHECK_EQ(buffer[kLen + 1], res);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Utf8ChunkBoundaries) {
|
||||
TEST(Ucs2ChunkBoundaries) {
|
||||
// Test utf-8 parsing at chunk boundaries.
|
||||
|
||||
// Split the test string at each byte and pass it to the stream. This way,
|
||||
@ -293,16 +301,14 @@ TEST(Utf8ChunkBoundaries) {
|
||||
buffer[len + 2] = '\0';
|
||||
const char* chunks[] = {buffer, buffer + i + 1, buffer + len + 2};
|
||||
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
ChunkSource<char> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
|
||||
for (size_t i = 0; unicode_ucs2[i]; i++) {
|
||||
CHECK_EQ(unicode_ucs2[i], stream->Advance());
|
||||
}
|
||||
CHECK_EQ(v8::internal::Utf16CharacterStream::kEndOfInput,
|
||||
stream->Advance());
|
||||
CHECK_EQ(i::ScannerStream::kEndOfInput, stream->Advance());
|
||||
}
|
||||
}
|
||||
|
||||
@ -322,22 +328,20 @@ TEST(Utf8SingleByteChunks) {
|
||||
const char* chunks[] = {buffer, buffer + i + 1, buffer + i + 3,
|
||||
buffer + len + 3};
|
||||
|
||||
ChunkSource chunk_source(chunks);
|
||||
std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
|
||||
v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
ChunkSource<char> chunk_source(chunks);
|
||||
std::unique_ptr<i::ScannerStream> stream(v8::internal::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
|
||||
for (size_t j = 0; unicode_ucs2[j]; j++) {
|
||||
CHECK_EQ(unicode_ucs2[j], stream->Advance());
|
||||
}
|
||||
CHECK_EQ(v8::internal::Utf16CharacterStream::kEndOfInput,
|
||||
stream->Advance());
|
||||
CHECK_EQ(i::ScannerStream::kEndOfInput, stream->Advance());
|
||||
}
|
||||
}
|
||||
|
||||
#define CHECK_EQU(v1, v2) CHECK_EQ(static_cast<int>(v1), static_cast<int>(v2))
|
||||
|
||||
void TestCharacterStream(const char* reference, i::Utf16CharacterStream* stream,
|
||||
void TestCharacterStream(const char* reference, i::ScannerStream* stream,
|
||||
unsigned length, unsigned start, unsigned end) {
|
||||
// Read streams one char at a time
|
||||
unsigned i;
|
||||
@ -346,7 +350,7 @@ void TestCharacterStream(const char* reference, i::Utf16CharacterStream* stream,
|
||||
CHECK_EQU(reference[i], stream->Advance());
|
||||
}
|
||||
CHECK_EQU(end, stream->pos());
|
||||
CHECK_EQU(i::Utf16CharacterStream::kEndOfInput, stream->Advance());
|
||||
CHECK_EQU(i::ScannerStream::kEndOfInput, stream->Advance());
|
||||
CHECK_EQU(end + 1, stream->pos());
|
||||
stream->Back();
|
||||
|
||||
@ -407,7 +411,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
|
||||
TestExternalResource resource(uc16_buffer.get(), length);
|
||||
i::Handle<i::String> uc16_string(
|
||||
factory->NewExternalStringFromTwoByte(&resource).ToHandleChecked());
|
||||
std::unique_ptr<i::Utf16CharacterStream> uc16_stream(
|
||||
std::unique_ptr<i::ScannerStream> uc16_stream(
|
||||
i::ScannerStream::For(isolate, uc16_string, start, end));
|
||||
TestCharacterStream(one_byte_source, uc16_stream.get(), length, start, end);
|
||||
|
||||
@ -427,7 +431,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
|
||||
i::Handle<i::String> ext_one_byte_string(
|
||||
factory->NewExternalStringFromOneByte(&one_byte_resource)
|
||||
.ToHandleChecked());
|
||||
std::unique_ptr<i::Utf16CharacterStream> one_byte_stream(
|
||||
std::unique_ptr<i::ScannerStream> one_byte_stream(
|
||||
i::ScannerStream::For(isolate, ext_one_byte_string, start, end));
|
||||
TestCharacterStream(one_byte_source, one_byte_stream.get(), length, start,
|
||||
end);
|
||||
@ -439,7 +443,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
|
||||
|
||||
// 1-byte generic i::String
|
||||
{
|
||||
std::unique_ptr<i::Utf16CharacterStream> string_stream(
|
||||
std::unique_ptr<i::ScannerStream> string_stream(
|
||||
i::ScannerStream::For(isolate, one_byte_string, start, end));
|
||||
TestCharacterStream(one_byte_source, string_stream.get(), length, start,
|
||||
end);
|
||||
@ -449,7 +453,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
|
||||
{
|
||||
i::Handle<i::String> two_byte_string =
|
||||
factory->NewStringFromTwoByte(two_byte_vector).ToHandleChecked();
|
||||
std::unique_ptr<i::Utf16CharacterStream> two_byte_string_stream(
|
||||
std::unique_ptr<i::ScannerStream> two_byte_string_stream(
|
||||
i::ScannerStream::For(isolate, two_byte_string, start, end));
|
||||
TestCharacterStream(one_byte_source, two_byte_string_stream.get(), length,
|
||||
start, end);
|
||||
@ -461,18 +465,18 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
|
||||
|
||||
// 1-byte streaming stream, single + many chunks.
|
||||
{
|
||||
const uint8_t* data = one_byte_vector.begin();
|
||||
const uint8_t* data_end = one_byte_vector.end();
|
||||
const char* data = reinterpret_cast<const char*>(one_byte_vector.begin());
|
||||
const char* data_end = reinterpret_cast<const char*>(one_byte_vector.end());
|
||||
|
||||
ChunkSource single_chunk(data, 1, data_end - data, false);
|
||||
std::unique_ptr<i::Utf16CharacterStream> one_byte_streaming_stream(
|
||||
ChunkSource<char> single_chunk(data, data_end - data, false);
|
||||
std::unique_ptr<i::ScannerStream> one_byte_streaming_stream(
|
||||
i::ScannerStream::For(&single_chunk,
|
||||
v8::ScriptCompiler::StreamedSource::ONE_BYTE,
|
||||
nullptr));
|
||||
TestCharacterStream(one_byte_source, one_byte_streaming_stream.get(),
|
||||
length, start, end);
|
||||
|
||||
ChunkSource many_chunks(data, 1, data_end - data, true);
|
||||
ChunkSource<char> many_chunks(data, data_end - data, true);
|
||||
one_byte_streaming_stream.reset(i::ScannerStream::For(
|
||||
&many_chunks, v8::ScriptCompiler::StreamedSource::ONE_BYTE, nullptr));
|
||||
TestCharacterStream(one_byte_source, one_byte_streaming_stream.get(),
|
||||
@ -481,16 +485,17 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
|
||||
|
||||
// UTF-8 streaming stream, single + many chunks.
|
||||
{
|
||||
const uint8_t* data = one_byte_vector.begin();
|
||||
const uint8_t* data_end = one_byte_vector.end();
|
||||
ChunkSource chunks(data, 1, data_end - data, false);
|
||||
std::unique_ptr<i::Utf16CharacterStream> utf8_streaming_stream(
|
||||
const char* data = reinterpret_cast<const char*>(one_byte_vector.begin());
|
||||
const char* data_end = reinterpret_cast<const char*>(one_byte_vector.end());
|
||||
|
||||
ChunkSource<char> chunks(data, data_end - data, false);
|
||||
std::unique_ptr<i::ScannerStream> utf8_streaming_stream(
|
||||
i::ScannerStream::For(&chunks, v8::ScriptCompiler::StreamedSource::UTF8,
|
||||
nullptr));
|
||||
TestCharacterStream(one_byte_source, utf8_streaming_stream.get(), length,
|
||||
start, end);
|
||||
|
||||
ChunkSource many_chunks(data, 1, data_end - data, true);
|
||||
ChunkSource<char> many_chunks(data, data_end - data, true);
|
||||
utf8_streaming_stream.reset(i::ScannerStream::For(
|
||||
&many_chunks, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
TestCharacterStream(one_byte_source, utf8_streaming_stream.get(), length,
|
||||
@ -499,18 +504,18 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
|
||||
|
||||
// 2-byte streaming stream, single + many chunks.
|
||||
{
|
||||
const uint8_t* data =
|
||||
reinterpret_cast<const uint8_t*>(two_byte_vector.begin());
|
||||
const uint8_t* data_end =
|
||||
reinterpret_cast<const uint8_t*>(two_byte_vector.end());
|
||||
ChunkSource chunks(data, 2, data_end - data, false);
|
||||
std::unique_ptr<i::Utf16CharacterStream> two_byte_streaming_stream(
|
||||
const uint16_t* data =
|
||||
reinterpret_cast<const uint16_t*>(two_byte_vector.begin());
|
||||
const uint16_t* data_end =
|
||||
reinterpret_cast<const uint16_t*>(two_byte_vector.end());
|
||||
ChunkSource<uint16_t> chunks(data, data_end - data, false);
|
||||
std::unique_ptr<i::ScannerStream> two_byte_streaming_stream(
|
||||
i::ScannerStream::For(
|
||||
&chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
|
||||
TestCharacterStream(one_byte_source, two_byte_streaming_stream.get(),
|
||||
length, start, end);
|
||||
|
||||
ChunkSource many_chunks(data, 2, data_end - data, true);
|
||||
ChunkSource<uint16_t> many_chunks(data, data_end - data, true);
|
||||
two_byte_streaming_stream.reset(i::ScannerStream::For(
|
||||
&many_chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
|
||||
TestCharacterStream(one_byte_source, two_byte_streaming_stream.get(),
|
||||
@ -541,7 +546,7 @@ TEST(CharacterStreams) {
|
||||
|
||||
// Regression test for crbug.com/651333. Read invalid utf-8.
|
||||
TEST(Regress651333) {
|
||||
const uint8_t bytes[] =
|
||||
const char bytes[] =
|
||||
"A\xf1"
|
||||
"ad"; // Anad, with n == n-with-tilde.
|
||||
const uint16_t unicode[] = {65, 65533, 97, 100};
|
||||
@ -552,13 +557,13 @@ TEST(Regress651333) {
|
||||
// Read len bytes from bytes, and compare against the expected unicode
|
||||
// characters. Expect kBadChar ( == Unicode replacement char == code point
|
||||
// 65533) instead of the incorrectly coded Latin1 char.
|
||||
ChunkSource chunks(bytes, 1, len, false);
|
||||
std::unique_ptr<i::Utf16CharacterStream> stream(i::ScannerStream::For(
|
||||
ChunkSource<char> chunks(bytes, len, false);
|
||||
std::unique_ptr<i::ScannerStream> stream(i::ScannerStream::For(
|
||||
&chunks, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
CHECK_EQ(unicode[i], stream->Advance());
|
||||
}
|
||||
CHECK_EQ(i::Utf16CharacterStream::kEndOfInput, stream->Advance());
|
||||
CHECK_EQ(i::ScannerStream::kEndOfInput, stream->Advance());
|
||||
}
|
||||
}
|
||||
|
||||
@ -566,18 +571,18 @@ void TestChunkStreamAgainstReference(
|
||||
const char* cases[],
|
||||
const std::vector<std::vector<uint16_t>>& unicode_expected) {
|
||||
for (size_t c = 0; c < unicode_expected.size(); ++c) {
|
||||
ChunkSource chunk_source(cases[c]);
|
||||
std::unique_ptr<i::Utf16CharacterStream> stream(i::ScannerStream::For(
|
||||
ChunkSource<char> chunk_source(cases[c]);
|
||||
std::unique_ptr<i::ScannerStream> stream(i::ScannerStream::For(
|
||||
&chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
|
||||
for (size_t i = 0; i < unicode_expected[c].size(); i++) {
|
||||
CHECK_EQ(unicode_expected[c][i], stream->Advance());
|
||||
}
|
||||
CHECK_EQ(i::Utf16CharacterStream::kEndOfInput, stream->Advance());
|
||||
CHECK_EQ(i::ScannerStream::kEndOfInput, stream->Advance());
|
||||
stream->Seek(0);
|
||||
for (size_t i = 0; i < unicode_expected[c].size(); i++) {
|
||||
CHECK_EQ(unicode_expected[c][i], stream->Advance());
|
||||
}
|
||||
CHECK_EQ(i::Utf16CharacterStream::kEndOfInput, stream->Advance());
|
||||
CHECK_EQ(i::ScannerStream::kEndOfInput, stream->Advance());
|
||||
}
|
||||
}
|
||||
|
||||
@ -669,7 +674,7 @@ TEST(RelocatingCharacterStream) {
|
||||
i_isolate->factory()
|
||||
->NewStringFromTwoByte(two_byte_vector, i::NOT_TENURED)
|
||||
.ToHandleChecked();
|
||||
std::unique_ptr<i::Utf16CharacterStream> two_byte_string_stream(
|
||||
std::unique_ptr<i::ScannerStream> two_byte_string_stream(
|
||||
i::ScannerStream::For(i_isolate, two_byte_string, 0, length));
|
||||
CHECK_EQ('a', two_byte_string_stream->Advance());
|
||||
CHECK_EQ('b', two_byte_string_stream->Advance());
|
||||
|
||||
@ -27,7 +27,7 @@ struct ScannerTestHelper {
|
||||
scanner(std::move(other.scanner)) {}
|
||||
|
||||
std::unique_ptr<UnicodeCache> unicode_cache;
|
||||
std::unique_ptr<Utf16CharacterStream> stream;
|
||||
std::unique_ptr<CharacterStream<uint16_t>> stream;
|
||||
std::unique_ptr<Scanner> scanner;
|
||||
|
||||
Scanner* operator->() const { return scanner.get(); }
|
||||
|
||||
@ -396,8 +396,7 @@ TEST(PreParseOverflow) {
|
||||
CHECK_EQ(i::PreParser::kPreParseStackOverflow, result);
|
||||
}
|
||||
|
||||
|
||||
void TestStreamScanner(i::Utf16CharacterStream* stream,
|
||||
void TestStreamScanner(i::CharacterStream<uint16_t>* stream,
|
||||
i::Token::Value* expected_tokens,
|
||||
int skip_pos = 0, // Zero means not skipping.
|
||||
int skip_to = 0) {
|
||||
@ -420,8 +419,7 @@ void TestStreamScanner(i::Utf16CharacterStream* stream,
|
||||
TEST(StreamScanner) {
|
||||
v8::V8::Initialize();
|
||||
const char* str1 = "{ foo get for : */ <- \n\n /*foo*/ bib";
|
||||
std::unique_ptr<i::Utf16CharacterStream> stream1(
|
||||
i::ScannerStream::ForTesting(str1));
|
||||
auto stream1(i::ScannerStream::ForTesting(str1));
|
||||
i::Token::Value expectations1[] = {
|
||||
i::Token::LBRACE,
|
||||
i::Token::IDENTIFIER,
|
||||
@ -439,8 +437,7 @@ TEST(StreamScanner) {
|
||||
TestStreamScanner(stream1.get(), expectations1, 0, 0);
|
||||
|
||||
const char* str2 = "case default const {THIS\nPART\nSKIPPED} do";
|
||||
std::unique_ptr<i::Utf16CharacterStream> stream2(
|
||||
i::ScannerStream::ForTesting(str2));
|
||||
auto stream2(i::ScannerStream::ForTesting(str2));
|
||||
i::Token::Value expectations2[] = {
|
||||
i::Token::CASE,
|
||||
i::Token::DEFAULT,
|
||||
@ -470,8 +467,7 @@ TEST(StreamScanner) {
|
||||
for (int i = 0; i <= 4; i++) {
|
||||
expectations3[6 - i] = i::Token::ILLEGAL;
|
||||
expectations3[5 - i] = i::Token::EOS;
|
||||
std::unique_ptr<i::Utf16CharacterStream> stream3(
|
||||
i::ScannerStream::ForTesting(str3));
|
||||
auto stream3(i::ScannerStream::ForTesting(str3));
|
||||
TestStreamScanner(stream3.get(), expectations3, 1, 1 + i);
|
||||
}
|
||||
}
|
||||
@ -1159,6 +1155,7 @@ void SetParserFlags(i::PreParser* parser, i::EnumSet<ParserFlag> flags) {
|
||||
flags.Contains(kAllowHarmonyNumericSeparator));
|
||||
}
|
||||
|
||||
template <typename Char>
|
||||
void TestParserSyncWithFlags(i::Handle<i::String> source,
|
||||
i::EnumSet<ParserFlag> flags,
|
||||
ParserSyncTestResult result,
|
||||
@ -1173,7 +1170,7 @@ void TestParserSyncWithFlags(i::Handle<i::String> source,
|
||||
i::PendingCompilationErrorHandler pending_error_handler;
|
||||
if (test_preparser) {
|
||||
i::Scanner scanner(isolate->unicode_cache());
|
||||
std::unique_ptr<i::Utf16CharacterStream> stream(
|
||||
std::unique_ptr<i::ScannerStream> stream(
|
||||
i::ScannerStream::For(isolate, source));
|
||||
i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
|
||||
i::AstValueFactory ast_value_factory(
|
||||
@ -1184,7 +1181,8 @@ void TestParserSyncWithFlags(i::Handle<i::String> source,
|
||||
isolate->counters()->runtime_call_stats(),
|
||||
isolate->logger(), -1, is_module);
|
||||
SetParserFlags(&preparser, flags);
|
||||
scanner.Initialize(stream.get(), is_module);
|
||||
scanner.Initialize(static_cast<CharacterStream<Char>*>(stream.get()),
|
||||
is_module);
|
||||
i::PreParser::PreParseResult result = preparser.PreParseProgram();
|
||||
CHECK_EQ(i::PreParser::kPreParseSuccess, result);
|
||||
}
|
||||
@ -1294,8 +1292,15 @@ void TestParserSync(const char* source, const ParserFlag* varying_flags,
|
||||
++flag_index) {
|
||||
flags.Remove(always_false_flags[flag_index]);
|
||||
}
|
||||
TestParserSyncWithFlags(str, flags, result, is_module, test_preparser,
|
||||
ignore_error_msg);
|
||||
if (str->IsSeqOneByteString()) {
|
||||
// TODO(verwaest): Switch to uint8_t.
|
||||
TestParserSyncWithFlags<uint16_t>(str, flags, result, is_module,
|
||||
test_preparser, ignore_error_msg);
|
||||
} else {
|
||||
DCHECK(str->IsSeqTwoByteString());
|
||||
TestParserSyncWithFlags<uint16_t>(str, flags, result, is_module,
|
||||
test_preparser, ignore_error_msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -17,7 +17,8 @@ class AsmJsScannerTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetupScanner(const char* source) {
|
||||
stream = ScannerStream::ForTesting(source);
|
||||
scanner.reset(new AsmJsScanner(stream.get()));
|
||||
scanner.reset(new AsmJsScanner(
|
||||
static_cast<CharacterStream<uint16_t>*>(stream.get()), 0));
|
||||
}
|
||||
|
||||
void Skip(AsmJsScanner::token_t t) {
|
||||
@ -41,7 +42,7 @@ class AsmJsScannerTest : public ::testing::Test {
|
||||
CHECK_EQ(scanner->Token(), AsmJsScanner::kParseError);
|
||||
}
|
||||
|
||||
std::unique_ptr<Utf16CharacterStream> stream;
|
||||
std::unique_ptr<ScannerStream> stream;
|
||||
std::unique_ptr<AsmJsScanner> scanner;
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user