AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[regexp] implement character classes for unicode regexps.

Browse Source 
We divide character ranges into
- BMP, matched normally.
- non-BMP, matched as alternatives of surrogate pair ranges.
- lone surrogates, matched with lookaround assertion that its indeed lone.

R=erik.corry@gmail.com
BUG=v8:2952
LOG=N

Committed: https://crrev.com/ea820ad5fa282a323a86fe20e64f83ee67ba5f04
Cr-Commit-Position: refs/heads/master@{#33432}

Review URL: https://codereview.chromium.org/1578253005

Cr-Commit-Position: refs/heads/master@{#33437}
This commit is contained in:
yangguo 2016-01-21 05:33:02 -08:00 committed by Commit bot
parent fe19b11e53
commit e709aa24c0
12 changed files with 929 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/objects.h
View File

@ -8880,6 +8880,7 @@ class String: public Name {
static const uint32_t kMaxOneByteCharCodeU = unibrow::Latin1::kMaxChar;
static const int kMaxUtf16CodeUnit = 0xffff;
static const uint32_t kMaxUtf16CodeUnitU = kMaxUtf16CodeUnit;
static const uc32 kMaxCodePoint = 0x10ffff;
// Value of hash field containing computed hash equal to zero.
static const int kEmptyStringHash = kIsNotArrayIndexMask;

16
src/ostreams.cc
View File

@ -3,6 +3,7 @@
// found in the LICENSE file.
#include "src/ostreams.h"
#include "src/objects.h"
#if V8_OS_WIN
#if _MSC_VER < 1900
@ -60,6 +61,16 @@ std::ostream& PrintUC16(std::ostream& os, uint16_t c, bool (*pred)(uint16_t)) {
return os << buf;
}
std::ostream& PrintUC32(std::ostream& os, int32_t c, bool (*pred)(uint16_t)) {
if (c <= String::kMaxUtf16CodeUnit) {
return PrintUC16(os, static_cast<uint16_t>(c), pred);
}
char buf[13];
snprintf(buf, sizeof(buf), "\\u{%06x}", c);
return os << buf;
}
} // namespace
@ -81,5 +92,10 @@ std::ostream& operator<<(std::ostream& os, const AsUC16& c) {
return PrintUC16(os, c.value, IsPrint);
}
std::ostream& operator<<(std::ostream& os, const AsUC32& c) {
return PrintUC32(os, c.value, IsPrint);
}
} // namespace internal
} // namespace v8

10
src/ostreams.h
View File

@ -50,6 +50,12 @@ struct AsUC16 {
};
struct AsUC32 {
explicit AsUC32(int32_t v) : value(v) {}
int32_t value;
};
struct AsReversiblyEscapedUC16 {
explicit AsReversiblyEscapedUC16(uint16_t v) : value(v) {}
uint16_t value;
@ -73,6 +79,10 @@ std::ostream& operator<<(std::ostream& os, const AsEscapedUC16ForJSON& c);
// of printable ASCII range.
std::ostream& operator<<(std::ostream& os, const AsUC16& c);
// Writes the given character to the output escaping everything outside
// of printable ASCII range.
std::ostream& operator<<(std::ostream& os, const AsUC32& c);
} // namespace internal
} // namespace v8

559
src/regexp/jsregexp.cc
View File

@ -72,7 +72,7 @@ ContainedInLattice AddRange(ContainedInLattice containment,
int ranges_length,
Interval new_range) {
DCHECK((ranges_length & 1) == 1);
DCHECK(ranges[ranges_length - 1] == String::kMaxUtf16CodeUnit + 1);
DCHECK(ranges[ranges_length - 1] == String::kMaxCodePoint + 1);
if (containment == kLatticeUnknown) return containment;
bool inside = false;
int last = 0;
@ -145,9 +145,8 @@ MaybeHandle<Object> RegExpImpl::Compile(Handle<JSRegExp> re,
PostponeInterruptsScope postpone(isolate);
RegExpCompileData parse_result;
FlatStringReader reader(isolate, pattern);
if (!RegExpParser::ParseRegExp(re->GetIsolate(), &zone, &reader,
flags & JSRegExp::kMultiline,
flags & JSRegExp::kUnicode, &parse_result)) {
if (!RegExpParser::ParseRegExp(re->GetIsolate(), &zone, &reader, flags,
&parse_result)) {
// Throw an exception if we fail to parse the pattern.
return ThrowRegExpException(re, pattern, parse_result.error);
}
@ -371,18 +370,16 @@ bool RegExpImpl::CompileIrregexp(Handle<JSRegExp> re,
pattern = String::Flatten(pattern);
RegExpCompileData compile_data;
FlatStringReader reader(isolate, pattern);
if (!RegExpParser::ParseRegExp(isolate, &zone, &reader,
flags & JSRegExp::kMultiline,
flags & JSRegExp::kUnicode, &compile_data)) {
if (!RegExpParser::ParseRegExp(isolate, &zone, &reader, flags,
&compile_data)) {
// Throw an exception if we fail to parse the pattern.
// THIS SHOULD NOT HAPPEN. We already pre-parsed it successfully once.
USE(ThrowRegExpException(re, pattern, compile_data.error));
return false;
}
RegExpEngine::CompilationResult result = RegExpEngine::Compile(
isolate, &zone, &compile_data, flags & JSRegExp::kIgnoreCase,
flags & JSRegExp::kGlobal, flags & JSRegExp::kMultiline,
flags & JSRegExp::kSticky, pattern, sample_subject, is_one_byte);
RegExpEngine::CompilationResult result =
RegExpEngine::Compile(isolate, &zone, &compile_data, flags, pattern,
sample_subject, is_one_byte);
if (result.error_message != NULL) {
// Unable to compile regexp.
Handle<String> error_message = isolate->factory()->NewStringFromUtf8(
@ -945,7 +942,7 @@ class FrequencyCollator {
class RegExpCompiler {
public:
RegExpCompiler(Isolate* isolate, Zone* zone, int capture_count,
bool ignore_case, bool is_one_byte);
JSRegExp::Flags flags, bool is_one_byte);
int AllocateRegister() {
if (next_register_ >= RegExpMacroAssembler::kMaxRegister) {
@ -955,6 +952,22 @@ class RegExpCompiler {
return next_register_++;
}
// Lookarounds to match lone surrogates for unicode character class matches
// are never nested. We can therefore reuse registers.
int UnicodeLookaroundStackRegister() {
if (unicode_lookaround_stack_register_ == kNoRegister) {
unicode_lookaround_stack_register_ = AllocateRegister();
}
return unicode_lookaround_stack_register_;
}
int UnicodeLookaroundPositionRegister() {
if (unicode_lookaround_position_register_ == kNoRegister) {
unicode_lookaround_position_register_ = AllocateRegister();
}
return unicode_lookaround_position_register_;
}
RegExpEngine::CompilationResult Assemble(RegExpMacroAssembler* assembler,
RegExpNode* start,
int capture_count,
@ -981,7 +994,8 @@ class RegExpCompiler {
void SetRegExpTooBig() { reg_exp_too_big_ = true; }
inline bool ignore_case() { return ignore_case_; }
inline bool ignore_case() { return (flags_ & JSRegExp::kIgnoreCase) != 0; }
inline bool unicode() { return (flags_ & JSRegExp::kUnicode) != 0; }
inline bool one_byte() { return one_byte_; }
inline bool optimize() { return optimize_; }
inline void set_optimize(bool value) { optimize_ = value; }
@ -1006,10 +1020,12 @@ class RegExpCompiler {
private:
EndNode* accept_;
int next_register_;
int unicode_lookaround_stack_register_;
int unicode_lookaround_position_register_;
List<RegExpNode*>* work_list_;
int recursion_depth_;
RegExpMacroAssembler* macro_assembler_;
bool ignore_case_;
JSRegExp::Flags flags_;
bool one_byte_;
bool reg_exp_too_big_;
bool limiting_recursion_;
@ -1041,11 +1057,13 @@ static RegExpEngine::CompilationResult IrregexpRegExpTooBig(Isolate* isolate) {
// Attempts to compile the regexp using an Irregexp code generator. Returns
// a fixed array or a null handle depending on whether it succeeded.
RegExpCompiler::RegExpCompiler(Isolate* isolate, Zone* zone, int capture_count,
bool ignore_case, bool one_byte)
JSRegExp::Flags flags, bool one_byte)
: next_register_(2 * (capture_count + 1)),
unicode_lookaround_stack_register_(kNoRegister),
unicode_lookaround_position_register_(kNoRegister),
work_list_(NULL),
recursion_depth_(0),
ignore_case_(ignore_case),
flags_(flags),
one_byte_(one_byte),
reg_exp_too_big_(false),
limiting_recursion_(false),
@ -2098,9 +2116,7 @@ static void EmitCharClass(RegExpMacroAssembler* macro_assembler,
Label* on_failure, int cp_offset, bool check_offset,
bool preloaded, Zone* zone) {
ZoneList<CharacterRange>* ranges = cc->ranges(zone);
if (!CharacterRange::IsCanonical(ranges)) {
CharacterRange::Canonicalize(ranges);
}
CharacterRange::Canonicalize(ranges);
int max_char;
if (one_byte) {
@ -2142,23 +2158,14 @@ static void EmitCharClass(RegExpMacroAssembler* macro_assembler,
}
return;
}
if (last_valid_range == 0 &&
!cc->is_negated() &&
ranges->at(0).IsEverything(max_char)) {
// This is a common case hit by non-anchored expressions.
if (check_offset) {
macro_assembler->CheckPosition(cp_offset, on_failure);
}
return;
}
if (!preloaded) {
macro_assembler->LoadCurrentCharacter(cp_offset, on_failure, check_offset);
}
if (cc->is_standard(zone) &&
macro_assembler->CheckSpecialCharacterClass(cc->standard_type(),
on_failure)) {
macro_assembler->CheckSpecialCharacterClass(cc->standard_type(),
on_failure)) {
return;
}
@ -2798,9 +2805,7 @@ RegExpNode* TextNode::FilterOneByte(int depth, bool ignore_case) {
DCHECK(elm.text_type() == TextElement::CHAR_CLASS);
RegExpCharacterClass* cc = elm.char_class();
ZoneList<CharacterRange>* ranges = cc->ranges(zone());
if (!CharacterRange::IsCanonical(ranges)) {
CharacterRange::Canonicalize(ranges);
}
CharacterRange::Canonicalize(ranges);
// Now they are in order so we only need to look at the first.
int range_count = ranges->length();
if (cc->is_negated()) {
@ -3289,6 +3294,36 @@ bool TextNode::SkipPass(int int_pass, bool ignore_case) {
}
TextNode* TextNode::CreateForCharacterRanges(Zone* zone,
ZoneList<CharacterRange>* ranges,
bool read_backward,
RegExpNode* on_success) {
DCHECK_NOT_NULL(ranges);
ZoneList<TextElement>* elms = new (zone) ZoneList<TextElement>(1, zone);
elms->Add(
TextElement::CharClass(new (zone) RegExpCharacterClass(ranges, false)),
zone);
return new (zone) TextNode(elms, read_backward, on_success);
}
TextNode* TextNode::CreateForSurrogatePair(Zone* zone, CharacterRange lead,
CharacterRange trail,
bool read_backward,
RegExpNode* on_success) {
ZoneList<CharacterRange>* lead_ranges = CharacterRange::List(zone, lead);
ZoneList<CharacterRange>* trail_ranges = CharacterRange::List(zone, trail);
ZoneList<TextElement>* elms = new (zone) ZoneList<TextElement>(2, zone);
elms->Add(TextElement::CharClass(
new (zone) RegExpCharacterClass(lead_ranges, false)),
zone);
elms->Add(TextElement::CharClass(
new (zone) RegExpCharacterClass(trail_ranges, false)),
zone);
return new (zone) TextNode(elms, read_backward, on_success);
}
// This generates the code to match a text node. A text node can contain
// straight character sequences (possibly to be matched in a case-independent
// way) and character classes. For efficiency we do not do this in a single
@ -3405,9 +3440,7 @@ RegExpNode* TextNode::GetSuccessorOfOmnivorousTextNode(
if (elm.text_type() != TextElement::CHAR_CLASS) return NULL;
RegExpCharacterClass* node = elm.char_class();
ZoneList<CharacterRange>* ranges = node->ranges(zone());
if (!CharacterRange::IsCanonical(ranges)) {
CharacterRange::Canonicalize(ranges);
}
CharacterRange::Canonicalize(ranges);
if (node->is_negated()) {
return ranges->length() == 0 ? on_success() : NULL;
}
@ -3554,27 +3587,35 @@ class AlternativeGenerationList {
};
static const uc32 kLeadSurrogateStart = 0xd800;
static const uc32 kLeadSurrogateEnd = 0xdbff;
static const uc32 kTrailSurrogateStart = 0xdc00;
static const uc32 kTrailSurrogateEnd = 0xdfff;
static const uc32 kNonBmpStart = 0x10000;
static const uc32 kNonBmpEnd = 0x10ffff;
static const uc32 kRangeEndMarker = 0x110000;
// The '2' variant is has inclusive from and exclusive to.
// This covers \s as defined in ECMA-262 5.1, 15.10.2.12,
// which include WhiteSpace (7.2) or LineTerminator (7.3) values.
static const int kSpaceRanges[] = { '\t', '\r' + 1, ' ', ' ' + 1,
0x00A0, 0x00A1, 0x1680, 0x1681, 0x180E, 0x180F, 0x2000, 0x200B,
0x2028, 0x202A, 0x202F, 0x2030, 0x205F, 0x2060, 0x3000, 0x3001,
0xFEFF, 0xFF00, 0x10000 };
static const int kSpaceRanges[] = {
'\t', '\r' + 1, ' ', ' ' + 1, 0x00A0, 0x00A1, 0x1680, 0x1681,
0x180E, 0x180F, 0x2000, 0x200B, 0x2028, 0x202A, 0x202F, 0x2030,
0x205F, 0x2060, 0x3000, 0x3001, 0xFEFF, 0xFF00, kRangeEndMarker};
static const int kSpaceRangeCount = arraysize(kSpaceRanges);
static const int kWordRanges[] = {
'0', '9' + 1, 'A', 'Z' + 1, '_', '_' + 1, 'a', 'z' + 1, 0x10000 };
'0', '9' + 1, 'A', 'Z' + 1, '_', '_' + 1, 'a', 'z' + 1, kRangeEndMarker};
static const int kWordRangeCount = arraysize(kWordRanges);
static const int kDigitRanges[] = { '0', '9' + 1, 0x10000 };
static const int kDigitRanges[] = {'0', '9' + 1, kRangeEndMarker};
static const int kDigitRangeCount = arraysize(kDigitRanges);
static const int kSurrogateRanges[] = { 0xd800, 0xe000, 0x10000 };
static const int kSurrogateRanges[] = {
kLeadSurrogateStart, kLeadSurrogateStart + 1, kRangeEndMarker};
static const int kSurrogateRangeCount = arraysize(kSurrogateRanges);
static const int kLineTerminatorRanges[] = { 0x000A, 0x000B, 0x000D, 0x000E,
0x2028, 0x202A, 0x10000 };
static const int kLineTerminatorRanges[] = {
0x000A, 0x000B, 0x000D, 0x000E, 0x2028, 0x202A, kRangeEndMarker};
static const int kLineTerminatorRangeCount = arraysize(kLineTerminatorRanges);
void BoyerMoorePositionInfo::Set(int character) {
SetInterval(Interval(character, character));
}
@ -4732,8 +4773,8 @@ RegExpNode* RegExpText::ToNode(RegExpCompiler* compiler,
static bool CompareInverseRanges(ZoneList<CharacterRange>* ranges,
const int* special_class,
int length) {
length--; // Remove final 0x10000.
DCHECK(special_class[length] == 0x10000);
length--; // Remove final marker.
DCHECK(special_class[length] == kRangeEndMarker);
DCHECK(ranges->length() != 0);
DCHECK(length != 0);
DCHECK(special_class[0] != 0);
@ -4763,8 +4804,8 @@ static bool CompareInverseRanges(ZoneList<CharacterRange>* ranges,
static bool CompareRanges(ZoneList<CharacterRange>* ranges,
const int* special_class,
int length) {
length--; // Remove final 0x10000.
DCHECK(special_class[length] == 0x10000);
length--; // Remove final marker.
DCHECK(special_class[length] == kRangeEndMarker);
if (ranges->length() * 2 != length) {
return false;
}
@ -4820,10 +4861,257 @@ bool RegExpCharacterClass::is_standard(Zone* zone) {
}
bool RegExpCharacterClass::NeedsDesugaringForUnicode(Zone* zone) {
ZoneList<CharacterRange>* ranges = this->ranges(zone);
CharacterRange::Canonicalize(ranges);
for (int i = ranges->length() - 1; i >= 0; i--) {
uc32 from = ranges->at(i).from();
uc32 to = ranges->at(i).to();
// Check for non-BMP characters.
if (to >= kNonBmpStart) return true;
// Check for lone surrogates.
if (from <= kTrailSurrogateEnd && to >= kLeadSurrogateStart) return true;
}
return false;
}
UnicodeRangeSplitter::UnicodeRangeSplitter(Zone* zone,
ZoneList<CharacterRange>* base)
: zone_(zone),
table_(zone),
bmp_(nullptr),
lead_surrogates_(nullptr),
trail_surrogates_(nullptr),
non_bmp_(nullptr) {
// The unicode range splitter categorizes given character ranges into:
// - Code points from the BMP representable by one code unit.
// - Code points outside the BMP that need to be split into surrogate pairs.
// - Lone lead surrogates.
// - Lone trail surrogates.
// Lone surrogates are valid code points, even though no actual characters.
// They require special matching to make sure we do not split surrogate pairs.
// We use the dispatch table to accomplish this. The base range is split up
// by the table by the overlay ranges, and the Call callback is used to
// filter and collect ranges for each category.
for (int i = 0; i < base->length(); i++) {
table_.AddRange(base->at(i), kBase, zone_);
}
// Add overlay ranges.
table_.AddRange(CharacterRange(0, kLeadSurrogateStart - 1), kBmpCodePoints,
zone_);
table_.AddRange(CharacterRange(kLeadSurrogateStart, kLeadSurrogateEnd),
kLeadSurrogates, zone_);
table_.AddRange(CharacterRange(kTrailSurrogateStart, kTrailSurrogateEnd),
kTrailSurrogates, zone_);
table_.AddRange(CharacterRange(kTrailSurrogateEnd, kNonBmpStart - 1),
kBmpCodePoints, zone_);
table_.AddRange(CharacterRange(kNonBmpStart, kNonBmpEnd), kNonBmpCodePoints,
zone_);
table_.ForEach(this);
}
void UnicodeRangeSplitter::Call(uc32 from, DispatchTable::Entry entry) {
OutSet* outset = entry.out_set();
if (!outset->Get(kBase)) return;
ZoneList<CharacterRange>** target = NULL;
if (outset->Get(kBmpCodePoints)) {
target = &bmp_;
} else if (outset->Get(kLeadSurrogates)) {
target = &lead_surrogates_;
} else if (outset->Get(kTrailSurrogates)) {
target = &trail_surrogates_;
} else {
DCHECK(outset->Get(kNonBmpCodePoints));
target = &non_bmp_;
}
if (*target == NULL) *target = new (zone_) ZoneList<CharacterRange>(2, zone_);
(*target)->Add(CharacterRange::Range(entry.from(), entry.to()), zone_);
}
void AddBmpCharacters(RegExpCompiler* compiler, ChoiceNode* result,
RegExpNode* on_success, UnicodeRangeSplitter* splitter) {
ZoneList<CharacterRange>* bmp = splitter->bmp();
if (bmp == nullptr) return;
result->AddAlternative(GuardedAlternative(TextNode::CreateForCharacterRanges(
compiler->zone(), bmp, compiler->read_backward(), on_success)));
}
void AddNonBmpSurrogatePairs(RegExpCompiler* compiler, ChoiceNode* result,
RegExpNode* on_success,
UnicodeRangeSplitter* splitter) {
ZoneList<CharacterRange>* non_bmp = splitter->non_bmp();
if (non_bmp == nullptr) return;
DCHECK(compiler->unicode());
DCHECK(!compiler->one_byte());
Zone* zone = compiler->zone();
CharacterRange::Canonicalize(non_bmp);
for (int i = 0; i < non_bmp->length(); i++) {
// Match surrogate pair.
// E.g. [\u10005-\u11005] becomes
// \ud800[\udc05-\udfff]|
// [\ud801-\ud803][\udc00-\udfff]|
// \ud804[\udc00-\udc05]
uc32 from = non_bmp->at(i).from();
uc32 to = non_bmp->at(i).to();
uc16 from_l = unibrow::Utf16::LeadSurrogate(from);
uc16 from_t = unibrow::Utf16::TrailSurrogate(from);
uc16 to_l = unibrow::Utf16::LeadSurrogate(to);
uc16 to_t = unibrow::Utf16::TrailSurrogate(to);
if (from_l == to_l) {
// The lead surrogate is the same.
result->AddAlternative(
GuardedAlternative(TextNode::CreateForSurrogatePair(
zone, CharacterRange::Singleton(from_l),
CharacterRange::Range(from_t, to_t), compiler->read_backward(),
on_success)));
} else {
if (from_t != kTrailSurrogateStart) {
// Add [from_l][from_t-\udfff]
result->AddAlternative(
GuardedAlternative(TextNode::CreateForSurrogatePair(
zone, CharacterRange::Singleton(from_l),
CharacterRange::Range(from_t, kTrailSurrogateEnd),
compiler->read_backward(), on_success)));
from_l++;
}
if (to_t != kTrailSurrogateEnd) {
// Add [to_l][\udc00-to_t]
result->AddAlternative(
GuardedAlternative(TextNode::CreateForSurrogatePair(
zone, CharacterRange::Singleton(to_l),
CharacterRange::Range(kTrailSurrogateStart, to_t),
compiler->read_backward(), on_success)));
to_l--;
}
if (from_l <= to_l) {
// Add [from_l-to_l][\udc00-\udfff]
result->AddAlternative(
GuardedAlternative(TextNode::CreateForSurrogatePair(
zone, CharacterRange::Range(from_l, to_l),
CharacterRange::Range(kTrailSurrogateStart, kTrailSurrogateEnd),
compiler->read_backward(), on_success)));
}
}
}
}
RegExpNode* NegativeLookaroundAgainstReadDirectionAndMatch(
RegExpCompiler* compiler, ZoneList<CharacterRange>* lookbehind,
ZoneList<CharacterRange>* match, RegExpNode* on_success,
bool read_backward) {
Zone* zone = compiler->zone();
RegExpNode* match_node = TextNode::CreateForCharacterRanges(
zone, match, read_backward, on_success);
int stack_register = compiler->UnicodeLookaroundStackRegister();
int position_register = compiler->UnicodeLookaroundPositionRegister();
RegExpLookaround::Builder lookaround(false, match_node, stack_register,
position_register);
RegExpNode* negative_match = TextNode::CreateForCharacterRanges(
zone, lookbehind, !read_backward, lookaround.on_match_success());
return lookaround.ForMatch(negative_match);
}
RegExpNode* MatchAndNegativeLookaroundInReadDirection(
RegExpCompiler* compiler, ZoneList<CharacterRange>* match,
ZoneList<CharacterRange>* lookahead, RegExpNode* on_success,
bool read_backward) {
Zone* zone = compiler->zone();
int stack_register = compiler->UnicodeLookaroundStackRegister();
int position_register = compiler->UnicodeLookaroundPositionRegister();
RegExpLookaround::Builder lookaround(false, on_success, stack_register,
position_register);
RegExpNode* negative_match = TextNode::CreateForCharacterRanges(
zone, lookahead, read_backward, lookaround.on_match_success());
return TextNode::CreateForCharacterRanges(
zone, match, read_backward, lookaround.ForMatch(negative_match));
}
void AddLoneLeadSurrogates(RegExpCompiler* compiler, ChoiceNode* result,
RegExpNode* on_success,
UnicodeRangeSplitter* splitter) {
ZoneList<CharacterRange>* lead_surrogates = splitter->lead_surrogates();
if (lead_surrogates == nullptr) return;
Zone* zone = compiler->zone();
// E.g. \ud801 becomes \ud801(?![\udc00-\udfff]).
ZoneList<CharacterRange>* trail_surrogates =
new (zone) ZoneList<CharacterRange>(1, zone);
trail_surrogates->Add(
CharacterRange::Range(kTrailSurrogateStart, kTrailSurrogateEnd), zone);
RegExpNode* match =
compiler->read_backward()
// Reading backward. Assert that reading forward, there is no trail
// surrogate, and then backward match the lead surrogate.
? NegativeLookaroundAgainstReadDirectionAndMatch(
compiler, trail_surrogates, lead_surrogates, on_success, true)
// Reading forward. Forwrad match the lead surrogate and assert that
// no
// trail surrogate follows.
: MatchAndNegativeLookaroundInReadDirection(
compiler, lead_surrogates, trail_surrogates, on_success, false);
result->AddAlternative(GuardedAlternative(match));
}
void AddLoneTrailSurrogates(RegExpCompiler* compiler, ChoiceNode* result,
RegExpNode* on_success,
UnicodeRangeSplitter* splitter) {
ZoneList<CharacterRange>* trail_surrogates = splitter->trail_surrogates();
if (trail_surrogates == nullptr) return;
Zone* zone = compiler->zone();
// E.g. \udc01 becomes (?<![\ud800-\udbff])\udc01
ZoneList<CharacterRange>* lead_surrogates =
new (zone) ZoneList<CharacterRange>(1, zone);
lead_surrogates->Add(
CharacterRange::Range(kLeadSurrogateStart, kLeadSurrogateEnd), zone);
RegExpNode* match =
compiler->read_backward()
// Reading backward. Backward match the trail surrogate and assert
// that no lead surrogate precedes it.
? MatchAndNegativeLookaroundInReadDirection(
compiler, trail_surrogates, lead_surrogates, on_success, true)
// Reading forward. Assert that reading backward, there is no lead
// surrogate, and then forward match the trail surrogate.
: NegativeLookaroundAgainstReadDirectionAndMatch(
compiler, lead_surrogates, trail_surrogates, on_success, false);
result->AddAlternative(GuardedAlternative(match));
}
RegExpNode* RegExpCharacterClass::ToNode(RegExpCompiler* compiler,
RegExpNode* on_success) {
return new (compiler->zone())
TextNode(this, compiler->read_backward(), on_success);
set_.Canonicalize();
Zone* zone = compiler->zone();
ZoneList<CharacterRange>* ranges = this->ranges(zone);
if (compiler->unicode() && !compiler->one_byte()) {
if (is_negated()) {
ZoneList<CharacterRange>* negated =
new (zone) ZoneList<CharacterRange>(2, zone);
CharacterRange::Negate(ranges, negated, zone);
ranges = negated;
}
if (ranges->length() == 0) {
// No matches possible.
return new (zone) EndNode(EndNode::BACKTRACK, zone);
}
UnicodeRangeSplitter splitter(zone, ranges);
ChoiceNode* result = new (compiler->zone()) ChoiceNode(2, compiler->zone());
AddBmpCharacters(compiler, result, on_success, &splitter);
AddNonBmpSurrogatePairs(compiler, result, on_success, &splitter);
AddLoneLeadSurrogates(compiler, result, on_success, &splitter);
AddLoneTrailSurrogates(compiler, result, on_success, &splitter);
return result;
} else {
return new (zone) TextNode(this, compiler->read_backward(), on_success);
}
}
@ -5338,6 +5626,47 @@ RegExpNode* RegExpEmpty::ToNode(RegExpCompiler* compiler,
}
RegExpLookaround::Builder::Builder(bool is_positive, RegExpNode* on_success,
int stack_pointer_register,
int position_register,
int capture_register_count,
int capture_register_start)
: is_positive_(is_positive),
on_success_(on_success),
stack_pointer_register_(stack_pointer_register),
position_register_(position_register) {
if (is_positive_) {
on_match_success_ = ActionNode::PositiveSubmatchSuccess(
stack_pointer_register, position_register, capture_register_count,
capture_register_start, on_success_);
} else {
Zone* zone = on_success_->zone();
on_match_success_ = new (zone) NegativeSubmatchSuccess(
stack_pointer_register, position_register, capture_register_count,
capture_register_start, zone);
}
}
RegExpNode* RegExpLookaround::Builder::ForMatch(RegExpNode* match) {
if (is_positive_) {
return ActionNode::BeginSubmatch(stack_pointer_register_,
position_register_, match);
} else {
Zone* zone = on_success_->zone();
// We use a ChoiceNode to represent the negative lookaround. The first
// alternative is the negative match. On success, the end node backtracks.
// On failure, the second alternative is tried and leads to success.
// NegativeLookaheadChoiceNode is a special ChoiceNode that ignores the
// first exit when calculating quick checks.
ChoiceNode* choice_node = new (zone) NegativeLookaroundChoiceNode(
GuardedAlternative(match), GuardedAlternative(on_success_), zone);
return ActionNode::BeginSubmatch(stack_pointer_register_,
position_register_, choice_node);
}
}
RegExpNode* RegExpLookaround::ToNode(RegExpCompiler* compiler,
RegExpNode* on_success) {
int stack_pointer_register = compiler->AllocateRegister();
@ -5352,35 +5681,10 @@ RegExpNode* RegExpLookaround::ToNode(RegExpCompiler* compiler,
RegExpNode* result;
bool was_reading_backward = compiler->read_backward();
compiler->set_read_backward(type() == LOOKBEHIND);
if (is_positive()) {
result = ActionNode::BeginSubmatch(
stack_pointer_register, position_register,
body()->ToNode(compiler,
ActionNode::PositiveSubmatchSuccess(
stack_pointer_register, position_register,
register_count, register_start, on_success)));
} else {
// We use a ChoiceNode for a negative lookahead because it has most of
// the characteristics we need. It has the body of the lookahead as its
// first alternative and the expression after the lookahead of the second
// alternative. If the first alternative succeeds then the
// NegativeSubmatchSuccess will unwind the stack including everything the
// choice node set up and backtrack. If the first alternative fails then
// the second alternative is tried, which is exactly the desired result
// for a negative lookahead. The NegativeLookaheadChoiceNode is a special
// ChoiceNode that knows to ignore the first exit when calculating quick
// checks.
Zone* zone = compiler->zone();
GuardedAlternative body_alt(
body()->ToNode(compiler, new (zone) NegativeSubmatchSuccess(
stack_pointer_register, position_register,
register_count, register_start, zone)));
ChoiceNode* choice_node = new (zone) NegativeLookaroundChoiceNode(
body_alt, GuardedAlternative(on_success), zone);
result = ActionNode::BeginSubmatch(stack_pointer_register,
position_register, choice_node);
}
Builder builder(is_positive(), on_success, stack_pointer_register,
position_register, register_count, register_start);
RegExpNode* match = body_->ToNode(compiler, builder.on_match_success());
result = builder.ForMatch(match);
compiler->set_read_backward(was_reading_backward);
return result;
}
@ -5428,7 +5732,7 @@ static void AddClass(const int* elmv,
ZoneList<CharacterRange>* ranges,
Zone* zone) {
elmc--;
DCHECK(elmv[elmc] == 0x10000);
DCHECK(elmv[elmc] == kRangeEndMarker);
for (int i = 0; i < elmc; i += 2) {
DCHECK(elmv[i] < elmv[i + 1]);
ranges->Add(CharacterRange(elmv[i], elmv[i + 1] - 1), zone);
@ -5441,9 +5745,9 @@ static void AddClassNegated(const int *elmv,
ZoneList<CharacterRange>* ranges,
Zone* zone) {
elmc--;
DCHECK(elmv[elmc] == 0x10000);
DCHECK(elmv[elmc] == kRangeEndMarker);
DCHECK(elmv[0] != 0x0000);
DCHECK(elmv[elmc-1] != String::kMaxUtf16CodeUnit);
DCHECK(elmv[elmc - 1] != String::kMaxCodePoint);
uc16 last = 0x0000;
for (int i = 0; i < elmc; i += 2) {
DCHECK(last <= elmv[i] - 1);
@ -5451,7 +5755,7 @@ static void AddClassNegated(const int *elmv,
ranges->Add(CharacterRange(last, elmv[i] - 1), zone);
last = elmv[i + 1];
}
ranges->Add(CharacterRange(last, String::kMaxUtf16CodeUnit), zone);
ranges->Add(CharacterRange(last, String::kMaxCodePoint), zone);
}
@ -5508,60 +5812,13 @@ Vector<const int> CharacterRange::GetWordBounds() {
}
class CharacterRangeSplitter {
public:
CharacterRangeSplitter(ZoneList<CharacterRange>** included,
ZoneList<CharacterRange>** excluded,
Zone* zone)
: included_(included),
excluded_(excluded),
zone_(zone) { }
void Call(uc16 from, DispatchTable::Entry entry);
static const int kInBase = 0;
static const int kInOverlay = 1;
private:
ZoneList<CharacterRange>** included_;
ZoneList<CharacterRange>** excluded_;
Zone* zone_;
};
void CharacterRangeSplitter::Call(uc16 from, DispatchTable::Entry entry) {
if (!entry.out_set()->Get(kInBase)) return;
ZoneList<CharacterRange>** target = entry.out_set()->Get(kInOverlay)
? included_
: excluded_;
if (*target == NULL) *target = new(zone_) ZoneList<CharacterRange>(2, zone_);
(*target)->Add(CharacterRange(entry.from(), entry.to()), zone_);
}
void CharacterRange::Split(ZoneList<CharacterRange>* base,
Vector<const int> overlay,
ZoneList<CharacterRange>** included,
ZoneList<CharacterRange>** excluded,
Zone* zone) {
DCHECK_NULL(*included);
DCHECK_NULL(*excluded);
DispatchTable table(zone);
for (int i = 0; i < base->length(); i++)
table.AddRange(base->at(i), CharacterRangeSplitter::kInBase, zone);
for (int i = 0; i < overlay.length(); i += 2) {
table.AddRange(CharacterRange(overlay[i], overlay[i + 1] - 1),
CharacterRangeSplitter::kInOverlay, zone);
}
CharacterRangeSplitter callback(included, excluded, zone);
table.ForEach(&callback);
}
void CharacterRange::AddCaseEquivalents(Isolate* isolate, Zone* zone,
ZoneList<CharacterRange>* ranges,
bool is_one_byte) {
uc16 bottom = from();
uc16 top = to();
uc32 bottom = from();
uc32 top = to();
// Nothing to be done for surrogates.
if (bottom >= kLeadSurrogateStart && top <= kTrailSurrogateEnd) return;
if (is_one_byte && !RangeContainsLatin1Equivalents(*this)) {
if (bottom > String::kMaxOneByteCharCode) return;
if (top > String::kMaxOneByteCharCode) top = String::kMaxOneByteCharCode;
@ -5599,7 +5856,7 @@ void CharacterRange::AddCaseEquivalents(Isolate* isolate, Zone* zone,
int pos = bottom;
while (pos <= top) {
int length = isolate->jsregexp_canonrange()->get(pos, '\0', range);
uc16 block_end;
uc32 block_end;
if (length == 0) {
block_end = pos;
} else {
@ -5610,8 +5867,8 @@ void CharacterRange::AddCaseEquivalents(Isolate* isolate, Zone* zone,
length = isolate->jsregexp_uncanonicalize()->get(block_end, '\0', range);
for (int i = 0; i < length; i++) {
uc32 c = range[i];
uc16 range_from = c - (block_end - pos);
uc16 range_to = c - (block_end - end);
uc32 range_from = c - (block_end - pos);
uc32 range_to = c - (block_end - end);
if (!(bottom <= range_from && range_to <= top)) {
ranges->Add(CharacterRange(range_from, range_to), zone);
}
@ -5672,8 +5929,8 @@ static int InsertRangeInCanonicalList(ZoneList<CharacterRange>* list,
// list[0..count] for the result. Returns the number of resulting
// canonicalized ranges. Inserting a range may collapse existing ranges into
// fewer ranges, so the return value can be anything in the range 1..count+1.
uc16 from = insert.from();
uc16 to = insert.to();
uc32 from = insert.from();
uc32 to = insert.to();
int start_pos = 0;
int end_pos = count;
for (int i = count - 1; i >= 0; i--) {
@ -5773,7 +6030,7 @@ void CharacterRange::Negate(ZoneList<CharacterRange>* ranges,
DCHECK(CharacterRange::IsCanonical(ranges));
DCHECK_EQ(0, negated_ranges->length());
int range_count = ranges->length();
uc16 from = 0;
uc32 from = 0;
int i = 0;
if (range_count > 0 && ranges->at(0).from() == 0) {
from = ranges->at(0).to();
@ -5785,9 +6042,8 @@ void CharacterRange::Negate(ZoneList<CharacterRange>* ranges,
from = range.to();
i++;
}
if (from < String::kMaxUtf16CodeUnit) {
negated_ranges->Add(CharacterRange(from + 1, String::kMaxUtf16CodeUnit),
zone);
if (from < String::kMaxCodePoint) {
negated_ranges->Add(CharacterRange(from + 1, String::kMaxCodePoint), zone);
}
}
@ -5838,7 +6094,7 @@ bool OutSet::Get(unsigned value) const {
}
const uc16 DispatchTable::Config::kNoKey = unibrow::Utf8::kBadChar;
const uc32 DispatchTable::Config::kNoKey = unibrow::Utf8::kBadChar;
void DispatchTable::AddRange(CharacterRange full_range, int value,
@ -5940,7 +6196,7 @@ void DispatchTable::AddRange(CharacterRange full_range, int value,
}
OutSet* DispatchTable::Get(uc16 value) {
OutSet* DispatchTable::Get(uc32 value) {
ZoneSplayTree<Config>::Locator loc;
if (!tree()->FindGreatestLessThan(value, &loc))
return empty();
@ -6258,13 +6514,16 @@ void DispatchTableConstructor::VisitAction(ActionNode* that) {
RegExpEngine::CompilationResult RegExpEngine::Compile(
Isolate* isolate, Zone* zone, RegExpCompileData* data, bool ignore_case,
bool is_global, bool is_multiline, bool is_sticky, Handle<String> pattern,
Isolate* isolate, Zone* zone, RegExpCompileData* data,
JSRegExp::Flags flags, Handle<String> pattern,
Handle<String> sample_subject, bool is_one_byte) {
if ((data->capture_count + 1) * 2 - 1 > RegExpMacroAssembler::kMaxRegister) {
return IrregexpRegExpTooBig(isolate);
}
RegExpCompiler compiler(isolate, zone, data->capture_count, ignore_case,
bool ignore_case = flags & JSRegExp::kIgnoreCase;
bool is_sticky = flags & JSRegExp::kSticky;
bool is_global = flags & JSRegExp::kGlobal;
RegExpCompiler compiler(isolate, zone, data->capture_count, flags,
is_one_byte);
if (compiler.optimize()) compiler.set_optimize(!TooMuchRegExpCode(pattern));

68
src/regexp/jsregexp.h
View File

@ -265,28 +265,28 @@ class DispatchTable : public ZoneObject {
class Entry {
public:
Entry() : from_(0), to_(0), out_set_(NULL) { }
Entry(uc16 from, uc16 to, OutSet* out_set)
: from_(from), to_(to), out_set_(out_set) { }
uc16 from() { return from_; }
uc16 to() { return to_; }
void set_to(uc16 value) { to_ = value; }
Entry(uc32 from, uc32 to, OutSet* out_set)
: from_(from), to_(to), out_set_(out_set) {}
uc32 from() { return from_; }
uc32 to() { return to_; }
void set_to(uc32 value) { to_ = value; }
void AddValue(int value, Zone* zone) {
out_set_ = out_set_->Extend(value, zone);
}
OutSet* out_set() { return out_set_; }
private:
uc16 from_;
uc16 to_;
uc32 from_;
uc32 to_;
OutSet* out_set_;
};
class Config {
public:
typedef uc16 Key;
typedef uc32 Key;
typedef Entry Value;
static const uc16 kNoKey;
static const uc32 kNoKey;
static const Entry NoValue() { return Value(); }
static inline int Compare(uc16 a, uc16 b) {
static inline int Compare(uc32 a, uc32 b) {
if (a == b)
return 0;
else if (a < b)
@ -297,7 +297,7 @@ class DispatchTable : public ZoneObject {
};
void AddRange(CharacterRange range, int value, Zone* zone);
OutSet* Get(uc16 value);
OutSet* Get(uc32 value);
void Dump();
template <typename Callback>
@ -315,6 +315,34 @@ class DispatchTable : public ZoneObject {
};
// Categorizes character ranges into BMP, non-BMP, lead, and trail surrogates.
class UnicodeRangeSplitter {
public:
UnicodeRangeSplitter(Zone* zone, ZoneList<CharacterRange>* base);
void Call(uc32 from, DispatchTable::Entry entry);
ZoneList<CharacterRange>* bmp() { return bmp_; }
ZoneList<CharacterRange>* lead_surrogates() { return lead_surrogates_; }
ZoneList<CharacterRange>* trail_surrogates() { return trail_surrogates_; }
ZoneList<CharacterRange>* non_bmp() const { return non_bmp_; }
private:
static const int kBase = 0;
// Separate ranges into
static const int kBmpCodePoints = 1;
static const int kLeadSurrogates = 2;
static const int kTrailSurrogates = 3;
static const int kNonBmpCodePoints = 4;
Zone* zone_;
DispatchTable table_;
ZoneList<CharacterRange>* bmp_;
ZoneList<CharacterRange>* lead_surrogates_;
ZoneList<CharacterRange>* trail_surrogates_;
ZoneList<CharacterRange>* non_bmp_;
};
#define FOR_EACH_NODE_TYPE(VISIT) \
VISIT(End) \
VISIT(Action) \
@ -690,6 +718,17 @@ class TextNode: public SeqRegExpNode {
read_backward_(read_backward) {
elms_->Add(TextElement::CharClass(that), zone());
}
// Create TextNode for a single character class for the given ranges.
static TextNode* CreateForCharacterRanges(Zone* zone,
ZoneList<CharacterRange>* ranges,
bool read_backward,
RegExpNode* on_success);
// Create TextNode for a surrogate pair with a range given for the
// lead and the trail surrogate each.
static TextNode* CreateForSurrogatePair(Zone* zone, CharacterRange lead,
CharacterRange trail,
bool read_backward,
RegExpNode* on_success);
virtual void Accept(NodeVisitor* visitor);
virtual void Emit(RegExpCompiler* compiler, Trace* trace);
virtual int EatsAtLeast(int still_to_find, int budget, bool not_at_start);
@ -813,8 +852,7 @@ class BackReferenceNode: public SeqRegExpNode {
class EndNode: public RegExpNode {
public:
enum Action { ACCEPT, BACKTRACK, NEGATIVE_SUBMATCH_SUCCESS };
explicit EndNode(Action action, Zone* zone)
: RegExpNode(zone), action_(action) { }
EndNode(Action action, Zone* zone) : RegExpNode(zone), action_(action) {}
virtual void Accept(NodeVisitor* visitor);
virtual void Emit(RegExpCompiler* compiler, Trace* trace);
virtual int EatsAtLeast(int still_to_find,
@ -1505,8 +1543,8 @@ class RegExpEngine: public AllStatic {
};
static CompilationResult Compile(Isolate* isolate, Zone* zone,
RegExpCompileData* input, bool ignore_case,
bool global, bool multiline, bool sticky,
RegExpCompileData* input,
JSRegExp::Flags flags,
Handle<String> pattern,
Handle<String> sample_subject,
bool is_one_byte);

4
src/regexp/regexp-ast.cc
View File

@ -172,9 +172,9 @@ void* RegExpUnparser::VisitAlternative(RegExpAlternative* that, void* data) {
void RegExpUnparser::VisitCharacterRange(CharacterRange that) {
os_ << AsUC16(that.from());
os_ << AsUC32(that.from());
if (!that.IsSingleton()) {
os_ << "-" << AsUC16(that.to());
os_ << "-" << AsUC32(that.to());
}
}

53
src/regexp/regexp-ast.h
View File

@ -5,6 +5,7 @@
#ifndef V8_REGEXP_REGEXP_AST_H_
#define V8_REGEXP_REGEXP_AST_H_
#include "src/objects.h"
#include "src/utils.h"
#include "src/zone.h"
@ -77,33 +78,38 @@ class CharacterRange {
CharacterRange() : from_(0), to_(0) {}
// For compatibility with the CHECK_OK macro
CharacterRange(void* null) { DCHECK_NULL(null); } // NOLINT
CharacterRange(uc16 from, uc16 to) : from_(from), to_(to) {}
CharacterRange(uc32 from, uc32 to) : from_(from), to_(to) {}
static void AddClassEscape(uc16 type, ZoneList<CharacterRange>* ranges,
Zone* zone);
static Vector<const int> GetWordBounds();
static inline CharacterRange Singleton(uc16 value) {
static inline CharacterRange Singleton(uc32 value) {
return CharacterRange(value, value);
}
static inline CharacterRange Range(uc16 from, uc16 to) {
DCHECK(from <= to);
static inline CharacterRange Range(uc32 from, uc32 to) {
DCHECK(0 <= from && to <= String::kMaxCodePoint);
DCHECK(static_cast<uint32_t>(from) <= static_cast<uint32_t>(to));
return CharacterRange(from, to);
}
static inline CharacterRange Everything() {
return CharacterRange(0, 0xFFFF);
return CharacterRange(0, String::kMaxCodePoint);
}
bool Contains(uc16 i) { return from_ <= i && i <= to_; }
uc16 from() const { return from_; }
void set_from(uc16 value) { from_ = value; }
uc16 to() const { return to_; }
void set_to(uc16 value) { to_ = value; }
static inline ZoneList<CharacterRange>* List(Zone* zone,
CharacterRange range) {
ZoneList<CharacterRange>* list =
new (zone) ZoneList<CharacterRange>(1, zone);
list->Add(range, zone);
return list;
}
bool Contains(uc32 i) { return from_ <= i && i <= to_; }
uc32 from() const { return from_; }
void set_from(uc32 value) { from_ = value; }
uc32 to() const { return to_; }
void set_to(uc32 value) { to_ = value; }
bool is_valid() { return from_ <= to_; }
bool IsEverything(uc16 max) { return from_ == 0 && to_ >= max; }
bool IsSingleton() { return (from_ == to_); }
void AddCaseEquivalents(Isolate* isolate, Zone* zone,
ZoneList<CharacterRange>* ranges, bool is_one_byte);
static void Split(ZoneList<CharacterRange>* base, Vector<const int> overlay,
ZoneList<CharacterRange>** included,
ZoneList<CharacterRange>** excluded, Zone* zone);
// Whether a range list is in canonical form: Ranges ordered by from value,
// and ranges non-overlapping and non-adjacent.
static bool IsCanonical(ZoneList<CharacterRange>* ranges);
@ -119,8 +125,8 @@ class CharacterRange {
static const int kPayloadMask = (1 << 24) - 1;
private:
uc16 from_;
uc16 to_;
uc32 from_;
uc32 to_;
};
@ -287,6 +293,7 @@ class RegExpCharacterClass final : public RegExpTree {
RegExpCharacterClass* AsCharacterClass() override;
bool IsCharacterClass() override;
bool IsTextElement() override { return true; }
bool NeedsDesugaringForUnicode(Zone* zone);
int min_match() override { return 1; }
int max_match() override { return 1; }
void AppendToText(RegExpText* text, Zone* zone) override;
@ -451,6 +458,22 @@ class RegExpLookaround final : public RegExpTree {
int capture_from() { return capture_from_; }
Type type() { return type_; }
class Builder {
public:
Builder(bool is_positive, RegExpNode* on_success,
int stack_pointer_register, int position_register,
int capture_register_count = 0, int capture_register_start = 0);
RegExpNode* on_match_success() { return on_match_success_; }
RegExpNode* ForMatch(RegExpNode* match);
private:
bool is_positive_;
RegExpNode* on_match_success_;
RegExpNode* on_success_;
int stack_pointer_register_;
int position_register_;
};
private:
RegExpTree* body_;
bool is_positive_;

190
src/regexp/regexp-parser.cc
View File

@ -15,20 +15,18 @@ namespace v8 {
namespace internal {
RegExpParser::RegExpParser(FlatStringReader* in, Handle<String>* error,
bool multiline, bool unicode, Isolate* isolate,
Zone* zone)
JSRegExp::Flags flags, Isolate* isolate, Zone* zone)
: isolate_(isolate),
zone_(zone),
error_(error),
captures_(NULL),
in_(in),
current_(kEndMarker),
flags_(flags),
next_pos_(0),
captures_started_(0),
capture_count_(0),
has_more_(true),
multiline_(multiline),
unicode_(unicode),
simple_(false),
contains_anchor_(false),
is_scanned_for_captures_(false),
@ -37,9 +35,28 @@ RegExpParser::RegExpParser(FlatStringReader* in, Handle<String>* error,
}
template <bool update_position>
uc32 RegExpParser::ReadNext() {
int position = next_pos_;
uc32 c0 = in()->Get(position);
position++;
// Read the whole surrogate pair in case of unicode flag, if possible.
if (unicode() && position < in()->length() &&
unibrow::Utf16::IsLeadSurrogate(static_cast<uc16>(c0))) {
uc16 c1 = in()->Get(position);
if (unibrow::Utf16::IsTrailSurrogate(c1)) {
c0 = unibrow::Utf16::CombineSurrogatePair(static_cast<uc16>(c0), c1);
position++;
}
}
if (update_position) next_pos_ = position;
return c0;
}
uc32 RegExpParser::Next() {
if (has_next()) {
return in()->Get(next_pos_);
return ReadNext<false>();
} else {
return kEndMarker;
}
@ -47,25 +64,14 @@ uc32 RegExpParser::Next() {
void RegExpParser::Advance() {
if (next_pos_ < in()->length()) {
if (has_next()) {
StackLimitCheck check(isolate());
if (check.HasOverflowed()) {
ReportError(CStrVector(Isolate::kStackOverflowMessage));
} else if (zone()->excess_allocation()) {
ReportError(CStrVector("Regular expression too large"));
} else {
current_ = in()->Get(next_pos_);
next_pos_++;
// Read the whole surrogate pair in case of unicode flag, if possible.
if (unicode_ && next_pos_ < in()->length() &&
unibrow::Utf16::IsLeadSurrogate(static_cast<uc16>(current_))) {
uc16 trail = in()->Get(next_pos_);
if (unibrow::Utf16::IsTrailSurrogate(trail)) {
current_ = unibrow::Utf16::CombineSurrogatePair(
static_cast<uc16>(current_), trail);
next_pos_++;
}
}
current_ = ReadNext<true>();
}
} else {
current_ = kEndMarker;
@ -142,7 +148,7 @@ RegExpTree* RegExpParser::ParsePattern() {
RegExpTree* RegExpParser::ParseDisjunction() {
// Used to store current state while parsing subexpressions.
RegExpParserState initial_state(NULL, INITIAL, RegExpLookaround::LOOKAHEAD, 0,
zone());
flags_, zone());
RegExpParserState* state = &initial_state;
// Cache the builder in a local variable for quick access.
RegExpBuilder* builder = initial_state.builder();
@ -206,7 +212,7 @@ RegExpTree* RegExpParser::ParseDisjunction() {
return ReportError(CStrVector("Nothing to repeat"));
case '^': {
Advance();
if (multiline_) {
if (multiline()) {
builder->AddAssertion(
new (zone()) RegExpAssertion(RegExpAssertion::START_OF_LINE));
} else {
@ -219,8 +225,8 @@ RegExpTree* RegExpParser::ParseDisjunction() {
case '$': {
Advance();
RegExpAssertion::AssertionType assertion_type =
multiline_ ? RegExpAssertion::END_OF_LINE
: RegExpAssertion::END_OF_INPUT;
multiline() ? RegExpAssertion::END_OF_LINE
: RegExpAssertion::END_OF_INPUT;
builder->AddAssertion(new (zone()) RegExpAssertion(assertion_type));
continue;
}
@ -230,8 +236,9 @@ RegExpTree* RegExpParser::ParseDisjunction() {
ZoneList<CharacterRange>* ranges =
new (zone()) ZoneList<CharacterRange>(2, zone());
CharacterRange::AddClassEscape('.', ranges, zone());
RegExpTree* atom = new (zone()) RegExpCharacterClass(ranges, false);
builder->AddAtom(atom);
RegExpCharacterClass* cc =
new (zone()) RegExpCharacterClass(ranges, false);
builder->AddCharacterClass(cc);
break;
}
case '(': {
@ -276,14 +283,15 @@ RegExpTree* RegExpParser::ParseDisjunction() {
captures_started_++;
}
// Store current state and begin new disjunction parsing.
state = new (zone()) RegExpParserState(
state, subexpr_type, lookaround_type, captures_started_, zone());
state =
new (zone()) RegExpParserState(state, subexpr_type, lookaround_type,
captures_started_, flags_, zone());
builder = state->builder();
continue;
}
case '[': {
RegExpTree* atom = ParseCharacterClass(CHECK_FAILED);
builder->AddAtom(atom);
RegExpTree* cc = ParseCharacterClass(CHECK_FAILED);
builder->AddCharacterClass(cc->AsCharacterClass());
break;
}
// Atom ::
@ -318,8 +326,9 @@ RegExpTree* RegExpParser::ParseDisjunction() {
ZoneList<CharacterRange>* ranges =
new (zone()) ZoneList<CharacterRange>(2, zone());
CharacterRange::AddClassEscape(c, ranges, zone());
RegExpTree* atom = new (zone()) RegExpCharacterClass(ranges, false);
builder->AddAtom(atom);
RegExpCharacterClass* cc =
new (zone()) RegExpCharacterClass(ranges, false);
builder->AddCharacterClass(cc);
break;
}
case '1':
@ -353,7 +362,7 @@ RegExpTree* RegExpParser::ParseDisjunction() {
// escaped,
// no other identity escapes are allowed. If the 'u' flag is not
// present, all identity escapes are allowed.
if (!unicode_) {
if (!unicode()) {
builder->AddCharacter(first_digit);
Advance(2);
} else {
@ -414,7 +423,7 @@ RegExpTree* RegExpParser::ParseDisjunction() {
uc32 value;
if (ParseHexEscape(2, &value)) {
builder->AddCharacter(value);
} else if (!unicode_) {
} else if (!unicode()) {
builder->AddCharacter('x');
} else {
// If the 'u' flag is present, invalid escapes are not treated as
@ -428,7 +437,7 @@ RegExpTree* RegExpParser::ParseDisjunction() {
uc32 value;
if (ParseUnicodeEscape(&value)) {
builder->AddUnicodeCharacter(value);
} else if (!unicode_) {
} else if (!unicode()) {
builder->AddCharacter('u');
} else {
// If the 'u' flag is present, invalid escapes are not treated as
@ -444,7 +453,7 @@ RegExpTree* RegExpParser::ParseDisjunction() {
// other identity escapes are allowed. If the 'u' flag is not
// present,
// all identity escapes are allowed.
if (!unicode_ || IsSyntaxCharacter(current())) {
if (!unicode() || IsSyntaxCharacter(current())) {
builder->AddCharacter(current());
Advance();
} else {
@ -745,7 +754,7 @@ bool RegExpParser::ParseUnicodeEscape(uc32* value) {
// Accept both \uxxxx and \u{xxxxxx} (if harmony unicode escapes are
// allowed). In the latter case, the number of hex digits between { } is
// arbitrary. \ and u have already been read.
if (current() == '{' && unicode_) {
if (current() == '{' && unicode()) {
int start = position();
Advance();
if (ParseUnlimitedLengthHexNumber(0x10ffff, value)) {
@ -840,7 +849,7 @@ uc32 RegExpParser::ParseClassCharacterEscape() {
if (ParseHexEscape(2, &value)) {
return value;
}
if (!unicode_) {
if (!unicode()) {
// If \x is not followed by a two-digit hexadecimal, treat it
// as an identity escape.
return 'x';
@ -856,7 +865,7 @@ uc32 RegExpParser::ParseClassCharacterEscape() {
if (ParseUnicodeEscape(&value)) {
return value;
}
if (!unicode_) {
if (!unicode()) {
return 'u';
}
// If the 'u' flag is present, invalid escapes are not treated as
@ -869,7 +878,7 @@ uc32 RegExpParser::ParseClassCharacterEscape() {
// If the 'u' flag is present, only syntax characters can be escaped, no
// other identity escapes are allowed. If the 'u' flag is not present, all
// identity escapes are allowed.
if (!unicode_ || IsSyntaxCharacter(result)) {
if (!unicode() || IsSyntaxCharacter(result)) {
Advance();
return result;
}
@ -899,13 +908,29 @@ CharacterRange RegExpParser::ParseClassAtom(uc16* char_class) {
case kEndMarker:
return ReportError(CStrVector("\\ at end of pattern"));
default:
uc32 c = ParseClassCharacterEscape(CHECK_FAILED);
return CharacterRange::Singleton(c);
first = ParseClassCharacterEscape(CHECK_FAILED);
}
} else {
Advance();
return CharacterRange::Singleton(first);
}
if (unicode() && unibrow::Utf16::IsLeadSurrogate(first)) {
// Combine with possibly following trail surrogate.
int start = position();
uc32 second = current();
if (second == '\\') {
second = ParseClassCharacterEscape(CHECK_FAILED);
} else {
Advance();
}
if (unibrow::Utf16::IsTrailSurrogate(second)) {
first = unibrow::Utf16::CombineSurrogatePair(first, second);
} else {
Reset(start);
}
}
return CharacterRange::Singleton(first);
}
@ -985,10 +1010,10 @@ RegExpTree* RegExpParser::ParseCharacterClass() {
bool RegExpParser::ParseRegExp(Isolate* isolate, Zone* zone,
FlatStringReader* input, bool multiline,
bool unicode, RegExpCompileData* result) {
FlatStringReader* input, JSRegExp::Flags flags,
RegExpCompileData* result) {
DCHECK(result != NULL);
RegExpParser parser(input, &result->error, multiline, unicode, isolate, zone);
RegExpParser parser(input, &result->error, flags, isolate, zone);
RegExpTree* tree = parser.ParsePattern();
if (parser.failed()) {
DCHECK(tree == NULL);
@ -1011,10 +1036,12 @@ bool RegExpParser::ParseRegExp(Isolate* isolate, Zone* zone,
}
RegExpBuilder::RegExpBuilder(Zone* zone)
RegExpBuilder::RegExpBuilder(Zone* zone, JSRegExp::Flags flags)
: zone_(zone),
pending_empty_(false),
flags_(flags),
characters_(NULL),
pending_surrogate_(kNoPendingSurrogate),
terms_(),
alternatives_()
#ifdef DEBUG
@ -1025,7 +1052,48 @@ RegExpBuilder::RegExpBuilder(Zone* zone)
}
void RegExpBuilder::AddLeadSurrogate(uc16 lead_surrogate) {
DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate));
FlushPendingSurrogate();
// Hold onto the lead surrogate, waiting for a trail surrogate to follow.
pending_surrogate_ = lead_surrogate;
}
void RegExpBuilder::AddTrailSurrogate(uc16 trail_surrogate) {
DCHECK(unibrow::Utf16::IsTrailSurrogate(trail_surrogate));
if (pending_surrogate_ != kNoPendingSurrogate) {
uc16 lead_surrogate = pending_surrogate_;
DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate));
ZoneList<uc16> surrogate_pair(2, zone());
surrogate_pair.Add(lead_surrogate, zone());
surrogate_pair.Add(trail_surrogate, zone());
RegExpAtom* atom = new (zone()) RegExpAtom(surrogate_pair.ToConstVector());
pending_surrogate_ = kNoPendingSurrogate;
AddAtom(atom);
} else {
pending_surrogate_ = trail_surrogate;
FlushPendingSurrogate();
}
}
void RegExpBuilder::FlushPendingSurrogate() {
if (pending_surrogate_ != kNoPendingSurrogate) {
// Use character class to desugar lone surrogate matching.
RegExpCharacterClass* cc = new (zone()) RegExpCharacterClass(
CharacterRange::List(zone(),
CharacterRange::Singleton(pending_surrogate_)),
false);
pending_surrogate_ = kNoPendingSurrogate;
DCHECK(unicode());
AddCharacterClass(cc);
}
}
void RegExpBuilder::FlushCharacters() {
FlushPendingSurrogate();
pending_empty_ = false;
if (characters_ != NULL) {
RegExpTree* atom = new (zone()) RegExpAtom(characters_->ToConstVector());
@ -1053,6 +1121,7 @@ void RegExpBuilder::FlushText() {
void RegExpBuilder::AddCharacter(uc16 c) {
FlushPendingSurrogate();
pending_empty_ = false;
if (characters_ == NULL) {
characters_ = new (zone()) ZoneList<uc16>(4, zone());
@ -1064,11 +1133,13 @@ void RegExpBuilder::AddCharacter(uc16 c) {
void RegExpBuilder::AddUnicodeCharacter(uc32 c) {
if (c > unibrow::Utf16::kMaxNonSurrogateCharCode) {
ZoneList<uc16> surrogate_pair(2, zone());
surrogate_pair.Add(unibrow::Utf16::LeadSurrogate(c), zone());
surrogate_pair.Add(unibrow::Utf16::TrailSurrogate(c), zone());
RegExpAtom* atom = new (zone()) RegExpAtom(surrogate_pair.ToConstVector());
AddAtom(atom);
DCHECK(unicode());
AddLeadSurrogate(unibrow::Utf16::LeadSurrogate(c));
AddTrailSurrogate(unibrow::Utf16::TrailSurrogate(c));
} else if (unicode() && unibrow::Utf16::IsLeadSurrogate(c)) {
AddLeadSurrogate(c);
} else if (unicode() && unibrow::Utf16::IsTrailSurrogate(c)) {
AddTrailSurrogate(c);
} else {
AddCharacter(static_cast<uc16>(c));
}
@ -1078,6 +1149,17 @@ void RegExpBuilder::AddUnicodeCharacter(uc32 c) {
void RegExpBuilder::AddEmpty() { pending_empty_ = true; }
void RegExpBuilder::AddCharacterClass(RegExpCharacterClass* cc) {
if (unicode() && cc->NeedsDesugaringForUnicode(zone())) {
// In unicode mode, character class needs to be desugared, so it
// must be a standalone term instead of being part of a RegExpText.
AddTerm(cc);
} else {
AddAtom(cc);
}
}
void RegExpBuilder::AddAtom(RegExpTree* term) {
if (term->IsEmpty()) {
AddEmpty();
@ -1094,6 +1176,13 @@ void RegExpBuilder::AddAtom(RegExpTree* term) {
}
void RegExpBuilder::AddTerm(RegExpTree* term) {
FlushText();
terms_.Add(term, zone());
LAST(ADD_ATOM);
}
void RegExpBuilder::AddAssertion(RegExpTree* assert) {
FlushText();
terms_.Add(assert, zone());
@ -1132,6 +1221,7 @@ RegExpTree* RegExpBuilder::ToRegExp() {
void RegExpBuilder::AddQuantifierToAtom(
int min, int max, RegExpQuantifier::QuantifierType quantifier_type) {
FlushPendingSurrogate();
if (pending_empty_) {
pending_empty_ = false;
return;

30
src/regexp/regexp-parser.h
View File

@ -99,13 +99,15 @@ class BufferedZoneList {
// Accumulates RegExp atoms and assertions into lists of terms and alternatives.
class RegExpBuilder : public ZoneObject {
public:
explicit RegExpBuilder(Zone* zone);
RegExpBuilder(Zone* zone, JSRegExp::Flags flags);
void AddCharacter(uc16 character);
void AddUnicodeCharacter(uc32 character);
// "Adds" an empty expression. Does nothing except consume a
// following quantifier
void AddEmpty();
void AddCharacterClass(RegExpCharacterClass* cc);
void AddAtom(RegExpTree* tree);
void AddTerm(RegExpTree* tree);
void AddAssertion(RegExpTree* tree);
void NewAlternative(); // '|'
void AddQuantifierToAtom(int min, int max,
@ -113,14 +115,21 @@ class RegExpBuilder : public ZoneObject {
RegExpTree* ToRegExp();
private:
static const uc16 kNoPendingSurrogate = 0;
void AddLeadSurrogate(uc16 lead_surrogate);
void AddTrailSurrogate(uc16 trail_surrogate);
void FlushPendingSurrogate();
void FlushCharacters();
void FlushText();
void FlushTerms();
Zone* zone() const { return zone_; }
bool unicode() const { return (flags_ & JSRegExp::kUnicode) != 0; }
Zone* zone_;
bool pending_empty_;
JSRegExp::Flags flags_;
ZoneList<uc16>* characters_;
uc16 pending_surrogate_;
BufferedZoneList<RegExpTree, 2> terms_;
BufferedZoneList<RegExpTree, 2> text_;
BufferedZoneList<RegExpTree, 2> alternatives_;
@ -135,12 +144,11 @@ class RegExpBuilder : public ZoneObject {
class RegExpParser BASE_EMBEDDED {
public:
RegExpParser(FlatStringReader* in, Handle<String>* error, bool multiline_mode,
bool unicode, Isolate* isolate, Zone* zone);
RegExpParser(FlatStringReader* in, Handle<String>* error,
JSRegExp::Flags flags, Isolate* isolate, Zone* zone);
static bool ParseRegExp(Isolate* isolate, Zone* zone, FlatStringReader* input,
bool multiline, bool unicode,
RegExpCompileData* result);
JSRegExp::Flags flags, RegExpCompileData* result);
RegExpTree* ParsePattern();
RegExpTree* ParseDisjunction();
@ -183,6 +191,8 @@ class RegExpParser BASE_EMBEDDED {
int captures_started() { return captures_started_; }
int position() { return next_pos_ - 1; }
bool failed() { return failed_; }
bool unicode() const { return (flags_ & JSRegExp::kUnicode) != 0; }
bool multiline() const { return (flags_ & JSRegExp::kMultiline) != 0; }
static bool IsSyntaxCharacter(uc32 c);
@ -203,9 +213,10 @@ class RegExpParser BASE_EMBEDDED {
RegExpParserState(RegExpParserState* previous_state,
SubexpressionType group_type,
RegExpLookaround::Type lookaround_type,
int disjunction_capture_index, Zone* zone)
int disjunction_capture_index, JSRegExp::Flags flags,
Zone* zone)
: previous_state_(previous_state),
builder_(new (zone) RegExpBuilder(zone)),
builder_(new (zone) RegExpBuilder(zone, flags)),
group_type_(group_type),
lookaround_type_(lookaround_type),
disjunction_capture_index_(disjunction_capture_index) {}
@ -249,6 +260,8 @@ class RegExpParser BASE_EMBEDDED {
bool has_more() { return has_more_; }
bool has_next() { return next_pos_ < in()->length(); }
uc32 Next();
template <bool update_position>
uc32 ReadNext();
FlatStringReader* in() { return in_; }
void ScanForCaptures();
@ -258,13 +271,12 @@ class RegExpParser BASE_EMBEDDED {
ZoneList<RegExpCapture*>* captures_;
FlatStringReader* in_;
uc32 current_;
JSRegExp::Flags flags_;
int next_pos_;
int captures_started_;
// The capture count is only valid after we have scanned for captures.
int capture_count_;
bool has_more_;
bool multiline_;
bool unicode_;
bool simple_;
bool contains_anchor_;
bool is_scanned_for_captures_;

91
test/cctest/test-regexp.cc
View File

@ -96,7 +96,7 @@ static bool CheckParse(const char* input) {
FlatStringReader reader(CcTest::i_isolate(), CStrVector(input));
RegExpCompileData result;
return v8::internal::RegExpParser::ParseRegExp(
CcTest::i_isolate(), &zone, &reader, false, false, &result);
CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result);
}
@ -106,8 +106,10 @@ static void CheckParseEq(const char* input, const char* expected,
Zone zone;
FlatStringReader reader(CcTest::i_isolate(), CStrVector(input));
RegExpCompileData result;
CHECK(v8::internal::RegExpParser::ParseRegExp(
CcTest::i_isolate(), &zone, &reader, false, unicode, &result));
JSRegExp::Flags flags = JSRegExp::kNone;
if (unicode) flags |= JSRegExp::kUnicode;
CHECK(v8::internal::RegExpParser::ParseRegExp(CcTest::i_isolate(), &zone,
&reader, flags, &result));
CHECK(result.tree != NULL);
CHECK(result.error.is_null());
std::ostringstream os;
@ -125,7 +127,7 @@ static bool CheckSimple(const char* input) {
FlatStringReader reader(CcTest::i_isolate(), CStrVector(input));
RegExpCompileData result;
CHECK(v8::internal::RegExpParser::ParseRegExp(
CcTest::i_isolate(), &zone, &reader, false, false, &result));
CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result));
CHECK(result.tree != NULL);
CHECK(result.error.is_null());
return result.simple;
@ -143,7 +145,7 @@ static MinMaxPair CheckMinMaxMatch(const char* input) {
FlatStringReader reader(CcTest::i_isolate(), CStrVector(input));
RegExpCompileData result;
CHECK(v8::internal::RegExpParser::ParseRegExp(
CcTest::i_isolate(), &zone, &reader, false, false, &result));
CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result));
CHECK(result.tree != NULL);
CHECK(result.error.is_null());
int min_match = result.tree->min_match();
@ -206,8 +208,8 @@ void TestRegExpParser(bool lookbehind) {
}
CheckParseEq("()", "(^ %)");
CheckParseEq("(?=)", "(-> + %)");
CheckParseEq("[]", "^[\\x00-\\uffff]"); // Doesn't compile on windows
CheckParseEq("[^]", "[\\x00-\\uffff]"); // \uffff isn't in codepage 1252
CheckParseEq("[]", "^[\\x00-\\u{10ffff}]"); // Doesn't compile on windows
CheckParseEq("[^]", "[\\x00-\\u{10ffff}]"); // \uffff isn't in codepage 1252
CheckParseEq("[x]", "[x]");
CheckParseEq("[xyz]", "[x y z]");
CheckParseEq("[a-zA-Z0-9]", "[a-z A-Z 0-9]");
@ -316,6 +318,10 @@ void TestRegExpParser(bool lookbehind) {
CheckParseEq("\\u{12345}{3}", "(# 3 3 g '\\ud808\\udf45')", true);
CheckParseEq("\\u{12345}*", "(# 0 - g '\\ud808\\udf45')", true);
CheckParseEq("\\ud808\\udf45*", "(# 0 - g '\\ud808\\udf45')", true);
CheckParseEq("[\\ud808\\udf45-\\ud809\\udccc]", "[\\u{012345}-\\u{0124cc}]",
true);
CHECK_SIMPLE("", false);
CHECK_SIMPLE("a", true);
CHECK_SIMPLE("a|b", false);
@ -454,7 +460,7 @@ static void ExpectError(const char* input,
FlatStringReader reader(CcTest::i_isolate(), CStrVector(input));
RegExpCompileData result;
CHECK(!v8::internal::RegExpParser::ParseRegExp(
CcTest::i_isolate(), &zone, &reader, false, false, &result));
CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result));
CHECK(result.tree == NULL);
CHECK(!result.error.is_null());
v8::base::SmartArrayPointer<char> str = result.error->ToCString(ALLOW_NULLS);
@ -523,7 +529,7 @@ static void TestCharacterClassEscapes(uc16 c, bool (pred)(uc16 c)) {
ZoneList<CharacterRange>* ranges =
new(&zone) ZoneList<CharacterRange>(2, &zone);
CharacterRange::AddClassEscape(c, ranges, &zone);
for (unsigned i = 0; i < (1 << 16); i++) {
for (uc32 i = 0; i < (1 << 16); i++) {
bool in_class = false;
for (int j = 0; !in_class && j < ranges->length(); j++) {
CharacterRange& range = ranges->at(j);
@ -550,17 +556,19 @@ static RegExpNode* Compile(const char* input, bool multiline, bool unicode,
Isolate* isolate = CcTest::i_isolate();
FlatStringReader reader(isolate, CStrVector(input));
RegExpCompileData compile_data;
JSRegExp::Flags flags = JSRegExp::kNone;
if (multiline) flags = JSRegExp::kMultiline;
if (unicode) flags = JSRegExp::kUnicode;
if (!v8::internal::RegExpParser::ParseRegExp(CcTest::i_isolate(), zone,
&reader, multiline, unicode,
&compile_data))
&reader, flags, &compile_data))
return NULL;
Handle<String> pattern = isolate->factory()
->NewStringFromUtf8(CStrVector(input))
.ToHandleChecked();
Handle<String> sample_subject =
isolate->factory()->NewStringFromUtf8(CStrVector("")).ToHandleChecked();
RegExpEngine::Compile(isolate, zone, &compile_data, false, false, multiline,
false, pattern, sample_subject, is_one_byte);
RegExpEngine::Compile(isolate, zone, &compile_data, flags, pattern,
sample_subject, is_one_byte);
return compile_data.node;
}
@ -1669,7 +1677,7 @@ TEST(CharacterRangeCaseIndependence) {
}
static bool InClass(uc16 c, ZoneList<CharacterRange>* ranges) {
static bool InClass(uc32 c, ZoneList<CharacterRange>* ranges) {
if (ranges == NULL)
return false;
for (int i = 0; i < ranges->length(); i++) {
@ -1681,29 +1689,46 @@ static bool InClass(uc16 c, ZoneList<CharacterRange>* ranges) {
}
TEST(CharClassDifference) {
TEST(UnicodeRangeSplitter) {
Zone zone;
ZoneList<CharacterRange>* base =
new(&zone) ZoneList<CharacterRange>(1, &zone);
base->Add(CharacterRange::Everything(), &zone);
Vector<const int> overlay = CharacterRange::GetWordBounds();
ZoneList<CharacterRange>* included = NULL;
ZoneList<CharacterRange>* excluded = NULL;
CharacterRange::Split(base, overlay, &included, &excluded, &zone);
for (int i = 0; i < (1 << 16); i++) {
bool in_base = InClass(i, base);
if (in_base) {
bool in_overlay = false;
for (int j = 0; !in_overlay && j < overlay.length(); j += 2) {
if (overlay[j] <= i && i < overlay[j+1])
in_overlay = true;
}
CHECK_EQ(in_overlay, InClass(i, included));
CHECK_EQ(!in_overlay, InClass(i, excluded));
} else {
CHECK(!InClass(i, included));
CHECK(!InClass(i, excluded));
}
UnicodeRangeSplitter splitter(&zone, base);
// BMP
for (uc32 c = 0; c < 0xd800; c++) {
CHECK(InClass(c, splitter.bmp()));
CHECK(!InClass(c, splitter.lead_surrogates()));
CHECK(!InClass(c, splitter.trail_surrogates()));
CHECK(!InClass(c, splitter.non_bmp()));
}
// Lead surrogates
for (uc32 c = 0xd800; c < 0xdbff; c++) {
CHECK(!InClass(c, splitter.bmp()));
CHECK(InClass(c, splitter.lead_surrogates()));
CHECK(!InClass(c, splitter.trail_surrogates()));
CHECK(!InClass(c, splitter.non_bmp()));
}
// Trail surrogates
for (uc32 c = 0xdc00; c < 0xdfff; c++) {
CHECK(!InClass(c, splitter.bmp()));
CHECK(!InClass(c, splitter.lead_surrogates()));
CHECK(InClass(c, splitter.trail_surrogates()));
CHECK(!InClass(c, splitter.non_bmp()));
}
// BMP
for (uc32 c = 0xe000; c < 0xffff; c++) {
CHECK(InClass(c, splitter.bmp()));
CHECK(!InClass(c, splitter.lead_surrogates()));
CHECK(!InClass(c, splitter.trail_surrogates()));
CHECK(!InClass(c, splitter.non_bmp()));
}
// Non-BMP
for (uc32 c = 0x10000; c < 0x10ffff; c++) {
CHECK(!InClass(c, splitter.bmp()));
CHECK(!InClass(c, splitter.lead_surrogates()));
CHECK(!InClass(c, splitter.trail_surrogates()));
CHECK(InClass(c, splitter.non_bmp()));
}
}

156
test/mjsunit/harmony/unicode-character-ranges.js Normal file
View File

@ -0,0 +1,156 @@
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Flags: --harmony-unicode-regexps --harmony-regexp-lookbehind
function execl(expectation, regexp, subject) {
if (regexp instanceof String) regexp = new RegExp(regexp, "u");
assertEquals(expectation, regexp.exec(subject));
}
function execs(expectation, regexp_source, subject) {
execl(expectation, new RegExp(regexp_source, "u"), subject);
}
// Character ranges.
execl(["A"], /[A-D]/u, "A");
execs(["A"], "[A-D]", "A");
execl(["ABCD"], /[A-D]+/u, "ZABCDEF");
execs(["ABCD"], "[A-D]+", "ZABCDEF");
execl(["\u{12345}"], /[\u1234-\u{12345}]/u, "\u{12345}");
execs(["\u{12345}"], "[\u1234-\u{12345}]", "\u{12345}");
execl(null, /[^\u1234-\u{12345}]/u, "\u{12345}");
execs(null, "[^\u1234-\u{12345}]", "\u{12345}");
execl(["\u{1234}"], /[\u1234-\u{12345}]/u, "\u{1234}");
execs(["\u{1234}"], "[\u1234-\u{12345}]", "\u{1234}");
execl(null, /[^\u1234-\u{12345}]/u, "\u{1234}");
execs(null, "[^\u1234-\u{12345}]", "\u{1234}");
execl(null, /[\u1234-\u{12345}]/u, "\u{1233}");
execs(null, "[\u1234-\u{12345}]", "\u{1233}");
execl(["\u{1233}"], /[^\u1234-\u{12345}]/u, "\u{1233}");
execs(["\u{1233}"], "[^\u1234-\u{12345}]", "\u{1233}");
execl(["\u{12346}"], /[^\u1234-\u{12345}]/u, "\u{12346}");
execs(["\u{12346}"], "[^\u1234-\u{12345}]", "\u{12346}");
execl(null, /[\u1234-\u{12345}]/u, "\u{12346}");
execs(null, "[\u1234-\u{12345}]", "\u{12346}");
execl(["\u{12342}"], /[\u{12340}-\u{12345}]/u, "\u{12342}");
execs(["\u{12342}"], "[\u{12340}-\u{12345}]", "\u{12342}");
execl(["\u{12342}"], /[\ud808\udf40-\ud808\udf45]/u, "\u{12342}");
execs(["\u{12342}"], "[\ud808\udf40-\ud808\udf45]", "\u{12342}");
execl(null, /[^\u{12340}-\u{12345}]/u, "\u{12342}");
execs(null, "[^\u{12340}-\u{12345}]", "\u{12342}");
execl(null, /[^\ud808\udf40-\ud808\udf45]/u, "\u{12342}");
execs(null, "[^\ud808\udf40-\ud808\udf45]", "\u{12342}");
execl(["\u{ffff}"], /[\u{ff80}-\u{12345}]/u, "\u{ffff}");
execs(["\u{ffff}"], "[\u{ff80}-\u{12345}]", "\u{ffff}");
execl(["\u{ffff}"], /[\u{ff80}-\ud808\udf45]/u, "\u{ffff}");
execs(["\u{ffff}"], "[\u{ff80}-\ud808\udf45]", "\u{ffff}");
execl(null, /[^\u{ff80}-\u{12345}]/u, "\u{ffff}");
execs(null, "[^\u{ff80}-\u{12345}]", "\u{ffff}");
execl(null, /[^\u{ff80}-\ud808\udf45]/u, "\u{ffff}");
execs(null, "[^\u{ff80}-\ud808\udf45]", "\u{ffff}");
// Lone surrogate
execl(["\ud800"], /[^\u{ff80}-\u{12345}]/u, "\uff99\u{d800}A");
execs(["\udc00"], "[^\u{ff80}-\u{12345}]", "\uff99\u{dc00}A");
execl(["\udc01"], /[\u0100-\u{10ffff}]/u, "A\udc01");
execl(["\udc03"], /[\udc01-\udc03]/u, "\ud801\udc02\udc03");
execl(["\ud801"], /[\ud801-\ud803]/u, "\ud802\udc01\ud801");
// Paired sorrogate.
execl(null, /[^\u{ff80}-\u{12345}]/u, "\u{d800}\u{dc00}");
execs(null, "[^\u{ff80}-\u{12345}]", "\u{d800}\u{dc00}");
execl(["\ud800\udc00"], /[\u{ff80}-\u{12345}]/u, "\u{d800}\u{dc00}");
execs(["\ud800\udc00"], "[\u{ff80}-\u{12345}]", "\u{d800}\u{dc00}");
execl(["foo\u{10e6d}bar"], /foo\ud803\ude6dbar/u, "foo\u{10e6d}bar");
// Lone surrogates
execl(["\ud801\ud801"], /\ud801+/u, "\ud801\udc01\ud801\ud801");
execl(["\udc01\udc01"], /\udc01+/u, "\ud801\ud801\udc01\udc01\udc01");
execl(["\udc02\udc03A"], /\W\WA/u, "\ud801\udc01A\udc02\udc03A");
execl(["\ud801\ud802"], /\ud801./u, "\ud801\udc01\ud801\ud802");
execl(["\udc02\udc03A"], /[\ud800-\udfff][\ud800-\udfff]A/u,
"\ud801\udc01A\udc02\udc03A");
// Character classes
execl(null, /\w/u, "\ud801\udc01");
execl(["\ud801"], /[^\w]/, "\ud801\udc01");
execl(["\ud801\udc01"], /[^\w]/u, "\ud801\udc01");
execl(["\ud801"], /\W/, "\ud801\udc01");
execl(["\ud801\udc01"], /\W/u, "\ud801\udc01");
execl(["\ud800X"], /.X/u, "\ud800XaX");
execl(["aX"], /.(?<!\ud800)X/u, "\ud800XaX");
execl(["aX"], /.(?<![\ud800-\ud900])X/u, "\ud800XaX");
execl(null, /[]/u, "\u1234");
execl(["0abc"], /[^]abc/u, "0abc");
execl(["\u1234abc"], /[^]abc/u, "\u1234abc");
execl(["\u{12345}abc"], /[^]abc/u, "\u{12345}abc");
// Backward matches of lone surrogates.
execl(["B", "\ud803A"], /(?<=([\ud800-\ud900]A))B/u,
"\ud801\udc00AB\udc00AB\ud802\ud803AB");
execl(["B", "\udc00A"], /(?<=([\ud800-\u{10300}]A))B/u,
"\ud801\udc00AB\udc00AB\ud802\ud803AB");
execl(["B", "\udc11A"], /(?<=([\udc00-\udd00]A))B/u,
"\ud801\udc00AB\udc11AB\ud802\ud803AB");
execl(["X", "\ud800C"], /(?<=(\ud800\w))X/u,
"\ud800\udc00AX\udc11BX\ud800\ud800CX");
execl(["C", "\ud800\ud800"], /(?<=(\ud800.))\w/u,
"\ud800\udc00AX\udc11BX\ud800\ud800CX");
execl(["X", "\udc01C"], /(?<=(\udc01\w))X/u,
"\ud800\udc01AX\udc11BX\udc01\udc01CX");
execl(["C", "\udc01\udc01"], /(?<=(\udc01.))./u,
"\ud800\udc01AX\udc11BX\udc01\udc01CX");
var L = "\ud800";
var T = "\udc00";
var X = "X";
// Test string contains only match.
function testw(expect, src, subject) {
var re = new RegExp("^" + src + "$", "u");
assertEquals(expect, re.test(subject));
}
// Test string starts with match.
function tests(expect, src, subject) {
var re = new RegExp("^" + src, "u");
assertEquals(expect, re.test(subject));
}
testw(true, X, X);
testw(true, L, L);
testw(true, T, T);
testw(true, L + T, L + T);
testw(true, T + L, T + L);
testw(false, T, L + T);
testw(false, L, L + T);
testw(true, ".(?<=" + L + ")", L);
testw(true, ".(?<=" + T + ")", T);
testw(true, ".(?<=" + L + T + ")", L + T);
testw(true, ".(?<=" + L + T + ")", L + T);
tests(true, ".(?<=" + T + ")", T + L);
tests(false, ".(?<=" + L + ")", L + T);
tests(false, ".(?<=" + T + ")", L + T);
tests(true, "..(?<=" + T + ")", T + T + L);
tests(true, "..(?<=" + T + ")", X + T + L);
tests(true, "...(?<=" + L + ")", X + T + L);
tests(false, "...(?<=" + T + ")", X + L + T)
tests(true, "..(?<=" + L + T + ")", X + L + T)
tests(true, "..(?<=" + L + T + "(?<=" + L + T + "))", X + L + T);
tests(false, "..(?<=" + L + "(" + T + "))", X + L + T);
tests(false, ".*" + L, X + L + T);
tests(true, ".*" + L, X + L + L + T);
tests(false, ".*" + L, X + L + T + L + T);
tests(false, ".*" + T, X + L + T + L + T);
tests(true, ".*" + T, X + L + T + T + L + T);

26
test/mjsunit/harmony/unicode-escapes-in-regexps.js
View File

@ -252,6 +252,30 @@ assertFalse(/(\u{12345}|\u{23456}).\1/u.test("\u{12345}b\u{23456}"));
assertTrue(new RegExp("\u{12345}{3}", "u").test("\u{12345}\u{12345}\u{12345}"));
assertTrue(/\u{12345}{3}/u.test("\u{12345}\u{12345}\u{12345}"));
assertTrue(new RegExp("\u{12345}{3}").test("\u{12345}\udf45\udf45"));
assertTrue(/\ud808\udf45{3}/u.test("\u{12345}\udf45\udf45"));
assertFalse(/\ud808\udf45{3}/u.test("\u{12345}\udf45\udf45"));
assertTrue(/\ud808\udf45{3}/u.test("\u{12345}\u{12345}\u{12345}"));
assertFalse(new RegExp("\u{12345}{3}", "u").test("\u{12345}\udf45\udf45"));
assertFalse(/\u{12345}{3}/u.test("\u{12345}\udf45\udf45"));
// Mixed escapes and literal surrogates.
assertEquals(["\u{10000}\u{10000}"],
new RegExp("\ud800\udc00+", "u").exec("\u{10000}\u{10000}"));
assertEquals(["\u{10000}\u{10000}"],
new RegExp("\\ud800\\udc00+", "u").exec("\u{10000}\u{10000}"));
assertEquals(["\u{10000}\u{10000}"],
new RegExp("\\ud800\udc00+", "u").exec("\u{10000}\u{10000}"));
assertEquals(["\u{10000}\u{10000}"],
new RegExp("\ud800\\udc00+", "u").exec("\u{10000}\u{10000}"));
assertEquals(["\u{10003}\u{50001}"],
new RegExp("[\\ud800\\udc03-\\ud900\\udc01\]+", "u").exec(
"\u{10003}\u{50001}"));
assertEquals(["\u{10003}\u{50001}"],
new RegExp("[\\ud800\udc03-\ud900\\udc01\]+", "u").exec(
"\u{10003}\u{50001}"));
assertEquals(["\u{50001}"],
new RegExp("[\\ud800\udc03-\ud900\\udc01\]+", "u").exec(
"\u{10002}\u{50001}"));
assertEquals(["\u{10003}\u{50001}"],
new RegExp("[\ud800\udc03-\u{50001}\]+", "u").exec(
"\u{10003}\u{50001}"));