diff --git a/src/objects.h b/src/objects.h index 9427227ac3..891d667bce 100644 --- a/src/objects.h +++ b/src/objects.h @@ -8880,6 +8880,7 @@ class String: public Name { static const uint32_t kMaxOneByteCharCodeU = unibrow::Latin1::kMaxChar; static const int kMaxUtf16CodeUnit = 0xffff; static const uint32_t kMaxUtf16CodeUnitU = kMaxUtf16CodeUnit; + static const uc32 kMaxCodePoint = 0x10ffff; // Value of hash field containing computed hash equal to zero. static const int kEmptyStringHash = kIsNotArrayIndexMask; diff --git a/src/ostreams.cc b/src/ostreams.cc index a7a67f5d2f..120db257cd 100644 --- a/src/ostreams.cc +++ b/src/ostreams.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. #include "src/ostreams.h" +#include "src/objects.h" #if V8_OS_WIN #if _MSC_VER < 1900 @@ -60,6 +61,16 @@ std::ostream& PrintUC16(std::ostream& os, uint16_t c, bool (*pred)(uint16_t)) { return os << buf; } + +std::ostream& PrintUC32(std::ostream& os, int32_t c, bool (*pred)(uint16_t)) { + if (c <= String::kMaxUtf16CodeUnit) { + return PrintUC16(os, static_cast(c), pred); + } + char buf[13]; + snprintf(buf, sizeof(buf), "\\u{%06x}", c); + return os << buf; +} + } // namespace @@ -81,5 +92,10 @@ std::ostream& operator<<(std::ostream& os, const AsUC16& c) { return PrintUC16(os, c.value, IsPrint); } + +std::ostream& operator<<(std::ostream& os, const AsUC32& c) { + return PrintUC32(os, c.value, IsPrint); +} + } // namespace internal } // namespace v8 diff --git a/src/ostreams.h b/src/ostreams.h index 56f4aa7e45..1c2f38a153 100644 --- a/src/ostreams.h +++ b/src/ostreams.h @@ -50,6 +50,12 @@ struct AsUC16 { }; +struct AsUC32 { + explicit AsUC32(int32_t v) : value(v) {} + int32_t value; +}; + + struct AsReversiblyEscapedUC16 { explicit AsReversiblyEscapedUC16(uint16_t v) : value(v) {} uint16_t value; @@ -73,6 +79,10 @@ std::ostream& operator<<(std::ostream& os, const AsEscapedUC16ForJSON& c); // of printable ASCII range. std::ostream& operator<<(std::ostream& os, const AsUC16& c); +// Writes the given character to the output escaping everything outside +// of printable ASCII range. +std::ostream& operator<<(std::ostream& os, const AsUC32& c); + } // namespace internal } // namespace v8 diff --git a/src/regexp/jsregexp.cc b/src/regexp/jsregexp.cc index 34d20fe781..3559bcd111 100644 --- a/src/regexp/jsregexp.cc +++ b/src/regexp/jsregexp.cc @@ -72,7 +72,7 @@ ContainedInLattice AddRange(ContainedInLattice containment, int ranges_length, Interval new_range) { DCHECK((ranges_length & 1) == 1); - DCHECK(ranges[ranges_length - 1] == String::kMaxUtf16CodeUnit + 1); + DCHECK(ranges[ranges_length - 1] == String::kMaxCodePoint + 1); if (containment == kLatticeUnknown) return containment; bool inside = false; int last = 0; @@ -145,9 +145,8 @@ MaybeHandle RegExpImpl::Compile(Handle re, PostponeInterruptsScope postpone(isolate); RegExpCompileData parse_result; FlatStringReader reader(isolate, pattern); - if (!RegExpParser::ParseRegExp(re->GetIsolate(), &zone, &reader, - flags & JSRegExp::kMultiline, - flags & JSRegExp::kUnicode, &parse_result)) { + if (!RegExpParser::ParseRegExp(re->GetIsolate(), &zone, &reader, flags, + &parse_result)) { // Throw an exception if we fail to parse the pattern. return ThrowRegExpException(re, pattern, parse_result.error); } @@ -371,18 +370,16 @@ bool RegExpImpl::CompileIrregexp(Handle re, pattern = String::Flatten(pattern); RegExpCompileData compile_data; FlatStringReader reader(isolate, pattern); - if (!RegExpParser::ParseRegExp(isolate, &zone, &reader, - flags & JSRegExp::kMultiline, - flags & JSRegExp::kUnicode, &compile_data)) { + if (!RegExpParser::ParseRegExp(isolate, &zone, &reader, flags, + &compile_data)) { // Throw an exception if we fail to parse the pattern. // THIS SHOULD NOT HAPPEN. We already pre-parsed it successfully once. USE(ThrowRegExpException(re, pattern, compile_data.error)); return false; } - RegExpEngine::CompilationResult result = RegExpEngine::Compile( - isolate, &zone, &compile_data, flags & JSRegExp::kIgnoreCase, - flags & JSRegExp::kGlobal, flags & JSRegExp::kMultiline, - flags & JSRegExp::kSticky, pattern, sample_subject, is_one_byte); + RegExpEngine::CompilationResult result = + RegExpEngine::Compile(isolate, &zone, &compile_data, flags, pattern, + sample_subject, is_one_byte); if (result.error_message != NULL) { // Unable to compile regexp. Handle error_message = isolate->factory()->NewStringFromUtf8( @@ -945,7 +942,7 @@ class FrequencyCollator { class RegExpCompiler { public: RegExpCompiler(Isolate* isolate, Zone* zone, int capture_count, - bool ignore_case, bool is_one_byte); + JSRegExp::Flags flags, bool is_one_byte); int AllocateRegister() { if (next_register_ >= RegExpMacroAssembler::kMaxRegister) { @@ -955,6 +952,22 @@ class RegExpCompiler { return next_register_++; } + // Lookarounds to match lone surrogates for unicode character class matches + // are never nested. We can therefore reuse registers. + int UnicodeLookaroundStackRegister() { + if (unicode_lookaround_stack_register_ == kNoRegister) { + unicode_lookaround_stack_register_ = AllocateRegister(); + } + return unicode_lookaround_stack_register_; + } + + int UnicodeLookaroundPositionRegister() { + if (unicode_lookaround_position_register_ == kNoRegister) { + unicode_lookaround_position_register_ = AllocateRegister(); + } + return unicode_lookaround_position_register_; + } + RegExpEngine::CompilationResult Assemble(RegExpMacroAssembler* assembler, RegExpNode* start, int capture_count, @@ -981,7 +994,8 @@ class RegExpCompiler { void SetRegExpTooBig() { reg_exp_too_big_ = true; } - inline bool ignore_case() { return ignore_case_; } + inline bool ignore_case() { return (flags_ & JSRegExp::kIgnoreCase) != 0; } + inline bool unicode() { return (flags_ & JSRegExp::kUnicode) != 0; } inline bool one_byte() { return one_byte_; } inline bool optimize() { return optimize_; } inline void set_optimize(bool value) { optimize_ = value; } @@ -1006,10 +1020,12 @@ class RegExpCompiler { private: EndNode* accept_; int next_register_; + int unicode_lookaround_stack_register_; + int unicode_lookaround_position_register_; List* work_list_; int recursion_depth_; RegExpMacroAssembler* macro_assembler_; - bool ignore_case_; + JSRegExp::Flags flags_; bool one_byte_; bool reg_exp_too_big_; bool limiting_recursion_; @@ -1041,11 +1057,13 @@ static RegExpEngine::CompilationResult IrregexpRegExpTooBig(Isolate* isolate) { // Attempts to compile the regexp using an Irregexp code generator. Returns // a fixed array or a null handle depending on whether it succeeded. RegExpCompiler::RegExpCompiler(Isolate* isolate, Zone* zone, int capture_count, - bool ignore_case, bool one_byte) + JSRegExp::Flags flags, bool one_byte) : next_register_(2 * (capture_count + 1)), + unicode_lookaround_stack_register_(kNoRegister), + unicode_lookaround_position_register_(kNoRegister), work_list_(NULL), recursion_depth_(0), - ignore_case_(ignore_case), + flags_(flags), one_byte_(one_byte), reg_exp_too_big_(false), limiting_recursion_(false), @@ -2098,9 +2116,7 @@ static void EmitCharClass(RegExpMacroAssembler* macro_assembler, Label* on_failure, int cp_offset, bool check_offset, bool preloaded, Zone* zone) { ZoneList* ranges = cc->ranges(zone); - if (!CharacterRange::IsCanonical(ranges)) { - CharacterRange::Canonicalize(ranges); - } + CharacterRange::Canonicalize(ranges); int max_char; if (one_byte) { @@ -2142,23 +2158,14 @@ static void EmitCharClass(RegExpMacroAssembler* macro_assembler, } return; } - if (last_valid_range == 0 && - !cc->is_negated() && - ranges->at(0).IsEverything(max_char)) { - // This is a common case hit by non-anchored expressions. - if (check_offset) { - macro_assembler->CheckPosition(cp_offset, on_failure); - } - return; - } if (!preloaded) { macro_assembler->LoadCurrentCharacter(cp_offset, on_failure, check_offset); } if (cc->is_standard(zone) && - macro_assembler->CheckSpecialCharacterClass(cc->standard_type(), - on_failure)) { + macro_assembler->CheckSpecialCharacterClass(cc->standard_type(), + on_failure)) { return; } @@ -2798,9 +2805,7 @@ RegExpNode* TextNode::FilterOneByte(int depth, bool ignore_case) { DCHECK(elm.text_type() == TextElement::CHAR_CLASS); RegExpCharacterClass* cc = elm.char_class(); ZoneList* ranges = cc->ranges(zone()); - if (!CharacterRange::IsCanonical(ranges)) { - CharacterRange::Canonicalize(ranges); - } + CharacterRange::Canonicalize(ranges); // Now they are in order so we only need to look at the first. int range_count = ranges->length(); if (cc->is_negated()) { @@ -3289,6 +3294,36 @@ bool TextNode::SkipPass(int int_pass, bool ignore_case) { } +TextNode* TextNode::CreateForCharacterRanges(Zone* zone, + ZoneList* ranges, + bool read_backward, + RegExpNode* on_success) { + DCHECK_NOT_NULL(ranges); + ZoneList* elms = new (zone) ZoneList(1, zone); + elms->Add( + TextElement::CharClass(new (zone) RegExpCharacterClass(ranges, false)), + zone); + return new (zone) TextNode(elms, read_backward, on_success); +} + + +TextNode* TextNode::CreateForSurrogatePair(Zone* zone, CharacterRange lead, + CharacterRange trail, + bool read_backward, + RegExpNode* on_success) { + ZoneList* lead_ranges = CharacterRange::List(zone, lead); + ZoneList* trail_ranges = CharacterRange::List(zone, trail); + ZoneList* elms = new (zone) ZoneList(2, zone); + elms->Add(TextElement::CharClass( + new (zone) RegExpCharacterClass(lead_ranges, false)), + zone); + elms->Add(TextElement::CharClass( + new (zone) RegExpCharacterClass(trail_ranges, false)), + zone); + return new (zone) TextNode(elms, read_backward, on_success); +} + + // This generates the code to match a text node. A text node can contain // straight character sequences (possibly to be matched in a case-independent // way) and character classes. For efficiency we do not do this in a single @@ -3405,9 +3440,7 @@ RegExpNode* TextNode::GetSuccessorOfOmnivorousTextNode( if (elm.text_type() != TextElement::CHAR_CLASS) return NULL; RegExpCharacterClass* node = elm.char_class(); ZoneList* ranges = node->ranges(zone()); - if (!CharacterRange::IsCanonical(ranges)) { - CharacterRange::Canonicalize(ranges); - } + CharacterRange::Canonicalize(ranges); if (node->is_negated()) { return ranges->length() == 0 ? on_success() : NULL; } @@ -3554,27 +3587,35 @@ class AlternativeGenerationList { }; +static const uc32 kLeadSurrogateStart = 0xd800; +static const uc32 kLeadSurrogateEnd = 0xdbff; +static const uc32 kTrailSurrogateStart = 0xdc00; +static const uc32 kTrailSurrogateEnd = 0xdfff; +static const uc32 kNonBmpStart = 0x10000; +static const uc32 kNonBmpEnd = 0x10ffff; +static const uc32 kRangeEndMarker = 0x110000; + // The '2' variant is has inclusive from and exclusive to. // This covers \s as defined in ECMA-262 5.1, 15.10.2.12, // which include WhiteSpace (7.2) or LineTerminator (7.3) values. -static const int kSpaceRanges[] = { '\t', '\r' + 1, ' ', ' ' + 1, - 0x00A0, 0x00A1, 0x1680, 0x1681, 0x180E, 0x180F, 0x2000, 0x200B, - 0x2028, 0x202A, 0x202F, 0x2030, 0x205F, 0x2060, 0x3000, 0x3001, - 0xFEFF, 0xFF00, 0x10000 }; +static const int kSpaceRanges[] = { + '\t', '\r' + 1, ' ', ' ' + 1, 0x00A0, 0x00A1, 0x1680, 0x1681, + 0x180E, 0x180F, 0x2000, 0x200B, 0x2028, 0x202A, 0x202F, 0x2030, + 0x205F, 0x2060, 0x3000, 0x3001, 0xFEFF, 0xFF00, kRangeEndMarker}; static const int kSpaceRangeCount = arraysize(kSpaceRanges); static const int kWordRanges[] = { - '0', '9' + 1, 'A', 'Z' + 1, '_', '_' + 1, 'a', 'z' + 1, 0x10000 }; + '0', '9' + 1, 'A', 'Z' + 1, '_', '_' + 1, 'a', 'z' + 1, kRangeEndMarker}; static const int kWordRangeCount = arraysize(kWordRanges); -static const int kDigitRanges[] = { '0', '9' + 1, 0x10000 }; +static const int kDigitRanges[] = {'0', '9' + 1, kRangeEndMarker}; static const int kDigitRangeCount = arraysize(kDigitRanges); -static const int kSurrogateRanges[] = { 0xd800, 0xe000, 0x10000 }; +static const int kSurrogateRanges[] = { + kLeadSurrogateStart, kLeadSurrogateStart + 1, kRangeEndMarker}; static const int kSurrogateRangeCount = arraysize(kSurrogateRanges); -static const int kLineTerminatorRanges[] = { 0x000A, 0x000B, 0x000D, 0x000E, - 0x2028, 0x202A, 0x10000 }; +static const int kLineTerminatorRanges[] = { + 0x000A, 0x000B, 0x000D, 0x000E, 0x2028, 0x202A, kRangeEndMarker}; static const int kLineTerminatorRangeCount = arraysize(kLineTerminatorRanges); - void BoyerMoorePositionInfo::Set(int character) { SetInterval(Interval(character, character)); } @@ -4732,8 +4773,8 @@ RegExpNode* RegExpText::ToNode(RegExpCompiler* compiler, static bool CompareInverseRanges(ZoneList* ranges, const int* special_class, int length) { - length--; // Remove final 0x10000. - DCHECK(special_class[length] == 0x10000); + length--; // Remove final marker. + DCHECK(special_class[length] == kRangeEndMarker); DCHECK(ranges->length() != 0); DCHECK(length != 0); DCHECK(special_class[0] != 0); @@ -4763,8 +4804,8 @@ static bool CompareInverseRanges(ZoneList* ranges, static bool CompareRanges(ZoneList* ranges, const int* special_class, int length) { - length--; // Remove final 0x10000. - DCHECK(special_class[length] == 0x10000); + length--; // Remove final marker. + DCHECK(special_class[length] == kRangeEndMarker); if (ranges->length() * 2 != length) { return false; } @@ -4820,10 +4861,257 @@ bool RegExpCharacterClass::is_standard(Zone* zone) { } +bool RegExpCharacterClass::NeedsDesugaringForUnicode(Zone* zone) { + ZoneList* ranges = this->ranges(zone); + CharacterRange::Canonicalize(ranges); + for (int i = ranges->length() - 1; i >= 0; i--) { + uc32 from = ranges->at(i).from(); + uc32 to = ranges->at(i).to(); + // Check for non-BMP characters. + if (to >= kNonBmpStart) return true; + // Check for lone surrogates. + if (from <= kTrailSurrogateEnd && to >= kLeadSurrogateStart) return true; + } + return false; +} + + +UnicodeRangeSplitter::UnicodeRangeSplitter(Zone* zone, + ZoneList* base) + : zone_(zone), + table_(zone), + bmp_(nullptr), + lead_surrogates_(nullptr), + trail_surrogates_(nullptr), + non_bmp_(nullptr) { + // The unicode range splitter categorizes given character ranges into: + // - Code points from the BMP representable by one code unit. + // - Code points outside the BMP that need to be split into surrogate pairs. + // - Lone lead surrogates. + // - Lone trail surrogates. + // Lone surrogates are valid code points, even though no actual characters. + // They require special matching to make sure we do not split surrogate pairs. + // We use the dispatch table to accomplish this. The base range is split up + // by the table by the overlay ranges, and the Call callback is used to + // filter and collect ranges for each category. + for (int i = 0; i < base->length(); i++) { + table_.AddRange(base->at(i), kBase, zone_); + } + // Add overlay ranges. + table_.AddRange(CharacterRange(0, kLeadSurrogateStart - 1), kBmpCodePoints, + zone_); + table_.AddRange(CharacterRange(kLeadSurrogateStart, kLeadSurrogateEnd), + kLeadSurrogates, zone_); + table_.AddRange(CharacterRange(kTrailSurrogateStart, kTrailSurrogateEnd), + kTrailSurrogates, zone_); + table_.AddRange(CharacterRange(kTrailSurrogateEnd, kNonBmpStart - 1), + kBmpCodePoints, zone_); + table_.AddRange(CharacterRange(kNonBmpStart, kNonBmpEnd), kNonBmpCodePoints, + zone_); + table_.ForEach(this); +} + + +void UnicodeRangeSplitter::Call(uc32 from, DispatchTable::Entry entry) { + OutSet* outset = entry.out_set(); + if (!outset->Get(kBase)) return; + ZoneList** target = NULL; + if (outset->Get(kBmpCodePoints)) { + target = &bmp_; + } else if (outset->Get(kLeadSurrogates)) { + target = &lead_surrogates_; + } else if (outset->Get(kTrailSurrogates)) { + target = &trail_surrogates_; + } else { + DCHECK(outset->Get(kNonBmpCodePoints)); + target = &non_bmp_; + } + if (*target == NULL) *target = new (zone_) ZoneList(2, zone_); + (*target)->Add(CharacterRange::Range(entry.from(), entry.to()), zone_); +} + + +void AddBmpCharacters(RegExpCompiler* compiler, ChoiceNode* result, + RegExpNode* on_success, UnicodeRangeSplitter* splitter) { + ZoneList* bmp = splitter->bmp(); + if (bmp == nullptr) return; + result->AddAlternative(GuardedAlternative(TextNode::CreateForCharacterRanges( + compiler->zone(), bmp, compiler->read_backward(), on_success))); +} + + +void AddNonBmpSurrogatePairs(RegExpCompiler* compiler, ChoiceNode* result, + RegExpNode* on_success, + UnicodeRangeSplitter* splitter) { + ZoneList* non_bmp = splitter->non_bmp(); + if (non_bmp == nullptr) return; + DCHECK(compiler->unicode()); + DCHECK(!compiler->one_byte()); + Zone* zone = compiler->zone(); + CharacterRange::Canonicalize(non_bmp); + for (int i = 0; i < non_bmp->length(); i++) { + // Match surrogate pair. + // E.g. [\u10005-\u11005] becomes + // \ud800[\udc05-\udfff]| + // [\ud801-\ud803][\udc00-\udfff]| + // \ud804[\udc00-\udc05] + uc32 from = non_bmp->at(i).from(); + uc32 to = non_bmp->at(i).to(); + uc16 from_l = unibrow::Utf16::LeadSurrogate(from); + uc16 from_t = unibrow::Utf16::TrailSurrogate(from); + uc16 to_l = unibrow::Utf16::LeadSurrogate(to); + uc16 to_t = unibrow::Utf16::TrailSurrogate(to); + if (from_l == to_l) { + // The lead surrogate is the same. + result->AddAlternative( + GuardedAlternative(TextNode::CreateForSurrogatePair( + zone, CharacterRange::Singleton(from_l), + CharacterRange::Range(from_t, to_t), compiler->read_backward(), + on_success))); + } else { + if (from_t != kTrailSurrogateStart) { + // Add [from_l][from_t-\udfff] + result->AddAlternative( + GuardedAlternative(TextNode::CreateForSurrogatePair( + zone, CharacterRange::Singleton(from_l), + CharacterRange::Range(from_t, kTrailSurrogateEnd), + compiler->read_backward(), on_success))); + from_l++; + } + if (to_t != kTrailSurrogateEnd) { + // Add [to_l][\udc00-to_t] + result->AddAlternative( + GuardedAlternative(TextNode::CreateForSurrogatePair( + zone, CharacterRange::Singleton(to_l), + CharacterRange::Range(kTrailSurrogateStart, to_t), + compiler->read_backward(), on_success))); + to_l--; + } + if (from_l <= to_l) { + // Add [from_l-to_l][\udc00-\udfff] + result->AddAlternative( + GuardedAlternative(TextNode::CreateForSurrogatePair( + zone, CharacterRange::Range(from_l, to_l), + CharacterRange::Range(kTrailSurrogateStart, kTrailSurrogateEnd), + compiler->read_backward(), on_success))); + } + } + } +} + + +RegExpNode* NegativeLookaroundAgainstReadDirectionAndMatch( + RegExpCompiler* compiler, ZoneList* lookbehind, + ZoneList* match, RegExpNode* on_success, + bool read_backward) { + Zone* zone = compiler->zone(); + RegExpNode* match_node = TextNode::CreateForCharacterRanges( + zone, match, read_backward, on_success); + int stack_register = compiler->UnicodeLookaroundStackRegister(); + int position_register = compiler->UnicodeLookaroundPositionRegister(); + RegExpLookaround::Builder lookaround(false, match_node, stack_register, + position_register); + RegExpNode* negative_match = TextNode::CreateForCharacterRanges( + zone, lookbehind, !read_backward, lookaround.on_match_success()); + return lookaround.ForMatch(negative_match); +} + + +RegExpNode* MatchAndNegativeLookaroundInReadDirection( + RegExpCompiler* compiler, ZoneList* match, + ZoneList* lookahead, RegExpNode* on_success, + bool read_backward) { + Zone* zone = compiler->zone(); + int stack_register = compiler->UnicodeLookaroundStackRegister(); + int position_register = compiler->UnicodeLookaroundPositionRegister(); + RegExpLookaround::Builder lookaround(false, on_success, stack_register, + position_register); + RegExpNode* negative_match = TextNode::CreateForCharacterRanges( + zone, lookahead, read_backward, lookaround.on_match_success()); + return TextNode::CreateForCharacterRanges( + zone, match, read_backward, lookaround.ForMatch(negative_match)); +} + + +void AddLoneLeadSurrogates(RegExpCompiler* compiler, ChoiceNode* result, + RegExpNode* on_success, + UnicodeRangeSplitter* splitter) { + ZoneList* lead_surrogates = splitter->lead_surrogates(); + if (lead_surrogates == nullptr) return; + Zone* zone = compiler->zone(); + // E.g. \ud801 becomes \ud801(?![\udc00-\udfff]). + ZoneList* trail_surrogates = + new (zone) ZoneList(1, zone); + trail_surrogates->Add( + CharacterRange::Range(kTrailSurrogateStart, kTrailSurrogateEnd), zone); + + RegExpNode* match = + compiler->read_backward() + // Reading backward. Assert that reading forward, there is no trail + // surrogate, and then backward match the lead surrogate. + ? NegativeLookaroundAgainstReadDirectionAndMatch( + compiler, trail_surrogates, lead_surrogates, on_success, true) + // Reading forward. Forwrad match the lead surrogate and assert that + // no + // trail surrogate follows. + : MatchAndNegativeLookaroundInReadDirection( + compiler, lead_surrogates, trail_surrogates, on_success, false); + result->AddAlternative(GuardedAlternative(match)); +} + + +void AddLoneTrailSurrogates(RegExpCompiler* compiler, ChoiceNode* result, + RegExpNode* on_success, + UnicodeRangeSplitter* splitter) { + ZoneList* trail_surrogates = splitter->trail_surrogates(); + if (trail_surrogates == nullptr) return; + Zone* zone = compiler->zone(); + // E.g. \udc01 becomes (?* lead_surrogates = + new (zone) ZoneList(1, zone); + lead_surrogates->Add( + CharacterRange::Range(kLeadSurrogateStart, kLeadSurrogateEnd), zone); + + RegExpNode* match = + compiler->read_backward() + // Reading backward. Backward match the trail surrogate and assert + // that no lead surrogate precedes it. + ? MatchAndNegativeLookaroundInReadDirection( + compiler, trail_surrogates, lead_surrogates, on_success, true) + // Reading forward. Assert that reading backward, there is no lead + // surrogate, and then forward match the trail surrogate. + : NegativeLookaroundAgainstReadDirectionAndMatch( + compiler, lead_surrogates, trail_surrogates, on_success, false); + result->AddAlternative(GuardedAlternative(match)); +} + + RegExpNode* RegExpCharacterClass::ToNode(RegExpCompiler* compiler, RegExpNode* on_success) { - return new (compiler->zone()) - TextNode(this, compiler->read_backward(), on_success); + set_.Canonicalize(); + Zone* zone = compiler->zone(); + ZoneList* ranges = this->ranges(zone); + if (compiler->unicode() && !compiler->one_byte()) { + if (is_negated()) { + ZoneList* negated = + new (zone) ZoneList(2, zone); + CharacterRange::Negate(ranges, negated, zone); + ranges = negated; + } + if (ranges->length() == 0) { + // No matches possible. + return new (zone) EndNode(EndNode::BACKTRACK, zone); + } + UnicodeRangeSplitter splitter(zone, ranges); + ChoiceNode* result = new (compiler->zone()) ChoiceNode(2, compiler->zone()); + AddBmpCharacters(compiler, result, on_success, &splitter); + AddNonBmpSurrogatePairs(compiler, result, on_success, &splitter); + AddLoneLeadSurrogates(compiler, result, on_success, &splitter); + AddLoneTrailSurrogates(compiler, result, on_success, &splitter); + return result; + } else { + return new (zone) TextNode(this, compiler->read_backward(), on_success); + } } @@ -5338,6 +5626,47 @@ RegExpNode* RegExpEmpty::ToNode(RegExpCompiler* compiler, } +RegExpLookaround::Builder::Builder(bool is_positive, RegExpNode* on_success, + int stack_pointer_register, + int position_register, + int capture_register_count, + int capture_register_start) + : is_positive_(is_positive), + on_success_(on_success), + stack_pointer_register_(stack_pointer_register), + position_register_(position_register) { + if (is_positive_) { + on_match_success_ = ActionNode::PositiveSubmatchSuccess( + stack_pointer_register, position_register, capture_register_count, + capture_register_start, on_success_); + } else { + Zone* zone = on_success_->zone(); + on_match_success_ = new (zone) NegativeSubmatchSuccess( + stack_pointer_register, position_register, capture_register_count, + capture_register_start, zone); + } +} + + +RegExpNode* RegExpLookaround::Builder::ForMatch(RegExpNode* match) { + if (is_positive_) { + return ActionNode::BeginSubmatch(stack_pointer_register_, + position_register_, match); + } else { + Zone* zone = on_success_->zone(); + // We use a ChoiceNode to represent the negative lookaround. The first + // alternative is the negative match. On success, the end node backtracks. + // On failure, the second alternative is tried and leads to success. + // NegativeLookaheadChoiceNode is a special ChoiceNode that ignores the + // first exit when calculating quick checks. + ChoiceNode* choice_node = new (zone) NegativeLookaroundChoiceNode( + GuardedAlternative(match), GuardedAlternative(on_success_), zone); + return ActionNode::BeginSubmatch(stack_pointer_register_, + position_register_, choice_node); + } +} + + RegExpNode* RegExpLookaround::ToNode(RegExpCompiler* compiler, RegExpNode* on_success) { int stack_pointer_register = compiler->AllocateRegister(); @@ -5352,35 +5681,10 @@ RegExpNode* RegExpLookaround::ToNode(RegExpCompiler* compiler, RegExpNode* result; bool was_reading_backward = compiler->read_backward(); compiler->set_read_backward(type() == LOOKBEHIND); - if (is_positive()) { - result = ActionNode::BeginSubmatch( - stack_pointer_register, position_register, - body()->ToNode(compiler, - ActionNode::PositiveSubmatchSuccess( - stack_pointer_register, position_register, - register_count, register_start, on_success))); - } else { - // We use a ChoiceNode for a negative lookahead because it has most of - // the characteristics we need. It has the body of the lookahead as its - // first alternative and the expression after the lookahead of the second - // alternative. If the first alternative succeeds then the - // NegativeSubmatchSuccess will unwind the stack including everything the - // choice node set up and backtrack. If the first alternative fails then - // the second alternative is tried, which is exactly the desired result - // for a negative lookahead. The NegativeLookaheadChoiceNode is a special - // ChoiceNode that knows to ignore the first exit when calculating quick - // checks. - Zone* zone = compiler->zone(); - - GuardedAlternative body_alt( - body()->ToNode(compiler, new (zone) NegativeSubmatchSuccess( - stack_pointer_register, position_register, - register_count, register_start, zone))); - ChoiceNode* choice_node = new (zone) NegativeLookaroundChoiceNode( - body_alt, GuardedAlternative(on_success), zone); - result = ActionNode::BeginSubmatch(stack_pointer_register, - position_register, choice_node); - } + Builder builder(is_positive(), on_success, stack_pointer_register, + position_register, register_count, register_start); + RegExpNode* match = body_->ToNode(compiler, builder.on_match_success()); + result = builder.ForMatch(match); compiler->set_read_backward(was_reading_backward); return result; } @@ -5428,7 +5732,7 @@ static void AddClass(const int* elmv, ZoneList* ranges, Zone* zone) { elmc--; - DCHECK(elmv[elmc] == 0x10000); + DCHECK(elmv[elmc] == kRangeEndMarker); for (int i = 0; i < elmc; i += 2) { DCHECK(elmv[i] < elmv[i + 1]); ranges->Add(CharacterRange(elmv[i], elmv[i + 1] - 1), zone); @@ -5441,9 +5745,9 @@ static void AddClassNegated(const int *elmv, ZoneList* ranges, Zone* zone) { elmc--; - DCHECK(elmv[elmc] == 0x10000); + DCHECK(elmv[elmc] == kRangeEndMarker); DCHECK(elmv[0] != 0x0000); - DCHECK(elmv[elmc-1] != String::kMaxUtf16CodeUnit); + DCHECK(elmv[elmc - 1] != String::kMaxCodePoint); uc16 last = 0x0000; for (int i = 0; i < elmc; i += 2) { DCHECK(last <= elmv[i] - 1); @@ -5451,7 +5755,7 @@ static void AddClassNegated(const int *elmv, ranges->Add(CharacterRange(last, elmv[i] - 1), zone); last = elmv[i + 1]; } - ranges->Add(CharacterRange(last, String::kMaxUtf16CodeUnit), zone); + ranges->Add(CharacterRange(last, String::kMaxCodePoint), zone); } @@ -5508,60 +5812,13 @@ Vector CharacterRange::GetWordBounds() { } -class CharacterRangeSplitter { - public: - CharacterRangeSplitter(ZoneList** included, - ZoneList** excluded, - Zone* zone) - : included_(included), - excluded_(excluded), - zone_(zone) { } - void Call(uc16 from, DispatchTable::Entry entry); - - static const int kInBase = 0; - static const int kInOverlay = 1; - - private: - ZoneList** included_; - ZoneList** excluded_; - Zone* zone_; -}; - - -void CharacterRangeSplitter::Call(uc16 from, DispatchTable::Entry entry) { - if (!entry.out_set()->Get(kInBase)) return; - ZoneList** target = entry.out_set()->Get(kInOverlay) - ? included_ - : excluded_; - if (*target == NULL) *target = new(zone_) ZoneList(2, zone_); - (*target)->Add(CharacterRange(entry.from(), entry.to()), zone_); -} - - -void CharacterRange::Split(ZoneList* base, - Vector overlay, - ZoneList** included, - ZoneList** excluded, - Zone* zone) { - DCHECK_NULL(*included); - DCHECK_NULL(*excluded); - DispatchTable table(zone); - for (int i = 0; i < base->length(); i++) - table.AddRange(base->at(i), CharacterRangeSplitter::kInBase, zone); - for (int i = 0; i < overlay.length(); i += 2) { - table.AddRange(CharacterRange(overlay[i], overlay[i + 1] - 1), - CharacterRangeSplitter::kInOverlay, zone); - } - CharacterRangeSplitter callback(included, excluded, zone); - table.ForEach(&callback); -} - - void CharacterRange::AddCaseEquivalents(Isolate* isolate, Zone* zone, ZoneList* ranges, bool is_one_byte) { - uc16 bottom = from(); - uc16 top = to(); + uc32 bottom = from(); + uc32 top = to(); + // Nothing to be done for surrogates. + if (bottom >= kLeadSurrogateStart && top <= kTrailSurrogateEnd) return; if (is_one_byte && !RangeContainsLatin1Equivalents(*this)) { if (bottom > String::kMaxOneByteCharCode) return; if (top > String::kMaxOneByteCharCode) top = String::kMaxOneByteCharCode; @@ -5599,7 +5856,7 @@ void CharacterRange::AddCaseEquivalents(Isolate* isolate, Zone* zone, int pos = bottom; while (pos <= top) { int length = isolate->jsregexp_canonrange()->get(pos, '\0', range); - uc16 block_end; + uc32 block_end; if (length == 0) { block_end = pos; } else { @@ -5610,8 +5867,8 @@ void CharacterRange::AddCaseEquivalents(Isolate* isolate, Zone* zone, length = isolate->jsregexp_uncanonicalize()->get(block_end, '\0', range); for (int i = 0; i < length; i++) { uc32 c = range[i]; - uc16 range_from = c - (block_end - pos); - uc16 range_to = c - (block_end - end); + uc32 range_from = c - (block_end - pos); + uc32 range_to = c - (block_end - end); if (!(bottom <= range_from && range_to <= top)) { ranges->Add(CharacterRange(range_from, range_to), zone); } @@ -5672,8 +5929,8 @@ static int InsertRangeInCanonicalList(ZoneList* list, // list[0..count] for the result. Returns the number of resulting // canonicalized ranges. Inserting a range may collapse existing ranges into // fewer ranges, so the return value can be anything in the range 1..count+1. - uc16 from = insert.from(); - uc16 to = insert.to(); + uc32 from = insert.from(); + uc32 to = insert.to(); int start_pos = 0; int end_pos = count; for (int i = count - 1; i >= 0; i--) { @@ -5773,7 +6030,7 @@ void CharacterRange::Negate(ZoneList* ranges, DCHECK(CharacterRange::IsCanonical(ranges)); DCHECK_EQ(0, negated_ranges->length()); int range_count = ranges->length(); - uc16 from = 0; + uc32 from = 0; int i = 0; if (range_count > 0 && ranges->at(0).from() == 0) { from = ranges->at(0).to(); @@ -5785,9 +6042,8 @@ void CharacterRange::Negate(ZoneList* ranges, from = range.to(); i++; } - if (from < String::kMaxUtf16CodeUnit) { - negated_ranges->Add(CharacterRange(from + 1, String::kMaxUtf16CodeUnit), - zone); + if (from < String::kMaxCodePoint) { + negated_ranges->Add(CharacterRange(from + 1, String::kMaxCodePoint), zone); } } @@ -5838,7 +6094,7 @@ bool OutSet::Get(unsigned value) const { } -const uc16 DispatchTable::Config::kNoKey = unibrow::Utf8::kBadChar; +const uc32 DispatchTable::Config::kNoKey = unibrow::Utf8::kBadChar; void DispatchTable::AddRange(CharacterRange full_range, int value, @@ -5940,7 +6196,7 @@ void DispatchTable::AddRange(CharacterRange full_range, int value, } -OutSet* DispatchTable::Get(uc16 value) { +OutSet* DispatchTable::Get(uc32 value) { ZoneSplayTree::Locator loc; if (!tree()->FindGreatestLessThan(value, &loc)) return empty(); @@ -6258,13 +6514,16 @@ void DispatchTableConstructor::VisitAction(ActionNode* that) { RegExpEngine::CompilationResult RegExpEngine::Compile( - Isolate* isolate, Zone* zone, RegExpCompileData* data, bool ignore_case, - bool is_global, bool is_multiline, bool is_sticky, Handle pattern, + Isolate* isolate, Zone* zone, RegExpCompileData* data, + JSRegExp::Flags flags, Handle pattern, Handle sample_subject, bool is_one_byte) { if ((data->capture_count + 1) * 2 - 1 > RegExpMacroAssembler::kMaxRegister) { return IrregexpRegExpTooBig(isolate); } - RegExpCompiler compiler(isolate, zone, data->capture_count, ignore_case, + bool ignore_case = flags & JSRegExp::kIgnoreCase; + bool is_sticky = flags & JSRegExp::kSticky; + bool is_global = flags & JSRegExp::kGlobal; + RegExpCompiler compiler(isolate, zone, data->capture_count, flags, is_one_byte); if (compiler.optimize()) compiler.set_optimize(!TooMuchRegExpCode(pattern)); diff --git a/src/regexp/jsregexp.h b/src/regexp/jsregexp.h index 0ad4b79c87..f4051fdf43 100644 --- a/src/regexp/jsregexp.h +++ b/src/regexp/jsregexp.h @@ -265,28 +265,28 @@ class DispatchTable : public ZoneObject { class Entry { public: Entry() : from_(0), to_(0), out_set_(NULL) { } - Entry(uc16 from, uc16 to, OutSet* out_set) - : from_(from), to_(to), out_set_(out_set) { } - uc16 from() { return from_; } - uc16 to() { return to_; } - void set_to(uc16 value) { to_ = value; } + Entry(uc32 from, uc32 to, OutSet* out_set) + : from_(from), to_(to), out_set_(out_set) {} + uc32 from() { return from_; } + uc32 to() { return to_; } + void set_to(uc32 value) { to_ = value; } void AddValue(int value, Zone* zone) { out_set_ = out_set_->Extend(value, zone); } OutSet* out_set() { return out_set_; } private: - uc16 from_; - uc16 to_; + uc32 from_; + uc32 to_; OutSet* out_set_; }; class Config { public: - typedef uc16 Key; + typedef uc32 Key; typedef Entry Value; - static const uc16 kNoKey; + static const uc32 kNoKey; static const Entry NoValue() { return Value(); } - static inline int Compare(uc16 a, uc16 b) { + static inline int Compare(uc32 a, uc32 b) { if (a == b) return 0; else if (a < b) @@ -297,7 +297,7 @@ class DispatchTable : public ZoneObject { }; void AddRange(CharacterRange range, int value, Zone* zone); - OutSet* Get(uc16 value); + OutSet* Get(uc32 value); void Dump(); template @@ -315,6 +315,34 @@ class DispatchTable : public ZoneObject { }; +// Categorizes character ranges into BMP, non-BMP, lead, and trail surrogates. +class UnicodeRangeSplitter { + public: + UnicodeRangeSplitter(Zone* zone, ZoneList* base); + void Call(uc32 from, DispatchTable::Entry entry); + + ZoneList* bmp() { return bmp_; } + ZoneList* lead_surrogates() { return lead_surrogates_; } + ZoneList* trail_surrogates() { return trail_surrogates_; } + ZoneList* non_bmp() const { return non_bmp_; } + + private: + static const int kBase = 0; + // Separate ranges into + static const int kBmpCodePoints = 1; + static const int kLeadSurrogates = 2; + static const int kTrailSurrogates = 3; + static const int kNonBmpCodePoints = 4; + + Zone* zone_; + DispatchTable table_; + ZoneList* bmp_; + ZoneList* lead_surrogates_; + ZoneList* trail_surrogates_; + ZoneList* non_bmp_; +}; + + #define FOR_EACH_NODE_TYPE(VISIT) \ VISIT(End) \ VISIT(Action) \ @@ -690,6 +718,17 @@ class TextNode: public SeqRegExpNode { read_backward_(read_backward) { elms_->Add(TextElement::CharClass(that), zone()); } + // Create TextNode for a single character class for the given ranges. + static TextNode* CreateForCharacterRanges(Zone* zone, + ZoneList* ranges, + bool read_backward, + RegExpNode* on_success); + // Create TextNode for a surrogate pair with a range given for the + // lead and the trail surrogate each. + static TextNode* CreateForSurrogatePair(Zone* zone, CharacterRange lead, + CharacterRange trail, + bool read_backward, + RegExpNode* on_success); virtual void Accept(NodeVisitor* visitor); virtual void Emit(RegExpCompiler* compiler, Trace* trace); virtual int EatsAtLeast(int still_to_find, int budget, bool not_at_start); @@ -813,8 +852,7 @@ class BackReferenceNode: public SeqRegExpNode { class EndNode: public RegExpNode { public: enum Action { ACCEPT, BACKTRACK, NEGATIVE_SUBMATCH_SUCCESS }; - explicit EndNode(Action action, Zone* zone) - : RegExpNode(zone), action_(action) { } + EndNode(Action action, Zone* zone) : RegExpNode(zone), action_(action) {} virtual void Accept(NodeVisitor* visitor); virtual void Emit(RegExpCompiler* compiler, Trace* trace); virtual int EatsAtLeast(int still_to_find, @@ -1505,8 +1543,8 @@ class RegExpEngine: public AllStatic { }; static CompilationResult Compile(Isolate* isolate, Zone* zone, - RegExpCompileData* input, bool ignore_case, - bool global, bool multiline, bool sticky, + RegExpCompileData* input, + JSRegExp::Flags flags, Handle pattern, Handle sample_subject, bool is_one_byte); diff --git a/src/regexp/regexp-ast.cc b/src/regexp/regexp-ast.cc index 31c93b114f..b5c2bb6d91 100644 --- a/src/regexp/regexp-ast.cc +++ b/src/regexp/regexp-ast.cc @@ -172,9 +172,9 @@ void* RegExpUnparser::VisitAlternative(RegExpAlternative* that, void* data) { void RegExpUnparser::VisitCharacterRange(CharacterRange that) { - os_ << AsUC16(that.from()); + os_ << AsUC32(that.from()); if (!that.IsSingleton()) { - os_ << "-" << AsUC16(that.to()); + os_ << "-" << AsUC32(that.to()); } } diff --git a/src/regexp/regexp-ast.h b/src/regexp/regexp-ast.h index f87778596a..ed91a82d49 100644 --- a/src/regexp/regexp-ast.h +++ b/src/regexp/regexp-ast.h @@ -5,6 +5,7 @@ #ifndef V8_REGEXP_REGEXP_AST_H_ #define V8_REGEXP_REGEXP_AST_H_ +#include "src/objects.h" #include "src/utils.h" #include "src/zone.h" @@ -77,33 +78,38 @@ class CharacterRange { CharacterRange() : from_(0), to_(0) {} // For compatibility with the CHECK_OK macro CharacterRange(void* null) { DCHECK_NULL(null); } // NOLINT - CharacterRange(uc16 from, uc16 to) : from_(from), to_(to) {} + CharacterRange(uc32 from, uc32 to) : from_(from), to_(to) {} static void AddClassEscape(uc16 type, ZoneList* ranges, Zone* zone); static Vector GetWordBounds(); - static inline CharacterRange Singleton(uc16 value) { + static inline CharacterRange Singleton(uc32 value) { return CharacterRange(value, value); } - static inline CharacterRange Range(uc16 from, uc16 to) { - DCHECK(from <= to); + static inline CharacterRange Range(uc32 from, uc32 to) { + DCHECK(0 <= from && to <= String::kMaxCodePoint); + DCHECK(static_cast(from) <= static_cast(to)); return CharacterRange(from, to); } static inline CharacterRange Everything() { - return CharacterRange(0, 0xFFFF); + return CharacterRange(0, String::kMaxCodePoint); } - bool Contains(uc16 i) { return from_ <= i && i <= to_; } - uc16 from() const { return from_; } - void set_from(uc16 value) { from_ = value; } - uc16 to() const { return to_; } - void set_to(uc16 value) { to_ = value; } + static inline ZoneList* List(Zone* zone, + CharacterRange range) { + ZoneList* list = + new (zone) ZoneList(1, zone); + list->Add(range, zone); + return list; + } + bool Contains(uc32 i) { return from_ <= i && i <= to_; } + uc32 from() const { return from_; } + void set_from(uc32 value) { from_ = value; } + uc32 to() const { return to_; } + void set_to(uc32 value) { to_ = value; } bool is_valid() { return from_ <= to_; } bool IsEverything(uc16 max) { return from_ == 0 && to_ >= max; } bool IsSingleton() { return (from_ == to_); } void AddCaseEquivalents(Isolate* isolate, Zone* zone, ZoneList* ranges, bool is_one_byte); - static void Split(ZoneList* base, Vector overlay, - ZoneList** included, - ZoneList** excluded, Zone* zone); // Whether a range list is in canonical form: Ranges ordered by from value, // and ranges non-overlapping and non-adjacent. static bool IsCanonical(ZoneList* ranges); @@ -119,8 +125,8 @@ class CharacterRange { static const int kPayloadMask = (1 << 24) - 1; private: - uc16 from_; - uc16 to_; + uc32 from_; + uc32 to_; }; @@ -287,6 +293,7 @@ class RegExpCharacterClass final : public RegExpTree { RegExpCharacterClass* AsCharacterClass() override; bool IsCharacterClass() override; bool IsTextElement() override { return true; } + bool NeedsDesugaringForUnicode(Zone* zone); int min_match() override { return 1; } int max_match() override { return 1; } void AppendToText(RegExpText* text, Zone* zone) override; @@ -451,6 +458,22 @@ class RegExpLookaround final : public RegExpTree { int capture_from() { return capture_from_; } Type type() { return type_; } + class Builder { + public: + Builder(bool is_positive, RegExpNode* on_success, + int stack_pointer_register, int position_register, + int capture_register_count = 0, int capture_register_start = 0); + RegExpNode* on_match_success() { return on_match_success_; } + RegExpNode* ForMatch(RegExpNode* match); + + private: + bool is_positive_; + RegExpNode* on_match_success_; + RegExpNode* on_success_; + int stack_pointer_register_; + int position_register_; + }; + private: RegExpTree* body_; bool is_positive_; diff --git a/src/regexp/regexp-parser.cc b/src/regexp/regexp-parser.cc index fa8900342c..07d5779675 100644 --- a/src/regexp/regexp-parser.cc +++ b/src/regexp/regexp-parser.cc @@ -15,20 +15,18 @@ namespace v8 { namespace internal { RegExpParser::RegExpParser(FlatStringReader* in, Handle* error, - bool multiline, bool unicode, Isolate* isolate, - Zone* zone) + JSRegExp::Flags flags, Isolate* isolate, Zone* zone) : isolate_(isolate), zone_(zone), error_(error), captures_(NULL), in_(in), current_(kEndMarker), + flags_(flags), next_pos_(0), captures_started_(0), capture_count_(0), has_more_(true), - multiline_(multiline), - unicode_(unicode), simple_(false), contains_anchor_(false), is_scanned_for_captures_(false), @@ -37,9 +35,28 @@ RegExpParser::RegExpParser(FlatStringReader* in, Handle* error, } +template +uc32 RegExpParser::ReadNext() { + int position = next_pos_; + uc32 c0 = in()->Get(position); + position++; + // Read the whole surrogate pair in case of unicode flag, if possible. + if (unicode() && position < in()->length() && + unibrow::Utf16::IsLeadSurrogate(static_cast(c0))) { + uc16 c1 = in()->Get(position); + if (unibrow::Utf16::IsTrailSurrogate(c1)) { + c0 = unibrow::Utf16::CombineSurrogatePair(static_cast(c0), c1); + position++; + } + } + if (update_position) next_pos_ = position; + return c0; +} + + uc32 RegExpParser::Next() { if (has_next()) { - return in()->Get(next_pos_); + return ReadNext(); } else { return kEndMarker; } @@ -47,25 +64,14 @@ uc32 RegExpParser::Next() { void RegExpParser::Advance() { - if (next_pos_ < in()->length()) { + if (has_next()) { StackLimitCheck check(isolate()); if (check.HasOverflowed()) { ReportError(CStrVector(Isolate::kStackOverflowMessage)); } else if (zone()->excess_allocation()) { ReportError(CStrVector("Regular expression too large")); } else { - current_ = in()->Get(next_pos_); - next_pos_++; - // Read the whole surrogate pair in case of unicode flag, if possible. - if (unicode_ && next_pos_ < in()->length() && - unibrow::Utf16::IsLeadSurrogate(static_cast(current_))) { - uc16 trail = in()->Get(next_pos_); - if (unibrow::Utf16::IsTrailSurrogate(trail)) { - current_ = unibrow::Utf16::CombineSurrogatePair( - static_cast(current_), trail); - next_pos_++; - } - } + current_ = ReadNext(); } } else { current_ = kEndMarker; @@ -142,7 +148,7 @@ RegExpTree* RegExpParser::ParsePattern() { RegExpTree* RegExpParser::ParseDisjunction() { // Used to store current state while parsing subexpressions. RegExpParserState initial_state(NULL, INITIAL, RegExpLookaround::LOOKAHEAD, 0, - zone()); + flags_, zone()); RegExpParserState* state = &initial_state; // Cache the builder in a local variable for quick access. RegExpBuilder* builder = initial_state.builder(); @@ -206,7 +212,7 @@ RegExpTree* RegExpParser::ParseDisjunction() { return ReportError(CStrVector("Nothing to repeat")); case '^': { Advance(); - if (multiline_) { + if (multiline()) { builder->AddAssertion( new (zone()) RegExpAssertion(RegExpAssertion::START_OF_LINE)); } else { @@ -219,8 +225,8 @@ RegExpTree* RegExpParser::ParseDisjunction() { case '$': { Advance(); RegExpAssertion::AssertionType assertion_type = - multiline_ ? RegExpAssertion::END_OF_LINE - : RegExpAssertion::END_OF_INPUT; + multiline() ? RegExpAssertion::END_OF_LINE + : RegExpAssertion::END_OF_INPUT; builder->AddAssertion(new (zone()) RegExpAssertion(assertion_type)); continue; } @@ -230,8 +236,9 @@ RegExpTree* RegExpParser::ParseDisjunction() { ZoneList* ranges = new (zone()) ZoneList(2, zone()); CharacterRange::AddClassEscape('.', ranges, zone()); - RegExpTree* atom = new (zone()) RegExpCharacterClass(ranges, false); - builder->AddAtom(atom); + RegExpCharacterClass* cc = + new (zone()) RegExpCharacterClass(ranges, false); + builder->AddCharacterClass(cc); break; } case '(': { @@ -276,14 +283,15 @@ RegExpTree* RegExpParser::ParseDisjunction() { captures_started_++; } // Store current state and begin new disjunction parsing. - state = new (zone()) RegExpParserState( - state, subexpr_type, lookaround_type, captures_started_, zone()); + state = + new (zone()) RegExpParserState(state, subexpr_type, lookaround_type, + captures_started_, flags_, zone()); builder = state->builder(); continue; } case '[': { - RegExpTree* atom = ParseCharacterClass(CHECK_FAILED); - builder->AddAtom(atom); + RegExpTree* cc = ParseCharacterClass(CHECK_FAILED); + builder->AddCharacterClass(cc->AsCharacterClass()); break; } // Atom :: @@ -318,8 +326,9 @@ RegExpTree* RegExpParser::ParseDisjunction() { ZoneList* ranges = new (zone()) ZoneList(2, zone()); CharacterRange::AddClassEscape(c, ranges, zone()); - RegExpTree* atom = new (zone()) RegExpCharacterClass(ranges, false); - builder->AddAtom(atom); + RegExpCharacterClass* cc = + new (zone()) RegExpCharacterClass(ranges, false); + builder->AddCharacterClass(cc); break; } case '1': @@ -353,7 +362,7 @@ RegExpTree* RegExpParser::ParseDisjunction() { // escaped, // no other identity escapes are allowed. If the 'u' flag is not // present, all identity escapes are allowed. - if (!unicode_) { + if (!unicode()) { builder->AddCharacter(first_digit); Advance(2); } else { @@ -414,7 +423,7 @@ RegExpTree* RegExpParser::ParseDisjunction() { uc32 value; if (ParseHexEscape(2, &value)) { builder->AddCharacter(value); - } else if (!unicode_) { + } else if (!unicode()) { builder->AddCharacter('x'); } else { // If the 'u' flag is present, invalid escapes are not treated as @@ -428,7 +437,7 @@ RegExpTree* RegExpParser::ParseDisjunction() { uc32 value; if (ParseUnicodeEscape(&value)) { builder->AddUnicodeCharacter(value); - } else if (!unicode_) { + } else if (!unicode()) { builder->AddCharacter('u'); } else { // If the 'u' flag is present, invalid escapes are not treated as @@ -444,7 +453,7 @@ RegExpTree* RegExpParser::ParseDisjunction() { // other identity escapes are allowed. If the 'u' flag is not // present, // all identity escapes are allowed. - if (!unicode_ || IsSyntaxCharacter(current())) { + if (!unicode() || IsSyntaxCharacter(current())) { builder->AddCharacter(current()); Advance(); } else { @@ -745,7 +754,7 @@ bool RegExpParser::ParseUnicodeEscape(uc32* value) { // Accept both \uxxxx and \u{xxxxxx} (if harmony unicode escapes are // allowed). In the latter case, the number of hex digits between { } is // arbitrary. \ and u have already been read. - if (current() == '{' && unicode_) { + if (current() == '{' && unicode()) { int start = position(); Advance(); if (ParseUnlimitedLengthHexNumber(0x10ffff, value)) { @@ -840,7 +849,7 @@ uc32 RegExpParser::ParseClassCharacterEscape() { if (ParseHexEscape(2, &value)) { return value; } - if (!unicode_) { + if (!unicode()) { // If \x is not followed by a two-digit hexadecimal, treat it // as an identity escape. return 'x'; @@ -856,7 +865,7 @@ uc32 RegExpParser::ParseClassCharacterEscape() { if (ParseUnicodeEscape(&value)) { return value; } - if (!unicode_) { + if (!unicode()) { return 'u'; } // If the 'u' flag is present, invalid escapes are not treated as @@ -869,7 +878,7 @@ uc32 RegExpParser::ParseClassCharacterEscape() { // If the 'u' flag is present, only syntax characters can be escaped, no // other identity escapes are allowed. If the 'u' flag is not present, all // identity escapes are allowed. - if (!unicode_ || IsSyntaxCharacter(result)) { + if (!unicode() || IsSyntaxCharacter(result)) { Advance(); return result; } @@ -899,13 +908,29 @@ CharacterRange RegExpParser::ParseClassAtom(uc16* char_class) { case kEndMarker: return ReportError(CStrVector("\\ at end of pattern")); default: - uc32 c = ParseClassCharacterEscape(CHECK_FAILED); - return CharacterRange::Singleton(c); + first = ParseClassCharacterEscape(CHECK_FAILED); } } else { Advance(); - return CharacterRange::Singleton(first); } + + if (unicode() && unibrow::Utf16::IsLeadSurrogate(first)) { + // Combine with possibly following trail surrogate. + int start = position(); + uc32 second = current(); + if (second == '\\') { + second = ParseClassCharacterEscape(CHECK_FAILED); + } else { + Advance(); + } + if (unibrow::Utf16::IsTrailSurrogate(second)) { + first = unibrow::Utf16::CombineSurrogatePair(first, second); + } else { + Reset(start); + } + } + + return CharacterRange::Singleton(first); } @@ -985,10 +1010,10 @@ RegExpTree* RegExpParser::ParseCharacterClass() { bool RegExpParser::ParseRegExp(Isolate* isolate, Zone* zone, - FlatStringReader* input, bool multiline, - bool unicode, RegExpCompileData* result) { + FlatStringReader* input, JSRegExp::Flags flags, + RegExpCompileData* result) { DCHECK(result != NULL); - RegExpParser parser(input, &result->error, multiline, unicode, isolate, zone); + RegExpParser parser(input, &result->error, flags, isolate, zone); RegExpTree* tree = parser.ParsePattern(); if (parser.failed()) { DCHECK(tree == NULL); @@ -1011,10 +1036,12 @@ bool RegExpParser::ParseRegExp(Isolate* isolate, Zone* zone, } -RegExpBuilder::RegExpBuilder(Zone* zone) +RegExpBuilder::RegExpBuilder(Zone* zone, JSRegExp::Flags flags) : zone_(zone), pending_empty_(false), + flags_(flags), characters_(NULL), + pending_surrogate_(kNoPendingSurrogate), terms_(), alternatives_() #ifdef DEBUG @@ -1025,7 +1052,48 @@ RegExpBuilder::RegExpBuilder(Zone* zone) } +void RegExpBuilder::AddLeadSurrogate(uc16 lead_surrogate) { + DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate)); + FlushPendingSurrogate(); + // Hold onto the lead surrogate, waiting for a trail surrogate to follow. + pending_surrogate_ = lead_surrogate; +} + + +void RegExpBuilder::AddTrailSurrogate(uc16 trail_surrogate) { + DCHECK(unibrow::Utf16::IsTrailSurrogate(trail_surrogate)); + if (pending_surrogate_ != kNoPendingSurrogate) { + uc16 lead_surrogate = pending_surrogate_; + DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate)); + ZoneList surrogate_pair(2, zone()); + surrogate_pair.Add(lead_surrogate, zone()); + surrogate_pair.Add(trail_surrogate, zone()); + RegExpAtom* atom = new (zone()) RegExpAtom(surrogate_pair.ToConstVector()); + pending_surrogate_ = kNoPendingSurrogate; + AddAtom(atom); + } else { + pending_surrogate_ = trail_surrogate; + FlushPendingSurrogate(); + } +} + + +void RegExpBuilder::FlushPendingSurrogate() { + if (pending_surrogate_ != kNoPendingSurrogate) { + // Use character class to desugar lone surrogate matching. + RegExpCharacterClass* cc = new (zone()) RegExpCharacterClass( + CharacterRange::List(zone(), + CharacterRange::Singleton(pending_surrogate_)), + false); + pending_surrogate_ = kNoPendingSurrogate; + DCHECK(unicode()); + AddCharacterClass(cc); + } +} + + void RegExpBuilder::FlushCharacters() { + FlushPendingSurrogate(); pending_empty_ = false; if (characters_ != NULL) { RegExpTree* atom = new (zone()) RegExpAtom(characters_->ToConstVector()); @@ -1053,6 +1121,7 @@ void RegExpBuilder::FlushText() { void RegExpBuilder::AddCharacter(uc16 c) { + FlushPendingSurrogate(); pending_empty_ = false; if (characters_ == NULL) { characters_ = new (zone()) ZoneList(4, zone()); @@ -1064,11 +1133,13 @@ void RegExpBuilder::AddCharacter(uc16 c) { void RegExpBuilder::AddUnicodeCharacter(uc32 c) { if (c > unibrow::Utf16::kMaxNonSurrogateCharCode) { - ZoneList surrogate_pair(2, zone()); - surrogate_pair.Add(unibrow::Utf16::LeadSurrogate(c), zone()); - surrogate_pair.Add(unibrow::Utf16::TrailSurrogate(c), zone()); - RegExpAtom* atom = new (zone()) RegExpAtom(surrogate_pair.ToConstVector()); - AddAtom(atom); + DCHECK(unicode()); + AddLeadSurrogate(unibrow::Utf16::LeadSurrogate(c)); + AddTrailSurrogate(unibrow::Utf16::TrailSurrogate(c)); + } else if (unicode() && unibrow::Utf16::IsLeadSurrogate(c)) { + AddLeadSurrogate(c); + } else if (unicode() && unibrow::Utf16::IsTrailSurrogate(c)) { + AddTrailSurrogate(c); } else { AddCharacter(static_cast(c)); } @@ -1078,6 +1149,17 @@ void RegExpBuilder::AddUnicodeCharacter(uc32 c) { void RegExpBuilder::AddEmpty() { pending_empty_ = true; } +void RegExpBuilder::AddCharacterClass(RegExpCharacterClass* cc) { + if (unicode() && cc->NeedsDesugaringForUnicode(zone())) { + // In unicode mode, character class needs to be desugared, so it + // must be a standalone term instead of being part of a RegExpText. + AddTerm(cc); + } else { + AddAtom(cc); + } +} + + void RegExpBuilder::AddAtom(RegExpTree* term) { if (term->IsEmpty()) { AddEmpty(); @@ -1094,6 +1176,13 @@ void RegExpBuilder::AddAtom(RegExpTree* term) { } +void RegExpBuilder::AddTerm(RegExpTree* term) { + FlushText(); + terms_.Add(term, zone()); + LAST(ADD_ATOM); +} + + void RegExpBuilder::AddAssertion(RegExpTree* assert) { FlushText(); terms_.Add(assert, zone()); @@ -1132,6 +1221,7 @@ RegExpTree* RegExpBuilder::ToRegExp() { void RegExpBuilder::AddQuantifierToAtom( int min, int max, RegExpQuantifier::QuantifierType quantifier_type) { + FlushPendingSurrogate(); if (pending_empty_) { pending_empty_ = false; return; diff --git a/src/regexp/regexp-parser.h b/src/regexp/regexp-parser.h index af9b765fba..dc0d943f86 100644 --- a/src/regexp/regexp-parser.h +++ b/src/regexp/regexp-parser.h @@ -99,13 +99,15 @@ class BufferedZoneList { // Accumulates RegExp atoms and assertions into lists of terms and alternatives. class RegExpBuilder : public ZoneObject { public: - explicit RegExpBuilder(Zone* zone); + RegExpBuilder(Zone* zone, JSRegExp::Flags flags); void AddCharacter(uc16 character); void AddUnicodeCharacter(uc32 character); // "Adds" an empty expression. Does nothing except consume a // following quantifier void AddEmpty(); + void AddCharacterClass(RegExpCharacterClass* cc); void AddAtom(RegExpTree* tree); + void AddTerm(RegExpTree* tree); void AddAssertion(RegExpTree* tree); void NewAlternative(); // '|' void AddQuantifierToAtom(int min, int max, @@ -113,14 +115,21 @@ class RegExpBuilder : public ZoneObject { RegExpTree* ToRegExp(); private: + static const uc16 kNoPendingSurrogate = 0; + void AddLeadSurrogate(uc16 lead_surrogate); + void AddTrailSurrogate(uc16 trail_surrogate); + void FlushPendingSurrogate(); void FlushCharacters(); void FlushText(); void FlushTerms(); Zone* zone() const { return zone_; } + bool unicode() const { return (flags_ & JSRegExp::kUnicode) != 0; } Zone* zone_; bool pending_empty_; + JSRegExp::Flags flags_; ZoneList* characters_; + uc16 pending_surrogate_; BufferedZoneList terms_; BufferedZoneList text_; BufferedZoneList alternatives_; @@ -135,12 +144,11 @@ class RegExpBuilder : public ZoneObject { class RegExpParser BASE_EMBEDDED { public: - RegExpParser(FlatStringReader* in, Handle* error, bool multiline_mode, - bool unicode, Isolate* isolate, Zone* zone); + RegExpParser(FlatStringReader* in, Handle* error, + JSRegExp::Flags flags, Isolate* isolate, Zone* zone); static bool ParseRegExp(Isolate* isolate, Zone* zone, FlatStringReader* input, - bool multiline, bool unicode, - RegExpCompileData* result); + JSRegExp::Flags flags, RegExpCompileData* result); RegExpTree* ParsePattern(); RegExpTree* ParseDisjunction(); @@ -183,6 +191,8 @@ class RegExpParser BASE_EMBEDDED { int captures_started() { return captures_started_; } int position() { return next_pos_ - 1; } bool failed() { return failed_; } + bool unicode() const { return (flags_ & JSRegExp::kUnicode) != 0; } + bool multiline() const { return (flags_ & JSRegExp::kMultiline) != 0; } static bool IsSyntaxCharacter(uc32 c); @@ -203,9 +213,10 @@ class RegExpParser BASE_EMBEDDED { RegExpParserState(RegExpParserState* previous_state, SubexpressionType group_type, RegExpLookaround::Type lookaround_type, - int disjunction_capture_index, Zone* zone) + int disjunction_capture_index, JSRegExp::Flags flags, + Zone* zone) : previous_state_(previous_state), - builder_(new (zone) RegExpBuilder(zone)), + builder_(new (zone) RegExpBuilder(zone, flags)), group_type_(group_type), lookaround_type_(lookaround_type), disjunction_capture_index_(disjunction_capture_index) {} @@ -249,6 +260,8 @@ class RegExpParser BASE_EMBEDDED { bool has_more() { return has_more_; } bool has_next() { return next_pos_ < in()->length(); } uc32 Next(); + template + uc32 ReadNext(); FlatStringReader* in() { return in_; } void ScanForCaptures(); @@ -258,13 +271,12 @@ class RegExpParser BASE_EMBEDDED { ZoneList* captures_; FlatStringReader* in_; uc32 current_; + JSRegExp::Flags flags_; int next_pos_; int captures_started_; // The capture count is only valid after we have scanned for captures. int capture_count_; bool has_more_; - bool multiline_; - bool unicode_; bool simple_; bool contains_anchor_; bool is_scanned_for_captures_; diff --git a/test/cctest/test-regexp.cc b/test/cctest/test-regexp.cc index a91058cc24..85616c45b8 100644 --- a/test/cctest/test-regexp.cc +++ b/test/cctest/test-regexp.cc @@ -96,7 +96,7 @@ static bool CheckParse(const char* input) { FlatStringReader reader(CcTest::i_isolate(), CStrVector(input)); RegExpCompileData result; return v8::internal::RegExpParser::ParseRegExp( - CcTest::i_isolate(), &zone, &reader, false, false, &result); + CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result); } @@ -106,8 +106,10 @@ static void CheckParseEq(const char* input, const char* expected, Zone zone; FlatStringReader reader(CcTest::i_isolate(), CStrVector(input)); RegExpCompileData result; - CHECK(v8::internal::RegExpParser::ParseRegExp( - CcTest::i_isolate(), &zone, &reader, false, unicode, &result)); + JSRegExp::Flags flags = JSRegExp::kNone; + if (unicode) flags |= JSRegExp::kUnicode; + CHECK(v8::internal::RegExpParser::ParseRegExp(CcTest::i_isolate(), &zone, + &reader, flags, &result)); CHECK(result.tree != NULL); CHECK(result.error.is_null()); std::ostringstream os; @@ -125,7 +127,7 @@ static bool CheckSimple(const char* input) { FlatStringReader reader(CcTest::i_isolate(), CStrVector(input)); RegExpCompileData result; CHECK(v8::internal::RegExpParser::ParseRegExp( - CcTest::i_isolate(), &zone, &reader, false, false, &result)); + CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result)); CHECK(result.tree != NULL); CHECK(result.error.is_null()); return result.simple; @@ -143,7 +145,7 @@ static MinMaxPair CheckMinMaxMatch(const char* input) { FlatStringReader reader(CcTest::i_isolate(), CStrVector(input)); RegExpCompileData result; CHECK(v8::internal::RegExpParser::ParseRegExp( - CcTest::i_isolate(), &zone, &reader, false, false, &result)); + CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result)); CHECK(result.tree != NULL); CHECK(result.error.is_null()); int min_match = result.tree->min_match(); @@ -206,8 +208,8 @@ void TestRegExpParser(bool lookbehind) { } CheckParseEq("()", "(^ %)"); CheckParseEq("(?=)", "(-> + %)"); - CheckParseEq("[]", "^[\\x00-\\uffff]"); // Doesn't compile on windows - CheckParseEq("[^]", "[\\x00-\\uffff]"); // \uffff isn't in codepage 1252 + CheckParseEq("[]", "^[\\x00-\\u{10ffff}]"); // Doesn't compile on windows + CheckParseEq("[^]", "[\\x00-\\u{10ffff}]"); // \uffff isn't in codepage 1252 CheckParseEq("[x]", "[x]"); CheckParseEq("[xyz]", "[x y z]"); CheckParseEq("[a-zA-Z0-9]", "[a-z A-Z 0-9]"); @@ -316,6 +318,10 @@ void TestRegExpParser(bool lookbehind) { CheckParseEq("\\u{12345}{3}", "(# 3 3 g '\\ud808\\udf45')", true); CheckParseEq("\\u{12345}*", "(# 0 - g '\\ud808\\udf45')", true); + CheckParseEq("\\ud808\\udf45*", "(# 0 - g '\\ud808\\udf45')", true); + CheckParseEq("[\\ud808\\udf45-\\ud809\\udccc]", "[\\u{012345}-\\u{0124cc}]", + true); + CHECK_SIMPLE("", false); CHECK_SIMPLE("a", true); CHECK_SIMPLE("a|b", false); @@ -454,7 +460,7 @@ static void ExpectError(const char* input, FlatStringReader reader(CcTest::i_isolate(), CStrVector(input)); RegExpCompileData result; CHECK(!v8::internal::RegExpParser::ParseRegExp( - CcTest::i_isolate(), &zone, &reader, false, false, &result)); + CcTest::i_isolate(), &zone, &reader, JSRegExp::kNone, &result)); CHECK(result.tree == NULL); CHECK(!result.error.is_null()); v8::base::SmartArrayPointer str = result.error->ToCString(ALLOW_NULLS); @@ -523,7 +529,7 @@ static void TestCharacterClassEscapes(uc16 c, bool (pred)(uc16 c)) { ZoneList* ranges = new(&zone) ZoneList(2, &zone); CharacterRange::AddClassEscape(c, ranges, &zone); - for (unsigned i = 0; i < (1 << 16); i++) { + for (uc32 i = 0; i < (1 << 16); i++) { bool in_class = false; for (int j = 0; !in_class && j < ranges->length(); j++) { CharacterRange& range = ranges->at(j); @@ -550,17 +556,19 @@ static RegExpNode* Compile(const char* input, bool multiline, bool unicode, Isolate* isolate = CcTest::i_isolate(); FlatStringReader reader(isolate, CStrVector(input)); RegExpCompileData compile_data; + JSRegExp::Flags flags = JSRegExp::kNone; + if (multiline) flags = JSRegExp::kMultiline; + if (unicode) flags = JSRegExp::kUnicode; if (!v8::internal::RegExpParser::ParseRegExp(CcTest::i_isolate(), zone, - &reader, multiline, unicode, - &compile_data)) + &reader, flags, &compile_data)) return NULL; Handle pattern = isolate->factory() ->NewStringFromUtf8(CStrVector(input)) .ToHandleChecked(); Handle sample_subject = isolate->factory()->NewStringFromUtf8(CStrVector("")).ToHandleChecked(); - RegExpEngine::Compile(isolate, zone, &compile_data, false, false, multiline, - false, pattern, sample_subject, is_one_byte); + RegExpEngine::Compile(isolate, zone, &compile_data, flags, pattern, + sample_subject, is_one_byte); return compile_data.node; } @@ -1669,7 +1677,7 @@ TEST(CharacterRangeCaseIndependence) { } -static bool InClass(uc16 c, ZoneList* ranges) { +static bool InClass(uc32 c, ZoneList* ranges) { if (ranges == NULL) return false; for (int i = 0; i < ranges->length(); i++) { @@ -1681,29 +1689,46 @@ static bool InClass(uc16 c, ZoneList* ranges) { } -TEST(CharClassDifference) { +TEST(UnicodeRangeSplitter) { Zone zone; ZoneList* base = new(&zone) ZoneList(1, &zone); base->Add(CharacterRange::Everything(), &zone); - Vector overlay = CharacterRange::GetWordBounds(); - ZoneList* included = NULL; - ZoneList* excluded = NULL; - CharacterRange::Split(base, overlay, &included, &excluded, &zone); - for (int i = 0; i < (1 << 16); i++) { - bool in_base = InClass(i, base); - if (in_base) { - bool in_overlay = false; - for (int j = 0; !in_overlay && j < overlay.length(); j += 2) { - if (overlay[j] <= i && i < overlay[j+1]) - in_overlay = true; - } - CHECK_EQ(in_overlay, InClass(i, included)); - CHECK_EQ(!in_overlay, InClass(i, excluded)); - } else { - CHECK(!InClass(i, included)); - CHECK(!InClass(i, excluded)); - } + UnicodeRangeSplitter splitter(&zone, base); + // BMP + for (uc32 c = 0; c < 0xd800; c++) { + CHECK(InClass(c, splitter.bmp())); + CHECK(!InClass(c, splitter.lead_surrogates())); + CHECK(!InClass(c, splitter.trail_surrogates())); + CHECK(!InClass(c, splitter.non_bmp())); + } + // Lead surrogates + for (uc32 c = 0xd800; c < 0xdbff; c++) { + CHECK(!InClass(c, splitter.bmp())); + CHECK(InClass(c, splitter.lead_surrogates())); + CHECK(!InClass(c, splitter.trail_surrogates())); + CHECK(!InClass(c, splitter.non_bmp())); + } + // Trail surrogates + for (uc32 c = 0xdc00; c < 0xdfff; c++) { + CHECK(!InClass(c, splitter.bmp())); + CHECK(!InClass(c, splitter.lead_surrogates())); + CHECK(InClass(c, splitter.trail_surrogates())); + CHECK(!InClass(c, splitter.non_bmp())); + } + // BMP + for (uc32 c = 0xe000; c < 0xffff; c++) { + CHECK(InClass(c, splitter.bmp())); + CHECK(!InClass(c, splitter.lead_surrogates())); + CHECK(!InClass(c, splitter.trail_surrogates())); + CHECK(!InClass(c, splitter.non_bmp())); + } + // Non-BMP + for (uc32 c = 0x10000; c < 0x10ffff; c++) { + CHECK(!InClass(c, splitter.bmp())); + CHECK(!InClass(c, splitter.lead_surrogates())); + CHECK(!InClass(c, splitter.trail_surrogates())); + CHECK(InClass(c, splitter.non_bmp())); } } diff --git a/test/mjsunit/harmony/unicode-character-ranges.js b/test/mjsunit/harmony/unicode-character-ranges.js new file mode 100644 index 0000000000..75a592f334 --- /dev/null +++ b/test/mjsunit/harmony/unicode-character-ranges.js @@ -0,0 +1,156 @@ +// Copyright 2016 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Flags: --harmony-unicode-regexps --harmony-regexp-lookbehind + +function execl(expectation, regexp, subject) { + if (regexp instanceof String) regexp = new RegExp(regexp, "u"); + assertEquals(expectation, regexp.exec(subject)); +} + +function execs(expectation, regexp_source, subject) { + execl(expectation, new RegExp(regexp_source, "u"), subject); +} + +// Character ranges. +execl(["A"], /[A-D]/u, "A"); +execs(["A"], "[A-D]", "A"); +execl(["ABCD"], /[A-D]+/u, "ZABCDEF"); +execs(["ABCD"], "[A-D]+", "ZABCDEF"); + +execl(["\u{12345}"], /[\u1234-\u{12345}]/u, "\u{12345}"); +execs(["\u{12345}"], "[\u1234-\u{12345}]", "\u{12345}"); +execl(null, /[^\u1234-\u{12345}]/u, "\u{12345}"); +execs(null, "[^\u1234-\u{12345}]", "\u{12345}"); + +execl(["\u{1234}"], /[\u1234-\u{12345}]/u, "\u{1234}"); +execs(["\u{1234}"], "[\u1234-\u{12345}]", "\u{1234}"); +execl(null, /[^\u1234-\u{12345}]/u, "\u{1234}"); +execs(null, "[^\u1234-\u{12345}]", "\u{1234}"); + +execl(null, /[\u1234-\u{12345}]/u, "\u{1233}"); +execs(null, "[\u1234-\u{12345}]", "\u{1233}"); +execl(["\u{1233}"], /[^\u1234-\u{12345}]/u, "\u{1233}"); +execs(["\u{1233}"], "[^\u1234-\u{12345}]", "\u{1233}"); + +execl(["\u{12346}"], /[^\u1234-\u{12345}]/u, "\u{12346}"); +execs(["\u{12346}"], "[^\u1234-\u{12345}]", "\u{12346}"); +execl(null, /[\u1234-\u{12345}]/u, "\u{12346}"); +execs(null, "[\u1234-\u{12345}]", "\u{12346}"); + +execl(["\u{12342}"], /[\u{12340}-\u{12345}]/u, "\u{12342}"); +execs(["\u{12342}"], "[\u{12340}-\u{12345}]", "\u{12342}"); +execl(["\u{12342}"], /[\ud808\udf40-\ud808\udf45]/u, "\u{12342}"); +execs(["\u{12342}"], "[\ud808\udf40-\ud808\udf45]", "\u{12342}"); +execl(null, /[^\u{12340}-\u{12345}]/u, "\u{12342}"); +execs(null, "[^\u{12340}-\u{12345}]", "\u{12342}"); +execl(null, /[^\ud808\udf40-\ud808\udf45]/u, "\u{12342}"); +execs(null, "[^\ud808\udf40-\ud808\udf45]", "\u{12342}"); + +execl(["\u{ffff}"], /[\u{ff80}-\u{12345}]/u, "\u{ffff}"); +execs(["\u{ffff}"], "[\u{ff80}-\u{12345}]", "\u{ffff}"); +execl(["\u{ffff}"], /[\u{ff80}-\ud808\udf45]/u, "\u{ffff}"); +execs(["\u{ffff}"], "[\u{ff80}-\ud808\udf45]", "\u{ffff}"); +execl(null, /[^\u{ff80}-\u{12345}]/u, "\u{ffff}"); +execs(null, "[^\u{ff80}-\u{12345}]", "\u{ffff}"); +execl(null, /[^\u{ff80}-\ud808\udf45]/u, "\u{ffff}"); +execs(null, "[^\u{ff80}-\ud808\udf45]", "\u{ffff}"); + +// Lone surrogate +execl(["\ud800"], /[^\u{ff80}-\u{12345}]/u, "\uff99\u{d800}A"); +execs(["\udc00"], "[^\u{ff80}-\u{12345}]", "\uff99\u{dc00}A"); +execl(["\udc01"], /[\u0100-\u{10ffff}]/u, "A\udc01"); +execl(["\udc03"], /[\udc01-\udc03]/u, "\ud801\udc02\udc03"); +execl(["\ud801"], /[\ud801-\ud803]/u, "\ud802\udc01\ud801"); + +// Paired sorrogate. +execl(null, /[^\u{ff80}-\u{12345}]/u, "\u{d800}\u{dc00}"); +execs(null, "[^\u{ff80}-\u{12345}]", "\u{d800}\u{dc00}"); +execl(["\ud800\udc00"], /[\u{ff80}-\u{12345}]/u, "\u{d800}\u{dc00}"); +execs(["\ud800\udc00"], "[\u{ff80}-\u{12345}]", "\u{d800}\u{dc00}"); +execl(["foo\u{10e6d}bar"], /foo\ud803\ude6dbar/u, "foo\u{10e6d}bar"); + +// Lone surrogates +execl(["\ud801\ud801"], /\ud801+/u, "\ud801\udc01\ud801\ud801"); +execl(["\udc01\udc01"], /\udc01+/u, "\ud801\ud801\udc01\udc01\udc01"); + +execl(["\udc02\udc03A"], /\W\WA/u, "\ud801\udc01A\udc02\udc03A"); +execl(["\ud801\ud802"], /\ud801./u, "\ud801\udc01\ud801\ud802"); +execl(["\udc02\udc03A"], /[\ud800-\udfff][\ud800-\udfff]A/u, + "\ud801\udc01A\udc02\udc03A"); + +// Character classes +execl(null, /\w/u, "\ud801\udc01"); +execl(["\ud801"], /[^\w]/, "\ud801\udc01"); +execl(["\ud801\udc01"], /[^\w]/u, "\ud801\udc01"); +execl(["\ud801"], /\W/, "\ud801\udc01"); +execl(["\ud801\udc01"], /\W/u, "\ud801\udc01"); + +execl(["\ud800X"], /.X/u, "\ud800XaX"); +execl(["aX"], /.(?