/******************************************************************** * COPYRIGHT: * Copyright (c) 2002, International Business Machines Corporation and * others. All Rights Reserved. ********************************************************************/ // // regextst.cpp // // ICU Regular Expressions test, part of intltest. // #include "unicode/utypes.h" #if !UCONFIG_NO_REGULAR_EXPRESSIONS #include "unicode/uchar.h" #include "unicode/ucnv.h" #include "intltest.h" #include "regextst.h" #include "uvector.h" #include "stdlib.h" #include "charstr.h" #include "util.h" //--------------------------------------------------------------------------- // // Test class boilerplate // //--------------------------------------------------------------------------- RegexTest::RegexTest() { }; RegexTest::~RegexTest() { }; void RegexTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ ) { if (exec) logln("TestSuite RegexTest: "); switch (index) { case 0: name = "Basic"; if (exec) Basic(); break; case 1: name = "API_Match"; if (exec) API_Match(); break; case 2: name = "API_Replace"; if (exec) API_Replace(); break; case 3: name = "API_Pattern"; if (exec) API_Pattern(); break; case 4: name = "Extended"; if (exec) Extended(); break; case 5: name = "Errors"; if (exec) Errors(); break; case 6: name = "PerlTests"; if (exec) PerlTests(); break; default: name = ""; break; //needed to end loop } } //--------------------------------------------------------------------------- // // Error Checking / Reporting macros used in all of the tests. // //--------------------------------------------------------------------------- #define REGEX_CHECK_STATUS {if (U_FAILURE(status)) {errln("RegexTest failure at line %d. status=%s\n", \ __LINE__, u_errorName(status)); return;}} #define REGEX_ASSERT(expr) {if ((expr)==FALSE) {errln("RegexTest failure at line %d.\n", __LINE__);};} #define REGEX_ASSERT_FAIL(expr, errcode) {UErrorCode status=U_ZERO_ERROR; (expr);\ if (status!=errcode) {errln("RegexTest failure at line %d. Expected status=%s, got %s\n", \ __LINE__, u_errorName(errcode), u_errorName(status));};} #define REGEX_CHECK_STATUS_L(line) {if (U_FAILURE(status)) {errln( \ "RegexTest failure at line %d, from %d. status=%d\n",__LINE__, (line), status); }} #define REGEX_ASSERT_L(expr, line) {if ((expr)==FALSE) { \ errln("RegexTest failure at line %d, from %d.", __LINE__, (line)); return;}} //--------------------------------------------------------------------------- // // REGEX_TESTLM Macro + invocation function to simplify writing quick tests // for the LookingAt() and Match() functions. // // usage: // REGEX_TESTLM("pattern", "input text", lookingAt expected, matches expected); // // The expected results are UBool - TRUE or FALSE. // The input text is unescaped. The pattern is not. // // //--------------------------------------------------------------------------- #define REGEX_TESTLM(pat, text, looking, match) doRegexLMTest(pat, text, looking, match, __LINE__); UBool RegexTest::doRegexLMTest(const char *pat, const char *text, UBool looking, UBool match, int line) { const UnicodeString pattern(pat); const UnicodeString inputText(text); UErrorCode status = U_ZERO_ERROR; UParseError pe; RegexPattern *REPattern = NULL; RegexMatcher *REMatcher = NULL; UBool retVal = TRUE; UnicodeString patString(pat); REPattern = RegexPattern::compile(patString, 0, pe, status); if (U_FAILURE(status)) { errln("RegexTest failure in RegexPattern::compile() at line %d. Status = %d\n", line, status); return FALSE; } if (line==376) { REPattern->dump();} UnicodeString inputString(inputText); UnicodeString unEscapedInput = inputString.unescape(); REMatcher = REPattern->matcher(unEscapedInput, status); if (U_FAILURE(status)) { errln("RegexTest failure in REPattern::matcher() at line %d. Status = %d\n", line, status); return FALSE; } UBool actualmatch; actualmatch = REMatcher->lookingAt(status); if (U_FAILURE(status)) { errln("RegexTest failure in lookingAt() at line %d. Status = %d\n", line, status); retVal = FALSE; } if (actualmatch != looking) { errln("RegexTest: wrong return from lookingAt() at line %d.\n", line); retVal = FALSE; } status = U_ZERO_ERROR; actualmatch = REMatcher->matches(status); if (U_FAILURE(status)) { errln("RegexTest failure in matches() at line %d. Status = %d\n", line, status); retVal = FALSE; } if (actualmatch != match) { errln("RegexTest: wrong return from matches() at line %d.\n", line); retVal = FALSE; } if (retVal == FALSE) { REPattern->dump(); } delete REPattern; delete REMatcher; return retVal; } //--------------------------------------------------------------------------- // // REGEX_FIND Macro + invocation function to simplify writing tests // regex tests. // // usage: // REGEX_FIND("pattern", "input text"); // REGEX_ERR("pattern", expected status); // // The input text is unescaped. The pattern is not. // The input text is marked with the expected match positions // <0>text <1> more text // The tags are removed before trying the match. // The tags mark the start and end of the match and of any capture groups. // // //--------------------------------------------------------------------------- // REGEX_FIND is invoked via a macro, which allows capturing the source file line // number for use in error messages. #define REGEX_FIND(pat, text) regex_find(pat, text, U_ZERO_ERROR, __LINE__); // Set a value into a UVector at position specified by a decimal number in // a UnicodeString. This is a utility function needed by the actual test function, // which follows. void set(UVector &vec, int val, UnicodeString index) { UErrorCode status=U_ZERO_ERROR; int idx = 0; for (int i=0; idump(); // // Find the tags in the input data, remove them, and record the group boundary // positions. // parsePat = RegexPattern::compile("<(/?)([0-9]+)>", 0, pe, status); REGEX_CHECK_STATUS_L(line); unEscapedInput = inputString.unescape(); parseMatcher = parsePat->matcher(unEscapedInput, status); REGEX_CHECK_STATUS_L(line); while(parseMatcher->find()) { parseMatcher->appendReplacement(deTaggedInput, "", status); REGEX_CHECK_STATUS; UnicodeString groupNum = parseMatcher->group(2, status); if (parseMatcher->group(1, status) == "/") { // close tag set(groupEnds, deTaggedInput.length(), groupNum); } else { set(groupStarts, deTaggedInput.length(), groupNum); } } parseMatcher->appendTail(deTaggedInput); REGEX_ASSERT_L(groupStarts.size() == groupEnds.size(), line); // // Do a find on the de-tagged input using the caller's pattern // matcher = callerPattern->matcher(deTaggedInput, status); REGEX_CHECK_STATUS_L(line); isMatch = matcher->find(); // // Match up the groups from the find() with the groups from the tags // // number of tags should match number of groups from find operation. // matcher->groupCount does not include group 0, the entire match, hence the +1. if (isMatch == FALSE && groupStarts.size() != 0) { errln("Error at line %d: Match expected, but none found.\n", line); failed = TRUE; goto cleanupAndReturn; } int i; for (i=0; i<=matcher->groupCount(); i++) { int32_t expectedStart = (i >= groupStarts.size()? -1 : groupStarts.elementAti(i)); if (matcher->start(i, status) != expectedStart) { errln("Error at line %d: incorrect start position for group %d. Expected %d, got %d", line, i, expectedStart, matcher->start(i, status)); failed = TRUE; goto cleanupAndReturn; // Good chance of subsequent bogus errors. Stop now. } int32_t expectedEnd = (i >= groupEnds.size()? -1 : groupEnds.elementAti(i)); if (matcher->end(i, status) != expectedEnd) { errln("Error at line %d: incorrect end position for group %d. Expected %d, got %d", line, i, expectedEnd, matcher->end(i, status)); failed = TRUE; // Error on end position; keep going; real error is probably yet to come as group // end positions work from end of the input data towards the front. } } if ( matcher->groupCount()+1 < groupStarts.size()) { errln("Error at line %d: Expected %d capture groups, found %d.", line, groupStarts.size()-1, matcher->groupCount()); failed = TRUE; } cleanupAndReturn: if (failed) { callerPattern->dump(); } delete parseMatcher; delete parsePat; delete matcher; delete callerPattern; } //--------------------------------------------------------------------------- // // REGEX_ERR Macro + invocation function to simplify writing tests // regex tests for incorrect patterns // // usage: // REGEX_ERR("pattern", expected error line, column, expected status); // //--------------------------------------------------------------------------- #define REGEX_ERR(pat, line, col, status) regex_err(pat, line, col, status, __LINE__); void RegexTest::regex_err(const char *pat, int32_t errLine, int32_t errCol, UErrorCode expectedStatus, int line) { UnicodeString pattern(pat); UErrorCode status = U_ZERO_ERROR; UParseError pe; RegexPattern *callerPattern = NULL; // // Compile the caller's pattern // UnicodeString patString(pat); callerPattern = RegexPattern::compile(patString, 0, pe, status); if (status != expectedStatus) { errln("Line %d: unexpected error %s compiling pattern.", line, u_errorName(status)); } else { if (status != U_ZERO_ERROR) { if (pe.line != errLine || pe.offset != errCol) { errln("Line %d: incorrect line/offset from UParseError. Expected %d/%d; got %d/%d.\n", line, errLine, errCol, pe.line, pe.offset); } } } delete callerPattern; } //--------------------------------------------------------------------------- // // Basic Check for basic functionality of regex pattern matching. // Avoid the use of REGEX_FIND test macro, which has // substantial dependencies on basic Regex functionality. // //--------------------------------------------------------------------------- void RegexTest::Basic() { // // Debug - slide failing test cases early // #if 0 { REGEX_TESTLM("(a)|\\1", "x", FALSE, FALSE); // REGEX_FIND("(?>(abc{2,4}?))(c*)", "<0>ab<1>cc<2>cccddd"); // REGEX_FIND("(X([abc=X]+)+X)|(y[abc=]+)", "=XX===================="); } exit(1); #endif // // Pattern with parentheses // REGEX_TESTLM("st(abc)ring", "stabcring thing", TRUE, FALSE); REGEX_TESTLM("st(abc)ring", "stabcring", TRUE, TRUE); REGEX_TESTLM("st(abc)ring", "stabcrung", FALSE, FALSE); // // Patterns with * // REGEX_TESTLM("st(abc)*ring", "string", TRUE, TRUE); REGEX_TESTLM("st(abc)*ring", "stabcring", TRUE, TRUE); REGEX_TESTLM("st(abc)*ring", "stabcabcring", TRUE, TRUE); REGEX_TESTLM("st(abc)*ring", "stabcabcdring", FALSE, FALSE); REGEX_TESTLM("st(abc)*ring", "stabcabcabcring etc.", TRUE, FALSE); REGEX_TESTLM("a*", "", TRUE, TRUE); REGEX_TESTLM("a*", "b", TRUE, FALSE); // // Patterns with "." // REGEX_TESTLM(".", "abc", TRUE, FALSE); REGEX_TESTLM("...", "abc", TRUE, TRUE); REGEX_TESTLM("....", "abc", FALSE, FALSE); REGEX_TESTLM(".*", "abcxyz123", TRUE, TRUE); REGEX_TESTLM("ab.*xyz", "abcdefghij", FALSE, FALSE); REGEX_TESTLM("ab.*xyz", "abcdefg...wxyz", TRUE, TRUE); REGEX_TESTLM("ab.*xyz", "abcde...wxyz...abc..xyz", TRUE, TRUE); REGEX_TESTLM("ab.*xyz", "abcde...wxyz...abc..xyz...", TRUE, FALSE); // // Patterns with * applied to chars at end of literal string // REGEX_TESTLM("abc*", "ab", TRUE, TRUE); REGEX_TESTLM("abc*", "abccccc", TRUE, TRUE); // // Supplemental chars match as single chars, not a pair of surrogates. // REGEX_TESTLM(".", "\\U00011000", TRUE, TRUE); REGEX_TESTLM("...", "\\U00011000x\\U00012002", TRUE, TRUE); REGEX_TESTLM("...", "\\U00011000x\\U00012002y", TRUE, FALSE); // // UnicodeSets in the pattern // REGEX_TESTLM("[1-6]", "1", TRUE, TRUE); REGEX_TESTLM("[1-6]", "3", TRUE, TRUE); REGEX_TESTLM("[1-6]", "7", FALSE, FALSE); REGEX_TESTLM("a[1-6]", "a3", TRUE, TRUE); REGEX_TESTLM("a[1-6]", "a3", TRUE, TRUE); REGEX_TESTLM("a[1-6]b", "a3b", TRUE, TRUE); REGEX_TESTLM("a[0-9]*b", "a123b", TRUE, TRUE); REGEX_TESTLM("a[0-9]*b", "abc", TRUE, FALSE); REGEX_TESTLM("[\\p{Nd}]*", "123456", TRUE, TRUE); REGEX_TESTLM("[\\p{Nd}]*", "a123456", TRUE, FALSE); // note that * matches 0 occurences. REGEX_TESTLM("[a][b][[:Zs:]]*", "ab ", TRUE, TRUE); // Set contains only a string, no individual chars. REGEX_TESTLM("[{ab}]", "a", FALSE, FALSE); // // OR operator in patterns // REGEX_TESTLM("(a|b)", "a", TRUE, TRUE); REGEX_TESTLM("(a|b)", "b", TRUE, TRUE); REGEX_TESTLM("(a|b)", "c", FALSE, FALSE); REGEX_TESTLM("a|b", "b", TRUE, TRUE); REGEX_TESTLM("(a|b|c)*", "aabcaaccbcabc", TRUE, TRUE); REGEX_TESTLM("(a|b|c)*", "aabcaaccbcabdc", TRUE, FALSE); REGEX_TESTLM("(a(b|c|d)(x|y|z)*|123)", "ac", TRUE, TRUE); REGEX_TESTLM("(a(b|c|d)(x|y|z)*|123)", "123", TRUE, TRUE); REGEX_TESTLM("(a|(1|2)*)(b|c|d)(x|y|z)*|123", "123", TRUE, TRUE); REGEX_TESTLM("(a|(1|2)*)(b|c|d)(x|y|z)*|123", "222211111czzzzw", TRUE, FALSE); // // + // REGEX_TESTLM("ab+", "abbc", TRUE, FALSE); REGEX_TESTLM("ab+c", "ac", FALSE, FALSE); REGEX_TESTLM("b+", "", FALSE, FALSE); REGEX_TESTLM("(abc|def)+", "defabc", TRUE, TRUE); REGEX_TESTLM(".+y", "zippity dooy dah ", TRUE, FALSE); REGEX_TESTLM(".+y", "zippity dooy", TRUE, TRUE); // // ? // REGEX_TESTLM("ab?", "ab", TRUE, TRUE); REGEX_TESTLM("ab?", "a", TRUE, TRUE); REGEX_TESTLM("ab?", "ac", TRUE, FALSE); REGEX_TESTLM("ab?", "abb", TRUE, FALSE); REGEX_TESTLM("a(b|c)?d", "abd", TRUE, TRUE); REGEX_TESTLM("a(b|c)?d", "acd", TRUE, TRUE); REGEX_TESTLM("a(b|c)?d", "ad", TRUE, TRUE); REGEX_TESTLM("a(b|c)?d", "abcd", FALSE, FALSE); REGEX_TESTLM("a(b|c)?d", "ab", FALSE, FALSE); // // Escape sequences that become single literal chars, handled internally // by ICU's Unescape. // // REGEX_TESTLM("\101\142", "Ab", TRUE, TRUE); // Octal TODO: not implemented yet. REGEX_TESTLM("\\a", "\\u0007", TRUE, TRUE); // BEL REGEX_TESTLM("\\cL", "\\u000c", TRUE, TRUE); // Control-L REGEX_TESTLM("\\e", "\\u001b", TRUE, TRUE); // Escape REGEX_TESTLM("\\f", "\\u000c", TRUE, TRUE); // Form Feed REGEX_TESTLM("\\n", "\\u000a", TRUE, TRUE); // new line REGEX_TESTLM("\\r", "\\u000d", TRUE, TRUE); // CR REGEX_TESTLM("\\t", "\\u0009", TRUE, TRUE); // Tab REGEX_TESTLM("\\u1234", "\\u1234", TRUE, TRUE); REGEX_TESTLM("\\U00001234", "\\u1234", TRUE, TRUE); REGEX_TESTLM(".*\\Ax", "xyz", TRUE, FALSE); // \A matches only at the beginning of input REGEX_TESTLM(".*\\Ax", " xyz", FALSE, FALSE); // \A matches only at the beginning of input // Escape of special chars in patterns REGEX_TESTLM("\\\\\\|\\(\\)\\[\\{\\~\\$\\*\\+\\?\\.", "\\\\|()[{~$*+?.", TRUE, TRUE); }; //--------------------------------------------------------------------------- // // API_Match Test that the API for class RegexMatcher // is present and nominally working, but excluding functions // implementing replace operations. // //--------------------------------------------------------------------------- void RegexTest::API_Match() { UParseError pe; UErrorCode status=U_ZERO_ERROR; int32_t flags = 0; // // Debug - slide failing test cases early // #if 0 { } return; #endif // // Simple pattern compilation // { UnicodeString re("abc"); RegexPattern *pat2; pat2 = RegexPattern::compile(re, flags, pe, status); REGEX_CHECK_STATUS; UnicodeString inStr1 = "abcdef this is a test"; UnicodeString instr2 = "not abc"; UnicodeString empty = ""; // // Matcher creation and reset. // RegexMatcher *m1 = pat2->matcher(inStr1, status); REGEX_CHECK_STATUS; REGEX_ASSERT(m1->lookingAt(status) == TRUE); REGEX_ASSERT(m1->input() == inStr1); m1->reset(instr2); REGEX_ASSERT(m1->lookingAt(status) == FALSE); REGEX_ASSERT(m1->input() == instr2); m1->reset(inStr1); REGEX_ASSERT(m1->input() == inStr1); REGEX_ASSERT(m1->lookingAt(status) == TRUE); m1->reset(empty); REGEX_ASSERT(m1->lookingAt(status) == FALSE); REGEX_ASSERT(m1->input() == empty); REGEX_ASSERT(&m1->pattern() == pat2); delete m1; delete pat2; } // // Capture Group. // RegexMatcher::start(); // RegexMatcher::end(); // RegexMatcher::groupCount(); // { int32_t flags=0; UParseError pe; UErrorCode status=U_ZERO_ERROR; UnicodeString re("01(23(45)67)(.*)"); RegexPattern *pat = RegexPattern::compile(re, flags, pe, status); REGEX_CHECK_STATUS; UnicodeString data = "0123456789"; RegexMatcher *matcher = pat->matcher(data, status); REGEX_CHECK_STATUS; REGEX_ASSERT(matcher->lookingAt(status) == TRUE); int matchStarts[] = {0, 2, 4, 8}; int matchEnds[] = {10, 8, 6, 10}; int i; for (i=0; i<4; i++) { int32_t actualStart = matcher->start(i, status); REGEX_CHECK_STATUS; if (actualStart != matchStarts[i]) { errln("RegexTest failure at line %d, index %d. Expected %d, got %d\n", __LINE__, i, matchStarts[i], actualStart); } int32_t actualEnd = matcher->end(i, status); REGEX_CHECK_STATUS; if (actualEnd != matchEnds[i]) { errln("RegexTest failure at line %d index %d. Expected %d, got %d\n", __LINE__, i, matchEnds[i], actualEnd); } } REGEX_ASSERT(matcher->start(0, status) == matcher->start(status)); REGEX_ASSERT(matcher->end(0, status) == matcher->end(status)); REGEX_ASSERT_FAIL(matcher->start(-1, status), U_INDEX_OUTOFBOUNDS_ERROR); REGEX_ASSERT_FAIL(matcher->start( 4, status), U_INDEX_OUTOFBOUNDS_ERROR); matcher->reset(); REGEX_ASSERT_FAIL(matcher->start( 0, status), U_REGEX_INVALID_STATE); matcher->lookingAt(status); REGEX_ASSERT(matcher->group(status) == "0123456789"); REGEX_ASSERT(matcher->group(0, status) == "0123456789"); REGEX_ASSERT(matcher->group(1, status) == "234567" ); REGEX_ASSERT(matcher->group(2, status) == "45" ); REGEX_ASSERT(matcher->group(3, status) == "89" ); REGEX_CHECK_STATUS; REGEX_ASSERT_FAIL(matcher->group(-1, status), U_INDEX_OUTOFBOUNDS_ERROR); REGEX_ASSERT_FAIL(matcher->group( 4, status), U_INDEX_OUTOFBOUNDS_ERROR); matcher->reset(); REGEX_ASSERT_FAIL(matcher->group( 0, status), U_REGEX_INVALID_STATE); delete matcher; delete pat; } // // find // { int32_t flags=0; UParseError pe; UErrorCode status=U_ZERO_ERROR; UnicodeString re("abc"); RegexPattern *pat = RegexPattern::compile(re, flags, pe, status); REGEX_CHECK_STATUS; UnicodeString data = ".abc..abc...abc.."; // 012345678901234567 RegexMatcher *matcher = pat->matcher(data, status); REGEX_CHECK_STATUS; REGEX_ASSERT(matcher->find()); REGEX_ASSERT(matcher->start(status) == 1); REGEX_ASSERT(matcher->find()); REGEX_ASSERT(matcher->start(status) == 6); REGEX_ASSERT(matcher->find()); REGEX_ASSERT(matcher->start(status) == 12); REGEX_ASSERT(matcher->find() == FALSE); REGEX_ASSERT(matcher->find() == FALSE); matcher->reset(); REGEX_ASSERT(matcher->find()); REGEX_ASSERT(matcher->start(status) == 1); REGEX_ASSERT(matcher->find(0, status)); REGEX_ASSERT(matcher->start(status) == 1); REGEX_ASSERT(matcher->find(1, status)); REGEX_ASSERT(matcher->start(status) == 1); REGEX_ASSERT(matcher->find(2, status)); REGEX_ASSERT(matcher->start(status) == 6); REGEX_ASSERT(matcher->find(12, status)); REGEX_ASSERT(matcher->start(status) == 12); REGEX_ASSERT(matcher->find(13, status) == FALSE); REGEX_ASSERT(matcher->find(16, status) == FALSE); REGEX_ASSERT_FAIL(matcher->start(status), U_REGEX_INVALID_STATE); REGEX_CHECK_STATUS; REGEX_ASSERT_FAIL(matcher->find(-1, status), U_INDEX_OUTOFBOUNDS_ERROR); REGEX_ASSERT_FAIL(matcher->find(17, status), U_INDEX_OUTOFBOUNDS_ERROR); REGEX_ASSERT(matcher->groupCount() == 0); delete matcher; delete pat; } // // find, with \G in pattern (true if at the end of a previous match). // { int32_t flags=0; UParseError pe; UErrorCode status=U_ZERO_ERROR; UnicodeString re(".*?(?:(\\Gabc)|(abc))"); RegexPattern *pat = RegexPattern::compile(re, flags, pe, status); REGEX_CHECK_STATUS; UnicodeString data = ".abcabc.abc.."; // 012345678901234567 RegexMatcher *matcher = pat->matcher(data, status); REGEX_CHECK_STATUS; REGEX_ASSERT(matcher->find()); REGEX_ASSERT(matcher->start(status) == 0); REGEX_ASSERT(matcher->start(1, status) == -1); REGEX_ASSERT(matcher->start(2, status) == 1); REGEX_ASSERT(matcher->find()); REGEX_ASSERT(matcher->start(status) == 4); REGEX_ASSERT(matcher->start(1, status) == 4); REGEX_ASSERT(matcher->start(2, status) == -1); REGEX_CHECK_STATUS; delete matcher; delete pat; } } //--------------------------------------------------------------------------- // // API_Replace API test for class RegexMatcher, testing the // Replace family of functions. // //--------------------------------------------------------------------------- void RegexTest::API_Replace() { // // Replace // int32_t flags=0; UParseError pe; UErrorCode status=U_ZERO_ERROR; UnicodeString re("abc"); RegexPattern *pat = RegexPattern::compile(re, flags, pe, status); REGEX_CHECK_STATUS; UnicodeString data = ".abc..abc...abc.."; // 012345678901234567 RegexMatcher *matcher = pat->matcher(data, status); // // Plain vanilla matches. // UnicodeString dest; dest = matcher->replaceFirst("yz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == ".yz..abc...abc.."); dest = matcher->replaceAll("yz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == ".yz..yz...yz.."); // // Plain vanilla non-matches. // UnicodeString d2 = ".abx..abx...abx.."; matcher->reset(d2); dest = matcher->replaceFirst("yz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == ".abx..abx...abx.."); dest = matcher->replaceAll("yz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == ".abx..abx...abx.."); // // Empty source string // UnicodeString d3 = ""; matcher->reset(d3); dest = matcher->replaceFirst("yz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == ""); dest = matcher->replaceAll("yz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == ""); // // Empty substitution string // matcher->reset(data); // ".abc..abc...abc.." dest = matcher->replaceFirst("", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "...abc...abc.."); dest = matcher->replaceAll("", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "........"); // // match whole string // UnicodeString d4 = "abc"; matcher->reset(d4); dest = matcher->replaceFirst("xyz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "xyz"); dest = matcher->replaceAll("xyz", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "xyz"); // // Capture Group, simple case // UnicodeString re2("a(..)"); RegexPattern *pat2 = RegexPattern::compile(re2, flags, pe, status); REGEX_CHECK_STATUS; UnicodeString d5 = "abcdefg"; RegexMatcher *matcher2 = pat2->matcher(d5, status); REGEX_CHECK_STATUS; dest = matcher2->replaceFirst("$1$1", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "bcbcdefg"); dest = matcher2->replaceFirst("The value of \\$1 is $1.", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "The value of $1 is bc.defg"); dest = matcher2->replaceFirst("$ by itself, no group number $$$", status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "$ by itself, no group number $$$defg"); UnicodeString replacement = "Supplemental Digit 1 $\\U0001D7CF."; replacement = replacement.unescape(); dest = matcher2->replaceFirst(replacement, status); REGEX_CHECK_STATUS; REGEX_ASSERT(dest == "Supplemental Digit 1 bc.defg"); REGEX_ASSERT_FAIL(matcher2->replaceFirst("bad capture group number $5...",status), U_INDEX_OUTOFBOUNDS_ERROR); // TODO: need more through testing of capture substitutions. delete matcher2; delete pat2; delete matcher; delete pat; } //--------------------------------------------------------------------------- // // API_Pattern Test that the API for class RegexPattern is // present and nominally working. // //--------------------------------------------------------------------------- void RegexTest::API_Pattern() { RegexPattern pata; // Test default constructor to not crash. RegexPattern patb; REGEX_ASSERT(pata == patb); REGEX_ASSERT(pata == pata); UnicodeString re1("abc[a-l][m-z]"); UnicodeString re2("def"); UErrorCode status = U_ZERO_ERROR; UParseError pe; RegexPattern *pat1 = RegexPattern::compile(re1, 0, pe, status); RegexPattern *pat2 = RegexPattern::compile(re2, 0, pe, status); REGEX_CHECK_STATUS; REGEX_ASSERT(*pat1 == *pat1); REGEX_ASSERT(*pat1 != pata); // Assign patb = *pat1; REGEX_ASSERT(patb == *pat1); // Copy Construct RegexPattern patc(*pat1); REGEX_ASSERT(patc == *pat1); REGEX_ASSERT(patb == patc); REGEX_ASSERT(pat1 != pat2); patb = *pat2; REGEX_ASSERT(patb != patc); REGEX_ASSERT(patb == *pat2); // Compile with no flags. RegexPattern *pat1a = RegexPattern::compile(re1, pe, status); REGEX_ASSERT(*pat1a == *pat1); REGEX_ASSERT(pat1a->flags() == 0); #if 0 // Compile with different flags should be not equal RegexPattern *pat1b = RegexPattern::compile(re1, UREGEX_CASE_INSENSITIVE, pe, status); REGEX_CHECK_STATUS; REGEX_ASSERT(*pat1b != *pat1a); REGEX_ASSERT(pat1b->flags() == UREGEX_CASE_INSENSITIVE); REGEX_ASSERT(pat1a->flags() == 0); delete pat1b; #endif // add test back in when we actually support flag settings. // clone RegexPattern *pat1c = pat1->clone(); REGEX_ASSERT(*pat1c == *pat1); REGEX_ASSERT(*pat1c != *pat2); // TODO: Actually do some matches with the cloned/copied/assigned patterns. delete pat1c; delete pat1a; delete pat1; delete pat2; // // matches convenience API // REGEX_ASSERT(RegexPattern::matches(".*", "random input", pe, status) == TRUE); REGEX_CHECK_STATUS; REGEX_ASSERT(RegexPattern::matches("abc", "random input", pe, status) == FALSE); REGEX_CHECK_STATUS; REGEX_ASSERT(RegexPattern::matches(".*nput", "random input", pe, status) == TRUE); REGEX_CHECK_STATUS; REGEX_ASSERT(RegexPattern::matches("random input", "random input", pe, status) == TRUE); REGEX_CHECK_STATUS; REGEX_ASSERT(RegexPattern::matches(".*u", "random input", pe, status) == FALSE); REGEX_CHECK_STATUS; status = U_INDEX_OUTOFBOUNDS_ERROR; REGEX_ASSERT(RegexPattern::matches("abc", "abc", pe, status) == FALSE); REGEX_ASSERT(status == U_INDEX_OUTOFBOUNDS_ERROR); // // Split() // status = U_ZERO_ERROR; pat1 = RegexPattern::compile(" +", pe, status); REGEX_CHECK_STATUS; UnicodeString fields[10]; int32_t n; n = pat1->split("Now is the time", fields, 10, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==4); REGEX_ASSERT(fields[0]=="Now"); REGEX_ASSERT(fields[1]=="is"); REGEX_ASSERT(fields[2]=="the"); REGEX_ASSERT(fields[3]=="time"); REGEX_ASSERT(fields[4]==""); n = pat1->split("Now is the time", fields, 2, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==2); REGEX_ASSERT(fields[0]=="Now"); REGEX_ASSERT(fields[1]=="is the time"); REGEX_ASSERT(fields[2]=="the"); // left over from previous test fields[1] = "*"; n = pat1->split("Now is the time", fields, 1, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==1); REGEX_ASSERT(fields[0]=="Now is the time"); REGEX_ASSERT(fields[1]=="*"); n = pat1->split(" Now is the time ", fields, 10, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==5); REGEX_ASSERT(fields[0]==""); REGEX_ASSERT(fields[1]=="Now"); REGEX_ASSERT(fields[2]=="is"); REGEX_ASSERT(fields[3]=="the"); REGEX_ASSERT(fields[4]=="time"); REGEX_ASSERT(fields[5]==""); n = pat1->split(" ", fields, 10, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==1); REGEX_ASSERT(fields[0]==""); fields[0] = "foo"; n = pat1->split("", fields, 10, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==0); REGEX_ASSERT(fields[0]=="foo"); delete pat1; // split, with a pattern with (capture) pat1 = RegexPattern::compile("<(\\w*)>", pe, status); REGEX_CHECK_STATUS; n = pat1->split("Now is the time", fields, 10, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==6); REGEX_ASSERT(fields[0]==""); REGEX_ASSERT(fields[1]=="a"); REGEX_ASSERT(fields[2]=="Now is "); REGEX_ASSERT(fields[3]=="b"); REGEX_ASSERT(fields[4]=="the time"); REGEX_ASSERT(fields[5]=="c"); REGEX_ASSERT(fields[6]==""); n = pat1->split(" Now is the time", fields, 10, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==6); REGEX_ASSERT(fields[0]==" "); REGEX_ASSERT(fields[1]=="a"); REGEX_ASSERT(fields[2]=="Now is "); REGEX_ASSERT(fields[3]=="b"); REGEX_ASSERT(fields[4]=="the time"); REGEX_ASSERT(fields[5]=="c"); REGEX_ASSERT(fields[6]==""); n = pat1->split(" Now is the time", fields, 4, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==4); REGEX_ASSERT(fields[0]==" "); REGEX_ASSERT(fields[1]=="a"); REGEX_ASSERT(fields[2]=="Now is "); REGEX_ASSERT(fields[3]=="the time"); delete pat1; pat1 = RegexPattern::compile("([-,])", pe, status); REGEX_CHECK_STATUS; n = pat1->split("1-10,20", fields, 10, status); REGEX_CHECK_STATUS; REGEX_ASSERT(n==5); REGEX_ASSERT(fields[0]=="1"); REGEX_ASSERT(fields[1]=="-"); REGEX_ASSERT(fields[2]=="10"); REGEX_ASSERT(fields[3]==","); REGEX_ASSERT(fields[4]=="20"); delete pat1; // // RegexPattern::pattern() // pat1 = new RegexPattern(); REGEX_ASSERT(pat1->pattern() == ""); delete pat1; pat1 = RegexPattern::compile("(Hello, world)*", pe, status); REGEX_CHECK_STATUS; REGEX_ASSERT(pat1->pattern() == "(Hello, world)*"); delete pat1; // // classID functions // pat1 = RegexPattern::compile("(Hello, world)*", pe, status); REGEX_CHECK_STATUS; REGEX_ASSERT(pat1->getDynamicClassID() == RegexPattern::getStaticClassID()); REGEX_ASSERT(pat1->getDynamicClassID() != NULL); RegexMatcher *m = pat1->matcher("Hello, World", status); REGEX_ASSERT(pat1->getDynamicClassID() != m->getDynamicClassID()); REGEX_ASSERT(m->getDynamicClassID() == RegexMatcher::getStaticClassID()); REGEX_ASSERT(m->getDynamicClassID() != NULL); delete m; delete pat1; } //--------------------------------------------------------------------------- // // Extended A more thorough check for features of regex patterns // //--------------------------------------------------------------------------- void RegexTest::Extended() { // Capturing parens REGEX_FIND(".(..).", "<0>a<1>bcd"); REGEX_FIND(".*\\A( +hello)", "<0><1> hello"); REGEX_FIND("(hello)|(goodbye)", "<0><1>hello"); REGEX_FIND("(hello)|(goodbye)", "<0><2>goodbye"); REGEX_FIND("abc( +( inner(X?) +) xyz)", "leading cruft <0>abc<1> <2> inner<3> xyz cruft"); // Non-capturing parens (?: stuff). Groups, but does not capture. REGEX_FIND("(?:abc)*(tail)", "<0>abcabcabc<1>tail"); // Non-greedy *? quantifier REGEX_FIND(".*?(abc)", "<0> abx <1>abc abc abc abc"); REGEX_FIND(".*(abc)", "<0> abx abc abc abc <1>abc"); REGEX_FIND( "((?:abc |xyz )*?)abc ", "<0><1>xyz abc abc abc "); REGEX_FIND( "((?:abc |xyz )*)abc ", "<0><1>xyz abc abc abc "); // Non-greedy +? quantifier REGEX_FIND( "(a+?)(a*)", "<0><1>a<2>aaaaaaaaaaaa"); REGEX_FIND( "(a+)(a*)", "<0><1>aaaaaaaaaaaaa<2>"); REGEX_FIND( "((ab)+?)((ab)*)", "<0><1><2>ab<3>ababababab<4>ab"); REGEX_FIND( "((ab)+)((ab)*)", "<0><1>abababababab<2>ab<3>"); // Non-greedy ?? quantifier REGEX_FIND( "(ab)(ab)\?\?(ab)\?\?(ab)\?\?(ab)\?\?c", "<0><1>ab<4>ab<5>abc"); // Unicode Properties as naked elements in a pattern REGEX_FIND( "\\p{Lu}+", "here we go ... <0>ABC and no more."); REGEX_FIND( "(\\p{L}+)(\\P{L}*?) (\\p{Zs}*)", "7999<0><1>letters<2>4949%^&*( <3> "); // \w and \W REGEX_FIND( "\\w+", " $%^&*( <0>hello123%^&*("); REGEX_FIND( "\\W+", "<0> $%^&*( hello123%^&*("); // \A match at beginning of input only. REGEX_FIND (".*\\Ahello", "<0>hello hello"); REGEX_FIND (".*hello", "<0>hello hello"); REGEX_FIND(".*\\Ahello", "stuff\nhello"); // don't match after embedded new-line. // \b \B REGEX_FIND( ".*?\\b(.).*", "<0> $%^&*( <1>hello123%^&*()gxx"); REGEX_FIND( "\\ba\\b", "-<0>a"); REGEX_FIND("\\by\\b", "xy"); // Finds first chars of up to 5 words REGEX_FIND( "(?:.*?\\b(\\w))?(?:.*?\\b(\\w))?(?:.*?\\b(\\w))?(?:.*?\\b(\\w))?(?:.*?\\b(\\w))?", "<0><1>Tthe <2>qick <3>brown <4>fox"); REGEX_FIND( "H.*?((?:\\B.)+)", "<0>H<1>ello "); REGEX_FIND( ".*?((?:\\B.)+).*?((?:\\B.)+).*?((?:\\B.)+)", "<0>H<1>ello <2> g<3>oodbye "); REGEX_FIND("(?:.*?\\b(.))?(?:.*?\\b(.))?(?:.*?\\b(.))?(?:.*?\\b(.))?(?:.*?\\b(.))?.*", "<0> \\u0301 \\u0301<1>A\\u0302BC\\u0303\\u0304<2> \\u0305 \\u0306" "<3>X\\u0307Y\\u0308"); // . does not match new-lines REGEX_FIND(".", "\\u000a\\u000d\\u0085\\u000c\\u2028\\u2029<0>X\\u000aY"); REGEX_FIND("A.", "A\\u000a "); // no match // \d for decimal digits REGEX_FIND("\\d*", "<0>0123456789\\u0660\\u06F9\\u0969\\u0A66\\u1369" "\\u17E2\\uFF10\\U0001D7CE\\U0001D7FFnon-digits"); REGEX_FIND("\\D+", "<0>non digits"); REGEX_FIND("\\D*(\\d*)(\\D*)", "<0>non-digits<1>3456666<2>more non digits"); // \Q...\E quote mode REGEX_FIND("hel\\Qlo, worl\\Ed", "<0>hello, world"); REGEX_FIND("\\Q$*^^(*)?\\A\\E(a*)", "<0>$*^^(*)?\\\\A<1>aaaaaaaaaaaaaaa"); // \S and \s space characters REGEX_FIND("\\s+", "not_space<0> \\t \\r \\n \\u3000 \\u2004 \\u2028 \\u2029xyz"); REGEX_FIND("(\\S+).*?(\\S+).*", "<0><1>Not-spaces <2>more-non-spaces "); // \X consume one combining char sequence. REGEX_FIND("(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?", "<0><1>A<2>B<3> <4>\\r\\n"); REGEX_FIND("(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?(\\X)?", "<0><1>A\\u0301<2>\n<3>\\u0305<4>a\\u0302\\u0303\\u0304"); // ^ matches only at beginning of line REGEX_FIND(".*^(Hello)", "<0><1>Hello Hello Hello Hello Goodbye"); REGEX_FIND(".*(Hello)", "<0>Hello Hello Hello <1>Hello Goodbye"); REGEX_FIND(".*^(Hello)", " Hello Hello Hello Hello Goodbye"); // No Match // $ matches only at end of line, or before a newline preceding the end of line REGEX_FIND(".*?(Goodbye)$", "<0>Hello Goodbye Goodbye <1>Goodbye"); REGEX_FIND(".*?(Goodbye)", "<0>Hello <1>Goodbye Goodbye Goodbye"); REGEX_FIND(".*?(Goodbye)$", "Hello Goodbye> Goodbye Goodbye "); // No Match REGEX_FIND(".*?(Goodbye)$", "<0>Hello Goodbye Goodbye <1>Goodbye\\n"); REGEX_FIND(".*?(Goodbye)$", "<0>Hello Goodbye Goodbye <1>Goodbye\\n"); REGEX_FIND(".*?(Goodbye)$", "<0>Hello Goodbye Goodbye <1>Goodbye\\r\\n"); REGEX_FIND(".*?(Goodbye)$", "Hello Goodbye Goodbye Goodbye\\n\\n"); // No Match // \Z matches at end of input, like $ with default flags. REGEX_FIND(".*?(Goodbye)\\Z", "<0>Hello Goodbye Goodbye <1>Goodbye"); REGEX_FIND(".*?(Goodbye)", "<0>Hello <1>Goodbye Goodbye Goodbye"); REGEX_FIND(".*?(Goodbye)\\Z", "Hello Goodbye> Goodbye Goodbye "); // No Match REGEX_FIND("here$", "here\\nthe end"); // No Match REGEX_FIND(".*?(Goodbye)\\Z", "<0>Hello Goodbye Goodbye <1>Goodbye\\n"); REGEX_FIND(".*?(Goodbye)\\Z", "<0>Hello Goodbye Goodbye <1>Goodbye\\n"); REGEX_FIND(".*?(Goodbye)\\Z", "<0>Hello Goodbye Goodbye <1>Goodbye\\r\\n"); REGEX_FIND(".*?(Goodbye)\\Z", "Hello Goodbye Goodbye Goodbye\\n\\n"); // No Match // \z matches only at the end of string. // no special treatment of new lines. // no dependencies on flag settings. REGEX_FIND(".*?(Goodbye)\\z", "<0>Hello Goodbye Goodbye <1>Goodbye"); REGEX_FIND(".*?(Goodbye)\\z", "Hello Goodbye Goodbye Goodbye "); // No Match REGEX_FIND("here$", "here\\nthe end"); // No Match REGEX_FIND(".*?(Goodbye)\\z", "Hello Goodbye Goodbye Goodbye\\n"); // No Match REGEX_FIND(".*?(Goodbye)\\n\\z", "<0>Hello Goodbye Goodbye <1>Goodbye\\n"); // (?# comment) doesn't muck up pattern REGEX_FIND("Hello (?# this is a comment) world", " <0>Hello world..."); // Check some implementation corner cases base on the way literal strings are compiled. REGEX_FIND("A", "<0>A"); REGEX_FIND("AB", "<0>ABABABAB"); REGEX_FIND("AB+", "<0>ABBBA"); REGEX_FIND("AB+", "<0>ABABAB"); REGEX_FIND("ABC+", "<0>ABCABC"); REGEX_FIND("ABC+", "<0>ABCCCCABC"); REGEX_FIND("(?:ABC)+", "<0>ABCABCABCD"); REGEX_FIND("(?:ABC)DEF+", "<0>ABCDEFFFD"); REGEX_FIND("AB\\.C\\eD\\u0666E", "<0>AB.C\\u001BD\\u0666EF"); // {min,max} iteration qualifier REGEX_TESTLM("A{3}BC", "AAABC", TRUE, TRUE); REGEX_FIND("(ABC){2,3}AB", "no matchAB"); REGEX_FIND("(ABC){2,3}AB", "ABCAB"); REGEX_FIND("(ABC){2,3}AB", "<0>ABC<1>ABCAB"); REGEX_FIND("(ABC){2,3}AB", "<0>ABCABC<1>ABCAB"); REGEX_FIND("(ABC){2,3}AB", "<0>ABCABC<1>ABCABCAB"); REGEX_FIND("(ABC){2}AB", "ABCAB"); REGEX_FIND("(ABC){2}AB", "<0>ABC<1>ABCAB"); REGEX_FIND("(ABC){2}AB", "<0>ABC<1>ABCABCAB"); REGEX_FIND("(ABC){2}AB", "<0>ABC<1>ABCABCABCAB"); REGEX_FIND("(ABC){2,}AB", "ABCAB"); REGEX_FIND("(ABC){2,}AB", "<0>ABC<1>ABCAB"); REGEX_FIND("(ABC){2,}AB", "<0>ABCABC<1>ABCAB"); REGEX_FIND("(ABC){2,}AB", "<0>ABCABCABC<1>ABCAB"); REGEX_FIND("X{0,0}ABC", "<0>ABC"); REGEX_FIND("X{0,1}ABC", "<0>ABC"); REGEX_FIND("(?:Hello(!{1,3}) there){1}", "Hello there"); REGEX_FIND("(?:Hello(!{1,3}) there){1}", "<0>Hello<1>! there"); REGEX_FIND("(?:Hello(!{1,3}) there){1}", "<0>Hello<1>!! there"); REGEX_FIND("(?:Hello(!{1,3}) there){1}", "<0>Hello<1>!!! there"); REGEX_FIND("(?:Hello(!{1,3}) there){1}", "Hello!!!! there"); // Nongreedy {min,max}? intervals REGEX_FIND("(ABC){2,3}?AB", "no matchAB"); REGEX_FIND("(ABC){2,3}?AB", "ABCAB"); REGEX_FIND("(ABC){2,3}?AB", "<0>ABC<1>ABCAB"); REGEX_FIND("(ABC){2,3}?AB", "<0>ABC<1>ABCABCAB"); REGEX_FIND("(ABC){2,3}?AB", "<0>ABC<1>ABCABCABCAB"); REGEX_FIND("(ABC){2,3}?AX", "<0>ABCABC<1>ABCAX"); REGEX_FIND("(ABC){2,3}?AX", "ABC<0>ABCABC<1>ABCAX"); // Atomic Grouping REGEX_FIND("(?>.*)abc", "abcabcabc"); // no match. .* consumed entire string. REGEX_FIND("(?>(abc{2,4}?))(c*)", "<0><1>abcc<2>cccddd"); REGEX_FIND("(\\.\\d\\d(?>[1-9]?))\\d+", "1.625"); REGEX_FIND("(\\.\\d\\d(?>[1-9]?))\\d+", "1<0><1>.6250"); // Possessive *+ REGEX_FIND("(abc)*+a", "abcabcabc"); REGEX_FIND("(abc)*+a", "<0>abc<1>abcab"); REGEX_FIND("(a*b)*+a", "<0><1>aaaabaaaa"); // Possessive ?+ REGEX_FIND("c?+ddd", "<0>cddd"); REGEX_FIND("c?+cddd", "cddd"); REGEX_FIND("c?cddd", "<0>cddd"); // Back Reference REGEX_FIND("(?:ab(..)cd\\1)*", "<0>ab23cd23ab<1>wwcdwwabxxcdyy"); REGEX_FIND("ab(?:c|(d?))(\\1)", "<0>ab<1><2>c"); REGEX_FIND("ab(?:c|(d?))(\\1)", "<0>ab<1>d<2>d"); REGEX_FIND("ab(?:c|(d?))(\\1)", "<0>ab<1><2>e"); REGEX_FIND("ab(?:c|(d?))(\\1)", "<0>ab<1><2>"); } //--------------------------------------------------------------------------- // // Errors Check for error handling in patterns. // //--------------------------------------------------------------------------- void RegexTest::Errors() { // \escape sequences that aren't implemented yet. REGEX_ERR("named chars \\N{GREEK CAPITAL LETTER ALPHA} not implementd", 1, 14, U_REGEX_UNIMPLEMENTED); REGEX_ERR("hex format \\x{abcd} not implemented", 1, 13, U_REGEX_UNIMPLEMENTED); // Missing close parentheses REGEX_ERR("Comment (?# with no close", 1, 25, U_REGEX_MISMATCHED_PAREN); REGEX_ERR("Capturing Parenthesis(...", 1, 25, U_REGEX_MISMATCHED_PAREN); REGEX_ERR("Grouping only parens (?: blah blah", 1, 34, U_REGEX_MISMATCHED_PAREN); // Extra close paren REGEX_ERR("Grouping only parens (?: blah)) blah", 1, 31, U_REGEX_MISMATCHED_PAREN); REGEX_ERR(")))))))", 1, 1, U_REGEX_MISMATCHED_PAREN); REGEX_ERR("(((((((", 1, 7, U_REGEX_MISMATCHED_PAREN); // Flag settings not yet implemented REGEX_ERR("(?i:stuff*)", 1, 3, U_REGEX_UNIMPLEMENTED); REGEX_ERR("(?-si) stuff", 1, 3, U_REGEX_UNIMPLEMENTED); // Look-ahead, Look-behind REGEX_ERR("abc(?<=xyz).*", 1, 7, U_REGEX_UNIMPLEMENTED); // look-behind REGEX_ERR("abc(?matcher(target, status); } if (mat != NULL) { target = mat->replaceFirst(replacement, status); } UBool retVal = (mat->start(0, status) != -1); delete mat; delete pat; return retVal; } static char *cstar(const UnicodeString &s) { UErrorCode status=U_ZERO_ERROR; static char buf[1000]; s.extract(buf, 1000, NULL, status); buf[999] = 0; return buf; } //------------------------------------------------------------------------------- // // Read a text data file, convert it to UChars, and return the data // in one big UChar * buffer, which the caller must delete. // //-------------------------------------------------------------------------------- UChar *RegexTest::ReadAndConvertFile(const char *fileName, int &ulen, UErrorCode &status) { UChar *retPtr = NULL; char *fileBuf = NULL; UConverter* conv = NULL; FILE *f = NULL; ulen = 0; { if (U_FAILURE(status)) { return retPtr; } // // Open the file. // f = fopen(fileName, "rb"); if (f == 0) { errln("Error opening test data file %s\n", fileName); goto cleanUpAndReturn; } // // Read it in // fseek( f, 0, SEEK_END); int fileSize = ftell(f); fileBuf = new char[fileSize]; fseek(f, 0, SEEK_SET); int amt_read = fread(fileBuf, 1, fileSize, f); if (amt_read != fileSize || fileSize <= 0) { errln("Error reading test data file."); goto cleanUpAndReturn; } // // Look for a Unicode Signature (BOM) on the data just read // int32_t signatureLength; const char * fileBufC = fileBuf; const char* encoding = ucnv_detectUnicodeSignature( fileBuf, fileSize, &signatureLength, &status); if(encoding!=NULL ){ fileBufC += signatureLength; fileSize -= signatureLength; } // // Open a converter to take the rule file to UTF-16 // conv = ucnv_open(encoding, &status); if (U_FAILURE(status)) { goto cleanUpAndReturn; } // // Convert the rules to UChar. // Preflight first to determine required buffer size. // ulen = ucnv_toUChars(conv, NULL, // dest, 0, // destCapacity, fileBufC, fileSize, &status); if (status == U_BUFFER_OVERFLOW_ERROR) { // Buffer Overflow is expected from the preflight operation. status = U_ZERO_ERROR; } retPtr = new UChar[ulen+1]; ucnv_toUChars(conv, retPtr, // dest, ulen+1, fileBufC, fileSize, &status); } cleanUpAndReturn: fclose(f); delete fileBuf; ucnv_close(conv); if (U_FAILURE(status)) { errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status)); delete retPtr; retPtr = 0; ulen = 0; }; return retPtr; } //------------------------------------------------------------------------------- // // PerlTests - Run Perl's regular expression tests // The input file for this test is re_tests, the standard regular // expression test data distributed with the Perl source code. // // Here is Perl's description of the test data file: // // # The tests are in a separate file 't/op/re_tests'. // # Each line in that file is a separate test. // # There are five columns, separated by tabs. // # // # Column 1 contains the pattern, optionally enclosed in C<''>. // # Modifiers can be put after the closing C<'>. // # // # Column 2 contains the string to be matched. // # // # Column 3 contains the expected result: // # y expect a match // # n expect no match // # c expect an error // # B test exposes a known bug in Perl, should be skipped // # b test exposes a known bug in Perl, should be skipped if noamp // # // # Columns 4 and 5 are used only if column 3 contains C or C. // # // # Column 4 contains a string, usually C<$&>. // # // # Column 5 contains the expected result of double-quote // # interpolating that string after the match, or start of error message. // # // # Column 6, if present, contains a reason why the test is skipped. // # This is printed with "skipped", for harness to pick up. // # // # \n in the tests are interpolated, as are variables of the form ${\w+}. // # // # If you want to add a regular expression test that can't be expressed // # in this format, don't add it here: put it in op/pat.t instead. // // For ICU, if field 3 contains an 'i', the test will be skipped. // The test exposes is some known incompatibility between ICU and Perl regexps. // (The i is in addition to whatever was there before.) // //------------------------------------------------------------------------------- void RegexTest::PerlTests() { UErrorCode status = U_ZERO_ERROR; UParseError pe; // // Open and read the test data file. // const char *testDataDirectory = loadTestData(status); UnicodeString tdd(testDataDirectory); ReplaceFirst(tdd, "([/\\\\])out[/\\\\]testdata", "$1re_tests.txt", status); int len; UChar *testData = ReadAndConvertFile(cstar(tdd), len, status); // // Put the test data into a UnicodeString // UnicodeString ruleSourceS(FALSE, testData, len); // // Regex to break the input file into lines, and strip the new lines. // One line per match, capture group one is the desired data. // RegexPattern* linePat = RegexPattern::compile("(.+?)[\\r\\n]+", 0, pe, status); RegexMatcher* lineMat = linePat->matcher(ruleSourceS, status); // // Regex to split a test file line into fields. // There are six fields, separated by tabs. // RegexPattern* fieldPat = RegexPattern::compile("\\t", 0, pe, status); // // Regex to identify test patterns with flag settings, and to separate them. // Test patterns with flags look like 'pattern'i // Test patterns without flags are not quoted: pattern // Coming out, capture group 2 is the pattern, capture group 3 is the flags. // RegexPattern *flagPat = RegexPattern::compile("('?)(.*)\\1(.*)", 0, pe, status); RegexMatcher* flagMat = flagPat->matcher("", status); // // The Perl tests reference several perl-isms, which are evaluated/substituted // in the test data. Not being perl, this must be done explicitly. Here // are string constants and REs for these constructs. // UnicodeString nulnulSrc("${nulnul}"); UnicodeString nulnul("\\u0000\\u0000"); nulnul = nulnul.unescape(); UnicodeString ffffSrc("${ffff}"); UnicodeString ffff("\\uffff"); ffff = ffff.unescape(); // regexp for $-[0], $+[2], etc. RegexPattern *groupsPat = RegexPattern::compile("\\$([+\\-])\\[(\\d+)\\]", 0, pe, status); RegexMatcher *groupsMat = groupsPat->matcher("", status); // regexp for $0, $1, $2, etc. RegexPattern *cgPat = RegexPattern::compile("\\$(\\d+)", 0, pe, status); RegexMatcher *cgMat = cgPat->matcher("", status); // // Main Loop for the Perl Tests, runs once per line from the // test data file. // int32_t lineNum = 0; int32_t skippedUnimplementedCount = 0; while (lineMat->find()) { lineNum++; // // Get a line, break it into its fields, do the Perl // variable substitutions. // UnicodeString line = lineMat->group(1, status); UnicodeString fields[7]; fieldPat->split(line, fields, 7, status); flagMat->reset(fields[0]); flagMat->matches(status); UnicodeString pattern = flagMat->group(2, status); pattern.findAndReplace("${bang}", "!"); pattern.findAndReplace(nulnulSrc, "\\u0000\\u0000"); pattern.findAndReplace(ffffSrc, ffff); // // Identify patterns that include match flag settings, // split off the flags, remove the extra quotes. // UnicodeString flagStr = flagMat->group(3, status); // printf("pattern = %s\n", cstar(pattern)); // printf(" flags = %s\n", cstar(flags)); if (U_FAILURE(status)) { errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status)); return; } int32_t flags = 0; const UChar UChar_c = 0x63; // Char constants for the flag letters. const UChar UChar_i = 0x69; // (Damn the lack of Unicode support in C) const UChar UChar_m = 0x6d; const UChar UChar_x = 0x78; const UChar UChar_y = 0x79; if (flagStr.indexOf(UChar_i) != -1) { flags |= UREGEX_CASE_INSENSITIVE; } if (flagStr.indexOf(UChar_m) != -1) { flags |= UREGEX_MULTILINE; } if (flagStr.indexOf(UChar_x) != -1) { flags |= UREGEX_COMMENTS; } // // Compile the test pattern. // status = U_ZERO_ERROR; RegexPattern *testPat = RegexPattern::compile(pattern, flags, pe, status); if (status == U_REGEX_UNIMPLEMENTED) { // // Test of a feature that is planned for ICU, but not yet implemented. // skip the test. skippedUnimplementedCount++; delete testPat; status = U_ZERO_ERROR; continue; } if (U_FAILURE(status)) { // Some tests are supposed to generate errors. // Only report an error for tests that are supposed to succeed. if (fields[2].indexOf(UChar_c) == -1 && // Compilation is not supposed to fail AND fields[2].indexOf(UChar_i) == -1) // it's not an accepted ICU incompatibility { errln("line %d: ICU Error \"%s\"\n", lineNum, u_errorName(status)); } status = U_ZERO_ERROR; delete testPat; continue; } if (fields[2].indexOf(UChar_i) >= 0) { // ICU should skip this test. delete testPat; continue; } if (fields[2].indexOf(UChar_c) >= 0) { // This pattern should have caused a compilation error, but didn't/ errln("line %d: Expected a pattern compile error, got success.", lineNum); delete testPat; continue; } // // replace the Perl variables that appear in some of the // match data strings. // UnicodeString matchString = fields[1]; matchString.findAndReplace(nulnulSrc, nulnul); matchString.findAndReplace(ffffSrc, ffff); // Replace any \n in the match string with an actual new-line char. // Don't do full unescape, as this unescapes more than Perl does, which // causes other spurious failures in the tests. matchString.findAndReplace("\\n", "\n"); // // Run the test, check for expected match/don't match result. // RegexMatcher *testMat = testPat->matcher(matchString, status); UBool found = testMat->find(); UBool expected = FALSE; if (fields[2].indexOf(UChar_y) >=0) { expected = TRUE; } if (expected != found) { errln("line %d: Expected %smatch, got %smatch", lineNum, expected?"":"no ", found?"":"no " ); } // // Interpret the Perl expression from the fourth field of the data file, // building up an ICU string from the results of the ICU match. // The Perl expression will contain references to the results of // a regex match, including the matched string, capture group strings, // group starting and ending indicies, etc. // UnicodeString resultString; UnicodeString perlExpr = fields[3]; groupsMat->reset(perlExpr); cgMat->reset(perlExpr); while (perlExpr.length() > 0) { if (perlExpr.startsWith("$&")) { resultString.append(testMat->group(status)); perlExpr.remove(0, 2); } else if (groupsMat->lookingAt(status)) { // $-[0] $+[2] etc. UnicodeString digitString = groupsMat->group(2, status); int32_t t = 0; int32_t groupNum = ICU_Utility::parseNumber(digitString, t, 10); UnicodeString plusOrMinus = groupsMat->group(1, status); int32_t matchPosition; if (plusOrMinus.compare("+") == 0) { matchPosition = testMat->end(groupNum, status); } else { matchPosition = testMat->start(groupNum, status); } if (matchPosition != -1) { ICU_Utility::appendNumber(resultString, matchPosition); } perlExpr.remove(0, groupsMat->end(status)); } else if (cgMat->lookingAt(status)) { // $1, $2, $3, etc. UnicodeString digitString = cgMat->group(1, status); int32_t t = 0; int32_t groupNum = ICU_Utility::parseNumber(digitString, t, 10); if (U_SUCCESS(status)) { resultString.append(testMat->group(groupNum, status)); status = U_ZERO_ERROR; } perlExpr.remove(0, cgMat->end(status)); } else if (perlExpr.startsWith("@-")) { int i; for (i=0; i<=testMat->groupCount(); i++) { if (i>0) { resultString.append(" "); } ICU_Utility::appendNumber(resultString, testMat->start(i, status)); } perlExpr.remove(0, 2); } else if (perlExpr.startsWith("@+")) { int i; for (i=0; i<=testMat->groupCount(); i++) { if (i>0) { resultString.append(" "); } ICU_Utility::appendNumber(resultString, testMat->end(i, status)); } perlExpr.remove(0, 2); } else if (perlExpr.startsWith("\\")) { // \Escape. Take following char as a literal. if (perlExpr.length() > 1) { perlExpr.remove(0, 1); // Remove the '\', but only if not last char. } resultString.append(perlExpr.charAt(0)); perlExpr.remove(0, 1); } else { // Any characters from the perl expression that we don't explicitly // recognize before here are assumed to be literals and copied // as-is to the expected results. resultString.append(perlExpr.charAt(0)); perlExpr.remove(0, 1); } if (U_FAILURE(status)) { errln("Line %d: ICU Error \"%s\"", lineNum, u_errorName(status)); break; } } // // Expected Results Compare // UnicodeString expectedS(fields[4]); expectedS.findAndReplace(nulnulSrc, nulnul); expectedS.findAndReplace(ffffSrc, ffff); if (expectedS.compare(resultString) != 0) { errln("Line %d: Incorrect perl expression results. Expected \"%s\"; got \"%s\"", lineNum, (const char *)CharString(expectedS), (const char *)CharString(resultString)); } delete testMat; delete testPat; } // // All done. Clean up allocated stuff. // delete cgMat; delete cgPat; delete groupsMat; delete groupsPat; delete flagMat; delete flagPat; delete lineMat; delete linePat; delete fieldPat; delete [] testData; logln("%d tests skipped because of unimplemented regexp features.", skippedUnimplementedCount); } #endif /* !UCONFIG_NO_REGULAR_EXPRESSIONS */