/* ********************************************************************** * Copyright (C) 2001, International Business Machines * Corporation and others. All Rights Reserved. ********************************************************************** * Date Name Description * 11/10/99 aliu Creation. ********************************************************************** */ #include "transtst.h" #include "unicode/utypes.h" #include "unicode/translit.h" #include "unicode/rbt.h" #include "unicode/unifilt.h" #include "unicode/cpdtrans.h" #include "unicode/dtfmtsym.h" #include "unicode/hextouni.h" #include "unicode/unitohex.h" #include "unicode/unicode.h" #include "unicode/uniset.h" #include "unicode/ucnv.h" #include "unicode/ucnv_err.h" // Define character constants thusly to be EBCDIC-friendly enum { LEFT_BRACE=((UChar)0x007B), /*{*/ PIPE =((UChar)0x007C), /*|*/ ZERO =((UChar)0x0030), /*0*/ UPPER_A =((UChar)0x0041) /*A*/ }; void TransliteratorTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) { switch (index) { TESTCASE(0,TestInstantiation); TESTCASE(1,TestSimpleRules); TESTCASE(2,TestRuleBasedInverse); TESTCASE(3,TestKeyboard); TESTCASE(4,TestKeyboard2); TESTCASE(5,TestKeyboard3); TESTCASE(6,TestArabic); TESTCASE(7,TestCompoundKana); TESTCASE(8,TestCompoundHex); TESTCASE(9,TestFiltering); TESTCASE(10,TestInlineSet); TESTCASE(11,TestPatternQuoting); TESTCASE(12,TestJ277); TESTCASE(13,TestJ243); TESTCASE(14,TestJ329); TESTCASE(15,TestSegments); TESTCASE(16,TestCursorOffset); TESTCASE(17,TestArbitraryVariableValues); TESTCASE(18,TestPositionHandling); TESTCASE(19,TestHiraganaKatakana); TESTCASE(20,TestCopyJ476); TESTCASE(21,TestAnchors); TESTCASE(22,TestInterIndic); TESTCASE(23,TestFilterIDs); TESTCASE(24,TestCaseMap); TESTCASE(25,TestNameMap); TESTCASE(26,TestLiberalizedID); TESTCASE(27,TestCreateInstance); TESTCASE(28,TestNormalizationTransliterator); TESTCASE(29,TestCompoundRBT); TESTCASE(30,TestCompoundFilter); default: name = ""; break; } } /** * Make sure every system transliterator can be instantiated. * * ALSO test that the result of toRules() for each rule is a valid * rule. Do this here so we don't have to have another test that * instantiates everything as well. */ void TransliteratorTest::TestInstantiation() { int32_t n = Transliterator::countAvailableIDs(); UnicodeString name; for (int32_t i=0; itoRules(rules, TRUE); Transliterator *u = Transliterator::createFromRules("x", rules, UTRANS_FORWARD, &parseError); if (u == 0) { errln(UnicodeString("FAIL: ") + id + ".toRules() => bad rules" + ", parse error " + parseError.code + ", line " + parseError.line + ", offset " + parseError.offset + ", context " + prettify(parseError.preContext) + ", rules: " + rules); } else { delete u; } delete t; } } // Now test the failure path UnicodeString id(""); Transliterator* t = Transliterator::createInstance(id); if (t != 0) { errln("FAIL: " + id + " returned a transliterator"); delete t; } else { logln("OK: Bogus ID handled properly"); } } void TransliteratorTest::TestSimpleRules(void) { /* Example: rules 1. ab>x|y * 2. yc>z * * []|eabcd start - no match, copy e to tranlated buffer * [e]|abcd match rule 1 - copy output & adjust cursor * [ex|y]cd match rule 2 - copy output & adjust cursor * [exz]|d no match, copy d to transliterated buffer * [exzd]| done */ expect(UnicodeString("ab>x|y;", "") + "yc>z", "eabcd", "exzd"); /* Another set of rules: * 1. ab>x|yzacw * 2. za>q * 3. qc>r * 4. cw>n * * []|ab Rule 1 * [x|yzacw] No match * [xy|zacw] Rule 2 * [xyq|cw] Rule 4 * [xyqn]| Done */ expect(UnicodeString("ab>x|yzacw;") + "za>q;" + "qc>r;" + "cw>n", "ab", "xyqn"); /* Test categories */ UErrorCode status = U_ZERO_ERROR; RuleBasedTransliterator t( "", UnicodeString("$dummy=").append((UChar)0xE100) + UnicodeString(";" "$vowel=[aeiouAEIOU];" "$lu=[:Lu:];" "$vowel } $lu > '!';" "$vowel > '&';" "'!' { $lu > '^';" "$lu > '*';" "a > ERROR", ""), status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor failed"); return; } expect(t, "abcdefgABCDEFGU", "&bcd&fg!^**!^*&"); } /** * Test inline set syntax and set variable syntax. */ void TransliteratorTest::TestInlineSet(void) { expect("{ [:Ll:] } x > y; [:Ll:] > z;", "aAbxq", "zAyzz"); expect("a[0-9]b > qrs", "1a7b9", "1qrs9"); expect(UnicodeString( "$digit = [0-9];" "$alpha = [a-zA-Z];" "$alphanumeric = [$digit $alpha];" // *** "$special = [^$alphanumeric];" // *** "$alphanumeric > '-';" "$special > '*';", ""), "thx-1138", "---*----"); } /** * Create some inverses and confirm that they work. We have to be * careful how we do this, since the inverses will not be true * inverses -- we can't throw any random string at the composition * of the transliterators and expect the identity function. F x * F' != I. However, if we are careful about the input, we will * get the expected results. */ void TransliteratorTest::TestRuleBasedInverse(void) { UnicodeString RULES = UnicodeString("abc>zyx;") + "ab>yz;" + "bc>zx;" + "ca>xy;" + "a>x;" + "b>y;" + "c>z;" + "abc", RULES, status); RuleBasedTransliterator rev("", RULES, UTRANS_REVERSE, status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor failed"); return; } for (int32_t i=0; i", UnicodeString("psch>Y;") +"ps>y;" +"ch>x;" +"a>A;", status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor failed"); return; } const char* DATA[] = { // insertion, buffer "a", "A", "p", "Ap", "s", "Aps", "c", "Apsc", "a", "AycA", "psch", "AycAY", 0, "AycAY", // null means finishKeyboardTransliteration }; keyboardAux(t, DATA, (int32_t)(sizeof(DATA)/sizeof(DATA[0]))); } /** * Basic test of keyboard with cursor. */ void TransliteratorTest::TestKeyboard2(void) { UErrorCode status = U_ZERO_ERROR; RuleBasedTransliterator t("", UnicodeString("ych>Y;") +"ps>|y;" +"ch>x;" +"a>A;", status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor failed"); return; } const char* DATA[] = { // insertion, buffer "a", "A", "p", "Ap", "s", "Ay", "c", "Ayc", "a", "AycA", "p", "AycAp", "s", "AycAy", "c", "AycAyc", "h", "AycAY", 0, "AycAY", // null means finishKeyboardTransliteration }; keyboardAux(t, DATA, (int32_t)(sizeof(DATA)/sizeof(DATA[0]))); } /** * Test keyboard transliteration with back-replacement. */ void TransliteratorTest::TestKeyboard3(void) { // We want th>z but t>y. Furthermore, during keyboard // transliteration we want t>y then yh>z if t, then h are // typed. UnicodeString RULES("t>|y;" "yh>z;"); const char* DATA[] = { // Column 1: characters to add to buffer (as if typed) // Column 2: expected appearance of buffer after // keyboard xliteration. "a", "a", "b", "ab", "t", "aby", "c", "abyc", "t", "abycy", "h", "abycz", 0, "abycz", // null means finishKeyboardTransliteration }; UErrorCode status = U_ZERO_ERROR; RuleBasedTransliterator t("", RULES, status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor failed"); return; } keyboardAux(t, DATA, (int32_t)(sizeof(DATA)/sizeof(DATA[0]))); } void TransliteratorTest::keyboardAux(const Transliterator& t, const char* DATA[], int32_t DATA_length) { UErrorCode status = U_ZERO_ERROR; UTransPosition index={0, 0, 0, 0}; UnicodeString s; for (int32_t i=0; i "; t.transliterate(s, index, DATA[i], status); } else { log = s + " => "; t.finishTransliteration(s, index); } // Show the start index '{' and the cursor '|' UnicodeString a, b, c; s.extractBetween(0, index.contextStart, a); s.extractBetween(index.contextStart, index.start, b); s.extractBetween(index.start, s.length(), c); log.append(a). append((UChar)LEFT_BRACE). append(b). append((UChar)PIPE). append(c); if (s == DATA[i+1] && U_SUCCESS(status)) { logln(log); } else { errln(UnicodeString("FAIL: ") + log + ", expected " + DATA[i+1]); } } } void TransliteratorTest::TestArabic(void) { /* const char* DATA[] = { "Arabic", "\u062a\u062a\u0645\u062a\u0639\u0020"+ "\u0627\u0644\u0644\u063a\u0629\u0020"+ "\u0627\u0644\u0639\u0631\u0628\u0628\u064a\u0629\u0020"+ "\u0628\u0628\u0646\u0638\u0645\u0020"+ "\u0643\u062a\u0627\u0628\u0628\u064a\u0629\u0020"+ "\u062c\u0645\u064a\u0644\u0629", }; */ UChar ar_raw[] = { 0x062a, 0x062a, 0x0645, 0x062a, 0x0639, 0x0020, 0x0627, 0x0644, 0x0644, 0x063a, 0x0629, 0x0020, 0x0627, 0x0644, 0x0639, 0x0631, 0x0628, 0x0628, 0x064a, 0x0629, 0x0020, 0x0628, 0x0628, 0x0646, 0x0638, 0x0645, 0x0020, 0x0643, 0x062a, 0x0627, 0x0628, 0x0628, 0x064a, 0x0629, 0x0020, 0x062c, 0x0645, 0x064a, 0x0644, 0x0629, 0 }; UnicodeString ar(ar_raw); Transliterator *t = Transliterator::createInstance("Latin-Arabic"); if (t == 0) { errln("FAIL: createInstance failed"); return; } expect(*t, "Arabic", ar); delete t; } /** * Compose the Kana transliterator forward and reverse and try * some strings that should come out unchanged. */ void TransliteratorTest::TestCompoundKana(void) { Transliterator* t = Transliterator::createInstance("Latin-Kana;Kana-Latin"); if (t == 0) { errln("FAIL: construction of Latin-Kana;Kana-Latin failed"); } else { expect(*t, "aaaaa", "aaaaa"); delete t; } } /** * Compose the hex transliterators forward and reverse. */ void TransliteratorTest::TestCompoundHex(void) { Transliterator* a = Transliterator::createInstance("Unicode-Hex"); Transliterator* b = Transliterator::createInstance("Hex-Unicode"); Transliterator* transab[] = { a, b }; Transliterator* transba[] = { b, a }; if (a == 0 || b == 0) { errln("FAIL: construction failed"); delete a; delete b; return; } // Do some basic tests of a expect(*a, "01", UnicodeString("\\u0030\\u0031", "")); // Do some basic tests of b expect(*b, UnicodeString("\\u0030\\u0031", ""), "01"); Transliterator* ab = new CompoundTransliterator(transab, 2); UnicodeString s("abcde", ""); expect(*ab, s, s); UnicodeString str(s); a->transliterate(str); Transliterator* ba = new CompoundTransliterator(transba, 2); expect(*ba, str, str); delete ab; delete ba; delete a; delete b; } /** * Used by TestFiltering(). */ class TestFilter : public UnicodeFilter { virtual UnicodeFilter* clone() const { return new TestFilter(*this); } virtual UBool contains(UChar c) const { return c != (UChar)0x0063 /*c*/; } }; /** * Do some basic tests of filtering. */ void TransliteratorTest::TestFiltering(void) { Transliterator* hex = Transliterator::createInstance("Unicode-Hex"); if (hex == 0) { errln("FAIL: createInstance(Unicode-Hex) failed"); return; } hex->adoptFilter(new TestFilter()); UnicodeString s("abcde"); hex->transliterate(s); UnicodeString exp("\\u0061\\u0062c\\u0064\\u0065", ""); if (s == exp) { logln(UnicodeString("Ok: \"") + exp + "\""); } else { logln(UnicodeString("FAIL: \"") + s + "\", wanted \"" + exp + "\""); } delete hex; } /** * Test anchors */ void TransliteratorTest::TestAnchors(void) { expect(UnicodeString("^ab > 01 ;" " ab > |8 ;" " b > k ;" " 8x$ > 45 ;" " 8x > 77 ;", ""), "ababbabxabx", "018k7745"); expect(UnicodeString("$s = [z$] ;" "$s{ab > 01 ;" " ab > |8 ;" " b > k ;" " 8x}$s > 45 ;" " 8x > 77 ;", ""), "abzababbabxzabxabx", "01z018k45z01x45"); } /** * Test pattern quoting and escape mechanisms. */ void TransliteratorTest::TestPatternQuoting(void) { // Array of 3n items // Each item is , , const UnicodeString DATA[] = { UnicodeString(UChar(0x4E01)) + ">'[male adult]'", UnicodeString(UChar(0x4E01)), "[male adult]" }; for (int32_t i=0; i<3; i+=3) { logln(UnicodeString("Pattern: ") + prettify(DATA[i])); UErrorCode status = U_ZERO_ERROR; RuleBasedTransliterator t("", DATA[i], status); if (U_FAILURE(status)) { errln("RBT constructor failed"); } else { expect(t, DATA[i+1], DATA[i+2]); } } } /** * Regression test for bugs found in Greek transliteration. */ void TransliteratorTest::TestJ277(void) { UErrorCode status = U_ZERO_ERROR; Transliterator *gl = Transliterator::createInstance("Greek-Latin"); if (gl == NULL) { errln("FAIL: createInstance(Greek-Latin) returned NULL"); return; } UChar sigma = 0x3C3; UChar upsilon = 0x3C5; UChar nu = 0x3BD; // UChar PHI = 0x3A6; UChar alpha = 0x3B1; // UChar omega = 0x3C9; // UChar omicron = 0x3BF; // UChar epsilon = 0x3B5; // sigma upsilon nu -> syn UnicodeString syn; syn.append(sigma).append(upsilon).append(nu); expect(*gl, syn, "syn"); // sigma alpha upsilon nu -> saun UnicodeString sayn; sayn.append(sigma).append(alpha).append(upsilon).append(nu); expect(*gl, sayn, "saun"); // Again, using a smaller rule set UnicodeString rules( "$alpha = \\u03B1;" "$nu = \\u03BD;" "$sigma = \\u03C3;" "$ypsilon = \\u03C5;" "$vowel = [aeiouAEIOU$alpha$ypsilon];" "s <> $sigma;" "a <> $alpha;" "u <> $vowel { $ypsilon;" "y <> $ypsilon;" "n <> $nu;", ""); RuleBasedTransliterator mini("mini", rules, UTRANS_REVERSE, status); if (U_FAILURE(status)) { errln("FAIL: Transliterator constructor failed"); return; } expect(mini, syn, "syn"); expect(mini, sayn, "saun"); // Transliterate the Greek locale data Locale el("el"); DateFormatSymbols syms(el, status); if (U_FAILURE(status)) { errln("FAIL: Transliterator constructor failed"); return; } int32_t i, count; const UnicodeString* data = syms.getMonths(count); for (i=0; itransliterate(out); UBool ok = TRUE; if (data[i].length() >= 2 && out.length() >= 2 && u_isupper(data[i].charAt(0)) && u_islower(data[i].charAt(1))) { if (!(u_isupper(out.charAt(0)) && u_islower(out.charAt(1)))) { ok = FALSE; } } if (ok) { logln(prettify(data[i] + " -> " + out)); } else { errln(UnicodeString("FAIL: ") + prettify(data[i] + " -> " + out)); } } delete gl; } /** * Prefix, suffix support in hex transliterators */ void TransliteratorTest::TestJ243(void) { UErrorCode status = U_ZERO_ERROR; #if !defined(HPUX) // Test default Hex-Unicode, which should handle // \u, \U, u+, and U+ HexToUnicodeTransliterator hex; expect(hex, UnicodeString("\\u0041+\\U0042,u+0043uu+0044z", ""), "A+B,CuDz"); // Try a custom Hex-Unicode // \uXXXX and &#xXXXX; status = U_ZERO_ERROR; HexToUnicodeTransliterator hex2(UnicodeString("\\\\u###0;&\\#x###0\\;", ""), status); expect(hex2, UnicodeString("\\u61\\u062\\u0063\\u00645\\u66x0123", ""), "abcd5fx0123"); // Try custom Unicode-Hex (default is tested elsewhere) status = U_ZERO_ERROR; UnicodeToHexTransliterator hex3(UnicodeString("&\\#x###0;", ""), status); expect(hex3, "012", "012"); #endif } /** * Parsers need better syntax error messages. */ void TransliteratorTest::TestJ329(void) { struct { UBool containsErrors; const char* rule; } DATA[] = { { FALSE, "a > b; c > d" }, { TRUE, "a > b; no operator; c > d" }, }; int32_t DATA_length = (int32_t)(sizeof(DATA) / sizeof(DATA[0])); for (int32_t i=0; i", DATA[i].rule, UTRANS_FORWARD, 0, parseError, status); UBool gotError = U_FAILURE(status); UnicodeString desc(DATA[i].rule); desc.append(gotError ? " -> error" : " -> no error"); if (gotError) { desc = desc + ", ParseError code=" + parseError.code + " line=" + parseError.line + " offset=" + parseError.offset + " context=" + parseError.preContext; } if (gotError == DATA[i].containsErrors) { logln(UnicodeString("Ok: ") + desc); } else { errln(UnicodeString("FAIL: ") + desc); } } } /** * Test segments and segment references. */ void TransliteratorTest::TestSegments(void) { // Array of 3n items // Each item is , , UnicodeString DATA[] = { "([a-z]) '.' ([0-9]) > $2 '-' $1", "abc.123.xyz.456", "ab1-c23.xy4-z56", }; int32_t DATA_length = (int32_t)(sizeof(DATA)/sizeof(*DATA)); for (int32_t i=0; i", DATA[i], status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor"); } else { expect(t, DATA[i+1], DATA[i+2]); } } } /** * Test cursor positioning outside of the key */ void TransliteratorTest::TestCursorOffset(void) { // Array of 3n items // Each item is , , UnicodeString DATA[] = { "pre {alpha} post > | @ ALPHA ;" "eALPHA > beta ;" "pre {beta} post > BETA @@ | ;" "post > xyz", "prealphapost prebetapost", "prbetaxyz preBETApost", }; int32_t DATA_length = (int32_t)(sizeof(DATA)/sizeof(*DATA)); for (int32_t i=0; i", DATA[i], status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor"); } else { expect(t, DATA[i+1], DATA[i+2]); } } } /** * Test zero length and > 1 char length variable values. Test * use of variable refs in UnicodeSets. */ void TransliteratorTest::TestArbitraryVariableValues(void) { // Array of 3n items // Each item is , , UnicodeString DATA[] = { "$abe = ab;" "$pat = x[yY]z;" "$ll = 'a-z';" "$llZ = [$ll];" "$llY = [$ll$pat];" "$emp = ;" "$abe > ABE;" "$pat > END;" "$llZ > 1;" "$llY > 2;" "7$emp 8 > 9;" "", "ab xYzxyz stY78", "ABE ENDEND 1129", }; int32_t DATA_length = (int32_t)(sizeof(DATA)/sizeof(*DATA)); for (int32_t i=0; i", DATA[i], status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor"); } else { expect(t, DATA[i+1], DATA[i+2]); } } } /** * Confirm that the contextStart, contextLimit, start, and limit * behave correctly. J474. */ void TransliteratorTest::TestPositionHandling(void) { // Array of 3n items // Each item is , , const char* DATA[] = { "a{t} > SS ; {t}b > UU ; {t} > TT ;", "xtat txtb", // pos 0,9,0,9 "xTTaSS TTxUUb", "a{t} > SS ; {t}b > UU ; {t} > TT ; a > A ; b > B ;", "xtat txtb", // pos 2,9,3,8 "xtaSS TTxUUb", "a{t} > SS ; {t}b > UU ; {t} > TT ; a > A ; b > B ;", "xtat txtb", // pos 3,8,3,8 "xtaTT TTxTTb", }; // Array of 4n positions -- these go with the DATA array // They are: contextStart, contextLimit, start, limit int32_t POS[] = { 0, 9, 0, 9, 2, 9, 3, 8, 3, 8, 3, 8, }; int32_t n = (int32_t)(sizeof(DATA) / sizeof(DATA[0])) / 3; for (int32_t i=0; i", DATA[3*i], status); if (U_FAILURE(status)) { delete t; errln("FAIL: RBT constructor"); return; } UTransPosition pos; pos.contextStart= POS[4*i]; pos.contextLimit = POS[4*i+1]; pos.start = POS[4*i+2]; pos.limit = POS[4*i+3]; UnicodeString rsource(DATA[3*i+1]); t->transliterate(rsource, pos, status); if (U_FAILURE(status)) { delete t; errln("FAIL: transliterate"); return; } t->finishTransliteration(rsource, pos); expectAux(DATA[3*i], DATA[3*i+1], rsource, DATA[3*i+2]); delete t; } } /** * Test the Hiragana-Katakana transliterator. */ void TransliteratorTest::TestHiraganaKatakana(void) { Transliterator* hk = Transliterator::createInstance("Hiragana-Katakana"); Transliterator* kh = Transliterator::createInstance("Katakana-Hiragana"); if (hk == 0 || kh == 0) { errln("FAIL: createInstance failed"); delete hk; delete kh; return; } // Array of 3n items // Each item is "hk"|"kh"|"both", , const char* DATA[] = { "both", "\\u3042\\u3090\\u3099\\u3092\\u3050", "\\u30A2\\u30F8\\u30F2\\u30B0", "kh", "\\u307C\\u3051\\u3060\\u3042\\u3093\\u30FC", "\\u30DC\\u30F6\\u30C0\\u30FC\\u30F3\\u30FC", }; int32_t DATA_length = (int32_t)(sizeof(DATA) / sizeof(DATA[0])); for (int32_t i=0; iA;b>B;", status); if (U_FAILURE(status)) { errln("FAIL: RBT constructor"); return; } t2 = new RuleBasedTransliterator(t1); expect(t1, "abc", "ABc"); } expect(*t2, "abc", "ABc"); delete t2; } /** * Test inter-Indic transliterators. These are composed. * ICU4C Jitterbug 483. */ void TransliteratorTest::TestInterIndic(void) { UnicodeString ID("Devanagari-Gujarati", ""); Transliterator* dg = Transliterator::createInstance(ID); if (dg == 0) { errln("FAIL: createInstance(" + ID + ") returned NULL"); return; } UnicodeString id = dg->getID(); if (id != ID) { errln("FAIL: createInstance(" + ID + ")->getID() => " + id); } UnicodeString dev = CharsToUnicodeString("\\u0901\\u090B\\u0925"); UnicodeString guj = CharsToUnicodeString("\\u0A81\\u0A8B\\u0AA5"); expect(*dg, dev, guj); delete dg; } /** * Test filter syntax in IDs. (J918) */ void TransliteratorTest::TestFilterIDs(void) { // Array of 3n strings: // , , , const char* DATA[] = { "Unicode[aeiou]-Hex", "Hex[aeiou]-Unicode", "quizzical", "q\\u0075\\u0069zz\\u0069c\\u0061l", "Unicode[aeiou]-Hex;Hex[^5]-Unicode", "Unicode[^5]-Hex;Hex[aeiou]-Unicode", "quizzical", "q\\u0075izzical", "Null[abc]", "Null[abc]", "xyz", "xyz", }; enum { DATA_length = sizeof(DATA) / sizeof(DATA[0]) }; for (int i=0; igetID()) { errln("FAIL: createInstance(" + ID + ").getID() => " + t->getID()); } // Check the inverse Transliterator *u = t->createInverse(); if (u == 0) { errln("FAIL: " + ID + ".createInverse() returned NULL"); } else if (u->getID() != uID) { errln("FAIL: " + ID + ".createInverse().getID() => " + u->getID() + ", expected " + uID); } delete t; delete u; } } /** * Test the case mapping transliterators. */ void TransliteratorTest::TestCaseMap(void) { Transliterator* toUpper = Transliterator::createInstance("Any-Upper[^xyzXYZ]"); Transliterator* toLower = Transliterator::createInstance("Any-Lower[^xyzXYZ]"); Transliterator* toTitle = Transliterator::createInstance("Any-Title[^xyzXYZ]"); if (toUpper==0 || toLower==0 || toTitle==0) { errln("FAIL: createInstance returned NULL"); delete toUpper; delete toLower; delete toTitle; return; } expect(*toUpper, "The quick brown fox jumped over the lazy dogs.", "THE QUICK BROWN FOx JUMPED OVER THE LAzy DOGS."); expect(*toLower, "The quIck brown fOX jUMPED OVER THE LAzY dogs.", "the quick brown foX jumped over the lazY dogs."); expect(*toTitle, "the quick brown foX can't jump over the laZy dogs.", "The Quick Brown FoX Can't Jump Over The LaZy Dogs."); delete toUpper; delete toLower; delete toTitle; } /** * Test the name mapping transliterators. */ void TransliteratorTest::TestNameMap(void) { Transliterator* uni2name = Transliterator::createInstance("Any-Name[^abc]"); Transliterator* name2uni = Transliterator::createInstance("Name-Any"); if (uni2name==0 || name2uni==0) { errln("FAIL: createInstance returned NULL"); delete uni2name; delete name2uni; return; } expect(*uni2name, CharsToUnicodeString("\\u00A0abc\\u4E01\\u00B5\\u0A81\\uFFFD\\uFFFF"), CharsToUnicodeString("{NO-BREAK SPACE}abc{CJK UNIFIED IDEOGRAPH-4E01}{MICRO SIGN}{GUJARATI SIGN CANDRABINDU}{REPLACEMENT CHARACTER}\\uFFFF")); expect(*name2uni, "{NO-BREAK SPACE}abc{CJK UNIFIED IDEOGRAPH-4E01}{x{MICRO SIGN}{GUJARATI SIGN CANDRABINDU}{REPLACEMENT CHARACTER}{", CharsToUnicodeString("\\u00A0abc\\u4E01{x\\u00B5\\u0A81\\uFFFD{")); delete uni2name; delete name2uni; } /** * Test liberalized ID syntax. 1006c */ void TransliteratorTest::TestLiberalizedID(void) { // Some test cases have an expected getID() value of NULL. This // means I have disabled the test case for now. This stuff is // still under development, and I haven't decided whether to make // getID() return canonical case yet. It will all get rewritten // with the move to Source-Target/Variant IDs anyway. [aliu] const char* DATA[] = { "latin-arabic", NULL /*"Latin-Arabic"*/, "case insensitivity", " Null ", "Null", "whitespace", " Latin[a-z]-Arabic ", "Latin[a-z]-Arabic", "inline filter", " null ; latin-arabic ", NULL /*"Null;Latin-Arabic"*/, "compound whitespace", }; const int32_t DATA_length = sizeof(DATA)/sizeof(DATA[0]); for (int32_t i=0; igetID()) { logln(UnicodeString("Ok: ") + DATA[i+2] + " create ID \"" + DATA[i] + "\" => \"" + exp + "\""); } else { errln(UnicodeString("FAIL: ") + DATA[i+2] + " create ID \"" + DATA[i] + "\" => \"" + t->getID() + "\", exp \"" + exp + "\""); } delete t; } } } /* test for Jitterbug 912 */ void TransliteratorTest::TestCreateInstance(){ UParseError *err = 0; Transliterator* myTrans = Transliterator::createInstance(UnicodeString("Latin-Hangul"),UTRANS_REVERSE,err); if (myTrans == 0) { errln("FAIL: createInstance failed"); return; } UnicodeString newID =myTrans->getID(); if(newID!=UnicodeString("Hangul-Latin")){ errln(UnicodeString("Test for Jitterbug 912 Transliterator::createInstance(id,UTRANS_REVERSE) failed")); } } /** * Test the normalization transliterator. */ void TransliteratorTest::TestNormalizationTransliterator() { // THE FOLLOWING TWO TABLES ARE COPIED FROM com.ibm.test.normalizer.BasicTest // PLEASE KEEP THEM IN SYNC WITH BasicTest. const char* CANON[] = { // Input Decomposed Composed "cat", "cat", "cat" , "\\u00e0ardvark", "a\\u0300ardvark", "\\u00e0ardvark" , "\\u1e0a", "D\\u0307", "\\u1e0a" , // D-dot_above "D\\u0307", "D\\u0307", "\\u1e0a" , // D dot_above "\\u1e0c\\u0307", "D\\u0323\\u0307", "\\u1e0c\\u0307" , // D-dot_below dot_above "\\u1e0a\\u0323", "D\\u0323\\u0307", "\\u1e0c\\u0307" , // D-dot_above dot_below "D\\u0307\\u0323", "D\\u0323\\u0307", "\\u1e0c\\u0307" , // D dot_below dot_above "\\u1e10\\u0307\\u0323", "D\\u0327\\u0323\\u0307","\\u1e10\\u0323\\u0307", // D dot_below cedilla dot_above "D\\u0307\\u0328\\u0323","D\\u0328\\u0323\\u0307","\\u1e0c\\u0328\\u0307", // D dot_above ogonek dot_below "\\u1E14", "E\\u0304\\u0300", "\\u1E14" , // E-macron-grave "\\u0112\\u0300", "E\\u0304\\u0300", "\\u1E14" , // E-macron + grave "\\u00c8\\u0304", "E\\u0300\\u0304", "\\u00c8\\u0304" , // E-grave + macron "\\u212b", "A\\u030a", "\\u00c5" , // angstrom_sign "\\u00c5", "A\\u030a", "\\u00c5" , // A-ring "\\u00fdffin", "y\\u0301ffin", "\\u00fdffin" , //updated with 3.0 "\\u00fd\\uFB03n", "y\\u0301\\uFB03n", "\\u00fd\\uFB03n" , //updated with 3.0 "Henry IV", "Henry IV", "Henry IV" , "Henry \\u2163", "Henry \\u2163", "Henry \\u2163" , "\\u30AC", "\\u30AB\\u3099", "\\u30AC" , // ga (Katakana) "\\u30AB\\u3099", "\\u30AB\\u3099", "\\u30AC" , // ka + ten "\\uFF76\\uFF9E", "\\uFF76\\uFF9E", "\\uFF76\\uFF9E" , // hw_ka + hw_ten "\\u30AB\\uFF9E", "\\u30AB\\uFF9E", "\\u30AB\\uFF9E" , // ka + hw_ten "\\uFF76\\u3099", "\\uFF76\\u3099", "\\uFF76\\u3099" , // hw_ka + ten "A\\u0300\\u0316", "A\\u0316\\u0300", "\\u00C0\\u0316" , 0 // end }; const char* COMPAT[] = { // Input Decomposed Composed "\\uFB4f", "\\u05D0\\u05DC", "\\u05D0\\u05DC" , // Alef-Lamed vs. Alef, Lamed "\\u00fdffin", "y\\u0301ffin", "\\u00fdffin" , //updated for 3.0 "\\u00fd\\uFB03n", "y\\u0301ffin", "\\u00fdffin" , // ffi ligature -> f + f + i "Henry IV", "Henry IV", "Henry IV" , "Henry \\u2163", "Henry IV", "Henry IV" , "\\u30AC", "\\u30AB\\u3099", "\\u30AC" , // ga (Katakana) "\\u30AB\\u3099", "\\u30AB\\u3099", "\\u30AC" , // ka + ten "\\uFF76\\u3099", "\\u30AB\\u3099", "\\u30AC" , // hw_ka + ten 0 // end }; int32_t i; Transliterator* NFD = Transliterator::createInstance("NFD"); Transliterator* NFC = Transliterator::createInstance("NFC"); if (!NFD || !NFC) { errln("FAIL: createInstance failed"); delete NFD; delete NFC; return; } for (i=0; CANON[i]; i+=3) { UnicodeString in = CharsToUnicodeString(CANON[i]); UnicodeString expd = CharsToUnicodeString(CANON[i+1]); UnicodeString expc = CharsToUnicodeString(CANON[i+2]); expect(*NFD, in, expd); expect(*NFC, in, expc); } delete NFD; delete NFC; Transliterator* NFKD = Transliterator::createInstance("NFKD"); Transliterator* NFKC = Transliterator::createInstance("NFKC"); if (!NFKD || !NFKC) { errln("FAIL: createInstance failed"); delete NFKD; delete NFKC; return; } for (i=0; COMPAT[i]; i+=3) { UnicodeString in = CharsToUnicodeString(COMPAT[i]); UnicodeString expkd = CharsToUnicodeString(COMPAT[i+1]); UnicodeString expkc = CharsToUnicodeString(COMPAT[i+2]); expect(*NFKD, in, expkd); expect(*NFKC, in, expkc); } delete NFKD; delete NFKC; } /** * Test compound RBT rules. */ void TransliteratorTest::TestCompoundRBT(void) { // Careful with spacing and ';' here: Phrase this exactly // as toRules() is going to return it. If toRules() changes // with regard to spacing or ';', then adjust this string. UnicodeString rule("::Hex-Unicode;\n" "::Any-Lower;\n" "a > '.A.';\n" "b > '.B.';\n" "::Any[^t]-Upper;", ""); Transliterator *t = Transliterator::createFromRules("Test", rule); if (t == 0) { errln("FAIL: createFromRules failed"); return; } expect(*t, "\\u0043at in the hat, bat on the mat", "C.A.t IN tHE H.A.t, .B..A.t ON tHE M.A.t"); UnicodeString r; t->toRules(r, TRUE); if (r == rule) { logln((UnicodeString)"OK: toRules() => " + r); } else { errln((UnicodeString)"FAIL: toRules() => " + r + ", expected " + rule); } delete t; // Now test toRules t = Transliterator::createInstance("Greek-Latin; Latin-Cyrillic"); if (t == 0) { errln("FAIL: createInstance failed"); return; } UnicodeString exp("::Greek-Latin;\n::Latin-Cyrillic;"); t->toRules(r, TRUE); if (r != exp) { errln((UnicodeString)"FAIL: toRules() => " + r + ", expected " + exp); } else { logln((UnicodeString)"OK: toRules() => " + r); } delete t; // Round trip the result of toRules t = Transliterator::createFromRules("Test", r); if (t == 0) { errln("FAIL: createFromRules #2 failed"); return; } else { logln((UnicodeString)"OK: createFromRules(" + r + ") succeeded"); } // Test toRules again t->toRules(r, TRUE); if (r != exp) { errln((UnicodeString)"FAIL: toRules() => " + r + ", expected " + exp); } else { logln((UnicodeString)"OK: toRules() => " + r); } delete t; // Test Foo(Bar) IDs. Careful with spacing in id; make it conform // to what the regenerated ID will look like. UnicodeString id("Upper(Lower);(NFKC)", ""); t = Transliterator::createInstance(id); if (t == 0) { errln("FAIL: createInstance #2 failed"); return; } if (t->getID() == id) { logln((UnicodeString)"OK: created " + id); } else { errln((UnicodeString)"FAIL: createInstance(" + id + ").getID() => " + t->getID()); } Transliterator *u = t->createInverse(); if (u == 0) { errln("FAIL: createInverse failed"); delete t; return; } exp = "NFKC();Lower(Upper)"; if (u->getID() == exp) { logln((UnicodeString)"OK: createInverse(" + id + ") => " + u->getID()); } else { errln((UnicodeString)"FAIL: createInverse(" + id + ") => " + u->getID()); } delete t; delete u; } /** * Compound filter semantics were orginially not implemented * correctly. Originally, each component filter f(i) is replaced by * f'(i) = f(i) && g, where g is the filter for the compound * transliterator. * * From Mark: * * Suppose and I have a transliterator X. Internally X is * "Greek-Latin; Latin-Cyrillic; Any-Lower". I use a filter [^A]. * * The compound should convert all greek characters (through latin) to * cyrillic, then lowercase the result. The filter should say "don't * touch 'A' in the original". But because an intermediate result * happens to go through "A", the Greek Alpha gets hung up. */ void TransliteratorTest::TestCompoundFilter(void) { Transliterator *t = Transliterator::createInstance ("Greek-Latin; Latin-Cyrillic; Lower"); if (t == 0) { errln("FAIL: createInstance failed"); return; } UErrorCode status = U_ZERO_ERROR; t->adoptFilter(new UnicodeSet("[^A]", status)); if (U_FAILURE(status)) { errln("FAIL: UnicodeSet ct failed"); delete t; return; } // Only the 'A' at index 1 should remain unchanged expect(*t, CharsToUnicodeString("CA\\u039A\\u0391"), CharsToUnicodeString("\\u043AA\\u043A\\u0430")); delete t; } //====================================================================== // Support methods //====================================================================== void TransliteratorTest::expect(const UnicodeString& rules, const UnicodeString& source, const UnicodeString& expectedResult) { UErrorCode status = U_ZERO_ERROR; Transliterator *t = new RuleBasedTransliterator("", rules, status); if (U_FAILURE(status)) { errln("FAIL: Transliterator constructor failed"); } else { expect(*t, source, expectedResult); } delete t; } void TransliteratorTest::expect(const Transliterator& t, const UnicodeString& source, const UnicodeString& expectedResult, const Transliterator& reverseTransliterator) { expect(t, source, expectedResult); expect(reverseTransliterator, expectedResult, source); } void TransliteratorTest::expect(const Transliterator& t, const UnicodeString& source, const UnicodeString& expectedResult) { UnicodeString result(source); t.transliterate(result); expectAux(t.getID() + ":String", source, result, expectedResult); UnicodeString rsource(source); t.transliterate(rsource); expectAux(t.getID() + ":Replaceable", source, rsource, expectedResult); // Test keyboard (incremental) transliteration -- this result // must be the same after we finalize (see below). rsource.remove(); UTransPosition index={0, 0, 0, 0}; UnicodeString log; for (int32_t i=0; i "); UErrorCode status = U_ZERO_ERROR; t.transliterate(rsource, index, source.charAt(i), status); // Append the string buffer with a vertical bar '|' where // the committed index is. UnicodeString left, right; rsource.extractBetween(0, index.start, left); rsource.extractBetween(index.start, rsource.length(), right); log.append(left).append((UChar)PIPE).append(right); } // As a final step in keyboard transliteration, we must call // transliterate to finish off any pending partial matches that // were waiting for more input. t.finishTransliteration(rsource, index); log.append(" => ").append(rsource); expectAux(t.getID() + ":Keyboard", log, rsource == expectedResult, expectedResult); } void TransliteratorTest::expectAux(const UnicodeString& tag, const UnicodeString& source, const UnicodeString& result, const UnicodeString& expectedResult) { expectAux(tag, source + " -> " + result, result == expectedResult, expectedResult); } void TransliteratorTest::expectAux(const UnicodeString& tag, const UnicodeString& summary, UBool pass, const UnicodeString& expectedResult) { if (pass) { logln(UnicodeString("(")+tag+") " + prettify(summary)); } else { errln(UnicodeString("FAIL: (")+tag+") " + prettify(summary) + ", expected " + prettify(expectedResult)); } }