/******************************************************************** * COPYRIGHT: * Copyright (c) 2001, International Business Machines Corporation and * others. All Rights Reserved. ********************************************************************/ /******************************************************************************* * * File cmsccoll.C * *******************************************************************************/ /** * These are the tests specific to ICU 1.8 and above, that I didn't know where * to fit. */ #include #include #include "unicode/utypes.h" #include "unicode/ucol.h" #include "unicode/ucoleitr.h" #include "unicode/uloc.h" #include "cintltst.h" #include "ccolltst.h" #include "callcoll.h" #include "unicode/ustring.h" #include "string.h" #include "ucol_imp.h" #include "ucol_tok.h" #include "cmemory.h" #define MAX_TOKEN_LEN 16 static UCollator *myCollation; const static UChar gRules[MAX_TOKEN_LEN] = /*" & 0 < 1,\u2461 size) { temp = (int32_t*)malloc(sizeof(int32_t) * size); memcpy(temp, orders, size * sizeof(int32_t)); free(orders); orders = temp; } *orderLength = size; return orders; } static void backAndForth(UCollationElements *iter) { /* Run through the iterator forwards and stick it into an array */ int32_t index, o; UErrorCode status = U_ZERO_ERROR; int32_t orderLength = 0; int32_t *orders; orders= getOrders(iter, &orderLength); /* Now go through it backwards and make sure we get the same values */ index = orderLength; ucol_reset(iter); /* synwee : changed */ while ((o = ucol_previous(iter, &status)) != UCOL_NULLORDER) { if (o != orders[-- index]) { if (o == 0) index ++; else { while (index > 0 && orders[-- index] == 0) { } if (o != orders[index]) { log_err("Mismatch at index : %d\n", index); break; } } } } while (index != 0 && orders[index - 1] == 0) { index --; } if (index != 0) { log_err("Didn't get back to beginning - index is %d\n", index); ucol_reset(iter); log_err("\nnext: "); while ((o = ucol_next(iter, &status)) != UCOL_NULLORDER) { log_err("Error at %d\n", o); } log_err("\nprev: "); while ((o = ucol_previous(iter, &status)) != UCOL_NULLORDER) { log_err("Error at %d\n", o); } log_verbose("\n"); } free(orders); } static void IncompleteCntTest( ) { UErrorCode status = U_ZERO_ERROR; UChar *temp=(UChar*)malloc(sizeof(UChar) * 90); UChar *t1 =(UChar*)malloc(sizeof(UChar) * 90); UChar *t2 =(UChar*)malloc(sizeof(UChar) * 90); UCollator *coll = NULL; uint32_t i = 0, j = 0; uint32_t size = 0; u_uastrcpy(temp, " & Z < ABC < Q < B"); coll = ucol_openRules(temp, u_strlen(temp), UCOL_NO_NORMALIZATION, UCOL_DEFAULT_STRENGTH, &status); if(U_SUCCESS(status)) { size = sizeof(cnt1)/sizeof(cnt1[0]); for(i = 0; i < size-1; i++) { for(j = i+1; j < size; j++) { UCollationElements *iter; u_uastrcpy(t1, cnt1[i]); u_uastrcpy(t2, cnt1[j]); doTest(coll, t1, t2, UCOL_LESS); /* synwee : added collation element iterator test */ iter = ucol_openElements(coll, t2, u_strlen(t2), &status); if (U_FAILURE(status)) { log_err("Creation of iterator failed\n"); break; } backAndForth(iter); free(iter); } } } ucol_close(coll); u_uastrcpy(temp, " & Z < DAVIS < MARK 0x0400 && *p<0x0500)?0x00e1:0x491; UChar preQ = (*p>0x0400 && *p<0x0500)?0x0041:0x413; */ /*log_verbose("Testing primary\n");*/ doTest(col, p, q, UCOL_LESS); /* UCollationResult result = ucol_strcoll(col,p,u_strlen(p),q,u_strlen(q)); if(result!=UCOL_LESS){ aescstrdup(p,utfSource,256); aescstrdup(q,utfTarget,256); fprintf(file,"Primary failed source: %s target: %s \n", utfSource,utfTarget); } */ source[0] = preP; u_strcpy(source+1,p); target[0] = preQ; u_strcpy(target+1,q); doTest(col, source, target, UCOL_LESS); /* fprintf(file,"Primary swamps 2nd failed source: %s target: %s \n", utfSource,utfTarget); */ } static void testSecondary(UCollator* col, const UChar* p,const UChar* q){ UChar source[256] = { '\0'}; UChar target[256] = { '\0'}; /*log_verbose("Testing secondary\n");*/ doTest(col, p, q, UCOL_LESS); /* fprintf(file,"secondary failed source: %s target: %s \n", utfSource,utfTarget); */ source[0] = 0x0053; u_strcpy(source+1,p); target[0]= 0x0073; u_strcpy(target+1,q); doTest(col, source, target, UCOL_LESS); /* fprintf(file,"secondary swamps 3rd failed source: %s target: %s \n",utfSource,utfTarget); */ u_strcpy(source,p); source[u_strlen(p)] = 0x62; source[u_strlen(p)+1] = 0; u_strcpy(target,q); target[u_strlen(q)] = 0x61; target[u_strlen(q)+1] = 0; doTest(col, source, target, UCOL_GREATER); /* fprintf(file,"secondary is swamped by 1 failed source: %s target: %s \n",utfSource,utfTarget); */ } static void testTertiary(UCollator* col, const UChar* p,const UChar* q){ UChar source[256] = { '\0'}; UChar target[256] = { '\0'}; /*log_verbose("Testing tertiary\n");*/ doTest(col, p, q, UCOL_LESS); /* fprintf(file,"Tertiary failed source: %s target: %s \n",utfSource,utfTarget); */ source[0] = 0x0020; u_strcpy(source+1,p); target[0]= 0x002D; u_strcpy(target+1,q); doTest(col, source, target, UCOL_LESS); /* fprintf(file,"Tertiary swamps 4th failed source: %s target: %s \n", utfSource,utfTarget); */ u_strcpy(source,p); source[u_strlen(p)] = 0xE0; source[u_strlen(p)+1] = 0; u_strcpy(target,q); target[u_strlen(q)] = 0x61; target[u_strlen(q)+1] = 0; doTest(col, source, target, UCOL_GREATER); /* fprintf(file,"Tertiary is swamped by 3rd failed source: %s target: %s \n",utfSource,utfTarget); */ } static void testEquality(UCollator* col, const UChar* p,const UChar* q){ /* UChar source[256] = { '\0'}; UChar target[256] = { '\0'}; */ doTest(col, p, q, UCOL_EQUAL); /* fprintf(file,"Primary failed source: %s target: %s \n", utfSource,utfTarget); */ } static void testCollator(UCollator *coll, UErrorCode *status) { const UChar *rules = NULL, *current = NULL; int32_t ruleLen = 0; uint32_t strength = 0; uint32_t chOffset = 0; uint32_t chLen = 0; uint32_t exOffset = 0; uint32_t exLen = 0; /* uint32_t rExpsLen = 0; */ uint32_t firstLen = 0; UBool varT = FALSE; UBool top_ = TRUE; UBool startOfRules = TRUE; UColTokenParser src; UColOptionSet opts; UChar first[256]; UChar second[256]; UChar *rulesCopy = NULL; src.opts = &opts; rules = ucol_getRules(coll, &ruleLen); if(U_SUCCESS(*status) && ruleLen > 0) { rulesCopy = (UChar *)uprv_malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar)); uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar)); src.source = src.current = rulesCopy; src.end = rulesCopy+ruleLen; src.extraCurrent = src.end; src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE; *first = *second = 0; while ((current = ucol_tok_parseNextToken(&src, &strength, &chOffset, &chLen, &exOffset, &exLen, &varT, &top_, startOfRules, status)) != NULL) { startOfRules = FALSE; u_strncpy(second,rulesCopy+chOffset, chLen); second[chLen] = 0; if(exLen > 0) { u_strncat(first, rulesCopy+exOffset, exLen); first[firstLen+exLen] = 0; } switch(strength){ case UCOL_IDENTICAL: testEquality(coll,first,second); break; case UCOL_PRIMARY: testPrimary(coll,first,second); break; case UCOL_SECONDARY: testSecondary(coll,first,second); break; case UCOL_TERTIARY: testTertiary(coll,first,second); break; case UCOL_TOK_RESET: default: break; } firstLen = chLen; u_strcpy(first, second); } uprv_free(rulesCopy); } } static UBool doTestUCA(UCollator *UCA, UChar *source, UChar *target, UCollationResult result) { uint32_t sLen = u_strlen(source); uint32_t tLen = u_strlen(target); UCollationResult r = ucol_strcoll(UCA, source, sLen, target, tLen); if(r != result) { return FALSE; } else { return TRUE; } } static void logUCAFailure (const char *comment, const UChar *source, const UChar *target) { uint32_t sLen = u_strlen(source); uint32_t tLen = u_strlen(target); uint32_t i = 0; char sEsc[256], tEsc[256], b[256]; *sEsc = *tEsc = 0; for(i = 0; i 0) { rulesCopy = (UChar *)uprv_malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar)); uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar)); src.source = src.current = rulesCopy; src.end = rulesCopy+ruleLen; src.extraCurrent = src.end; src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE; *first = *second = 0; while ((current = ucol_tok_parseNextToken(&src, &strength, &chOffset, &chLen, &exOffset, &exLen, &varT, &top_, startOfRules, status)) != NULL) { startOfRules = FALSE; u_strncpy(second,rulesCopy+chOffset, chLen); second[chLen] = 0; secondLen = chLen; if(exLen > 0) { u_strncat(first, rulesCopy+exOffset, exLen); first[firstLen+exLen] = 0; firstLen += exLen; } switch(strength){ case UCOL_IDENTICAL: if(!doTestUCA(UCA, first, second, UCOL_EQUAL)) { logUCAFailure ("Rules says equal, but not UCA", first, second); UCAdiff++; goto EndOfLoop; } break; case UCOL_PRIMARY: if(!doTestUCA(UCA, first, second, UCOL_LESS)) { logUCAFailure("Wrong order, primary level", first, second); UCAdiff++; goto EndOfLoop; } source[0] = 0x0491; u_strcpy(source+1,first); target[0] = 0x0413; u_strcpy(target+1,second); if(!doTestUCA(UCA, source, target, UCOL_LESS)) { /* UCA has non primary difference */ source[0] = 0x0053; u_strcpy(source+1,first); target[0]= 0x0073; u_strcpy(target+1,second); if(!doTestUCA(UCA, source, target, UCOL_LESS)) { /* test if it is tertiary */ logUCAFailure("Wrong strength: rules:primary UCA:tertiary", first, second); } else { logUCAFailure("Wrong strength - rules:primary UCA:secondary", first, second); } UCAdiff++; goto EndOfLoop; } break; case UCOL_SECONDARY: if(!doTestUCA(UCA, first, second, UCOL_LESS)) { logUCAFailure("Wrong order, secondary level", first, second); UCAdiff++; goto EndOfLoop; } source[0] = 0x0053; u_strcpy(source+1,first); target[0]= 0x0073; u_strcpy(target+1,second); if(!doTestUCA(UCA, source, target, UCOL_LESS)) { logUCAFailure("Wrong strength - rules:secondary UCA:tertiary", first, second); UCAdiff++; goto EndOfLoop; } u_strcpy(source,first); source[u_strlen(first)] = 0x62; source[u_strlen(first)+1] = 0; u_strcpy(target,second); target[u_strlen(second)] = 0x61; target[u_strlen(second)+1] = 0; if(!doTestUCA(UCA, source, target, UCOL_GREATER)) { logUCAFailure("Wrong strength - rules:secondary UCA:primary", first, second); UCAdiff++; goto EndOfLoop; } break; case UCOL_TERTIARY: if(!doTestUCA(UCA, first, second, UCOL_LESS)) { logUCAFailure("Wrong order, tertiary level", first, second); UCAdiff++; goto EndOfLoop; } source[0] = 0x0020; u_strcpy(source+1,first); target[0]= 0x002D; u_strcpy(target+1,second); if(!doTestUCA(UCA, source, target, UCOL_LESS)) { logUCAFailure("Wrong strength - rules:tertiary UCA:quad", first, second); UCAdiff++; goto EndOfLoop; } u_strcpy(source,first); source[u_strlen(first)] = 0xE0; source[u_strlen(first)+1] = 0; u_strcpy(target,second); target[u_strlen(second)] = 0x61; target[u_strlen(second)+1] = 0; if(!doTestUCA(UCA, source, target, UCOL_GREATER)) { u_strcpy(source,first); source[u_strlen(first)] = 0x62; source[u_strlen(first)+1] = 0; u_strcpy(target,second); target[u_strlen(second)] = 0x61; target[u_strlen(second)+1] = 0; if(!doTestUCA(UCA, source, target, UCOL_GREATER)) { logUCAFailure("Wrong strength - rules:tertiary UCA:primary", first, second); } else { logUCAFailure("Wrong strength - rules:tertiary UCA:secondary", first, second); } UCAdiff++; goto EndOfLoop; } break; case UCOL_TOK_RESET: default: break; } EndOfLoop: firstLen = chLen; u_strcpy(first, second); } if(UCAdiff == 0) { log_verbose("No immediate conflict with UCA!\n"); } uprv_free(rulesCopy); } } static void testCEs(UCollator *coll, UErrorCode *status) { const UChar *rules = NULL, *current = NULL; int32_t ruleLen = 0; uint32_t strength = 0; uint32_t maxStrength = UCOL_IDENTICAL; uint32_t baseCE, baseContCE, nextCE, nextContCE, currCE, currContCE; uint32_t lastCE; uint32_t lastContCE; int32_t result = 0; uint32_t chOffset = 0; uint32_t chLen = 0; uint32_t exOffset = 0; uint32_t exLen = 0; uint32_t oldOffset = 0; /* uint32_t rExpsLen = 0; */ /* uint32_t firstLen = 0; */ UBool varT = FALSE; UBool top_ = TRUE; UBool startOfRules = TRUE; UColTokenParser src; UColOptionSet opts; UChar *rulesCopy = NULL; collIterate c; baseCE=baseContCE=nextCE=nextContCE=currCE=currContCE=lastCE=lastContCE = UCOL_NOT_FOUND; src.opts = &opts; rules = ucol_getRules(coll, &ruleLen); ucol_initInverseUCA(status); if(U_SUCCESS(*status) && ruleLen > 0) { rulesCopy = (UChar *)uprv_malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar)); uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar)); src.source = src.current = rulesCopy; src.end = rulesCopy+ruleLen; src.extraCurrent = src.end; src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE; while ((current = ucol_tok_parseNextToken(&src, &strength, &chOffset, &chLen, &exOffset, &exLen, &varT, &top_, startOfRules, status)) != NULL) { startOfRules = FALSE; init_collIterate(coll, rulesCopy+chOffset, chLen, &c, FALSE); currCE = ucol_getNextCE(coll, &c, status); if(currCE == 0 && UCOL_ISTHAIPREVOWEL(*(rulesCopy+chOffset))) { log_verbose("Thai prevowel detected. Will pick next CE\n"); currCE = ucol_getNextCE(coll, &c, status); } currContCE = ucol_getNextCE(coll, &c, status); if(!isContinuation(currContCE)) { currContCE = 0; } if(strength == UCOL_TOK_RESET) { if(top_ == TRUE) { nextCE = baseCE = currCE = 0x9FFF0000; nextContCE = baseContCE = currContCE = 0; } else { nextCE = baseCE = currCE; nextContCE = baseContCE = currContCE; } maxStrength = UCOL_IDENTICAL; } else { if(strength < maxStrength) { maxStrength = strength; if(baseCE == 0x9FFF0000) { log_verbose("Resetting to [top]\n"); nextCE = 0xD0000000; nextContCE = 0; } else { result = ucol_inv_getNextCE(baseCE, baseContCE, &nextCE, &nextContCE, maxStrength); } if(result < 0) { if(isTailored(coll, *(rulesCopy+oldOffset), status)) { log_verbose("Reset is tailored codepoint %04X, don't know how to continue, taking next test\n", *(rulesCopy+oldOffset)); return; } else { log_err("couldn't find the CE\n"); return; } } } currCE &= 0xFFFFFF3F; currContCE &= 0xFFFFFFBF; if(maxStrength == UCOL_IDENTICAL) { if(baseCE != currCE || baseContCE != currContCE) { log_err("current CE (initial strength UCOL_EQUAL)\n"); } } else { if(strength == UCOL_IDENTICAL) { if(lastCE != currCE || lastContCE != currContCE) { log_err("current CE (initial strength UCOL_EQUAL)\n"); } } else { if(currCE > nextCE || (currCE == nextCE && currContCE >= nextContCE)) { log_err("current CE is not less than base CE\n"); } if(currCE < lastCE || (currCE == lastCE && currContCE <= lastContCE)) { log_err("sequence of generated CEs is broken\n"); } } } } oldOffset = chOffset; lastCE = currCE & 0xFFFFFF3F; lastContCE = currContCE & 0xFFFFFFBF; } uprv_free(rulesCopy); } } static const char* localesToTest[] = { "ar", "bg", "ca", "cs", "da", "el", "en_BE", "en_US_POSIX", "es", "et", "fi", "fr", "hi", "hr", "hu", "is", "iw", "ja", "ko", "lt", "lv", "mk", "mt", "nb", "nn", "nn_NO", "pl", "ro", "ru", /*"sh",*/ "sk", "sl", "sq", /* sh test is turned off until we do closure in contractions */ "sr", "sv", "th", "tr", /*"uk", */ "vi", "zh", "zh_TW" }; static const char* rulesToTest[] = { /*"& Z < p, P",*/ /* Cui Mins rules */ "