ICU-3381 fix the length bug
X-SVN-Rev: 13720
This commit is contained in:
Ram Viswanadha
parent
09d453333b
commit
8eda2a2927
@ -5,8 +5,8 @@
|
||||
*******************************************************************************
|
||||
*
|
||||
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/icu/dev/test/normalizer/BasicTest.java,v $
|
||||
* $Date: 2003/06/11 17:51:25 $
|
||||
* $Revision: 1.33 $
|
||||
* $Date: 2003/11/14 00:07:07 $
|
||||
* $Revision: 1.34 $
|
||||
*
|
||||
*****************************************************************************************
|
||||
*/
|
||||
@ -291,8 +291,8 @@ public class BasicTest extends TestFmwk {
|
||||
// (!uinfo.hasCanonicalDecomposition((char)i) &&
|
||||
// !uinfo.hasCompatibilityDecomposition((char)i))) continue;
|
||||
// String a = String.valueOf((char)i);
|
||||
// String b = NewNormalizer.normalize(a,NewNormalizer.DECOMP_COMPAT,0);
|
||||
// String c = NewNormalizer.normalize(b,NewNormalizer.COMPOSE,0);
|
||||
// String b = Normalizer.normalize(a,Normalizer.DECOMP_COMPAT,0);
|
||||
// String c = Normalizer.normalize(b,Normalizer.COMPOSE,0);
|
||||
// if (c.equals(a)) {
|
||||
// errln("FAIL: " + hex(a) + " x DECOMP_COMPAT => " +
|
||||
// hex(b) + " x COMPOSE => " +
|
||||
@ -793,32 +793,38 @@ public class BasicTest extends TestFmwk {
|
||||
}
|
||||
}
|
||||
output = new char[1];
|
||||
for (int i = 0; i < tests.length; i++)
|
||||
{
|
||||
char[] input = Utility.unescape(tests[i][0]).toCharArray();
|
||||
String expect = Utility.unescape(tests[i][outCol]);
|
||||
|
||||
logln("Normalizing '" + new String(input) + "' (" +
|
||||
hex(new String(input)) + ")" );
|
||||
int reqLength=0;
|
||||
while(true){
|
||||
try{
|
||||
reqLength=Normalizer.decompose(input,0,input.length,output,0,output.length, mode==Normalizer.NFKD,0);
|
||||
if(reqLength<=output.length ){
|
||||
break;
|
||||
}
|
||||
}catch(IndexOutOfBoundsException e){
|
||||
output= new char[Integer.parseInt(e.getMessage())];
|
||||
continue;
|
||||
for (int i = 0; i < tests.length; i++)
|
||||
{
|
||||
char[] input = Utility.unescape(tests[i][0]).toCharArray();
|
||||
String expect = Utility.unescape(tests[i][outCol]);
|
||||
|
||||
logln("Normalizing '" + new String(input) + "' (" +
|
||||
hex(new String(input)) + ")" );
|
||||
int reqLength=0;
|
||||
while(true){
|
||||
try{
|
||||
reqLength=Normalizer.decompose(input,0,input.length,output,0,output.length, mode==Normalizer.NFKD,0);
|
||||
if(reqLength<=output.length ){
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!expect.equals(new String(output,0,reqLength))) {
|
||||
errln("FAIL: case " + i
|
||||
+ " expected '" + expect + "' (" + hex(expect) + ")"
|
||||
+ " but got '" + new String(output)
|
||||
+ "' (" + hex(new String(output)) + ")" );
|
||||
}catch(IndexOutOfBoundsException e){
|
||||
output= new char[Integer.parseInt(e.getMessage())];
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (!expect.equals(new String(output,0,reqLength))) {
|
||||
errln("FAIL: case " + i
|
||||
+ " expected '" + expect + "' (" + hex(expect) + ")"
|
||||
+ " but got '" + new String(output)
|
||||
+ "' (" + hex(new String(output)) + ")" );
|
||||
}
|
||||
char[] output2 = new char[reqLength * 2];
|
||||
System.arraycopy(output, 0, output2, 0, reqLength);
|
||||
int retLength = Normalizer.decompose(input,0,input.length, output2, reqLength, output2.length, mode==Normalizer.NFKC,0);
|
||||
if(retLength != reqLength){
|
||||
logln("FAIL: Normalizer.compose did not return the expected length. Expected: " +reqLength + " Got: " + retLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void composeTest(Normalizer.Mode mode,
|
||||
@ -891,6 +897,13 @@ public class BasicTest extends TestFmwk {
|
||||
+ " but got '" + new String(output)
|
||||
+ "' (" + hex(new String(output)) + ")" );
|
||||
}
|
||||
|
||||
char[] output2 = new char[reqLength * 2];
|
||||
System.arraycopy(output, 0, output2, 0, reqLength);
|
||||
int retLength = Normalizer.compose(input,0,input.length, output2, reqLength, output2.length, mode==Normalizer.NFKC,0);
|
||||
if(retLength != reqLength){
|
||||
logln("FAIL: Normalizer.compose did not return the expected length. Expected: " +reqLength + " Got: " + retLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
private void iterateTest(Normalizer iter, String[][] tests, int outCol){
|
||||
@ -1557,18 +1570,18 @@ public class BasicTest extends TestFmwk {
|
||||
int count = 0;
|
||||
|
||||
if (Normalizer.quickCheck(FAST,0,FAST.length, Normalizer.FCD,0) != Normalizer.YES)
|
||||
errln("NewNormalizer.quickCheck(FCD) failed: expected value for fast NewNormalizer.quickCheck is NewNormalizer.YES\n");
|
||||
errln("Normalizer.quickCheck(FCD) failed: expected value for fast Normalizer.quickCheck is Normalizer.YES\n");
|
||||
if (Normalizer.quickCheck(FALSE,0, FALSE.length,Normalizer.FCD,0) != Normalizer.NO)
|
||||
errln("NewNormalizer.quickCheck(FCD) failed: expected value for error NewNormalizer.quickCheck is NewNormalizer.NO\n");
|
||||
errln("Normalizer.quickCheck(FCD) failed: expected value for error Normalizer.quickCheck is Normalizer.NO\n");
|
||||
if (Normalizer.quickCheck(TRUE,0,TRUE.length,Normalizer.FCD,0) != Normalizer.YES)
|
||||
errln("NewNormalizer.quickCheck(FCD) failed: expected value for correct NewNormalizer.quickCheck is NewNormalizer.YES\n");
|
||||
errln("Normalizer.quickCheck(FCD) failed: expected value for correct Normalizer.quickCheck is Normalizer.YES\n");
|
||||
|
||||
|
||||
while (count < 4)
|
||||
{
|
||||
Normalizer.QuickCheckResult fcdresult = Normalizer.quickCheck(datastr[count],0,datastr[count].length, Normalizer.FCD,0);
|
||||
if (result[count] != fcdresult) {
|
||||
errln("NewNormalizer.quickCheck(FCD) failed: Data set "+ count
|
||||
errln("Normalizer.quickCheck(FCD) failed: Data set "+ count
|
||||
+ " expected value "+ result[count]);
|
||||
}
|
||||
count ++;
|
||||
@ -1590,7 +1603,7 @@ public class BasicTest extends TestFmwk {
|
||||
while (size != 19) {
|
||||
data[size] = datachar[rand.nextInt(RAND_MAX)*50/RAND_MAX];
|
||||
logln("0x"+data[size]);
|
||||
normStart = Normalizer.normalize(data,size,size+1,
|
||||
normStart += Normalizer.normalize(data,size,size+1,
|
||||
norm,normStart,100,
|
||||
Normalizer.NFD,0);
|
||||
size ++;
|
||||
@ -1604,14 +1617,14 @@ public class BasicTest extends TestFmwk {
|
||||
testresult = Normalizer.NO;
|
||||
}
|
||||
if (testresult == Normalizer.YES) {
|
||||
logln("result NewNormalizer.YES\n");
|
||||
logln("result Normalizer.YES\n");
|
||||
}
|
||||
else {
|
||||
logln("result NewNormalizer.NO\n");
|
||||
logln("result Normalizer.NO\n");
|
||||
}
|
||||
|
||||
if (Normalizer.quickCheck(data,0,data.length, Normalizer.FCD,0) != testresult) {
|
||||
errln("NewNormalizer.quickCheck(FCD) failed: expected "+ testresult+" for random data\n" );
|
||||
errln("Normalizer.quickCheck(FCD) failed: expected "+ testresult +" for random data: "+hex(new String(data)) );
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2774,4 +2787,14 @@ public class BasicTest extends TestFmwk {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TestReturnFailure(){
|
||||
char[] term = {'r','\u00e9','s','u','m','\u00e9' };
|
||||
char[] decomposed_term = new char[10 + term.length + 2];
|
||||
int rc = Normalizer.decompose(term,0,term.length, decomposed_term,0,decomposed_term.length,true, 0);
|
||||
int rc1 = Normalizer.decompose(term,0,term.length, decomposed_term,10,decomposed_term.length,true, 0);
|
||||
if(rc!=rc1){
|
||||
errln("Normalizer decompose did not return correct length");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,8 +5,8 @@
|
||||
*******************************************************************************
|
||||
*
|
||||
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/icu/dev/test/normalizer/ConformanceTest.java,v $
|
||||
* $Date: 2003/06/03 18:49:30 $
|
||||
* $Revision: 1.14 $
|
||||
* $Date: 2003/11/14 00:07:08 $
|
||||
* $Revision: 1.15 $
|
||||
*
|
||||
*****************************************************************************************
|
||||
*/
|
||||
@ -47,7 +47,7 @@ public class ConformanceTest extends TestFmwk {
|
||||
};
|
||||
|
||||
/**
|
||||
* Test the conformance of NewNormalizer to
|
||||
* Test the conformance of Normalizer to
|
||||
* http://www.unicode.org/unicode/reports/tr15/conformance/Draft-TestSuite.txt.* http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
|
||||
* This file must be located at the path specified as TEST_SUITE_FILE.
|
||||
*/
|
||||
@ -229,65 +229,74 @@ public class ConformanceTest extends TestFmwk {
|
||||
compare(field[0],field[2]);
|
||||
// test quick checks
|
||||
if(Normalizer.NO == Normalizer.quickCheck(field[1], Normalizer.NFC,options)) {
|
||||
errln("Normalizer error: quickCheck(NFC(s), NewNormalizer.NFC) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(NFC(s), Normalizer.NFC) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
if(Normalizer.NO == Normalizer.quickCheck(field[2], Normalizer.NFD,options)) {
|
||||
errln("Normalizer error: quickCheck(NFD(s), NewNormalizer.NFD) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(NFD(s), Normalizer.NFD) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
if(Normalizer.NO == Normalizer.quickCheck(field[3], Normalizer.NFKC,options)) {
|
||||
errln("Normalizer error: quickCheck(NFKC(s), NewNormalizer.NFKC) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(NFKC(s), Normalizer.NFKC) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
if(Normalizer.NO == Normalizer.quickCheck(field[4], Normalizer.NFKD,options)) {
|
||||
errln("Normalizer error: quickCheck(NFKD(s), NewNormalizer.NFKD) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(NFKD(s), Normalizer.NFKD) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
|
||||
if(!Normalizer.isNormalized(field[1], Normalizer.NFC, options)) {
|
||||
errln("Normalizer error: isNormalized(NFC(s), NewNormalizer.NFC) is false");
|
||||
errln("Normalizer error: isNormalized(NFC(s), Normalizer.NFC) is false");
|
||||
pass = false;
|
||||
}
|
||||
if(!field[0].equals(field[1]) && Normalizer.isNormalized(field[0], Normalizer.NFC, options)) {
|
||||
errln("Normalizer error: isNormalized(s, NewNormalizer.NFC) is TRUE");
|
||||
errln("Normalizer error: isNormalized(s, Normalizer.NFC) is TRUE");
|
||||
pass = false;
|
||||
}
|
||||
if(!Normalizer.isNormalized(field[3], Normalizer.NFKC, options)) {
|
||||
errln("Normalizer error: isNormalized(NFKC(s), NewNormalizer.NFKC) is false");
|
||||
errln("Normalizer error: isNormalized(NFKC(s), Normalizer.NFKC) is false");
|
||||
pass = false;
|
||||
}
|
||||
if(!field[0].equals(field[3]) && Normalizer.isNormalized(field[0], Normalizer.NFKC, options)) {
|
||||
errln("Normalizer error: isNormalized(s, NewNormalizer.NFKC) is TRUE");
|
||||
errln("Normalizer error: isNormalized(s, Normalizer.NFKC) is TRUE");
|
||||
pass = false;
|
||||
}
|
||||
// test api that takes a char[]
|
||||
if(!Normalizer.isNormalized(field[1].toCharArray(),0,field[1].length(), Normalizer.NFC,options)) {
|
||||
errln("Normalizer error: isNormalized(NFC(s), NewNormalizer.NFC) is false");
|
||||
errln("Normalizer error: isNormalized(NFC(s), Normalizer.NFC) is false");
|
||||
pass = false;
|
||||
}
|
||||
// test api that takes a codepoint
|
||||
if(!Normalizer.isNormalized(UTF16.charAt(field[1],0), Normalizer.NFC,options)) {
|
||||
errln("Normalizer error: isNormalized(NFC(s), NewNormalizer.NFC) is false");
|
||||
errln("Normalizer error: isNormalized(NFC(s), Normalizer.NFC) is false");
|
||||
pass = false;
|
||||
}
|
||||
// test FCD quick check and "makeFCD"
|
||||
fcd=Normalizer.normalize(field[0], Normalizer.FCD);
|
||||
if(Normalizer.NO == Normalizer.quickCheck(fcd, Normalizer.FCD,options)) {
|
||||
errln("Normalizer error: quickCheck(FCD(s), NewNormalizer.FCD) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(FCD(s), Normalizer.FCD) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
// check FCD return length
|
||||
{
|
||||
char[] fcd2 = new char[ fcd.length() * 2 ];
|
||||
char[] src = field[0].toCharArray();
|
||||
int fcdLen = Normalizer.normalize(src, 0, src.length, fcd2, fcd.length(), fcd2.length,Normalizer.FCD, 0);
|
||||
if(fcdLen != fcd.length()){
|
||||
errln("makeFCD did not return the correct length");
|
||||
}
|
||||
}
|
||||
if(Normalizer.NO == Normalizer.quickCheck(fcd, Normalizer.FCD, options)) {
|
||||
errln("Normalizer error: quickCheck(FCD(s), NewNormalizer.FCD) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(FCD(s), Normalizer.FCD) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
if(Normalizer.NO == Normalizer.quickCheck(field[2], Normalizer.FCD, options)) {
|
||||
errln("Normalizer error: quickCheck(NFD(s), NewNormalizer.FCD) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(NFD(s), Normalizer.FCD) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
|
||||
if(Normalizer.NO == Normalizer.quickCheck(field[4], Normalizer.FCD, options)) {
|
||||
errln("Normalizer error: quickCheck(NFKD(s), NewNormalizer.FCD) is NewNormalizer.NO");
|
||||
errln("Normalizer error: quickCheck(NFKD(s), Normalizer.FCD) is Normalizer.NO");
|
||||
pass = false;
|
||||
}
|
||||
|
||||
|
||||
@ -5,8 +5,8 @@
|
||||
*******************************************************************************
|
||||
*
|
||||
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/icu/impl/NormalizerImpl.java,v $
|
||||
* $Date: 2003/06/09 23:31:09 $
|
||||
* $Revision: 1.22 $
|
||||
* $Date: 2003/11/14 00:06:08 $
|
||||
* $Revision: 1.23 $
|
||||
*******************************************************************************
|
||||
*/
|
||||
|
||||
@ -1131,8 +1131,8 @@ public final class NormalizerImpl {
|
||||
}
|
||||
|
||||
|
||||
public static int decompose(char[] src,int srcIndex,int srcLimit,
|
||||
char[] dest,int destIndex,int destLimit,
|
||||
public static int decompose(char[] src,int srcStart,int srcLimit,
|
||||
char[] dest,int destStart,int destLimit,
|
||||
boolean compat,int[] outTrailCC,
|
||||
UnicodeSet nx) {
|
||||
|
||||
@ -1145,6 +1145,8 @@ public final class NormalizerImpl {
|
||||
int/*unsigned byte*/ cc, prevCC, trailCC;
|
||||
char[] p;
|
||||
int pStart;
|
||||
int destIndex = destStart;
|
||||
int srcIndex = srcStart;
|
||||
if(!compat) {
|
||||
minNoMaybe=(char)indexes[INDEX_MIN_NFD_NO_MAYBE];
|
||||
qcMask=QC_NFD;
|
||||
@ -1337,7 +1339,7 @@ public final class NormalizerImpl {
|
||||
|
||||
outTrailCC[0]=prevCC;
|
||||
|
||||
return destIndex;
|
||||
return destIndex - destStart;
|
||||
}
|
||||
|
||||
/* make NFC & NFKC ------------------------------------------------------ */
|
||||
@ -1913,8 +1915,8 @@ public final class NormalizerImpl {
|
||||
}
|
||||
*/
|
||||
|
||||
public static int compose(char[] src, int srcIndex, int srcLimit,
|
||||
char[] dest,int destIndex,int destLimit,
|
||||
public static int compose(char[] src, int srcStart, int srcLimit,
|
||||
char[] dest,int destStart,int destLimit,
|
||||
boolean compat,UnicodeSet nx) {
|
||||
|
||||
int prevSrc, prevStarter;
|
||||
@ -1924,6 +1926,8 @@ public final class NormalizerImpl {
|
||||
char c, c2, minNoMaybe;
|
||||
int/*unsigned byte*/ cc, prevCC;
|
||||
int[] ioIndex = new int[1];
|
||||
int destIndex = destStart;
|
||||
int srcIndex = srcStart;
|
||||
|
||||
if(!compat) {
|
||||
minNoMaybe=(char)indexes[INDEX_MIN_NFC_NO_MAYBE];
|
||||
@ -2182,7 +2186,7 @@ public final class NormalizerImpl {
|
||||
}
|
||||
}
|
||||
|
||||
return destIndex;
|
||||
return destIndex - destStart;
|
||||
}
|
||||
/* make FCD --------------------------------------------------------------*/
|
||||
|
||||
@ -2529,7 +2533,7 @@ public final class NormalizerImpl {
|
||||
}
|
||||
}
|
||||
|
||||
return destIndex;
|
||||
return destIndex - destStart;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -5,8 +5,8 @@
|
||||
*******************************************************************************
|
||||
*
|
||||
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/icu/text/Normalizer.java,v $
|
||||
* $Date: 2003/06/11 17:51:25 $
|
||||
* $Revision: 1.34 $
|
||||
* $Date: 2003/11/14 00:02:47 $
|
||||
* $Revision: 1.35 $
|
||||
*
|
||||
*******************************************************************************
|
||||
*/
|
||||
@ -711,7 +711,7 @@ public final class Normalizer implements Cloneable{
|
||||
* options>>COMPARE_NORM_OPTIONS_SHIFT will be passed on to the
|
||||
* internal normalization functions.)
|
||||
*
|
||||
* @see compare
|
||||
* @see #compare
|
||||
* @draft ICU 2.6
|
||||
*/
|
||||
public static final int COMPARE_NORM_OPTIONS_SHIFT = 20;
|
||||
@ -1146,7 +1146,7 @@ public final class Normalizer implements Cloneable{
|
||||
* The only option that is currently recognized is UNICODE_3_2
|
||||
* @return String The normalized string
|
||||
* @draft ICU 2.6
|
||||
* @see UNICODE_3_2
|
||||
* @see #UNICODE_3_2
|
||||
*/
|
||||
// TODO: actually do the optimization when the guts of Normalizer are
|
||||
// upgraded --has just dumb implementation for now
|
||||
@ -1158,7 +1158,7 @@ public final class Normalizer implements Cloneable{
|
||||
* @param char32 The input string to be normalized.
|
||||
* @param aMode The normalization mode
|
||||
* @return String The normalized string
|
||||
* @see UNICODE_3_2
|
||||
* @see #UNICODE_3_2
|
||||
* @draft ICU 2.6
|
||||
*/
|
||||
// TODO: actually do the optimization when the guts of Normalizer are
|
||||
|
||||
Loading…
Reference in New Issue
Block a user