AuroraMiddleware/scuffed-code
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ICU-6165 Test reading past the end of a buffer a little more. Speed up some code. Format some code for easier code review.

Browse Source 
X-SVN-Rev: 23342
This commit is contained in:
George Rhoten 2008-02-01 10:34:40 +00:00
parent d170952252
commit 1565c2fcba
5 changed files with 146 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
icu4c/source/i18n/csrmbcs.cpp
View File

@ -19,7 +19,7 @@ U_NAMESPACE_BEGIN
#define min(x,y) (((x)<(y))?(x):(y))
const uint16_t commonChars_sjis [] = {
static const uint16_t commonChars_sjis [] = {
// TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
@ -30,7 +30,7 @@ const uint16_t commonChars_sjis [] = {
0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389,
0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa};
const uint16_t commonChars_euc_jp[] = {
static const uint16_t commonChars_euc_jp[] = {
// TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
@ -45,7 +45,7 @@ const uint16_t commonChars_euc_jp[] = {
0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee,
0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1};
const uint16_t commonChars_euc_kr[] = {
static const uint16_t commonChars_euc_kr[] = {
// TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
@ -60,7 +60,7 @@ const uint16_t commonChars_euc_kr[] = {
0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6,
0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad};
const uint16_t commonChars_big5[] = {
static const uint16_t commonChars_big5[] = {
// TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
@ -75,7 +75,7 @@ const uint16_t commonChars_big5[] = {
0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44,
0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f};
const uint16_t commonChars_gb_18030[] = {
static const uint16_t commonChars_gb_18030[] = {
// TODO: This set of data comes from the character frequency-
// of-occurence analysis tool. The data needs to be moved
// into a resource and loaded from there.
@ -112,19 +112,20 @@ static int32_t binarySearch(const uint16_t *array, int32_t len, uint16_t value)
return -1;
}
IteratedChar::IteratedChar():charValue(0), index(0), nextIndex(0), error(FALSE), done(FALSE)
IteratedChar::IteratedChar() :
charValue(0), index(-1), nextIndex(0), error(FALSE), done(FALSE)
{
// nothing else to do.
}
void IteratedChar::reset()
/*void IteratedChar::reset()
{
charValue = 0;
index = -1;
nextIndex = 0;
error = FALSE;
done = FALSE;
}
}*/
int32_t IteratedChar::nextByte(InputText *det)
{
@ -143,28 +144,27 @@ CharsetRecog_mbcs::~CharsetRecog_mbcs()
}
int32_t CharsetRecog_mbcs::match_mbcs(InputText *det, const uint16_t commonChars[], int32_t commonCharsLen) {
int singleByteCharCount = 0;
int doubleByteCharCount = 0;
int commonCharCount = 0;
int badCharCount = 0;
int totalCharCount = 0;
int confidence = 0;
IteratedChar *iter = new IteratedChar();
int32_t singleByteCharCount = 0;
int32_t doubleByteCharCount = 0;
int32_t commonCharCount = 0;
int32_t badCharCount = 0;
int32_t totalCharCount = 0;
int32_t confidence = 0;
IteratedChar iter;
// {
for (iter->reset(); nextChar(iter, det);) {
totalCharCount += 1;
while (nextChar(&iter, det)) {
totalCharCount++;
if (iter->error) {
badCharCount += 1;
if (iter.error) {
badCharCount++;
} else {
if (iter->charValue <= 0xFF) {
singleByteCharCount += 1;
if (iter.charValue <= 0xFF) {
singleByteCharCount++;
} else {
doubleByteCharCount += 1;
doubleByteCharCount++;
if (commonChars != 0) {
if (binarySearch(commonChars, commonCharsLen, iter->charValue) >= 0){
if (binarySearch(commonChars, commonCharsLen, iter.charValue) >= 0){
commonCharCount += 1;
}
}
@ -175,18 +175,23 @@ int32_t CharsetRecog_mbcs::match_mbcs(InputText *det, const uint16_t commonChars
if (badCharCount >= 2 && badCharCount*5 >= doubleByteCharCount) {
// Bail out early if the byte data is not matching the encoding scheme.
// break detectBlock;
delete iter;
return confidence;
}
}
delete iter;
if (doubleByteCharCount <= 10 && badCharCount == 0) {
// Not many multi-byte chars.
// ASCII or ISO file? It's probably not our encoding,
// but is not incompatible with our encoding, so don't give it a zero.
confidence = 10;
if (doubleByteCharCount == 0 && totalCharCount < 10) {
// There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes.
// We don't have enough data to have any confidence.
// Statistical analysis of single byte non-ASCII charcters would probably help here.
confidence = 0;
}
else {
// ASCII or ISO file? It's probably not our encoding,
// but is not incompatible with our encoding, so don't give it a zero.
confidence = 10;
}
return confidence;
}
@ -249,11 +254,11 @@ UBool CharsetRecog_sjis::nextChar(IteratedChar* it, InputText* det) {
}
int32_t secondByte = it->nextByte(det);
if (secondByte < 0) {
return FALSE;
if (secondByte >= 0) {
it->charValue = (firstByte << 8) | secondByte;
}
it->charValue = (firstByte << 8) | secondByte;
// else we'll handle the error later.
if (! ((secondByte >= 0x40 && secondByte <= 0x7F) || (secondByte >= 0x80 && secondByte <= 0xFE))) {
// Illegal second byte value.
it->error = TRUE;
@ -286,7 +291,6 @@ UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) {
int32_t firstByte = 0;
int32_t secondByte = 0;
int32_t thirdByte = 0;
// int32_t fourthByte = 0;
it->index = it->nextIndex;
it->error = FALSE;
@ -294,18 +298,19 @@ UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) {
if (firstByte < 0) {
// Ran off the end of the input data
it->done = TRUE;
return (! it->done);
return FALSE;
}
if (firstByte <= 0x8D) {
// single byte char
return (! it->done);
return TRUE;
}
secondByte = it->nextByte(det);
it->charValue = (it->charValue << 8) | secondByte;
if (secondByte >= 0) {
it->charValue = (it->charValue << 8) | secondByte;
}
// else we'll handle the error later.
if (firstByte >= 0xA1 && firstByte <= 0xFE) {
// Two byte Char
@ -313,7 +318,7 @@ UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) {
it->error = TRUE;
}
return (! it->done);
return TRUE;
}
if (firstByte == 0x8E) {
@ -327,7 +332,7 @@ UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) {
it->error = TRUE;
}
return (! it->done);
return TRUE;
}
if (firstByte == 0x8F) {
@ -337,11 +342,12 @@ UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) {
it->charValue = (it->charValue << 8) | thirdByte;
if (thirdByte < 0xa1) {
// Bad second byte or ran off the end of the input data with a non-ASCII first byte.
it->error = TRUE;
}
}
return (! it->done);
return TRUE;
}
@ -408,17 +414,13 @@ UBool CharsetRecog_big5::nextChar(IteratedChar* it, InputText* det)
}
int32_t secondByte = it->nextByte(det);
if (secondByte < 0) {
return FALSE;
if (secondByte >= 0) {
it->charValue = (it->charValue << 8) | secondByte;
}
// else we'll handle the error later.
it->charValue = (it->charValue << 8) | secondByte;
if (secondByte < 0x40 ||
secondByte == 0x7F ||
secondByte == 0xFF) {
it->error = TRUE;
if (secondByte < 0x40 || secondByte == 0x7F || secondByte == 0xFF) {
it->error = TRUE;
}
return TRUE;
@ -456,23 +458,24 @@ UBool CharsetRecog_gb_18030::nextChar(IteratedChar* it, InputText* det) {
if (firstByte < 0) {
// Ran off the end of the input data
it->done = TRUE;
return (! it->done);
return FALSE;
}
if (firstByte <= 0x80) {
// single byte char
return (! it->done);
return TRUE;
}
secondByte = it->nextByte(det);
it->charValue = (it->charValue << 8) | secondByte;
if (secondByte >= 0) {
it->charValue = (it->charValue << 8) | secondByte;
}
// else we'll handle the error later.
if (firstByte >= 0x81 && firstByte <= 0xFE) {
// Two byte Char
if ((secondByte >= 0x40 && secondByte <= 0x7E) || (secondByte >=80 && secondByte <= 0xFE)) {
return (! it->done);
return TRUE;
}
// Four byte char
@ -485,17 +488,16 @@ UBool CharsetRecog_gb_18030::nextChar(IteratedChar* it, InputText* det) {
if (fourthByte >= 0x30 && fourthByte <= 0x39) {
it->charValue = (it->charValue << 16) | (thirdByte << 8) | fourthByte;
return (! it->done);
return TRUE;
}
}
}
// Something wasn't valid, or we ran out of data (-1).
it->error = TRUE;
return (! it->done);
}
return (! it->done);
return TRUE;
}
const char *CharsetRecog_gb_18030::getName() const

2
icu4c/source/i18n/csrmbcs.h
View File

@ -38,7 +38,7 @@ public:
public:
IteratedChar();
void reset();
//void reset();
int32_t nextByte(InputText* det);
};

9
icu4c/source/i18n/csrsbcs.cpp
View File

@ -1,6 +1,6 @@
/*
**********************************************************************
* Copyright (C) 2005-2006, International Business Machines
* Copyright (C) 2005-2008, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
@ -10,8 +10,6 @@
#if !UCONFIG_NO_CONVERSION
#include "csrsbcs.h"
#include <stdio.h>
#define N_GRAM_SIZE 3
#define N_GRAM_MASK 0xFFFFFF
@ -143,12 +141,11 @@ CharsetRecog_sbcs::~CharsetRecog_sbcs()
int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[])
{
NGramParser *parser = new NGramParser(ngrams, byteMap);
NGramParser parser(ngrams, byteMap);
int32_t result;
haveC1Bytes = det->fC1Bytes;
result = parser->parse(det);
delete parser;
result = parser.parse(det);
return result;
}

118
icu4c/source/i18n/csrutf8.cpp
View File

@ -1,6 +1,6 @@
/*
**********************************************************************
* Copyright (C) 2005-2006, International Business Machines
* Copyright (C) 2005-2008, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
@ -35,76 +35,76 @@ int32_t CharsetRecog_UTF8::match(InputText* det) {
if (det->fRawLength >= 3 &&
input[0] == 0xEF && input[1] == 0xBB && input[2] == 0xBF) {
hasBOM = TRUE;
}
// Scan for multi-byte sequences
for (i=0; i < det->fRawLength; i += 1) {
int32_t b = input[i];
if ((b & 0x80) == 0) {
continue; // ASCII
}
// Scan for multi-byte sequences
for (i=0; i < det->fRawLength; i += 1) {
int32_t b = input[i];
// Hi bit on char found. Figure out how long the sequence should be
if ((b & 0x0E0) == 0x0C0) {
trailBytes = 1;
} else if ((b & 0x0F0) == 0x0E0) {
trailBytes = 2;
} else if ((b & 0x0F8) == 0xF0) {
trailBytes = 3;
} else {
numInvalid += 1;
if ((b & 0x80) == 0) {
continue; // ASCII
if (numInvalid > 5) {
break;
}
// Hi bit on char found. Figure out how long the sequence should be
if ((b & 0x0E0) == 0x0C0) {
trailBytes = 1;
} else if ((b & 0x0F0) == 0x0E0) {
trailBytes = 2;
} else if ((b & 0x0F8) == 0xF0) {
trailBytes = 3;
} else {
trailBytes = 0;
}
// Verify that we've got the right number of trail bytes in the sequence
for (;;) {
i += 1;
if (i >= det->fRawLength) {
break;
}
b = input[i];
if ((b & 0xC0) != 0x080) {
numInvalid += 1;
if (numInvalid > 5) {
break;
}
trailBytes = 0;
break;
}
// Verify that we've got the right number of trail bytes in the sequence
for (;;) {
i += 1;
if (i >= det->fRawLength) {
break;
}
b = input[i];
if ((b & 0xC0) != 0x080) {
numInvalid += 1;
break;
}
if (--trailBytes == 0) {
numValid += 1;
break;
}
if (--trailBytes == 0) {
numValid += 1;
break;
}
}
// Cook up some sort of confidence score, based on presense of a BOM
// and the existence of valid and/or invalid multi-byte sequences.
confidence = 0;
if (hasBOM && numInvalid == 0) {
confidence = 100;
} else if (hasBOM && numValid > numInvalid*10) {
confidence = 80;
} else if (numValid > 3 && numInvalid == 0) {
confidence = 100;
} else if (numValid > 0 && numInvalid == 0) {
confidence = 80;
} else if (numValid == 0 && numInvalid == 0) {
// Plain ASCII.
confidence = 10;
} else if (numValid > numInvalid*10) {
// Probably corruput utf-8 data. Valid sequences aren't likely by chance.
confidence = 25;
}
}
return confidence;
// Cook up some sort of confidence score, based on presense of a BOM
// and the existence of valid and/or invalid multi-byte sequences.
confidence = 0;
if (hasBOM && numInvalid == 0) {
confidence = 100;
} else if (hasBOM && numValid > numInvalid*10) {
confidence = 80;
} else if (numValid > 3 && numInvalid == 0) {
confidence = 100;
} else if (numValid > 0 && numInvalid == 0) {
confidence = 80;
} else if (numValid == 0 && numInvalid == 0) {
// Plain ASCII.
confidence = 10;
} else if (numValid > numInvalid*10) {
// Probably corruput utf-8 data. Valid sequences aren't likely by chance.
confidence = 25;
}
return confidence;
}
U_NAMESPACE_END

22
icu4c/source/test/cintltst/ucsdetst.c
View File

@ -363,14 +363,20 @@ static void TestBufferOverflow(void) {
"\x80 This is English \x1b\x24", /* A partial ISO-2022 shift state at the end */
"\x80 This is English \x1b\x24\x28", /* A partial ISO-2022 shift state at the end */
"\x80 This is English \x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end with a bad one at the start */
"\x1b\x24\x28\x44" /* A complete ISO-2022 shift state at the end */
"\x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end */
"\xa1", /* Could be a single byte shift-jis at the end */
"th\xa1", /* Could be a single byte shift-jis at the end */
"the\xa1" /* Could be a single byte shift-jis at the end, but now we have English creeping in. */
};
static const char *testResults[] = {
"windows-1252",
"windows-1252",
"windows-1252",
"windows-1252",
"ISO-2022-JP"
"ISO-2022-JP",
NULL,
NULL,
"ISO-8859-1"
};
int32_t idx = 0;
UCharsetDetector *csd = ucsdet_open(&status);
@ -387,7 +393,17 @@ static void TestBufferOverflow(void) {
ucsdet_setText(csd, testStrings[idx], -1, &status);
match = ucsdet_detect(csd, &status);
if (strcmp(ucsdet_getName(match, &status), testResults[idx]) != 0) {
if (match == NULL) {
if (testResults[idx] != NULL) {
log_err("Unexpectedly got no results at index %d.\n", idx);
}
else {
log_verbose("Got no result as expected at index %d.\n", idx);
}
continue;
}
if (testResults[idx] == NULL || strcmp(ucsdet_getName(match, &status), testResults[idx]) != 0) {
log_err("Unexpectedly got %s instead of %s at index %d with confidence %d.\n",
ucsdet_getName(match, &status), testResults[idx], idx, ucsdet_getConfidence(match, &status));
goto bail;