AuroraMiddleware/scuffed-code
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ICU-507 use common implementation for getNextUChar() where appropriate

Browse Source 
X-SVN-Rev: 3261
This commit is contained in:
Markus Scherer 2000-12-19 00:29:27 +00:00
parent 76dc0104b5
commit b7c791ad75
6 changed files with 88 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
icu4c/source/common/ucnv.c
View File

@ -986,8 +986,8 @@ UChar32 ucnv_getNextUChar(UConverter * converter,
UTF_NEXT_CHAR(converter->UCharErrorBuffer, i, sizeof(converter->UCharErrorBuffer), myUChar);
/*In this memmove we update the internal buffer by
*popping the first character.
*Note that in the call itself we decrement
*UCharErrorBufferLength
*Note that in the call itself we decrement
*UCharErrorBufferLength
*/
uprv_memmove (converter->UCharErrorBuffer,
converter->UCharErrorBuffer + i,
@ -1005,7 +1005,13 @@ UChar32 ucnv_getNextUChar(UConverter * converter,
args.target = NULL;
args.targetLimit = NULL;
args.size = sizeof(args);
ch = converter->sharedData->impl->getNextUChar(&args, err);
if (converter->sharedData->impl->getNextUChar != NULL)
{
ch = converter->sharedData->impl->getNextUChar(&args, err);
} else {
/* default implementation */
ch = ucnv_getNextUCharFromToUImpl(&args, converter->sharedData->impl->toUnicode, FALSE, err);
}
*source = args.source;
return ch;
}

126
icu4c/source/common/ucnv2022.c
View File

@ -152,9 +152,6 @@ U_CFUNC void UConverter_toUnicode_ISO_2022_JP(UConverterToUnicodeArgs* args,
U_CFUNC void UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC(UConverterToUnicodeArgs* args,
UErrorCode* err);
U_CFUNC UChar32 UConverter_getNextUChar_ISO_2022_JP (UConverterToUnicodeArgs * args,
UErrorCode * err);
/***************** ISO-2022-KR ********************************/
U_CFUNC void UConverter_fromUnicode_ISO_2022_KR(UConverterFromUnicodeArgs* args,
UErrorCode* err);
@ -168,9 +165,6 @@ U_CFUNC void UConverter_toUnicode_ISO_2022_KR(UConverterToUnicodeArgs* args,
U_CFUNC void UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC(UConverterToUnicodeArgs* args,
UErrorCode* err);
U_CFUNC UChar32 UConverter_getNextUChar_ISO_2022_KR (UConverterToUnicodeArgs * args,
UErrorCode * err);
/***************** ISO-2022-CN ********************************/
U_CFUNC void UConverter_fromUnicode_ISO_2022_CN(UConverterFromUnicodeArgs* args,
UErrorCode* err);
@ -184,9 +178,6 @@ U_CFUNC void UConverter_toUnicode_ISO_2022_CN(UConverterToUnicodeArgs* args,
U_CFUNC void UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC(UConverterToUnicodeArgs* args,
UErrorCode* err);
U_CFUNC UChar32 UConverter_getNextUChar_ISO_2022_CN (UConverterToUnicodeArgs * args,
UErrorCode * err);
#define ESC_2022 0x1B /*ESC*/
typedef enum
@ -416,7 +407,7 @@ static const UConverterImpl _ISO2022JPImpl={
UConverter_toUnicode_ISO_2022_JP_OFFSETS_LOGIC,
UConverter_fromUnicode_ISO_2022_JP,
UConverter_fromUnicode_ISO_2022_JP_OFFSETS_LOGIC,
UConverter_getNextUChar_ISO_2022_JP,
NULL,
NULL,
_ISO2022getName
@ -447,7 +438,7 @@ static const UConverterImpl _ISO2022KRImpl={
UConverter_toUnicode_ISO_2022_KR_OFFSETS_LOGIC,
UConverter_fromUnicode_ISO_2022_KR,
UConverter_fromUnicode_ISO_2022_KR_OFFSETS_LOGIC,
UConverter_getNextUChar_ISO_2022_KR,
NULL,
NULL,
_ISO2022getName
@ -479,7 +470,7 @@ static const UConverterImpl _ISO2022CNImpl={
UConverter_toUnicode_ISO_2022_CN_OFFSETS_LOGIC,
UConverter_fromUnicode_ISO_2022_CN,
UConverter_fromUnicode_ISO_2022_CN_OFFSETS_LOGIC,
UConverter_getNextUChar_ISO_2022_CN,
NULL,
NULL,
_ISO2022getName
@ -1899,43 +1890,6 @@ static void concatChar(UConverterFromUnicodeArgs* args, int32_t *targetIndex, in
/*************** to unicode *******************/
/*
* This is a simple, interim implementation of GetNextUChar()
* that allows to concentrate on testing one single implementation
* of the ToUnicode conversion before it gets copied to
* multiple version that are then optimized for their needs
* (with vs. without offsets and getNextUChar).
*/
U_CFUNC UChar32
UConverter_getNextUChar_ISO_2022_JP(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UChar buffer[UTF_MAX_CHAR_LENGTH];
const char *realLimit=pArgs->sourceLimit;
pArgs->target=buffer;
pArgs->targetLimit=buffer+UTF_MAX_CHAR_LENGTH;
while(pArgs->source<realLimit) {
/* feed in one byte at a time to make sure to get only one character out */
pArgs->sourceLimit=pArgs->source+1;
pArgs->flush= (UBool)(pArgs->sourceLimit==realLimit);
UConverter_toUnicode_ISO_2022_JP(pArgs, pErrorCode);
if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) {
return 0xffff;
} else if(pArgs->target!=buffer) {
if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
*pErrorCode=U_ZERO_ERROR;
}
return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, pArgs->target-buffer);
}
}
/* no output because of empty input or only state changes and skipping callbacks */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
return 0xffff;
}
/****************************************************************************
* Recognized escape sequences are
* <ESC>(B ASCII
@ -3243,43 +3197,6 @@ END_LOOP:
args->source = mySource;
}
/*
* This is a simple, interim implementation of GetNextUChar()
* that allows to concentrate on testing one single implementation
* of the ToUnicode conversion before it gets copied to
* multiple version that are then optimized for their needs
* (with vs. without offsets and getNextUChar).
*/
U_CFUNC UChar32
UConverter_getNextUChar_ISO_2022_KR(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UChar buffer[UTF_MAX_CHAR_LENGTH];
const char *realLimit=pArgs->sourceLimit;
pArgs->target=buffer;
pArgs->targetLimit=buffer+UTF_MAX_CHAR_LENGTH;
while(pArgs->source<realLimit) {
/* feed in one byte at a time to make sure to get only one character out */
pArgs->sourceLimit=pArgs->source+1;
pArgs->flush= (UBool)(pArgs->sourceLimit==realLimit);
UConverter_toUnicode_ISO_2022_KR(pArgs, pErrorCode);
if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) {
return 0xffff;
} else if(pArgs->target!=buffer) {
if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
*pErrorCode=U_ZERO_ERROR;
}
return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, pArgs->target-buffer);
}
}
/* no output because of empty input or only state changes and skipping callbacks */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
return 0xffff;
}
/*************************** END ISO2022-KR *********************************/
@ -4779,40 +4696,3 @@ END_LOOP:
args->target = myTarget;
args->source = mySource;
}
/*
* This is a simple, interim implementation of GetNextUChar()
* that allows to concentrate on testing one single implementation
* of the ToUnicode conversion before it gets copied to
* multiple version that are then optimized for their needs
* (with vs. without offsets and getNextUChar).
*/
U_CFUNC UChar32
UConverter_getNextUChar_ISO_2022_CN(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UChar buffer[UTF_MAX_CHAR_LENGTH];
const char *realLimit=pArgs->sourceLimit;
pArgs->target=buffer;
pArgs->targetLimit=buffer+UTF_MAX_CHAR_LENGTH;
while(pArgs->source<realLimit) {
/* feed in one byte at a time to make sure to get only one character out */
pArgs->sourceLimit=pArgs->source+1;
pArgs->flush= (UBool)(pArgs->sourceLimit==realLimit);
UConverter_toUnicode_ISO_2022_CN(pArgs, pErrorCode);
if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) {
return 0xffff;
} else if(pArgs->target!=buffer) {
if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
*pErrorCode=U_ZERO_ERROR;
}
return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, pArgs->target-buffer);
}
}
/* no output because of empty input or only state changes and skipping callbacks */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
return 0xffff;
}

48
icu4c/source/common/ucnv_cnv.c
View File

@ -193,3 +193,51 @@ ucnv_updateCallbackOffsets(int32_t *offsets, int32_t length, int32_t sourceIndex
return NULL;
}
}
/*
* This is a simple implementation of ucnv_getNextUChar() that uses the
* converter's toUnicode() function. See ucnv_cnv.h for details.
*/
U_CFUNC UChar32
ucnv_getNextUCharFromToUImpl(UConverterToUnicodeArgs *pArgs,
T_ToUnicodeFunction toU,
UBool collectPairs,
UErrorCode *pErrorCode) {
UChar buffer[UTF_MAX_CHAR_LENGTH];
const char *realLimit=pArgs->sourceLimit;
pArgs->target=buffer;
pArgs->targetLimit=buffer+UTF_MAX_CHAR_LENGTH;
while(pArgs->source<realLimit) {
/* feed in one byte at a time to make sure to get only one character out */
pArgs->sourceLimit=pArgs->source+1;
pArgs->flush= (UBool)(pArgs->sourceLimit==realLimit);
/* convert this byte and check the result */
toU(pArgs, pErrorCode);
if(U_SUCCESS(*pErrorCode)) {
int32_t length=pArgs->target-buffer;
/* this test is UTF-16 specific */
if(/* some output and
(source consumed or don't collect surrogate pairs or not a surrogate or a surrogate pair) */
length>0 &&
(pArgs->flush || !collectPairs || !UTF_IS_FIRST_SURROGATE(buffer[0]) || length==2)
) {
return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, length);
}
/* else continue with the loop */
} else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
*pErrorCode=U_ZERO_ERROR;
return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, UTF_MAX_CHAR_LENGTH);
} else {
/* U_FAILURE() */
return 0xffff;
}
}
/* no output because of empty input or only state changes and skipping callbacks */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
return 0xffff;
}

26
icu4c/source/common/ucnv_cnv.h
View File

@ -260,4 +260,30 @@ ucnv_updateCallbackOffsets(int32_t *offsets, int32_t length, int32_t sourceIndex
#define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || (uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000)
#define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c)
/**
* This is a simple implementation of ucnv_getNextUChar() that uses the
* converter's toUnicode() function.
*
* \par
* A surrogate pair from a single byte sequence is always
* combined to a supplementary code point.
* A surrogate pair from consecutive byte sequences is only combined
* if collectPairs is set. This is necessary for SCSU
* but not allowed for most legacy codepages.
*
* @param pArgs The argument structure supplied by ucnv_getNextUChar()
* @param toU A function pointer to the converter's toUnicode() function
* @param collectPairs indicates whether separate surrogate results from
* consecutive byte sequences should be combined into
* a single code point
* @param pErrorCode An ICU error code parameter
* @return The Unicode code point as a result of a conversion of a minimal
* number of input bytes
*/
U_CFUNC UChar32
ucnv_getNextUCharFromToUImpl(UConverterToUnicodeArgs *pArgs,
T_ToUnicodeFunction toU,
UBool collectPairs,
UErrorCode *pErrorCode);
#endif /* UCNV_CNV */

33
icu4c/source/common/ucnvhz.c
View File

@ -60,9 +60,6 @@ U_CFUNC void UConverter_fromUnicode_HZ(UConverterFromUnicodeArgs *args,
U_CFUNC void UConverter_fromUnicode_HZ_OFFSETS_LOGIC (UConverterFromUnicodeArgs *args,
UErrorCode *err);
U_CFUNC UChar32 UConverter_getNextUChar_HZ (UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode);
static UConverterImpl _HZImpl={
UCNV_HZ,
@ -77,7 +74,7 @@ static UConverterImpl _HZImpl={
UConverter_toUnicode_HZ_OFFSETS_LOGIC,
UConverter_fromUnicode_HZ,
UConverter_fromUnicode_HZ_OFFSETS_LOGIC,
UConverter_getNextUChar_HZ,
NULL,
NULL,
NULL
@ -998,31 +995,3 @@ CALLBACK:
return;
}
U_CFUNC UChar32 UConverter_getNextUChar_HZ (UConverterToUnicodeArgs * pArgs,
UErrorCode *pErrorCode){
UChar buffer[UTF_MAX_CHAR_LENGTH];
const char *realLimit=pArgs->sourceLimit;
pArgs->target=buffer;
pArgs->targetLimit=buffer+UTF_MAX_CHAR_LENGTH;
while(pArgs->source<realLimit) {
/* feed in one byte at a time to make sure to get only one character out */
pArgs->sourceLimit=pArgs->source+1;
pArgs->flush= (UBool)(pArgs->sourceLimit==realLimit);
UConverter_toUnicode_HZ(pArgs, pErrorCode);
if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) {
return 0xffff;
} else if(pArgs->target!=buffer) {
if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
*pErrorCode=U_ZERO_ERROR;
}
return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, pArgs->target-buffer);
}
}
/* no output because of empty input or only state changes and skipping callbacks */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
return 0xffff;
}

63
icu4c/source/common/ucnvmbcs.c
View File

@ -908,73 +908,12 @@ endloop:
}
/*
* This is a simple, interim implementation of GetNextUChar()
* that allows to concentrate on testing one single implementation
* of the ToUnicode conversion before it gets copied to
* multiple version that are then optimized for their needs
* (with vs. without offsets and getNextUChar).
* ### TODO: implement this directly similar to ToUnicode()
*/
U_CFUNC UChar32
_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs,
UErrorCode *pErrorCode) {
UChar buffer[UTF_MAX_CHAR_LENGTH];
const char *realLimit=pArgs->sourceLimit;
pArgs->target=buffer;
pArgs->targetLimit=buffer+UTF_MAX_CHAR_LENGTH;
while(pArgs->source<realLimit) {
/* feed in one byte at a time to make sure to get only one character out */
pArgs->sourceLimit=pArgs->source+1;
pArgs->flush= (UBool)(pArgs->sourceLimit==realLimit);
_MBCSToUnicode(pArgs, pErrorCode);
if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) {
return 0xffff;
} else {
int32_t length=pArgs->target-buffer;
#if 0
/*
* markus 2000-oct-26
*
* This version of the exit condition is commented out because of
* a clarification of the semantics of ucnv_getNextUChar() (see updated javadoc):
*
* Codepages that provide direct encodings of supplementary Unicode code points (U+10000 and up)
* should return single surrogates without combining them into pairs if single surrogates
* are encoded. This group of codepages includes UTF-8, UTF-32, and GB 18030.
*
* Codepages that provide direct encodings only of single surrogates
* must attempt to match pairs of them into supplementary code points.
* Single surrogates are returned only if they are not part of matched pairs.
* This group of codepages includes SCSU, LMBCS, and UTF-16.
*
* Currently, there is no MBCS codepage in the second group. SCSU, LMBCS, and UTF-16
* are implemented with separate code.
*
* Therefore, this feature is removed here.
* It might need to be added back in later when some MBCS codepages are created that
* fall into the second group. In this case, a flag in the .cnv file will be necessary
* to indicate this. makeconv would need to set this flag based on whether the codepage
* contains only mappings for single surrogates but
* not directly for any supplementary code points.
*/
if(/* some output and (source consumed or not a surrogate or a surrogate pair [UTF-16 specific]) */
length>0 &&
(pArgs->flush || !UTF_IS_FIRST_SURROGATE(buffer[0]) || length==2)
#endif
if(length>0) {
if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
*pErrorCode=U_ZERO_ERROR;
}
return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, length);
}
}
}
/* no output because of empty input or only state changes and skipping callbacks */
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
return 0xffff;
return ucnv_getNextUCharFromToUImpl(pArgs, _MBCSToUnicode, FALSE, pErrorCode);
}
/*