1144 lines
39 KiB
C++
1144 lines
39 KiB
C++
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*
|
|
******************************************************************************
|
|
*
|
|
* Copyright (C) 2003-2016, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
*
|
|
******************************************************************************
|
|
* file name: ucnv_ext.cpp
|
|
* encoding: UTF-8
|
|
* tab size: 8 (not used)
|
|
* indentation:4
|
|
*
|
|
* created on: 2003jun13
|
|
* created by: Markus W. Scherer
|
|
*
|
|
* Conversion extensions
|
|
*/
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
|
|
|
|
#include "unicode/uset.h"
|
|
#include "unicode/ustring.h"
|
|
#include "ucnv_bld.h"
|
|
#include "ucnv_cnv.h"
|
|
#include "ucnv_ext.h"
|
|
#include "cmemory.h"
|
|
#include "uassert.h"
|
|
|
|
/* to Unicode --------------------------------------------------------------- */
|
|
|
|
/*
|
|
* @return lookup value for the byte, if found; else 0
|
|
*/
|
|
static inline uint32_t
|
|
ucnv_extFindToU(const uint32_t *toUSection, int32_t length, uint8_t byte) {
|
|
uint32_t word0, word;
|
|
int32_t i, start, limit;
|
|
|
|
/* check the input byte against the lowest and highest section bytes */
|
|
start=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[0]);
|
|
limit=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[length-1]);
|
|
if(byte<start || limit<byte) {
|
|
return 0; /* the byte is out of range */
|
|
}
|
|
|
|
if(length==((limit-start)+1)) {
|
|
/* direct access on a linear array */
|
|
return UCNV_EXT_TO_U_GET_VALUE(toUSection[byte-start]); /* could be 0 */
|
|
}
|
|
|
|
/* word0 is suitable for <=toUSection[] comparison, word for <toUSection[] */
|
|
word0=UCNV_EXT_TO_U_MAKE_WORD(byte, 0);
|
|
|
|
/*
|
|
* Shift byte once instead of each section word and add 0xffffff.
|
|
* We will compare the shifted/added byte (bbffffff) against
|
|
* section words which have byte values in the same bit position.
|
|
* If and only if byte bb < section byte ss then bbffffff<ssvvvvvv
|
|
* for all v=0..f
|
|
* so we need not mask off the lower 24 bits of each section word.
|
|
*/
|
|
word=word0|UCNV_EXT_TO_U_VALUE_MASK;
|
|
|
|
/* binary search */
|
|
start=0;
|
|
limit=length;
|
|
for(;;) {
|
|
i=limit-start;
|
|
if(i<=1) {
|
|
break; /* done */
|
|
}
|
|
/* start<limit-1 */
|
|
|
|
if(i<=4) {
|
|
/* linear search for the last part */
|
|
if(word0<=toUSection[start]) {
|
|
break;
|
|
}
|
|
if(++start<limit && word0<=toUSection[start]) {
|
|
break;
|
|
}
|
|
if(++start<limit && word0<=toUSection[start]) {
|
|
break;
|
|
}
|
|
/* always break at start==limit-1 */
|
|
++start;
|
|
break;
|
|
}
|
|
|
|
i=(start+limit)/2;
|
|
if(word<toUSection[i]) {
|
|
limit=i;
|
|
} else {
|
|
start=i;
|
|
}
|
|
}
|
|
|
|
/* did we really find it? */
|
|
if(start<limit && byte==UCNV_EXT_TO_U_GET_BYTE(word=toUSection[start])) {
|
|
return UCNV_EXT_TO_U_GET_VALUE(word); /* never 0 */
|
|
} else {
|
|
return 0; /* not found */
|
|
}
|
|
}
|
|
|
|
/*
|
|
* TRUE if not an SI/SO stateful converter,
|
|
* or if the match length fits with the current converter state
|
|
*/
|
|
#define UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, match) \
|
|
((sisoState)<0 || ((sisoState)==0) == (match==1))
|
|
|
|
/*
|
|
* this works like ucnv_extMatchFromU() except
|
|
* - the first character is in pre
|
|
* - no trie is used
|
|
* - the returned matchLength is not offset by 2
|
|
*/
|
|
static int32_t
|
|
ucnv_extMatchToU(const int32_t *cx, int8_t sisoState,
|
|
const char *pre, int32_t preLength,
|
|
const char *src, int32_t srcLength,
|
|
uint32_t *pMatchValue,
|
|
UBool /*useFallback*/, UBool flush) {
|
|
const uint32_t *toUTable, *toUSection;
|
|
|
|
uint32_t value, matchValue;
|
|
int32_t i, j, idx, length, matchLength;
|
|
uint8_t b;
|
|
|
|
if(cx==NULL || cx[UCNV_EXT_TO_U_LENGTH]<=0) {
|
|
return 0; /* no extension data, no match */
|
|
}
|
|
|
|
/* initialize */
|
|
toUTable=UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_INDEX, uint32_t);
|
|
idx=0;
|
|
|
|
matchValue=0;
|
|
i=j=matchLength=0;
|
|
|
|
if(sisoState==0) {
|
|
/* SBCS state of an SI/SO stateful converter, look at only exactly 1 byte */
|
|
if(preLength>1) {
|
|
return 0; /* no match of a DBCS sequence in SBCS mode */
|
|
} else if(preLength==1) {
|
|
srcLength=0;
|
|
} else /* preLength==0 */ {
|
|
if(srcLength>1) {
|
|
srcLength=1;
|
|
}
|
|
}
|
|
flush=TRUE;
|
|
}
|
|
|
|
/* we must not remember fallback matches when not using fallbacks */
|
|
|
|
/* match input units until there is a full match or the input is consumed */
|
|
for(;;) {
|
|
/* go to the next section */
|
|
toUSection=toUTable+idx;
|
|
|
|
/* read first pair of the section */
|
|
value=*toUSection++;
|
|
length=UCNV_EXT_TO_U_GET_BYTE(value);
|
|
value=UCNV_EXT_TO_U_GET_VALUE(value);
|
|
if( value!=0 &&
|
|
(UCNV_EXT_TO_U_IS_ROUNDTRIP(value) ||
|
|
TO_U_USE_FALLBACK(useFallback)) &&
|
|
UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j)
|
|
) {
|
|
/* remember longest match so far */
|
|
matchValue=value;
|
|
matchLength=i+j;
|
|
}
|
|
|
|
/* match pre[] then src[] */
|
|
if(i<preLength) {
|
|
b=(uint8_t)pre[i++];
|
|
} else if(j<srcLength) {
|
|
b=(uint8_t)src[j++];
|
|
} else {
|
|
/* all input consumed, partial match */
|
|
if(flush || (length=(i+j))>UCNV_EXT_MAX_BYTES) {
|
|
/*
|
|
* end of the entire input stream, stop with the longest match so far
|
|
* or: partial match must not be longer than UCNV_EXT_MAX_BYTES
|
|
* because it must fit into state buffers
|
|
*/
|
|
break;
|
|
} else {
|
|
/* continue with more input next time */
|
|
return -length;
|
|
}
|
|
}
|
|
|
|
/* search for the current UChar */
|
|
value=ucnv_extFindToU(toUSection, length, b);
|
|
if(value==0) {
|
|
/* no match here, stop with the longest match so far */
|
|
break;
|
|
} else {
|
|
if(UCNV_EXT_TO_U_IS_PARTIAL(value)) {
|
|
/* partial match, continue */
|
|
idx=(int32_t)UCNV_EXT_TO_U_GET_PARTIAL_INDEX(value);
|
|
} else {
|
|
if( (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) ||
|
|
TO_U_USE_FALLBACK(useFallback)) &&
|
|
UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j)
|
|
) {
|
|
/* full match, stop with result */
|
|
matchValue=value;
|
|
matchLength=i+j;
|
|
} else {
|
|
/* full match on fallback not taken, stop with the longest match so far */
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if(matchLength==0) {
|
|
/* no match at all */
|
|
return 0;
|
|
}
|
|
|
|
/* return result */
|
|
*pMatchValue=UCNV_EXT_TO_U_MASK_ROUNDTRIP(matchValue);
|
|
return matchLength;
|
|
}
|
|
|
|
static inline void
|
|
ucnv_extWriteToU(UConverter *cnv, const int32_t *cx,
|
|
uint32_t value,
|
|
UChar **target, const UChar *targetLimit,
|
|
int32_t **offsets, int32_t srcIndex,
|
|
UErrorCode *pErrorCode) {
|
|
/* output the result */
|
|
if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) {
|
|
/* output a single code point */
|
|
ucnv_toUWriteCodePoint(
|
|
cnv, UCNV_EXT_TO_U_GET_CODE_POINT(value),
|
|
target, targetLimit,
|
|
offsets, srcIndex,
|
|
pErrorCode);
|
|
} else {
|
|
/* output a string - with correct data we have resultLength>0 */
|
|
ucnv_toUWriteUChars(
|
|
cnv,
|
|
UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_UCHARS_INDEX, UChar)+
|
|
UCNV_EXT_TO_U_GET_INDEX(value),
|
|
UCNV_EXT_TO_U_GET_LENGTH(value),
|
|
target, targetLimit,
|
|
offsets, srcIndex,
|
|
pErrorCode);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* get the SI/SO toU state (state 0 is for SBCS, 1 for DBCS),
|
|
* or 1 for DBCS-only,
|
|
* or -1 if the converter is not SI/SO stateful
|
|
*
|
|
* Note: For SI/SO stateful converters getting here,
|
|
* cnv->mode==0 is equivalent to firstLength==1.
|
|
*/
|
|
#define UCNV_SISO_STATE(cnv) \
|
|
((cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO ? (int8_t)(cnv)->mode : \
|
|
(cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? 1 : -1)
|
|
|
|
/*
|
|
* target<targetLimit; set error code for overflow
|
|
*/
|
|
U_CFUNC UBool
|
|
ucnv_extInitialMatchToU(UConverter *cnv, const int32_t *cx,
|
|
int32_t firstLength,
|
|
const char **src, const char *srcLimit,
|
|
UChar **target, const UChar *targetLimit,
|
|
int32_t **offsets, int32_t srcIndex,
|
|
UBool flush,
|
|
UErrorCode *pErrorCode) {
|
|
uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */
|
|
int32_t match;
|
|
|
|
/* try to match */
|
|
match=ucnv_extMatchToU(cx, (int8_t)UCNV_SISO_STATE(cnv),
|
|
(const char *)cnv->toUBytes, firstLength,
|
|
*src, (int32_t)(srcLimit-*src),
|
|
&value,
|
|
cnv->useFallback, flush);
|
|
if(match>0) {
|
|
/* advance src pointer for the consumed input */
|
|
*src+=match-firstLength;
|
|
|
|
/* write result to target */
|
|
ucnv_extWriteToU(cnv, cx,
|
|
value,
|
|
target, targetLimit,
|
|
offsets, srcIndex,
|
|
pErrorCode);
|
|
return TRUE;
|
|
} else if(match<0) {
|
|
/* save state for partial match */
|
|
const char *s;
|
|
int32_t j;
|
|
|
|
/* copy the first code point */
|
|
s=(const char *)cnv->toUBytes;
|
|
cnv->preToUFirstLength=(int8_t)firstLength;
|
|
for(j=0; j<firstLength; ++j) {
|
|
cnv->preToU[j]=*s++;
|
|
}
|
|
|
|
/* now copy the newly consumed input */
|
|
s=*src;
|
|
match=-match;
|
|
for(; j<match; ++j) {
|
|
cnv->preToU[j]=*s++;
|
|
}
|
|
*src=s; /* same as *src=srcLimit; because we reached the end of input */
|
|
cnv->preToULength=(int8_t)match;
|
|
return TRUE;
|
|
} else /* match==0 no match */ {
|
|
return FALSE;
|
|
}
|
|
}
|
|
|
|
U_CFUNC UChar32
|
|
ucnv_extSimpleMatchToU(const int32_t *cx,
|
|
const char *source, int32_t length,
|
|
UBool useFallback) {
|
|
uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */
|
|
int32_t match;
|
|
|
|
if(length<=0) {
|
|
return 0xffff;
|
|
}
|
|
|
|
/* try to match */
|
|
match=ucnv_extMatchToU(cx, -1,
|
|
source, length,
|
|
NULL, 0,
|
|
&value,
|
|
useFallback, TRUE);
|
|
if(match==length) {
|
|
/* write result for simple, single-character conversion */
|
|
if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) {
|
|
return UCNV_EXT_TO_U_GET_CODE_POINT(value);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* return no match because
|
|
* - match>0 && value points to string: simple conversion cannot handle multiple code points
|
|
* - match>0 && match!=length: not all input consumed, forbidden for this function
|
|
* - match==0: no match found in the first place
|
|
* - match<0: partial match, not supported for simple conversion (and flush==TRUE)
|
|
*/
|
|
return 0xfffe;
|
|
}
|
|
|
|
/*
|
|
* continue partial match with new input
|
|
* never called for simple, single-character conversion
|
|
*/
|
|
U_CFUNC void
|
|
ucnv_extContinueMatchToU(UConverter *cnv,
|
|
UConverterToUnicodeArgs *pArgs, int32_t srcIndex,
|
|
UErrorCode *pErrorCode) {
|
|
uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */
|
|
int32_t match, length;
|
|
|
|
match=ucnv_extMatchToU(cnv->sharedData->mbcs.extIndexes, (int8_t)UCNV_SISO_STATE(cnv),
|
|
cnv->preToU, cnv->preToULength,
|
|
pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source),
|
|
&value,
|
|
cnv->useFallback, pArgs->flush);
|
|
if(match>0) {
|
|
if(match>=cnv->preToULength) {
|
|
/* advance src pointer for the consumed input */
|
|
pArgs->source+=match-cnv->preToULength;
|
|
cnv->preToULength=0;
|
|
} else {
|
|
/* the match did not use all of preToU[] - keep the rest for replay */
|
|
length=cnv->preToULength-match;
|
|
uprv_memmove(cnv->preToU, cnv->preToU+match, length);
|
|
cnv->preToULength=(int8_t)-length;
|
|
}
|
|
|
|
/* write result */
|
|
ucnv_extWriteToU(cnv, cnv->sharedData->mbcs.extIndexes,
|
|
value,
|
|
&pArgs->target, pArgs->targetLimit,
|
|
&pArgs->offsets, srcIndex,
|
|
pErrorCode);
|
|
} else if(match<0) {
|
|
/* save state for partial match */
|
|
const char *s;
|
|
int32_t j;
|
|
|
|
/* just _append_ the newly consumed input to preToU[] */
|
|
s=pArgs->source;
|
|
match=-match;
|
|
for(j=cnv->preToULength; j<match; ++j) {
|
|
cnv->preToU[j]=*s++;
|
|
}
|
|
pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */
|
|
cnv->preToULength=(int8_t)match;
|
|
} else /* match==0 */ {
|
|
/*
|
|
* no match
|
|
*
|
|
* We need to split the previous input into two parts:
|
|
*
|
|
* 1. The first codepage character is unmappable - that's how we got into
|
|
* trying the extension data in the first place.
|
|
* We need to move it from the preToU buffer
|
|
* to the error buffer, set an error code,
|
|
* and prepare the rest of the previous input for 2.
|
|
*
|
|
* 2. The rest of the previous input must be converted once we
|
|
* come back from the callback for the first character.
|
|
* At that time, we have to try again from scratch to convert
|
|
* these input characters.
|
|
* The replay will be handled by the ucnv.c conversion code.
|
|
*/
|
|
|
|
/* move the first codepage character to the error field */
|
|
uprv_memcpy(cnv->toUBytes, cnv->preToU, cnv->preToUFirstLength);
|
|
cnv->toULength=cnv->preToUFirstLength;
|
|
|
|
/* move the rest up inside the buffer */
|
|
length=cnv->preToULength-cnv->preToUFirstLength;
|
|
if(length>0) {
|
|
uprv_memmove(cnv->preToU, cnv->preToU+cnv->preToUFirstLength, length);
|
|
}
|
|
|
|
/* mark preToU for replay */
|
|
cnv->preToULength=(int8_t)-length;
|
|
|
|
/* set the error code for unassigned */
|
|
*pErrorCode=U_INVALID_CHAR_FOUND;
|
|
}
|
|
}
|
|
|
|
/* from Unicode ------------------------------------------------------------- */
|
|
|
|
// Use roundtrips, "good one-way" mappings, and some normal fallbacks.
|
|
static inline UBool
|
|
extFromUUseMapping(UBool useFallback, uint32_t value, UChar32 firstCP) {
|
|
return
|
|
((value&UCNV_EXT_FROM_U_STATUS_MASK)!=0 ||
|
|
FROM_U_USE_FALLBACK(useFallback, firstCP)) &&
|
|
(value&UCNV_EXT_FROM_U_RESERVED_MASK)==0;
|
|
}
|
|
|
|
/*
|
|
* @return index of the UChar, if found; else <0
|
|
*/
|
|
static inline int32_t
|
|
ucnv_extFindFromU(const UChar *fromUSection, int32_t length, UChar u) {
|
|
int32_t i, start, limit;
|
|
|
|
/* binary search */
|
|
start=0;
|
|
limit=length;
|
|
for(;;) {
|
|
i=limit-start;
|
|
if(i<=1) {
|
|
break; /* done */
|
|
}
|
|
/* start<limit-1 */
|
|
|
|
if(i<=4) {
|
|
/* linear search for the last part */
|
|
if(u<=fromUSection[start]) {
|
|
break;
|
|
}
|
|
if(++start<limit && u<=fromUSection[start]) {
|
|
break;
|
|
}
|
|
if(++start<limit && u<=fromUSection[start]) {
|
|
break;
|
|
}
|
|
/* always break at start==limit-1 */
|
|
++start;
|
|
break;
|
|
}
|
|
|
|
i=(start+limit)/2;
|
|
if(u<fromUSection[i]) {
|
|
limit=i;
|
|
} else {
|
|
start=i;
|
|
}
|
|
}
|
|
|
|
/* did we really find it? */
|
|
if(start<limit && u==fromUSection[start]) {
|
|
return start;
|
|
} else {
|
|
return -1; /* not found */
|
|
}
|
|
}
|
|
|
|
/*
|
|
* @param cx pointer to extension data; if NULL, returns 0
|
|
* @param firstCP the first code point before all the other UChars
|
|
* @param pre UChars that must match; !initialMatch: partial match with them
|
|
* @param preLength length of pre, >=0
|
|
* @param src UChars that can be used to complete a match
|
|
* @param srcLength length of src, >=0
|
|
* @param pMatchValue [out] output result value for the match from the data structure
|
|
* @param useFallback "use fallback" flag, usually from cnv->useFallback
|
|
* @param flush TRUE if the end of the input stream is reached
|
|
* @return >1: matched, return value=total match length (number of input units matched)
|
|
* 1: matched, no mapping but request for <subchar1>
|
|
* (only for the first code point)
|
|
* 0: no match
|
|
* <0: partial match, return value=negative total match length
|
|
* (partial matches are never returned for flush==TRUE)
|
|
* (partial matches are never returned as being longer than UCNV_EXT_MAX_UCHARS)
|
|
* the matchLength is 2 if only firstCP matched, and >2 if firstCP and
|
|
* further code units matched
|
|
*/
|
|
static int32_t
|
|
ucnv_extMatchFromU(const int32_t *cx,
|
|
UChar32 firstCP,
|
|
const UChar *pre, int32_t preLength,
|
|
const UChar *src, int32_t srcLength,
|
|
uint32_t *pMatchValue,
|
|
UBool useFallback, UBool flush) {
|
|
const uint16_t *stage12, *stage3;
|
|
const uint32_t *stage3b;
|
|
|
|
const UChar *fromUTableUChars, *fromUSectionUChars;
|
|
const uint32_t *fromUTableValues, *fromUSectionValues;
|
|
|
|
uint32_t value, matchValue;
|
|
int32_t i, j, idx, length, matchLength;
|
|
UChar c;
|
|
|
|
if(cx==NULL) {
|
|
return 0; /* no extension data, no match */
|
|
}
|
|
|
|
/* trie lookup of firstCP */
|
|
idx=firstCP>>10; /* stage 1 index */
|
|
if(idx>=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH]) {
|
|
return 0; /* the first code point is outside the trie */
|
|
}
|
|
|
|
stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t);
|
|
stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t);
|
|
idx=UCNV_EXT_FROM_U(stage12, stage3, idx, firstCP);
|
|
|
|
stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t);
|
|
value=stage3b[idx];
|
|
if(value==0) {
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Tests for (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0:
|
|
* Do not interpret values with reserved bits used, for forward compatibility,
|
|
* and do not even remember intermediate results with reserved bits used.
|
|
*/
|
|
|
|
if(UCNV_EXT_TO_U_IS_PARTIAL(value)) {
|
|
/* partial match, enter the loop below */
|
|
idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value);
|
|
|
|
/* initialize */
|
|
fromUTableUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar);
|
|
fromUTableValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t);
|
|
|
|
matchValue=0;
|
|
i=j=matchLength=0;
|
|
|
|
/* we must not remember fallback matches when not using fallbacks */
|
|
|
|
/* match input units until there is a full match or the input is consumed */
|
|
for(;;) {
|
|
/* go to the next section */
|
|
fromUSectionUChars=fromUTableUChars+idx;
|
|
fromUSectionValues=fromUTableValues+idx;
|
|
|
|
/* read first pair of the section */
|
|
length=*fromUSectionUChars++;
|
|
value=*fromUSectionValues++;
|
|
if(value!=0 && extFromUUseMapping(useFallback, value, firstCP)) {
|
|
/* remember longest match so far */
|
|
matchValue=value;
|
|
matchLength=2+i+j;
|
|
}
|
|
|
|
/* match pre[] then src[] */
|
|
if(i<preLength) {
|
|
c=pre[i++];
|
|
} else if(j<srcLength) {
|
|
c=src[j++];
|
|
} else {
|
|
/* all input consumed, partial match */
|
|
if(flush || (length=(i+j))>UCNV_EXT_MAX_UCHARS) {
|
|
/*
|
|
* end of the entire input stream, stop with the longest match so far
|
|
* or: partial match must not be longer than UCNV_EXT_MAX_UCHARS
|
|
* because it must fit into state buffers
|
|
*/
|
|
break;
|
|
} else {
|
|
/* continue with more input next time */
|
|
return -(2+length);
|
|
}
|
|
}
|
|
|
|
/* search for the current UChar */
|
|
idx=ucnv_extFindFromU(fromUSectionUChars, length, c);
|
|
if(idx<0) {
|
|
/* no match here, stop with the longest match so far */
|
|
break;
|
|
} else {
|
|
value=fromUSectionValues[idx];
|
|
if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
|
|
/* partial match, continue */
|
|
idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value);
|
|
} else {
|
|
if(extFromUUseMapping(useFallback, value, firstCP)) {
|
|
/* full match, stop with result */
|
|
matchValue=value;
|
|
matchLength=2+i+j;
|
|
} else {
|
|
/* full match on fallback not taken, stop with the longest match so far */
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if(matchLength==0) {
|
|
/* no match at all */
|
|
return 0;
|
|
}
|
|
} else /* result from firstCP trie lookup */ {
|
|
if(extFromUUseMapping(useFallback, value, firstCP)) {
|
|
/* full match, stop with result */
|
|
matchValue=value;
|
|
matchLength=2;
|
|
} else {
|
|
/* fallback not taken */
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/* return result */
|
|
if(matchValue==UCNV_EXT_FROM_U_SUBCHAR1) {
|
|
return 1; /* assert matchLength==2 */
|
|
}
|
|
|
|
*pMatchValue=matchValue;
|
|
return matchLength;
|
|
}
|
|
|
|
/*
|
|
* @param value fromUnicode mapping table value; ignores roundtrip and reserved bits
|
|
*/
|
|
static inline void
|
|
ucnv_extWriteFromU(UConverter *cnv, const int32_t *cx,
|
|
uint32_t value,
|
|
char **target, const char *targetLimit,
|
|
int32_t **offsets, int32_t srcIndex,
|
|
UErrorCode *pErrorCode) {
|
|
uint8_t buffer[1+UCNV_EXT_MAX_BYTES];
|
|
const uint8_t *result;
|
|
int32_t length, prevLength;
|
|
|
|
length=UCNV_EXT_FROM_U_GET_LENGTH(value);
|
|
value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value);
|
|
|
|
/* output the result */
|
|
if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) {
|
|
/*
|
|
* Generate a byte array and then write it below.
|
|
* This is not the fastest possible way, but it should be ok for
|
|
* extension mappings, and it is much simpler.
|
|
* Offset and overflow handling are only done once this way.
|
|
*/
|
|
uint8_t *p=buffer+1; /* reserve buffer[0] for shiftByte below */
|
|
switch(length) {
|
|
case 3:
|
|
*p++=(uint8_t)(value>>16);
|
|
U_FALLTHROUGH;
|
|
case 2:
|
|
*p++=(uint8_t)(value>>8);
|
|
U_FALLTHROUGH;
|
|
case 1:
|
|
*p++=(uint8_t)value;
|
|
U_FALLTHROUGH;
|
|
default:
|
|
break; /* will never occur */
|
|
}
|
|
result=buffer+1;
|
|
} else {
|
|
result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value;
|
|
}
|
|
|
|
/* with correct data we have length>0 */
|
|
|
|
if((prevLength=cnv->fromUnicodeStatus)!=0) {
|
|
/* handle SI/SO stateful output */
|
|
uint8_t shiftByte;
|
|
|
|
if(prevLength>1 && length==1) {
|
|
/* change from double-byte mode to single-byte */
|
|
shiftByte=(uint8_t)UCNV_SI;
|
|
cnv->fromUnicodeStatus=1;
|
|
} else if(prevLength==1 && length>1) {
|
|
/* change from single-byte mode to double-byte */
|
|
shiftByte=(uint8_t)UCNV_SO;
|
|
cnv->fromUnicodeStatus=2;
|
|
} else {
|
|
shiftByte=0;
|
|
}
|
|
|
|
if(shiftByte!=0) {
|
|
/* prepend the shift byte to the result bytes */
|
|
buffer[0]=shiftByte;
|
|
if(result!=buffer+1) {
|
|
uprv_memcpy(buffer+1, result, length);
|
|
}
|
|
result=buffer;
|
|
++length;
|
|
}
|
|
}
|
|
|
|
ucnv_fromUWriteBytes(cnv, (const char *)result, length,
|
|
target, targetLimit,
|
|
offsets, srcIndex,
|
|
pErrorCode);
|
|
}
|
|
|
|
/*
|
|
* target<targetLimit; set error code for overflow
|
|
*/
|
|
U_CFUNC UBool
|
|
ucnv_extInitialMatchFromU(UConverter *cnv, const int32_t *cx,
|
|
UChar32 cp,
|
|
const UChar **src, const UChar *srcLimit,
|
|
char **target, const char *targetLimit,
|
|
int32_t **offsets, int32_t srcIndex,
|
|
UBool flush,
|
|
UErrorCode *pErrorCode) {
|
|
uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */
|
|
int32_t match;
|
|
|
|
/* try to match */
|
|
match=ucnv_extMatchFromU(cx, cp,
|
|
NULL, 0,
|
|
*src, (int32_t)(srcLimit-*src),
|
|
&value,
|
|
cnv->useFallback, flush);
|
|
|
|
/* reject a match if the result is a single byte for DBCS-only */
|
|
if( match>=2 &&
|
|
!(UCNV_EXT_FROM_U_GET_LENGTH(value)==1 &&
|
|
cnv->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY)
|
|
) {
|
|
/* advance src pointer for the consumed input */
|
|
*src+=match-2; /* remove 2 for the initial code point */
|
|
|
|
/* write result to target */
|
|
ucnv_extWriteFromU(cnv, cx,
|
|
value,
|
|
target, targetLimit,
|
|
offsets, srcIndex,
|
|
pErrorCode);
|
|
return TRUE;
|
|
} else if(match<0) {
|
|
/* save state for partial match */
|
|
const UChar *s;
|
|
int32_t j;
|
|
|
|
/* copy the first code point */
|
|
cnv->preFromUFirstCP=cp;
|
|
|
|
/* now copy the newly consumed input */
|
|
s=*src;
|
|
match=-match-2; /* remove 2 for the initial code point */
|
|
for(j=0; j<match; ++j) {
|
|
cnv->preFromU[j]=*s++;
|
|
}
|
|
*src=s; /* same as *src=srcLimit; because we reached the end of input */
|
|
cnv->preFromULength=(int8_t)match;
|
|
return TRUE;
|
|
} else if(match==1) {
|
|
/* matched, no mapping but request for <subchar1> */
|
|
cnv->useSubChar1=TRUE;
|
|
return FALSE;
|
|
} else /* match==0 no match */ {
|
|
return FALSE;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Used by ISO 2022 implementation.
|
|
* @return number of bytes in *pValue; negative number if fallback; 0 for no mapping
|
|
*/
|
|
U_CFUNC int32_t
|
|
ucnv_extSimpleMatchFromU(const int32_t *cx,
|
|
UChar32 cp, uint32_t *pValue,
|
|
UBool useFallback) {
|
|
uint32_t value;
|
|
int32_t match;
|
|
|
|
/* try to match */
|
|
match=ucnv_extMatchFromU(cx,
|
|
cp,
|
|
NULL, 0,
|
|
NULL, 0,
|
|
&value,
|
|
useFallback, TRUE);
|
|
if(match>=2) {
|
|
/* write result for simple, single-character conversion */
|
|
int32_t length;
|
|
int isRoundtrip;
|
|
|
|
isRoundtrip=UCNV_EXT_FROM_U_IS_ROUNDTRIP(value);
|
|
length=UCNV_EXT_FROM_U_GET_LENGTH(value);
|
|
value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value);
|
|
|
|
if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) {
|
|
*pValue=value;
|
|
return isRoundtrip ? length : -length;
|
|
#if 0 /* not currently used */
|
|
} else if(length==4) {
|
|
/* de-serialize a 4-byte result */
|
|
const uint8_t *result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value;
|
|
*pValue=
|
|
((uint32_t)result[0]<<24)|
|
|
((uint32_t)result[1]<<16)|
|
|
((uint32_t)result[2]<<8)|
|
|
result[3];
|
|
return isRoundtrip ? 4 : -4;
|
|
#endif
|
|
}
|
|
}
|
|
|
|
/*
|
|
* return no match because
|
|
* - match>1 && resultLength>4: result too long for simple conversion
|
|
* - match==1: no match found, <subchar1> preferred
|
|
* - match==0: no match found in the first place
|
|
* - match<0: partial match, not supported for simple conversion (and flush==TRUE)
|
|
*/
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* continue partial match with new input, requires cnv->preFromUFirstCP>=0
|
|
* never called for simple, single-character conversion
|
|
*/
|
|
U_CFUNC void
|
|
ucnv_extContinueMatchFromU(UConverter *cnv,
|
|
UConverterFromUnicodeArgs *pArgs, int32_t srcIndex,
|
|
UErrorCode *pErrorCode) {
|
|
uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */
|
|
int32_t match;
|
|
|
|
match=ucnv_extMatchFromU(cnv->sharedData->mbcs.extIndexes,
|
|
cnv->preFromUFirstCP,
|
|
cnv->preFromU, cnv->preFromULength,
|
|
pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source),
|
|
&value,
|
|
cnv->useFallback, pArgs->flush);
|
|
if(match>=2) {
|
|
match-=2; /* remove 2 for the initial code point */
|
|
|
|
if(match>=cnv->preFromULength) {
|
|
/* advance src pointer for the consumed input */
|
|
pArgs->source+=match-cnv->preFromULength;
|
|
cnv->preFromULength=0;
|
|
} else {
|
|
/* the match did not use all of preFromU[] - keep the rest for replay */
|
|
int32_t length=cnv->preFromULength-match;
|
|
u_memmove(cnv->preFromU, cnv->preFromU+match, length);
|
|
cnv->preFromULength=(int8_t)-length;
|
|
}
|
|
|
|
/* finish the partial match */
|
|
cnv->preFromUFirstCP=U_SENTINEL;
|
|
|
|
/* write result */
|
|
ucnv_extWriteFromU(cnv, cnv->sharedData->mbcs.extIndexes,
|
|
value,
|
|
&pArgs->target, pArgs->targetLimit,
|
|
&pArgs->offsets, srcIndex,
|
|
pErrorCode);
|
|
} else if(match<0) {
|
|
/* save state for partial match */
|
|
const UChar *s;
|
|
int32_t j;
|
|
|
|
/* just _append_ the newly consumed input to preFromU[] */
|
|
s=pArgs->source;
|
|
match=-match-2; /* remove 2 for the initial code point */
|
|
for(j=cnv->preFromULength; j<match; ++j) {
|
|
U_ASSERT(j>=0);
|
|
cnv->preFromU[j]=*s++;
|
|
}
|
|
pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */
|
|
cnv->preFromULength=(int8_t)match;
|
|
} else /* match==0 or 1 */ {
|
|
/*
|
|
* no match
|
|
*
|
|
* We need to split the previous input into two parts:
|
|
*
|
|
* 1. The first code point is unmappable - that's how we got into
|
|
* trying the extension data in the first place.
|
|
* We need to move it from the preFromU buffer
|
|
* to the error buffer, set an error code,
|
|
* and prepare the rest of the previous input for 2.
|
|
*
|
|
* 2. The rest of the previous input must be converted once we
|
|
* come back from the callback for the first code point.
|
|
* At that time, we have to try again from scratch to convert
|
|
* these input characters.
|
|
* The replay will be handled by the ucnv.c conversion code.
|
|
*/
|
|
|
|
if(match==1) {
|
|
/* matched, no mapping but request for <subchar1> */
|
|
cnv->useSubChar1=TRUE;
|
|
}
|
|
|
|
/* move the first code point to the error field */
|
|
cnv->fromUChar32=cnv->preFromUFirstCP;
|
|
cnv->preFromUFirstCP=U_SENTINEL;
|
|
|
|
/* mark preFromU for replay */
|
|
cnv->preFromULength=-cnv->preFromULength;
|
|
|
|
/* set the error code for unassigned */
|
|
*pErrorCode=U_INVALID_CHAR_FOUND;
|
|
}
|
|
}
|
|
|
|
static UBool
|
|
extSetUseMapping(UConverterUnicodeSet which, int32_t minLength, uint32_t value) {
|
|
if(which==UCNV_ROUNDTRIP_SET) {
|
|
// Add only code points for which the roundtrip flag is set.
|
|
// Do not add any fallbacks, even if ucnv_fromUnicode() would use them
|
|
// (fallbacks from PUA). See the API docs for ucnv_getUnicodeSet().
|
|
//
|
|
// By analogy, also do not add "good one-way" mappings.
|
|
//
|
|
// Do not add entries with reserved bits set.
|
|
if(((value&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG|UCNV_EXT_FROM_U_RESERVED_MASK))!=
|
|
UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)) {
|
|
return FALSE;
|
|
}
|
|
} else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ {
|
|
// Do not add entries with reserved bits set.
|
|
if((value&UCNV_EXT_FROM_U_RESERVED_MASK)!=0) {
|
|
return FALSE;
|
|
}
|
|
}
|
|
// Do not add <subchar1> entries or other (future?) pseudo-entries
|
|
// with an output length of 0.
|
|
return UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength;
|
|
}
|
|
|
|
static void
|
|
ucnv_extGetUnicodeSetString(const UConverterSharedData *sharedData,
|
|
const int32_t *cx,
|
|
const USetAdder *sa,
|
|
UConverterUnicodeSet which,
|
|
int32_t minLength,
|
|
UChar32 firstCP,
|
|
UChar s[UCNV_EXT_MAX_UCHARS], int32_t length,
|
|
int32_t sectionIndex,
|
|
UErrorCode *pErrorCode) {
|
|
const UChar *fromUSectionUChars;
|
|
const uint32_t *fromUSectionValues;
|
|
|
|
uint32_t value;
|
|
int32_t i, count;
|
|
|
|
fromUSectionUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar)+sectionIndex;
|
|
fromUSectionValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t)+sectionIndex;
|
|
|
|
/* read first pair of the section */
|
|
count=*fromUSectionUChars++;
|
|
value=*fromUSectionValues++;
|
|
|
|
if(extSetUseMapping(which, minLength, value)) {
|
|
if(length==U16_LENGTH(firstCP)) {
|
|
/* add the initial code point */
|
|
sa->add(sa->set, firstCP);
|
|
} else {
|
|
/* add the string so far */
|
|
sa->addString(sa->set, s, length);
|
|
}
|
|
}
|
|
|
|
for(i=0; i<count; ++i) {
|
|
/* append this code unit and recurse or add the string */
|
|
s[length]=fromUSectionUChars[i];
|
|
value=fromUSectionValues[i];
|
|
|
|
if(value==0) {
|
|
/* no mapping, do nothing */
|
|
} else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
|
|
ucnv_extGetUnicodeSetString(
|
|
sharedData, cx, sa, which, minLength,
|
|
firstCP, s, length+1,
|
|
(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value),
|
|
pErrorCode);
|
|
} else if(extSetUseMapping(which, minLength, value)) {
|
|
sa->addString(sa->set, s, length+1);
|
|
}
|
|
}
|
|
}
|
|
|
|
U_CFUNC void
|
|
ucnv_extGetUnicodeSet(const UConverterSharedData *sharedData,
|
|
const USetAdder *sa,
|
|
UConverterUnicodeSet which,
|
|
UConverterSetFilter filter,
|
|
UErrorCode *pErrorCode) {
|
|
const int32_t *cx;
|
|
const uint16_t *stage12, *stage3, *ps2, *ps3;
|
|
const uint32_t *stage3b;
|
|
|
|
uint32_t value;
|
|
int32_t st1, stage1Length, st2, st3, minLength;
|
|
|
|
UChar s[UCNV_EXT_MAX_UCHARS];
|
|
UChar32 c;
|
|
int32_t length;
|
|
|
|
cx=sharedData->mbcs.extIndexes;
|
|
if(cx==NULL) {
|
|
return;
|
|
}
|
|
|
|
stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t);
|
|
stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t);
|
|
stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t);
|
|
|
|
stage1Length=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH];
|
|
|
|
/* enumerate the from-Unicode trie table */
|
|
c=0; /* keep track of the current code point while enumerating */
|
|
|
|
if(filter==UCNV_SET_FILTER_2022_CN) {
|
|
minLength=3;
|
|
} else if( sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ||
|
|
filter!=UCNV_SET_FILTER_NONE
|
|
) {
|
|
/* DBCS-only, ignore single-byte results */
|
|
minLength=2;
|
|
} else {
|
|
minLength=1;
|
|
}
|
|
|
|
/*
|
|
* the trie enumeration is almost the same as
|
|
* in MBCSGetUnicodeSet() for MBCS_OUTPUT_1
|
|
*/
|
|
for(st1=0; st1<stage1Length; ++st1) {
|
|
st2=stage12[st1];
|
|
if(st2>stage1Length) {
|
|
ps2=stage12+st2;
|
|
for(st2=0; st2<64; ++st2) {
|
|
if((st3=(int32_t)ps2[st2]<<UCNV_EXT_STAGE_2_LEFT_SHIFT)!=0) {
|
|
/* read the stage 3 block */
|
|
ps3=stage3+st3;
|
|
|
|
do {
|
|
value=stage3b[*ps3++];
|
|
if(value==0) {
|
|
/* no mapping, do nothing */
|
|
} else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
|
|
// Recurse for partial results.
|
|
length=0;
|
|
U16_APPEND_UNSAFE(s, length, c);
|
|
ucnv_extGetUnicodeSetString(
|
|
sharedData, cx, sa, which, minLength,
|
|
c, s, length,
|
|
(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value),
|
|
pErrorCode);
|
|
} else if(extSetUseMapping(which, minLength, value)) {
|
|
switch(filter) {
|
|
case UCNV_SET_FILTER_2022_CN:
|
|
if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==3 && UCNV_EXT_FROM_U_GET_DATA(value)<=0x82ffff)) {
|
|
continue;
|
|
}
|
|
break;
|
|
case UCNV_SET_FILTER_SJIS:
|
|
if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 && (value=UCNV_EXT_FROM_U_GET_DATA(value))>=0x8140 && value<=0xeffc)) {
|
|
continue;
|
|
}
|
|
break;
|
|
case UCNV_SET_FILTER_GR94DBCS:
|
|
if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 &&
|
|
(uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfefe - 0xa1a1) &&
|
|
(uint8_t)(value-0xa1)<=(0xfe - 0xa1))) {
|
|
continue;
|
|
}
|
|
break;
|
|
case UCNV_SET_FILTER_HZ:
|
|
if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 &&
|
|
(uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfdfe - 0xa1a1) &&
|
|
(uint8_t)(value-0xa1)<=(0xfe - 0xa1))) {
|
|
continue;
|
|
}
|
|
break;
|
|
default:
|
|
/*
|
|
* UCNV_SET_FILTER_NONE,
|
|
* or UCNV_SET_FILTER_DBCS_ONLY which is handled via minLength
|
|
*/
|
|
break;
|
|
}
|
|
sa->add(sa->set, c);
|
|
}
|
|
} while((++c&0xf)!=0);
|
|
} else {
|
|
c+=16; /* empty stage 3 block */
|
|
}
|
|
}
|
|
} else {
|
|
c+=1024; /* empty stage 2 block */
|
|
}
|
|
}
|
|
}
|
|
|
|
#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
|