/* ******************************************************************************* * * Copyright (C) 2000, International Business Machines * Corporation and others. All Rights Reserved. * ******************************************************************************* * file name: ucnvmbcs.c * encoding: US-ASCII * tab size: 8 (not used) * indentation:4 * * created on: 2000jul03 * created by: Markus W. Scherer * * The current code in this file replaces the previous implementation * of conversion code from multi-byte codepages to Unicode and back. * This implementation supports the following: * - legacy variable-length codepages with up to 4 bytes per character * - all Unicode code points (up to 0x10ffff) * - efficient distinction of unassigned vs. illegal byte sequences * - it is possible in fromUnicode() to directly deal with simple * stateful encodings * - it is possible to convert Unicode code points other than U+0000 * to a single zero byte (but not as a fallback) * * Remaining limitations in fromUnicode: * - byte sequences must not have leading zero bytes * - no fallback mapping from Unicode to a zero byte * - limitation to up to 4 bytes per character */ #include "unicode/utypes.h" #include "unicode/ucnv.h" #include "unicode/ucnv_cb.h" #include "ucnv_bld.h" #include "ucnvmbcs.h" #include "ucnv_cnv.h" #include "cstring.h" /* * Converting stateless codepage data * (or codepage data with simple states) to Unicode. * * Data structure and algorithm for converting from complex legacy codepages * to Unicode. (Designed before 2000-may-22.) * * The basic idea is that the structure of legacy codepages can be described * with state tables. * When reading a byte stream, each input byte causes a state transition. * Some transitions result in the output of a code point, some result in * "unassigned" or "illegal" output. * This is used here for character conversion. * * The data structure begins with a state table consisting of a row * per state, with 256 entries (columns) per row for each possible input * byte value. * Each entry is 32 bits wide, with the lower 7 bits containing the next state. * State 0 is the initial state. * * Bit 31 of each entry indicates whether the state is * terminal (bit 31 set) or not. * * Most of the time, the offset values of subsequent states are added * up to a scalar value. This value will eventually be the index of * the Unicode code point in a table that follows the state table. * The effect is that the code points for final state table rows * are contiguous. The code points of final state rows follow each other * in the order of the references to those final states by previous * states, etc. * * For some terminal states, the offset is itself the output Unicode * code point (16 bits for a BMP code point or 20 bits for a code point * that is written as a surrogate pair). * For others, the code point in the Unicode table is stored with either * one or two code units: one for BMP code points, two for a pair of * surrogates. * All code points for a final table take up the same number of code * units, regardless of whether they all actually _use_ the same number * of code units. This is necessary for simple array access. * * An additional feature comes in with what in ICU is called "fallback" * mappings: * In addition to round-trippable, precise, 1:1 mappings, there are often * mappings defined between similar, though not the same, characters. * Typically, such mappings occur only in fromUnicode mapping tables because * Unicode has a superset repertoire of most other codepages. However, it * is possible to provide such mappings in the toUnicode tables, too. * In this case, the fallback mappings are partly integrated into the * general state tables because the structure of the encoding includes their * byte sequences. They are optional mappings when the main mapping is * "unassigned", and are looked up by the scalar offset of the main mapping * in a separate table. Only when the main mapping does not have such a * scalar offset, i.e., in the case of action codes 5 of 6 below (valid-direct), * would there need to be some different mechanism. Therefore, there are * separate action codes 3 and 4 (fallback-direct) especially for that. * The "unassigned" action code 2 cannot be used for fallback lookups because * it also does not result in a scalar offset. This means that fallback mappings * require to fit into either fallback-direct action codes or valid-single or * valid-pair codes that result in scalar offsets. * "Unassigned" really means "structurally unassigned". * * The interpretation of the bits in each entry is as follows: * * Bit 31 not set, not a terminal entry: * 30..7 offset delta, to be added up * 6..0 next state * * Bit 31 set, terminal entry: * 30..27 action code: * 0 illegal byte sequence * 26..7 not used, 0 * 1 state change only * 26..7 not used, 0 * useful for state changes in simple stateful encodings, * at Shift-In/Shift-Out codes * 2 unassigned byte sequence * 26..7 not used, 0 * this does not contain a final offset delta because the main * purpose of this action code is to save scalar offset values; * therefore, fallback values cannot be assigned to byte * sequences that result in this action code - use codes 5 or 6 * 3 valid byte sequence (fallback) * 22..7 16-bit Unicode BMP code point as fallback result * 4 valid byte sequence (fallback) * 26..7 20-bit Unicode surrogate code point as fallback result * * action codes 5, 6, 7, and 8 result in precise-mapping Unicode code points * 5 valid byte sequence * 22..7 16-bit Unicode BMP code point * never U+fffe or U+ffff (use action codes 0, 2, 3 or 4 for that) * 6 valid byte sequence * 26..7 20-bit Unicode surrogate code point * never U+fffe or U+ffff (use action codes 0, 2, 3 or 4 for that) * * action codes 7 and 8 may result in U+fffe (unassigned), in which case the * final offset is to be looked up in a special fallback table * 7 valid byte sequence * 26..16 not used, 0 * 15..7 final offset delta * pointing to one 16-bit code unit * which may be U+fffe (unassigned) or U+ffff (illegal) * 8 valid byte sequence * 26..16 not used, 0 * 15..7 final offset delta * pointing to two 16-bit code units * (UTF-16 surrogates) * the first code unit either is a lead surrogate and indicates * an assigned surrogate pair, or it is a single unit * which may be U+fffe (unassigned) or U+ffff (illegal) * (the final offset deltas are at most 255 * 2, * times 2 because of storing code unit pairs) * 9..15 reserved for future use * current implementations will only perform a state change * and ignore bits 26..7 * 6..0 next state (regardless of action code) * * An encoding with contiguous ranges of unassigned byte sequences, like * Shift-JIS and especially EUC-TW, can be stored efficiently by having * at least two states for the trail bytes: * One trail byte state that results in code points, and one that only * has "unassigned" and "illegal" terminal states. * * Note: partly by accident, this data structure supports simple stateless * encodings without any additional logic. * Especially simple Shift-In/Shift-Out schemes could be handled with * appropriate state tables (especially EBCDIC_STATEFUL!). */ /* prototypes --------------------------------------------------------------- */ U_CFUNC void _MBCSSingleToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, UErrorCode *pErrorCode); U_CFUNC UChar32 _MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, uint8_t b, UBool useFallback); U_CFUNC void _MBCSSingleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, UErrorCode *pErrorCode); static void fromUCallback(UConverter *cnv, void *context, UConverterFromUnicodeArgs *pArgs, const UChar *codeUnits, int32_t length, UChar32 codePoint, UConverterCallbackReason reason, UErrorCode *pErrorCode); static void toUCallback(UConverter *cnv, void *context, UConverterToUnicodeArgs *pArgs, const char *codeUnits, int32_t length, UConverterCallbackReason reason, UErrorCode *pErrorCode); /* GB 18030 data ------------------------------------------------------------ */ /* helper macros for linear values for GB 18030 four-byte sequences */ #define LINEAR_18030(a, b, c, d) ((((a)*10+(b))*126L+(c))*10L+(d)) #define LINEAR_18030_BASE LINEAR_18030(0x81, 0x30, 0x81, 0x30) #define LINEAR(x) LINEAR_18030(x>>24, (x>>16)&0xff, (x>>8)&0xff, x&0xff) /* * Some ranges of GB 18030 where both the Unicode code points and the * GB four-byte sequences are contiguous and are handled algorithmically by * the special callback functions below. * The values are start & end of Unicode & GB codes. * * Note that single surrogates are not mapped by GB 18030 * as of the re-released mapping tables from 2000-nov-30. */ static const uint32_t gb18030Ranges[13][4]={ 0x10000, 0x10FFFF, LINEAR(0x90308130), LINEAR(0xE3329A35), 0x9FA6, 0xD7FF, LINEAR(0x82358F33), LINEAR(0x8336C738), 0x0452, 0x200F, LINEAR(0x8130D330), LINEAR(0x8136A531), 0xE865, 0xF92B, LINEAR(0x8336D030), LINEAR(0x84308534), 0x2643, 0x2E80, LINEAR(0x8137A839), LINEAR(0x8138FD38), 0xFA2A, 0xFE2F, LINEAR(0x84309C38), LINEAR(0x84318537), 0x3CE1, 0x4055, LINEAR(0x8231D438), LINEAR(0x8232AF32), 0x361B, 0x3917, LINEAR(0x8230A633), LINEAR(0x8230F237), 0x49B8, 0x4C76, LINEAR(0x8234A131), LINEAR(0x8234E733), 0x4160, 0x4336, LINEAR(0x8232C937), LINEAR(0x8232F837), 0x478E, 0x4946, LINEAR(0x8233E838), LINEAR(0x82349638), 0x44D7, 0x464B, LINEAR(0x8233A339), LINEAR(0x8233C931), 0xFFE6, 0xFFFF, LINEAR(0x8431A234), LINEAR(0x8431A439) }; /* MBCS setup functions ----------------------------------------------------- */ U_CFUNC void _MBCSLoad(UConverterSharedData *sharedData, const uint8_t *raw, UErrorCode *pErrorCode) { UConverterMBCSTable *mbcsTable=&sharedData->table->mbcs; _MBCSHeader *header=(_MBCSHeader *)raw; if(header->version[0]!=1) { *pErrorCode=U_INVALID_TABLE_FORMAT; return; } mbcsTable->countStates=(uint8_t)header->countStates; mbcsTable->countToUFallbacks=header->countToUFallbacks; mbcsTable->stateTable=(const int32_t (*)[256])(raw+sizeof(_MBCSHeader)); mbcsTable->toUFallbacks=(const _MBCSToUFallback *)(mbcsTable->stateTable+header->countStates); mbcsTable->unicodeCodeUnits=(const uint16_t *)(raw+header->offsetToUCodeUnits); mbcsTable->fromUnicodeTable=(const uint16_t *)(raw+header->offsetFromUTable); mbcsTable->fromUnicodeBytes=(const uint8_t *)(raw+header->offsetFromUBytes); mbcsTable->outputType=(uint8_t)header->flags; } U_CFUNC void _MBCSReset(UConverter *cnv) { /* toUnicode */ cnv->toUnicodeStatus=0; cnv->mode=0; cnv->toULength=0; /* fromUnicode */ cnv->fromUSurrogateLead=0; } U_CFUNC void _MBCSOpen(UConverter *cnv, const char *name, const char *locale, uint32_t options, UErrorCode *pErrorCode) { _MBCSReset(cnv); if(uprv_strstr(name, "gb18030")!=NULL || uprv_strstr(name, "GB18030")!=NULL) { /* set a flag for GB 18030 mode, which changes the callback behavior */ cnv->extraInfo=(void *)gb18030Ranges; } } /* MBCS-to-Unicode conversion functions ------------------------------------- */ static UChar32 _MBCSGetFallback(UConverterMBCSTable *mbcsTable, uint32_t offset) { const _MBCSToUFallback *toUFallbacks; uint32_t i, start, limit; limit=mbcsTable->countToUFallbacks; if(limit>0) { /* do a binary search for the fallback mapping */ toUFallbacks=mbcsTable->toUFallbacks; start=0; while(startconverter; if(cnv->sharedData->table->mbcs.countStates==1) { _MBCSSingleToUnicodeWithOffsets(pArgs, pErrorCode); return; } /* set up the local pointers */ source=(const uint8_t *)pArgs->source; sourceLimit=(const uint8_t *)pArgs->sourceLimit; target=pArgs->target; targetLimit=pArgs->targetLimit; offsets=pArgs->offsets; stateTable=cnv->sharedData->table->mbcs.stateTable; unicodeCodeUnits=cnv->sharedData->table->mbcs.unicodeCodeUnits; /* get the converter state from UConverter */ offset=cnv->toUnicodeStatus; state=(uint8_t)(cnv->mode); byteIndex=cnv->toULength; bytes=cnv->toUBytes; /* sourceIndex=-1 if the current character began in the previous buffer */ sourceIndex=byteIndex==0 ? 0 : -1; nextSourceIndex=0; /* conversion loop */ while(source=0) { /* * bit 31 is not set, bits: * 30..7 offset delta * 6..0 next state */ state=(uint8_t)(entry&0x7f); offset+=entry>>7; } else { /* * bit 31 is set, bits: * 30..27 action code * (do not mask out bit 31 for speed, include it in action values) * 26..7 depend on the action code * 6..0 next state */ /* set the next state early so that we can reuse the entry variable */ state=(uint8_t)(entry&0x7f); /* typically 0 */ /* switch per action code */ switch((uint32_t)entry>>27U) { case 16|MBCS_STATE_ILLEGAL: /* bits 26..7 are not used, 0 */ /* callback(illegal) */ reason=UCNV_ILLEGAL; *pErrorCode=U_ILLEGAL_CHAR_FOUND; goto callback; case 16|MBCS_STATE_CHANGE_ONLY: /* bits 26..7 are not used, 0 */ /* * This serves as a state change without any output. * It is useful for reading simple stateful encodings, * for example using just Shift-In/Shift-Out codes. * The 21 unused bits may later be used for more sophisticated * state transitions. */ break; case 16|MBCS_STATE_UNASSIGNED: /* bits 26..7 are not used, 0 */ /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; case 16|MBCS_STATE_FALLBACK_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ if(!UCNV_TO_U_USE_FALLBACK(cnv)) { /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } /* fall through to the MBCS_STATE_VALID_DIRECT_16 branch */ case 16|MBCS_STATE_VALID_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ /* output BMP code point */ *target++=(UChar)(entry>>7); if(offsets!=NULL) { *offsets++=sourceIndex; } break; case 16|MBCS_STATE_FALLBACK_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ if(!UCNV_TO_U_USE_FALLBACK(cnv)) { /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } /* fall through to the MBCS_STATE_VALID_DIRECT_20 branch */ case 16|MBCS_STATE_VALID_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ entry=(entry>>7)&0xfffff; /* output surrogate pair */ *target++=(UChar)(0xd800|(UChar)(entry>>10)); if(offsets!=NULL) { *offsets++=sourceIndex; } c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); if(targetUCharErrorBuffer[0]=c; cnv->UCharErrorBufferLength=1; *pErrorCode=U_BUFFER_OVERFLOW_ERROR; offset=0; byteIndex=0; goto endloop; } break; case 16|MBCS_STATE_VALID_16: /* bits 26..16 are not used, 0 */ /* bits 15..7 contain the final offset delta to one 16-bit code unit */ offset+=(uint16_t)entry>>7; c=unicodeCodeUnits[offset]; if(c<0xfffe) { /* output BMP code point */ *target++=c; if(offsets!=NULL) { *offsets++=sourceIndex; } } else if(c==0xfffe) { if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)_MBCSGetFallback(&cnv->sharedData->table->mbcs, offset))!=0xfffe) { goto output32; } /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } else { /* callback(illegal) */ reason=UCNV_ILLEGAL; *pErrorCode=U_ILLEGAL_CHAR_FOUND; goto callback; } break; case 16|MBCS_STATE_VALID_16_PAIR: /* bits 26..16 are not used, 0 */ /* bits 15..7 contain the final offset delta to two 16-bit code units */ offset+=(uint16_t)entry>>7; c=unicodeCodeUnits[offset++]; if(UTF_IS_FIRST_SURROGATE(c)) { *target++=c; if(offsets!=NULL) { *offsets++=sourceIndex; } if(targetUCharErrorBuffer[0]=unicodeCodeUnits[offset]; cnv->UCharErrorBufferLength=1; *pErrorCode=U_BUFFER_OVERFLOW_ERROR; offset=0; byteIndex=0; goto endloop; } } else if(c<0xfffe) { /* output BMP code point */ *target++=c; if(offsets!=NULL) { *offsets++=sourceIndex; } } else if(c==0xfffe) { /* * For the fallback, we need to restore the offset that * we had before the unicodeCodeUnits[offset++] above that incremented it! */ if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)_MBCSGetFallback(&cnv->sharedData->table->mbcs, offset-1))!=0xfffe) { goto output32; } /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } else { /* callback(illegal) */ reason=UCNV_ILLEGAL; *pErrorCode=U_ILLEGAL_CHAR_FOUND; goto callback; } break; default: /* reserved, must never occur */ /* bits 26..7 are not used, 0 */ break; } /* normal end of action codes: prepare for a new character */ offset=0; byteIndex=0; sourceIndex=nextSourceIndex; continue; /* * Markus Scherer 2000-jul-05 * * The following is extremely ugly, and I apologize for it: * Several places in the above switch statement need to call * a callback function or output a 32-bit code point, * each of which is an involved process with * a couple dozen of statements. * * I could do this in a function call, but I fear that then * the compiler does not keep the frequently used variables in * registers because the function call would need them on the stack * for input and output. * * I could do this with a macro, but that is harder to debug and * bloats the compiled code. * * I could just copy and paste the code, but that would also bloat * the program size, make the pieces harder to maintain, and make * the switch statement extremely long and clumsy. * * Therefore, those places goto here and do it all in one place, * while the normal processing has a continue above and skips this * part. * This actually _saves_ goto statements, too: * Since it is not possible in C to break a loop from within a switch * statement, the callback code in the switch statement would have to * goto behind the loop. Here, it can break if necessary. */ output32: /* output a 32-bit (21-bit) Unicode code point stored in entry */ if(entry<=0xffff) { /* output BMP code point */ *target++=(UChar)entry; if(offsets!=NULL) { *offsets++=sourceIndex; } } else { /* output surrogate pair */ *target++=(UChar)(0xd7c0+(entry>>10)); if(offsets!=NULL) { *offsets++=sourceIndex; } c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); if(targetUCharErrorBuffer[0]=c; cnv->UCharErrorBufferLength=1; *pErrorCode=U_BUFFER_OVERFLOW_ERROR; offset=0; byteIndex=0; break; } } /* same as normal end of action codes: prepare for a new character */ offset=0; byteIndex=0; sourceIndex=nextSourceIndex; continue; callback: /* call the callback function with all the preparations and post-processing */ /* update the arguments structure */ pArgs->source=(const char *)source; pArgs->target=target; pArgs->offsets=offsets; /* copy the current bytes to invalidCharBuffer */ for(b=0; b<(uint8_t)byteIndex; ++b) { cnv->invalidCharBuffer[b]=(char)bytes[b]; } cnv->invalidCharLength=byteIndex; /* set the converter state in UConverter to deal with the next character */ cnv->toUnicodeStatus=0; cnv->mode=state; cnv->toULength=0; /* call the callback function */ toUCallback(cnv, cnv->toUContext, pArgs, (const char *)bytes, byteIndex, reason, pErrorCode); /* get the converter state from UConverter */ offset=cnv->toUnicodeStatus; state=(uint8_t)cnv->mode; byteIndex=cnv->toULength; /* update target and deal with offsets if necessary */ offsets=ucnv_updateCallbackOffsets(offsets, pArgs->target-target, sourceIndex); target=pArgs->target; /* update the source pointer and index */ sourceIndex=nextSourceIndex+((const uint8_t *)pArgs->source-source); source=(const uint8_t *)pArgs->source; /* * If the callback overflowed the target, then we need to * stop here with an overflow indication. */ if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { break; } else if(cnv->UCharErrorBufferLength>0) { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } else if(U_FAILURE(*pErrorCode)) { /* break on error */ offset=0; state=0; byteIndex=0; break; } /* * We do not need to repeat the statements from the normal * end of the action codes because we already updated all the * necessary variables. */ } } else { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } } endloop: if(pArgs->flush && source>=sourceLimit) { /* reset the state for the next conversion */ if(byteIndex>0 && U_SUCCESS(*pErrorCode)) { /* a character byte sequence remains incomplete */ *pErrorCode=U_TRUNCATED_CHAR_FOUND; } cnv->toUnicodeStatus=0; cnv->mode=0; cnv->toULength=0; } else { /* set the converter state back into UConverter */ cnv->toUnicodeStatus=offset; cnv->mode=state; cnv->toULength=byteIndex; } /* write back the updated pointers */ pArgs->source=(const char *)source; pArgs->target=target; pArgs->offsets=offsets; } U_CFUNC void _MBCSToUnicode(UConverterToUnicodeArgs *pArgs, UErrorCode *pErrorCode) { _MBCSToUnicodeWithOffsets(pArgs, pErrorCode); } /* This version of _MBCSToUnicode() is optimized for single-byte, single-state codepages. */ U_CFUNC void _MBCSSingleToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, UErrorCode *pErrorCode) { /* set up the local pointers */ UConverter *cnv; const uint8_t *source, *sourceLimit; UChar *target; const UChar *targetLimit; int32_t *offsets; const int32_t (*stateTable)[256]; int32_t sourceIndex, nextSourceIndex; int32_t entry; UChar c; uint8_t b; UConverterCallbackReason reason; /* set up the local pointers */ cnv=pArgs->converter; source=(const uint8_t *)pArgs->source; sourceLimit=(const uint8_t *)pArgs->sourceLimit; target=pArgs->target; targetLimit=pArgs->targetLimit; offsets=pArgs->offsets; stateTable=cnv->sharedData->table->mbcs.stateTable; /* sourceIndex=-1 if the current character began in the previous buffer */ sourceIndex=0; nextSourceIndex=0; /* conversion loop */ while(source>27U) { case 16|MBCS_STATE_ILLEGAL: /* bits 26..7 are not used, 0 */ /* callback(illegal) */ reason=UCNV_ILLEGAL; *pErrorCode=U_ILLEGAL_CHAR_FOUND; goto callback; case 16|MBCS_STATE_UNASSIGNED: /* bits 26..7 are not used, 0 */ /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; case 16|MBCS_STATE_FALLBACK_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ if(!UCNV_TO_U_USE_FALLBACK(cnv)) { /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } /* fall through to the MBCS_STATE_VALID_DIRECT_16 branch */ case 16|MBCS_STATE_VALID_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ /* output BMP code point */ *target++=(UChar)(entry>>7); if(offsets!=NULL) { *offsets++=sourceIndex; } break; case 16|MBCS_STATE_FALLBACK_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ if(!UCNV_TO_U_USE_FALLBACK(cnv)) { /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } /* fall through to the MBCS_STATE_VALID_DIRECT_20 branch */ case 16|MBCS_STATE_VALID_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ entry=(entry>>7)&0xfffff; /* output surrogate pair */ *target++=(UChar)(0xd800|(UChar)(entry>>10)); if(offsets!=NULL) { *offsets++=sourceIndex; } c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); if(targetUCharErrorBuffer[0]=c; cnv->UCharErrorBufferLength=1; *pErrorCode=U_BUFFER_OVERFLOW_ERROR; goto endloop; } break; default: /* reserved, must never occur */ /* bits 26..7 are not used, 0 */ break; } /* normal end of action codes: prepare for a new character */ sourceIndex=nextSourceIndex; continue; callback: /* call the callback function with all the preparations and post-processing */ /* update the arguments structure */ pArgs->source=(const char *)source; pArgs->target=target; pArgs->offsets=offsets; /* copy the current bytes to invalidCharBuffer */ cnv->invalidCharBuffer[0]=b; cnv->invalidCharLength=1; /* call the callback function */ toUCallback(cnv, cnv->toUContext, pArgs, (const char *)&b, 1, reason, pErrorCode); /* update target and deal with offsets if necessary */ offsets=ucnv_updateCallbackOffsets(offsets, pArgs->target-target, sourceIndex); target=pArgs->target; /* update the source pointer and index */ sourceIndex=nextSourceIndex+((const uint8_t *)pArgs->source-source); source=(const uint8_t *)pArgs->source; /* * If the callback overflowed the target, then we need to * stop here with an overflow indication. */ if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { break; } else if(cnv->UCharErrorBufferLength>0) { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } else if(U_FAILURE(*pErrorCode)) { /* break on error */ break; } /* * We do not need to repeat the statements from the normal * end of the action codes because we already updated all the * necessary variables. */ } else { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } } endloop: /* write back the updated pointers */ pArgs->source=(const char *)source; pArgs->target=target; pArgs->offsets=offsets; } /* * This is a simple, interim implementation of GetNextUChar() * that allows to concentrate on testing one single implementation * of the ToUnicode conversion before it gets copied to * multiple version that are then optimized for their needs * (with vs. without offsets and getNextUChar). * ### TODO: implement this directly similar to ToUnicode() */ U_CFUNC UChar32 _MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, UErrorCode *pErrorCode) { UChar buffer[UTF_MAX_CHAR_LENGTH]; const char *realLimit=pArgs->sourceLimit; pArgs->target=buffer; pArgs->targetLimit=buffer+UTF_MAX_CHAR_LENGTH; while(pArgs->sourcesourceLimit=pArgs->source+1; pArgs->flush= (UBool)(pArgs->sourceLimit==realLimit); _MBCSToUnicode(pArgs, pErrorCode); if(U_FAILURE(*pErrorCode) && *pErrorCode!=U_BUFFER_OVERFLOW_ERROR) { return 0xffff; } else { int32_t length=pArgs->target-buffer; #if 0 /* * markus 2000-oct-26 * * This version of the exit condition is commented out because of * a clarification of the semantics of ucnv_getNextUChar() (see updated javadoc): * * Codepages that provide direct encodings of supplementary Unicode code points (U+10000 and up) * should return single surrogates without combining them into pairs if single surrogates * are encoded. This group of codepages includes UTF-8, UTF-32, and GB 18030. * * Codepages that provide direct encodings only of single surrogates * must attempt to match pairs of them into supplementary code points. * Single surrogates are returned only if they are not part of matched pairs. * This group of codepages includes SCSU, LMBCS, and UTF-16. * * Currently, there is no MBCS codepage in the second group. SCSU, LMBCS, and UTF-16 * are implemented with separate code. * * Therefore, this feature is removed here. * It might need to be added back in later when some MBCS codepages are created that * fall into the second group. In this case, a flag in the .cnv file will be necessary * to indicate this. makeconv would need to set this flag based on whether the codepage * contains only mappings for single surrogates but * not directly for any supplementary code points. */ if(/* some output and (source consumed or not a surrogate or a surrogate pair [UTF-16 specific]) */ length>0 && (pArgs->flush || !UTF_IS_FIRST_SURROGATE(buffer[0]) || length==2) #endif if(length>0) { if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { *pErrorCode=U_ZERO_ERROR; } return ucnv_getUChar32KeepOverflow(pArgs->converter, buffer, length); } } } /* no output because of empty input or only state changes and skipping callbacks */ *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; return 0xffff; } /* * This is a simple version of getNextUChar() that is used * by other converter implementations. * It does not use state from the converter, nor error codes. * * Return value: * U+fffe unassigned * U+ffff illegal * otherwise the Unicode code point */ U_CFUNC UChar32 _MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, const char **pSource, const char *sourceLimit, UBool useFallback) { const uint8_t *source; const int32_t (*stateTable)[256]; const uint16_t *unicodeCodeUnits; uint32_t offset; uint8_t state; int32_t entry; /* set up the local pointers */ source=(const uint8_t *)*pSource; if(source>=(const uint8_t *)sourceLimit) { /* no input at all: "unassigned" */ return 0xfffe; } /* use optimized function if possible */ if(sharedData->table->mbcs.countStates==1) { return _MBCSSingleSimpleGetNextUChar(sharedData, (uint8_t)(*(*pSource)++), useFallback); } stateTable=sharedData->table->mbcs.stateTable; unicodeCodeUnits=sharedData->table->mbcs.unicodeCodeUnits; /* converter state */ offset=0; state=0; /* conversion loop */ do { entry=stateTable[state][*source++]; if(entry>=0) { /* * bit 31 is not set, bits: * 30..7 offset delta * 6..0 next state */ state=(uint8_t)(entry&0x7f); offset+=entry>>7; } else { /* * bit 31 is set, bits: * 30..27 action code * (do not mask out bit 31 for speed, include it in action values) * 26..7 depend on the action code * 6..0 next state */ *pSource=(const char *)source; /* switch per action code */ switch((uint32_t)entry>>27U) { case 16|MBCS_STATE_ILLEGAL: /* bits 26..7 are not used, 0 */ return 0xffff; case 16|MBCS_STATE_CHANGE_ONLY: /* bits 26..7 are not used, 0 */ /* * This serves as a state change without any output. * It is useful for reading simple stateful encodings, * for example using just Shift-In/Shift-Out codes. * The 21 unused bits may later be used for more sophisticated * state transitions. */ if(source==(const uint8_t *)sourceLimit) { /* if there are only state changes, then return "unassigned" */ return 0xfffe; } break; case 16|MBCS_STATE_UNASSIGNED: /* bits 26..7 are not used, 0 */ return 0xfffe; case 16|MBCS_STATE_FALLBACK_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ if(!TO_U_USE_FALLBACK(useFallback)) { return 0xfffe; } /* fall through to the MBCS_STATE_VALID_DIRECT_16 branch */ case 16|MBCS_STATE_VALID_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ /* output BMP code point */ return (UChar)(entry>>7); case 16|MBCS_STATE_FALLBACK_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ if(!TO_U_USE_FALLBACK(useFallback)) { return 0xfffe; } /* fall through to the MBCS_STATE_VALID_DIRECT_20 branch */ case 16|MBCS_STATE_VALID_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ return 0x10000+((entry>>7)&0xfffff); case 16|MBCS_STATE_VALID_16: /* bits 26..16 are not used, 0 */ /* bits 15..7 contain the final offset delta to one 16-bit code unit */ offset+=(uint16_t)entry>>7; entry=unicodeCodeUnits[offset]; if(entry!=0xfffe) { return (UChar32)entry; } else { return _MBCSGetFallback(&sharedData->table->mbcs, offset); } case 16|MBCS_STATE_VALID_16_PAIR: /* bits 26..16 are not used, 0 */ /* bits 15..7 contain the final offset delta to two 16-bit code units */ offset+=(uint16_t)entry>>7; entry=unicodeCodeUnits[offset++]; if(UTF_IS_FIRST_SURROGATE(entry)) { return UTF16_GET_PAIR_VALUE(entry, unicodeCodeUnits[offset]); } else if(entry!=0xfffe) { /* output BMP code point */ return (UChar32)entry; } else { /* * For the fallback, we need to restore the offset that * we had before the unicodeCodeUnits[offset++] above that incremented it! */ return _MBCSGetFallback(&sharedData->table->mbcs, offset-1); } default: /* reserved, must never occur */ /* bits 26..7 are not used, 0 */ break; } /* state change only - prepare for a new character */ state=(uint8_t)(entry&0x7f); /* typically 0 */ offset=0; } } while(source<(const uint8_t *)sourceLimit); *pSource=(const char *)source; return 0xffff; } /* This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. */ U_CFUNC UChar32 _MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, uint8_t b, UBool useFallback) { int32_t entry; entry=sharedData->table->mbcs.stateTable[0][b]; /* entry<0 */ /* * bit 31 is set, bits: * 30..27 action code * (do not mask out bit 31 for speed, include it in action values) * 26..7 depend on the action code * 6..0 next state */ /* switch per action code */ switch((uint32_t)entry>>27U) { case 16|MBCS_STATE_ILLEGAL: /* bits 26..7 are not used, 0 */ return 0xffff; case 16|MBCS_STATE_UNASSIGNED: /* bits 26..7 are not used, 0 */ return 0xfffe; case 16|MBCS_STATE_FALLBACK_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ if(!TO_U_USE_FALLBACK(useFallback)) { return 0xfffe; } /* fall through to the MBCS_STATE_VALID_DIRECT_16 branch */ case 16|MBCS_STATE_VALID_DIRECT_16: /* bits 26..23 are not used, 0 */ /* bits 22..7 contain the Unicode BMP code point */ /* output BMP code point */ return (UChar)(entry>>7); case 16|MBCS_STATE_FALLBACK_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ if(!TO_U_USE_FALLBACK(useFallback)) { return 0xfffe; } /* fall through to the MBCS_STATE_VALID_DIRECT_20 branch */ case 16|MBCS_STATE_VALID_DIRECT_20: /* bits 26..7 contain the Unicode surrogate code point minus 0x10000 */ return 0x10000+((entry>>7)&0xfffff); default: /* reserved, must never occur */ /* bits 26..7 are not used, 0 */ return 0xffff; } } /* MBCS-from-Unicode conversion functions ----------------------------------- */ U_CFUNC void _MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, UErrorCode *pErrorCode) { UConverter *cnv; const UChar *source, *sourceLimit; uint8_t *target; int32_t targetCapacity; int32_t *offsets; const uint16_t *table; const uint8_t *bytes; uint8_t outputType; UChar32 c; int32_t sourceIndex, nextSourceIndex; UConverterCallbackReason reason; uint32_t i; uint32_t value; int32_t length; /* use optimized function if possible */ cnv=pArgs->converter; outputType=cnv->sharedData->table->mbcs.outputType; if(outputType==MBCS_OUTPUT_1) { _MBCSSingleFromUnicodeWithOffsets(pArgs, pErrorCode); return; } /* set up the local pointers */ source=pArgs->source; sourceLimit=pArgs->sourceLimit; target=(uint8_t *)pArgs->target; targetCapacity=pArgs->targetLimit-pArgs->target; offsets=pArgs->offsets; table=cnv->sharedData->table->mbcs.fromUnicodeTable; bytes=cnv->sharedData->table->mbcs.fromUnicodeBytes; /* get the converter state from UConverter */ c=cnv->fromUSurrogateLead; /* sourceIndex=-1 if the current character began in the previous buffer */ sourceIndex= c==0 ? 0 : -1; nextSourceIndex=0; /* conversion loop */ /* * This is another piece of ugly code: * A goto into the loop if the converter state contains a first surrogate * from the previous function call. * It saves me to check in each loop iteration a check of if(c==0) * and duplicating the trail-surrogate-handling code in the else * branch of that check. * I could not find any other way to get around this other than * using a function call for the conversion and callback, which would * be even more inefficient. * * Markus Scherer 2000-jul-19 */ if(c!=0 && targetCapacity>0) { goto getTrail; } while(source0) { /* * Get a correct Unicode code point: * a single UChar for a BMP code point or * a matched surrogate pair for a "surrogate code point". */ c=*source++; ++nextSourceIndex; if(UTF_IS_SURROGATE(c)) { if(UTF_IS_SURROGATE_FIRST(c)) { getTrail: if(source>10]+((c>>4)&0x3f)); /* is this code point assigned, or do we use fallbacks? */ if((table[i++]&(1<<(c&0xf)))!=0 || UCNV_FROM_U_USE_FALLBACK(cnv, c)) { const uint8_t *p=bytes; /* get the bytes and the length for the output */ switch(outputType) { case MBCS_OUTPUT_1: p+=(16*(uint32_t)table[i]+(c&0xf)); value=*p; length=1; break; case MBCS_OUTPUT_2: p+=(16*(uint32_t)table[i]+(c&0xf))*2; # if U_IS_BIG_ENDIAN value=*(uint16_t *)p; # else value=((uint32_t)*p<<8)|p[1]; # endif if(value<=0xff) { length=1; } else { length=2; } break; case MBCS_OUTPUT_3: p+=(16*(uint32_t)table[i]+(c&0xf))*3; value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; if(value<=0xff) { length=1; } else if(value<=0xffff) { length=2; } else { length=3; } break; case MBCS_OUTPUT_4: p+=(16*(uint32_t)table[i]+(c&0xf))*4; # if U_IS_BIG_ENDIAN value=*(uint32_t *)p; # else value=((uint32_t)*p<<24)|((uint32_t)p[1]<<16)|((uint32_t)p[2]<<8)|p[3]; # endif if(value<=0xff) { length=1; } else if(value<=0xffff) { length=2; } else if(value<=0xffffff) { length=3; } else { length=4; } break; case MBCS_OUTPUT_3_EUC: p+=(16*(uint32_t)table[i]+(c&0xf))*2; # if U_IS_BIG_ENDIAN value=*(uint16_t *)p; # else value=((uint32_t)*p<<8)|p[1]; # endif /* EUC 16-bit fixed-length representation */ if(value<=0xff) { length=1; } else if((value&0x8000)==0) { value|=0x8e8000; length=3; } else if((value&0x80)==0) { value|=0x8f0080; length=3; } else { length=2; } break; case MBCS_OUTPUT_4_EUC: p+=(16*(uint32_t)table[i]+(c&0xf))*3; value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; /* EUC 16-bit fixed-length representation applied to the first two bytes */ if(value<=0xff) { length=1; } else if(value<=0xffff) { length=2; } else if((value&0x800000)==0) { value|=0x8e800000; length=4; } else if((value&0x8000)==0) { value|=0x8f008000; length=4; } else { length=3; } break; default: /* must not occur */ /* * To avoid compiler warnings that value & length may be * used without having been initialized, we set them here. * In reality, this is unreachable code. * Not having a default branch also causes warnings with * some compilers. */ value=0; length=0; break; } /* is the codepage value really an "unassigned" indicator? */ if(value==0 && c!=0 && (table[i-1]&(1<<(c&0xf)))==0) { /* * We allow a 0 byte output if the Unicode code point is * U+0000 and also if the "assigned" bit is set for this entry. * There is no way with this data structure for fallback output * for other than U+0000 to be a zero byte. */ /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } } else { /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } /* write the output character bytes from value and length */ /* from the first if in the loop we know that targetCapacity>0 */ if(length<=targetCapacity) { if(offsets==NULL) { switch(length) { /* each branch falls through to the next one */ case 4: *target++=(uint8_t)(value>>24); case 3: *target++=(uint8_t)(value>>16); case 2: *target++=(uint8_t)(value>>8); case 1: *target++=(uint8_t)value; default: /* will never occur */ break; } } else { switch(length) { /* each branch falls through to the next one */ case 4: *target++=(uint8_t)(value>>24); *offsets++=sourceIndex; case 3: *target++=(uint8_t)(value>>16); *offsets++=sourceIndex; case 2: *target++=(uint8_t)(value>>8); *offsets++=sourceIndex; case 1: *target++=(uint8_t)value; *offsets++=sourceIndex; default: /* will never occur */ break; } } targetCapacity-=length; } else { uint8_t *p; /* * We actually do this backwards here: * In order to save an intermediate variable, we output * first to the overflow buffer what does not fit into the * regular target. */ /* we know that 1<=targetCapacitycharErrorBuffer; switch(length) { /* each branch falls through to the next one */ case 3: *p++=(uint8_t)(value>>16); case 2: *p++=(uint8_t)(value>>8); case 1: *p=(uint8_t)value; default: /* will never occur */ break; } cnv->charErrorBufferLength=(int8_t)length; /* now output what fits into the regular target */ value>>=8*length; /* length was reduced by targetCapacity */ switch(targetCapacity) { /* each branch falls through to the next one */ case 3: *target++=(uint8_t)(value>>16); if(offsets!=NULL) { *offsets++=sourceIndex; } case 2: *target++=(uint8_t)(value>>8); if(offsets!=NULL) { *offsets++=sourceIndex; } case 1: *target++=(uint8_t)value; if(offsets!=NULL) { *offsets++=sourceIndex; } default: /* will never occur */ break; } /* target overflow */ targetCapacity=0; *pErrorCode=U_BUFFER_OVERFLOW_ERROR; c=0; break; } /* normal end of conversion: prepare for a new character */ c=0; sourceIndex=nextSourceIndex; continue; /* * This is the same ugly trick as in ToUnicode(), for the * same reasons... */ callback: /* call the callback function with all the preparations and post-processing */ /* update the arguments structure */ pArgs->source=source; pArgs->target=(char *)target; pArgs->offsets=offsets; /* set the converter state in UConverter to deal with the next character */ cnv->fromUSurrogateLead=0; /* write the code point as code units */ i=0; UTF_APPEND_CHAR_UNSAFE(cnv->invalidUCharBuffer, i, c); cnv->invalidUCharLength=(int8_t)i; /* call the callback function */ fromUCallback(cnv, cnv->fromUContext, pArgs, cnv->invalidUCharBuffer, i, c, reason, pErrorCode); /* get the converter state from UConverter */ c=cnv->fromUSurrogateLead; /* update target and deal with offsets if necessary */ offsets=ucnv_updateCallbackOffsets(offsets, ((uint8_t *)pArgs->target)-target, sourceIndex); target=(uint8_t *)pArgs->target; /* update the source pointer and index */ sourceIndex=nextSourceIndex+(pArgs->source-source); source=pArgs->source; targetCapacity=(uint8_t *)pArgs->targetLimit-target; /* * If the callback overflowed the target, then we need to * stop here with an overflow indication. */ if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { break; } else if(cnv->charErrorBufferLength>0) { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } else if(U_FAILURE(*pErrorCode)) { /* break on error */ c=0; break; } /* * We do not need to repeat the statements from the normal * end of the conversion because we already updated all the * necessary variables. */ } else { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } } if(pArgs->flush && source>=sourceLimit) { /* reset the state for the next conversion */ if(c!=0 && U_SUCCESS(*pErrorCode)) { /* a character byte sequence remains incomplete */ *pErrorCode=U_TRUNCATED_CHAR_FOUND; } cnv->fromUSurrogateLead=0; } else { /* set the converter state back into UConverter */ cnv->fromUSurrogateLead=(UChar)c; } /* write back the updated pointers */ pArgs->source=source; pArgs->target=(char *)target; pArgs->offsets=offsets; } U_CFUNC void _MBCSFromUnicode(UConverterFromUnicodeArgs *pArgs, UErrorCode *pErrorCode) { _MBCSFromUnicodeWithOffsets(pArgs, pErrorCode); } /* This version of _MBCSFromUnicode() is optimized for single-byte codepages. */ U_CFUNC void _MBCSSingleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, UErrorCode *pErrorCode) { UConverter *cnv; const UChar *source, *sourceLimit; uint8_t *target; int32_t targetCapacity; int32_t *offsets; const uint16_t *table; const uint8_t *bytes; uint8_t outputType; UChar32 c; int32_t sourceIndex, nextSourceIndex; UConverterCallbackReason reason; uint32_t i; uint32_t value; /* set up the local pointers */ cnv=pArgs->converter; source=pArgs->source; sourceLimit=pArgs->sourceLimit; target=(uint8_t *)pArgs->target; targetCapacity=pArgs->targetLimit-pArgs->target; offsets=pArgs->offsets; table=cnv->sharedData->table->mbcs.fromUnicodeTable; bytes=cnv->sharedData->table->mbcs.fromUnicodeBytes; outputType=cnv->sharedData->table->mbcs.outputType; /* get the converter state from UConverter */ c=cnv->fromUSurrogateLead; /* sourceIndex=-1 if the current character began in the previous buffer */ sourceIndex= c==0 ? 0 : -1; nextSourceIndex=0; /* conversion loop */ if(c!=0 && targetCapacity>0) { goto getTrail; } while(source0) { /* * Get a correct Unicode code point: * a single UChar for a BMP code point or * a matched surrogate pair for a "surrogate code point". */ c=*source++; ++nextSourceIndex; if(UTF_IS_SURROGATE(c)) { if(UTF_IS_SURROGATE_FIRST(c)) { getTrail: if(source>10]+((c>>4)&0x3f)); /* is this code point assigned, or do we use fallbacks? */ if((table[i++]&(1<<(c&0xf)))!=0 || UCNV_FROM_U_USE_FALLBACK(cnv, c)) { const uint8_t *p=bytes; /* MBCS_OUTPUT_1 */ p+=(16*(uint32_t)table[i]+(c&0xf)); value=*p; /* is the codepage value really an "unassigned" indicator? */ if(value==0 && c!=0 && (table[i-1]&(1<<(c&0xf)))==0) { /* * We allow a 0 byte output if the Unicode code point is * U+0000 and also if the "assigned" bit is set for this entry. * There is no way with this data structure for fallback output * for other than U+0000 to be a zero byte. */ /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } } else { /* callback(unassigned) */ reason=UCNV_UNASSIGNED; *pErrorCode=U_INVALID_CHAR_FOUND; goto callback; } /* write the output character bytes from value and length */ /* length==1 */ /* this is easy because we know that there is enough space */ *target++=(uint8_t)value; if(offsets!=NULL) { *offsets++=sourceIndex; } --targetCapacity; /* normal end of conversion: prepare for a new character */ c=0; sourceIndex=nextSourceIndex; continue; callback: /* call the callback function with all the preparations and post-processing */ /* update the arguments structure */ pArgs->source=source; pArgs->target=(char *)target; pArgs->offsets=offsets; /* set the converter state in UConverter to deal with the next character */ cnv->fromUSurrogateLead=0; /* write the code point as code units */ i=0; UTF_APPEND_CHAR_UNSAFE(cnv->invalidUCharBuffer, i, c); cnv->invalidUCharLength=(int8_t)i; /* call the callback function */ fromUCallback(cnv, cnv->fromUContext, pArgs, cnv->invalidUCharBuffer, i, c, reason, pErrorCode); /* get the converter state from UConverter */ c=cnv->fromUSurrogateLead; /* update target and deal with offsets if necessary */ offsets=ucnv_updateCallbackOffsets(offsets, ((uint8_t *)pArgs->target)-target, sourceIndex); target=(uint8_t *)pArgs->target; /* update the source pointer and index */ sourceIndex=nextSourceIndex+(pArgs->source-source); source=pArgs->source; targetCapacity=(uint8_t *)pArgs->targetLimit-target; /* * If the callback overflowed the target, then we need to * stop here with an overflow indication. */ if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { break; } else if(cnv->charErrorBufferLength>0) { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } else if(U_FAILURE(*pErrorCode)) { /* break on error */ c=0; break; } /* * We do not need to repeat the statements from the normal * end of the conversion because we already updated all the * necessary variables. */ } else { /* target is full */ *pErrorCode=U_BUFFER_OVERFLOW_ERROR; break; } } if(pArgs->flush && source>=sourceLimit) { /* reset the state for the next conversion */ if(c!=0 && U_SUCCESS(*pErrorCode)) { /* a character byte sequence remains incomplete */ *pErrorCode=U_TRUNCATED_CHAR_FOUND; } cnv->fromUSurrogateLead=0; } else { /* set the converter state back into UConverter */ cnv->fromUSurrogateLead=(UChar)c; } /* write back the updated pointers */ pArgs->source=source; pArgs->target=(char *)target; pArgs->offsets=offsets; } /* * This is another simple conversion function for internal use by other * conversion implementations. * It does not use the converter state nor call callbacks. * It converts one single Unicode code point into codepage bytes, encoded * as one 32-bit value. The function returns the number of bytes in *pValue: * 1..4 the number of bytes in *pValue * 0 unassigned (*pValue undefined) * -1 illegal (currently not used, *pValue undefined) * * *pValue will contain the resulting bytes with the last byte in bits 7..0, * the second to last byte in bits 15..8, etc. * Currently, the function assumes but does not check that 0<=c<=0x10ffff. */ U_CFUNC int32_t _MBCSFromUChar32(UConverterSharedData *sharedData, UChar32 c, uint32_t *pValue, UBool useFallback) { const uint16_t *table=sharedData->table->mbcs.fromUnicodeTable; uint32_t i; uint32_t value; int32_t length; /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ i=0x440+2*((uint32_t)table[c>>10]+((c>>4)&0x3f)); /* is this code point assigned, or do we use fallbacks? */ if((table[i++]&(1<<(c&0xf)))!=0 || FROM_U_USE_FALLBACK(useFallback, c)) { const uint8_t *p=sharedData->table->mbcs.fromUnicodeBytes; /* get the bytes and the length for the output */ switch(sharedData->table->mbcs.outputType) { case MBCS_OUTPUT_1: p+=(16*(uint32_t)table[i]+(c&0xf)); value=*p; length=1; break; case MBCS_OUTPUT_2: p+=(16*(uint32_t)table[i]+(c&0xf))*2; # if U_IS_BIG_ENDIAN value=*(uint16_t *)p; # else value=((uint32_t)*p<<8)|p[1]; # endif if(value<=0xff) { length=1; } else { length=2; } break; case MBCS_OUTPUT_3: p+=(16*(uint32_t)table[i]+(c&0xf))*3; value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; if(value<=0xff) { length=1; } else if(value<=0xffff) { length=2; } else { length=3; } break; case MBCS_OUTPUT_4: p+=(16*(uint32_t)table[i]+(c&0xf))*4; # if U_IS_BIG_ENDIAN value=*(uint32_t *)p; # else value=((uint32_t)*p<<24)|((uint32_t)p[1]<<16)|((uint32_t)p[2]<<8)|p[3]; # endif if(value<=0xff) { length=1; } else if(value<=0xffff) { length=2; } else if(value<=0xffffff) { length=3; } else { length=4; } break; case MBCS_OUTPUT_3_EUC: p+=(16*(uint32_t)table[i]+(c&0xf))*2; # if U_IS_BIG_ENDIAN value=*(uint16_t *)p; # else value=((uint32_t)*p<<8)|p[1]; # endif /* EUC 16-bit fixed-length representation */ if(value<=0xff) { length=1; } else if((value&0x8000)==0) { value|=0x8e8000; length=3; } else if((value&0x80)==0) { value|=0x8f0080; length=3; } else { length=2; } break; case MBCS_OUTPUT_4_EUC: p+=(16*(uint32_t)table[i]+(c&0xf))*3; value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; /* EUC 16-bit fixed-length representation applied to the first two bytes */ if(value<=0xff) { length=1; } else if(value<=0xffff) { length=2; } else if((value&0x800000)==0) { value|=0x8e800000; length=4; } else if((value&0x8000)==0) { value|=0x8f008000; length=4; } else { length=3; } break; default: /* must not occur */ return -1; } /* is the codepage value really an "unassigned" indicator? */ if(value==0 && c!=0 && (table[i-1]&(1<<(c&0xf)))==0) { /* * We allow a 0 byte output if the Unicode code point is * U+0000 and also if the "assigned" bit is set for this entry. * There is no way with this data structure for fallback output * for other than U+0000 to be a zero byte. */ return 0; } else { *pValue=value; return length; } } else { return 0; } } /* miscellaneous ------------------------------------------------------------ */ static void _MBCSGetStarters(const UConverter* cnv, UBool starters[256], UErrorCode *pErrorCode) { const int32_t *state0=cnv->sharedData->table->mbcs.stateTable[0]; int i; for(i=0; i<256; ++i) { /* all bytes that cause a state transition from state 0 are lead bytes */ starters[i]= (UBool)(state0[i]>=0); } } /* * This is an internal function that allows other converter implementations * to check whether a byte is a lead byte. */ U_CFUNC UBool _MBCSIsLeadByte(UConverterSharedData *sharedData, char byte) { return (UBool)(sharedData->table->mbcs.stateTable[0][(uint8_t)byte]>=0); } static const UConverterImpl _MBCSImpl={ UCNV_MBCS, _MBCSLoad, NULL, _MBCSOpen, NULL, _MBCSReset, _MBCSToUnicode, _MBCSToUnicodeWithOffsets, _MBCSFromUnicode, _MBCSFromUnicodeWithOffsets, _MBCSGetNextUChar, _MBCSGetStarters, NULL }; /* Static data is in tools/makeconv/ucnvstat.c for data-based * converters. Be sure to update it as well. */ const UConverterSharedData _MBCSData={ sizeof(UConverterSharedData), 1, NULL, NULL, NULL, FALSE, &_MBCSImpl, 0 }; /* GB 18030 special handling ------------------------------------------------ */ /* ### IMPORTANT: THIS IS ALPHA-VERSION SUPPORT CODE FOR GB 18030 AND MAY CHANGE WITHOUT NOTICE */ /* definition of LINEAR macros and gb18030Ranges see near the beginning of the file */ /* the callback functions handle GB 18030 specially */ static void fromUCallback(UConverter *cnv, void *context, UConverterFromUnicodeArgs *pArgs, const UChar *codeUnits, int32_t length, UChar32 codePoint, UConverterCallbackReason reason, UErrorCode *pErrorCode) { if(cnv->extraInfo==gb18030Ranges && reason==UCNV_UNASSIGNED) { const uint32_t *range; int i; range=gb18030Ranges[0]; for(i=0; ifromUCharErrorBehaviour(context, pArgs, codeUnits, length, codePoint, reason, pErrorCode); } static void toUCallback(UConverter *cnv, void *context, UConverterToUnicodeArgs *pArgs, const char *codeUnits, int32_t length, UConverterCallbackReason reason, UErrorCode *pErrorCode) { if(cnv->extraInfo==gb18030Ranges && reason==UCNV_UNASSIGNED && length==4) { const uint32_t *range; uint32_t linear; int i; linear=LINEAR_18030((uint8_t)codeUnits[0], (uint8_t)codeUnits[1], (uint8_t)codeUnits[2], (uint8_t)codeUnits[3]); range=gb18030Ranges[0]; for(i=0; ifromCharErrorBehaviour(context, pArgs, codeUnits, length, reason, pErrorCode); }