1160 lines
36 KiB
C
1160 lines
36 KiB
C
/*
|
|
**********************************************************************
|
|
* Copyright (C) 2000, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
**********************************************************************
|
|
* file name: ucnv_utf.cpp
|
|
* encoding: US-ASCII
|
|
* tab size: 8 (not used)
|
|
* indentation:4
|
|
*
|
|
* created on: 2000feb03
|
|
* created by: Markus W. Scherer
|
|
*
|
|
* Change history:
|
|
*
|
|
* 06/29/2000 helena Major rewrite of the callback APIs.
|
|
* 07/20/2000 george Change the coding style to conform to the coding guidelines,
|
|
* and a few miscellaneous bug fixes.
|
|
*/
|
|
|
|
#include "cmemory.h"
|
|
#include "unicode/utypes.h"
|
|
#include "ucmp16.h"
|
|
#include "ucmp8.h"
|
|
#include "unicode/ucnv_err.h"
|
|
#include "ucnv_bld.h"
|
|
#include "unicode/ucnv.h"
|
|
#include "ucnv_cnv.h"
|
|
|
|
/* UTF-8 -------------------------------------------------------------------- */
|
|
|
|
/* UTF-8 Conversion DATA
|
|
* for more information see Unicode Strandard 2.0 , Transformation Formats Appendix A-9
|
|
*/
|
|
/*static const uint32_t REPLACEMENT_CHARACTER = 0x0000FFFD;*/
|
|
static const uint32_t MAXIMUM_UCS2 = 0x0000FFFF;
|
|
static const uint32_t MAXIMUM_UTF16 = 0x0010FFFF;
|
|
static const uint32_t MAXIMUM_UCS4 = 0x7FFFFFFF;
|
|
static const int8_t HALF_SHIFT = 10;
|
|
static const uint32_t HALF_BASE = 0x0010000;
|
|
static const uint32_t HALF_MASK = 0x3FF;
|
|
static const uint32_t SURROGATE_HIGH_START = 0xD800;
|
|
static const uint32_t SURROGATE_HIGH_END = 0xDBFF;
|
|
static const uint32_t SURROGATE_LOW_START = 0xDC00;
|
|
static const uint32_t SURROGATE_LOW_END = 0xDFFF;
|
|
static const uint32_t SURROGATE_LOW_BASE = 9216; /* -SURROGATE_LOW_START + HALF_BASE */
|
|
|
|
static const uint32_t offsetsFromUTF8[7] = {0,
|
|
(uint32_t) 0x00000000, (uint32_t) 0x00003080, (uint32_t) 0x000E2080,
|
|
(uint32_t) 0x03C82080, (uint32_t) 0xFA082080, (uint32_t) 0x82082080
|
|
};
|
|
|
|
/* END OF UTF-8 Conversion DATA */
|
|
|
|
static const int8_t bytesFromUTF8[256] = {
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
|
|
};
|
|
|
|
/* static const unsigned char firstByteMark[7] = {0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC};*/
|
|
|
|
#define INVALID_UTF8_TAIL(utf8) (((utf8) & 0xC0) != 0x80)
|
|
|
|
/**
|
|
* Calls invalid char callback when an invalid character sequence is encountered.
|
|
* It presumes that the converter has a callback to call.
|
|
*
|
|
* @returns true when callback fails
|
|
*/
|
|
UBool T_UConverter_toUnicode_InvalidChar_Callback(UConverterToUnicodeArgs * args,
|
|
UErrorCode *err)
|
|
{
|
|
UConverter *converter = args->converter;
|
|
|
|
if (U_SUCCESS(*err))
|
|
{
|
|
*err = U_ILLEGAL_CHAR_FOUND;
|
|
}
|
|
|
|
/* Make the toUBytes invalid */
|
|
uprv_memcpy(converter->invalidCharBuffer,
|
|
converter->toUBytes,
|
|
converter->invalidCharLength);
|
|
|
|
/* Call the ErrorFunction */
|
|
args->converter->fromCharErrorBehaviour(converter->toUContext,
|
|
args,
|
|
converter->invalidCharBuffer,
|
|
converter->invalidCharLength,
|
|
UCNV_ILLEGAL,
|
|
err);
|
|
|
|
return (UBool)U_FAILURE(*err);
|
|
}
|
|
|
|
UBool T_UConverter_toUnicode_InvalidChar_OffsetCallback(UConverterToUnicodeArgs * args,
|
|
int32_t currentOffset,
|
|
UErrorCode *err)
|
|
{
|
|
int32_t *saveOffsets = args->offsets;
|
|
UBool result;
|
|
|
|
result = T_UConverter_toUnicode_InvalidChar_Callback(args, err);
|
|
|
|
while (saveOffsets < args->offsets)
|
|
{
|
|
*(saveOffsets++) = currentOffset;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
U_CFUNC void T_UConverter_toUnicode_UTF8 (UConverterToUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const unsigned char *mySource = (unsigned char *) args->source;
|
|
UChar *myTarget = args->target;
|
|
const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit;
|
|
const UChar *targetLimit = args->targetLimit;
|
|
unsigned char *toUBytes = args->converter->toUBytes;
|
|
UBool invalidTailChar = FALSE;
|
|
uint32_t ch, ch2 = 0, i;
|
|
uint32_t inBytes; /* Total number of bytes in the current UTF8 sequence */
|
|
|
|
if (U_FAILURE(*err))
|
|
{
|
|
return;
|
|
}
|
|
|
|
/* Restore size of current sequence */
|
|
if (args->converter->toUnicodeStatus && myTarget < targetLimit)
|
|
{
|
|
inBytes = args->converter->toULength; /* restore # of bytes to consume */
|
|
i = args->converter->invalidCharLength; /* restore # of bytes consumed */
|
|
|
|
ch = args->converter->toUnicodeStatus;/*Stores the previously calculated ch from a previous call*/
|
|
args->converter->toUnicodeStatus = 0;
|
|
goto morebytes;
|
|
}
|
|
|
|
|
|
while (mySource < sourceLimit && myTarget < targetLimit)
|
|
{
|
|
ch = *(mySource++);
|
|
if (ch < 0x80) /* Simple case */
|
|
{
|
|
*(myTarget++) = (UChar) ch;
|
|
}
|
|
else
|
|
{
|
|
/* store the first char */
|
|
toUBytes[0] = (char)ch;
|
|
inBytes = bytesFromUTF8[ch]; /* lookup current sequence length */
|
|
i = 1;
|
|
|
|
morebytes:
|
|
while (i < inBytes)
|
|
{
|
|
if (mySource < sourceLimit)
|
|
{
|
|
toUBytes[i] = (char) (ch2 = *(mySource++));
|
|
if (INVALID_UTF8_TAIL(ch2))
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
invalidTailChar = TRUE;
|
|
break;
|
|
}
|
|
ch = (ch << 6) + ch2;
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
if (args->flush)
|
|
{
|
|
if (U_SUCCESS(*err))
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
}
|
|
}
|
|
else
|
|
{ /* stores a partially calculated target*/
|
|
args->converter->toUnicodeStatus = ch;
|
|
args->converter->toULength = (int8_t) inBytes;
|
|
args->converter->invalidCharLength = (int8_t) i;
|
|
}
|
|
goto donefornow;
|
|
}
|
|
}
|
|
|
|
/* Remove the acummulated high bits */
|
|
ch -= offsetsFromUTF8[inBytes];
|
|
|
|
if (i == inBytes && ch <= MAXIMUM_UTF16)
|
|
{
|
|
/* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */
|
|
if (ch <= MAXIMUM_UCS2)
|
|
{
|
|
/* fits in 16 bits */
|
|
*(myTarget++) = (UChar) ch;
|
|
}
|
|
else
|
|
{
|
|
/* write out the surrogates */
|
|
ch -= HALF_BASE;
|
|
*(myTarget++) = (UChar) ((ch >> HALF_SHIFT) + SURROGATE_HIGH_START);
|
|
ch = (ch & HALF_MASK) + SURROGATE_LOW_START;
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myTarget++) = (UChar)ch;
|
|
}
|
|
else
|
|
{
|
|
/* Put in overflow buffer (not handled here) */
|
|
args->converter->UCharErrorBuffer[0] = (UChar) ch;
|
|
args->converter->UCharErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
args->source = (const char *) mySource;
|
|
args->target = myTarget;
|
|
args->converter->invalidCharLength = (int8_t)i;
|
|
if (T_UConverter_toUnicode_InvalidChar_Callback(args, err))
|
|
{
|
|
/* Stop if the error wasn't handled */
|
|
break;
|
|
}
|
|
args->converter->invalidCharLength = 0;
|
|
mySource = (unsigned char *) args->source;
|
|
myTarget = args->target;
|
|
if (invalidTailChar)
|
|
{
|
|
/* Treat the tail as ASCII*/
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myTarget++) = (UChar) ch2;
|
|
invalidTailChar = FALSE;
|
|
}
|
|
else
|
|
{
|
|
/* Put in overflow buffer (not handled here) */
|
|
args->converter->UCharErrorBuffer[0] = (UChar) ch2;
|
|
args->converter->UCharErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
donefornow:
|
|
if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
|
|
{
|
|
/* End of target buffer */
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
|
|
args->target = myTarget;
|
|
args->source = (const char *) mySource;
|
|
}
|
|
|
|
U_CFUNC void T_UConverter_toUnicode_UTF8_OFFSETS_LOGIC (UConverterToUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const unsigned char *mySource = (unsigned char *) args->source;
|
|
UChar *myTarget = args->target;
|
|
int32_t *myOffsets = args->offsets;
|
|
int32_t offsetNum = 0;
|
|
const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit;
|
|
const UChar *targetLimit = args->targetLimit;
|
|
unsigned char *toUBytes = args->converter->toUBytes;
|
|
UBool invalidTailChar = FALSE;
|
|
uint32_t ch, ch2 = 0, i;
|
|
uint32_t inBytes;
|
|
|
|
/* Restore size of current sequence */
|
|
if (args->converter->toUnicodeStatus && myTarget < targetLimit)
|
|
{
|
|
inBytes = args->converter->toULength; /* restore # of bytes to consume */
|
|
i = args->converter->invalidCharLength; /* restore # of bytes consumed */
|
|
|
|
ch = args->converter->toUnicodeStatus;/*Stores the previously calculated ch from a previous call*/
|
|
args->converter->toUnicodeStatus = 0;
|
|
goto morebytes;
|
|
}
|
|
|
|
while (mySource < sourceLimit && myTarget < targetLimit)
|
|
{
|
|
ch = *(mySource++);
|
|
if (ch < 0x80) /* Simple case */
|
|
{
|
|
*(myTarget++) = (UChar) ch;
|
|
*(myOffsets++) = offsetNum++;
|
|
}
|
|
else
|
|
{
|
|
toUBytes[0] = (char)ch;
|
|
inBytes = bytesFromUTF8[ch];
|
|
i = 1;
|
|
|
|
morebytes:
|
|
while (i < inBytes)
|
|
{
|
|
if (mySource < sourceLimit)
|
|
{
|
|
toUBytes[i] = (char) (ch2 = *(mySource++));
|
|
if (INVALID_UTF8_TAIL(ch2))
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
invalidTailChar = TRUE;
|
|
break;
|
|
}
|
|
ch = (ch << 6) + ch2;
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
if (args->flush)
|
|
{
|
|
if (U_SUCCESS(*err))
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
args->converter->toUnicodeStatus = 0;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
args->converter->toUnicodeStatus = ch;
|
|
args->converter->toULength = (int8_t)inBytes;
|
|
args->converter->invalidCharLength = (int8_t)i;
|
|
}
|
|
goto donefornow;
|
|
}
|
|
}
|
|
|
|
/* Remove the acummulated high bits */
|
|
ch -= offsetsFromUTF8[inBytes];
|
|
|
|
if (i == inBytes && ch <= MAXIMUM_UTF16)
|
|
{
|
|
/* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */
|
|
if (ch <= MAXIMUM_UCS2)
|
|
{
|
|
/* fits in 16 bits */
|
|
*(myTarget++) = (UChar) ch;
|
|
*(myOffsets++) = offsetNum;
|
|
}
|
|
else
|
|
{
|
|
/* write out the surrogates */
|
|
*(myOffsets++) = offsetNum;
|
|
ch -= HALF_BASE;
|
|
*(myTarget++) = (UChar) ((ch >> HALF_SHIFT) + SURROGATE_HIGH_START);
|
|
ch = (ch & HALF_MASK) + SURROGATE_LOW_START;
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myTarget++) = (UChar)ch;
|
|
*(myOffsets++) = offsetNum;
|
|
}
|
|
else
|
|
{
|
|
args->converter->UCharErrorBuffer[0] = (UChar) ch;
|
|
args->converter->UCharErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
}
|
|
offsetNum += i;
|
|
}
|
|
else
|
|
{
|
|
UBool useOffset;
|
|
|
|
args->source = (const char *) mySource;
|
|
args->target = myTarget;
|
|
args->offsets = myOffsets;
|
|
args->converter->invalidCharLength = (int8_t)i;
|
|
if (T_UConverter_toUnicode_InvalidChar_OffsetCallback(args,
|
|
offsetNum, err))
|
|
{
|
|
/* Stop if the error wasn't handled */
|
|
break;
|
|
}
|
|
|
|
args->converter->invalidCharLength = 0;
|
|
mySource = (unsigned char *) args->source;
|
|
myTarget = args->target;
|
|
|
|
useOffset = (UBool)(myOffsets != args->offsets);
|
|
myOffsets = args->offsets;
|
|
offsetNum += i;
|
|
|
|
if (invalidTailChar)
|
|
{
|
|
/* Treat the tail as ASCII*/
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myTarget++) = (UChar) ch2;
|
|
*myOffsets = offsetNum++;
|
|
if (useOffset)
|
|
{
|
|
/* Increment when the target was consumed */
|
|
myOffsets++;
|
|
}
|
|
invalidTailChar = FALSE;
|
|
}
|
|
else
|
|
{
|
|
/* Put in overflow buffer (not handled here) */
|
|
args->converter->UCharErrorBuffer[0] = (UChar) ch2;
|
|
args->converter->UCharErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
donefornow:
|
|
if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
|
|
{ /* End of target buffer */
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
|
|
args->target = myTarget;
|
|
args->source = (const char *) mySource;
|
|
args->offsets = myOffsets;
|
|
}
|
|
|
|
U_CFUNC void T_UConverter_fromUnicode_UTF8 (UConverterFromUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const UChar *mySource = args->source;
|
|
unsigned char *myTarget = (unsigned char *) args->target;
|
|
const UChar *sourceLimit = args->sourceLimit;
|
|
const unsigned char *targetLimit = (unsigned char *) args->targetLimit;
|
|
uint32_t ch, ch2;
|
|
int16_t indexToWrite;
|
|
char temp[4];
|
|
|
|
if (args->converter->fromUnicodeStatus && myTarget < targetLimit)
|
|
{
|
|
ch = args->converter->fromUnicodeStatus;
|
|
args->converter->fromUnicodeStatus = 0;
|
|
goto lowsurogate;
|
|
}
|
|
|
|
while (mySource < sourceLimit && myTarget < targetLimit)
|
|
{
|
|
ch = *(mySource++);
|
|
|
|
if (ch < 0x80) /* Single byte */
|
|
{
|
|
*(myTarget++) = (char) ch;
|
|
}
|
|
else if (ch < 0x800) /* Double byte */
|
|
{
|
|
*(myTarget++) = (char) ((ch >> 6) | 0xc0);
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myTarget++) = (char) ((ch & 0x3f) | 0x80);
|
|
}
|
|
else
|
|
{
|
|
args->converter->charErrorBuffer[0] = (char) ((ch & 0x3f) | 0x80);
|
|
args->converter->charErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
}
|
|
else
|
|
/* Check for surogates */
|
|
{
|
|
if ((ch >= SURROGATE_HIGH_START) && (ch <= SURROGATE_HIGH_END))
|
|
{
|
|
lowsurogate:
|
|
if (mySource < sourceLimit)
|
|
{
|
|
ch2 = *mySource;
|
|
if ((ch2 >= SURROGATE_LOW_START) && (ch2 <= SURROGATE_LOW_END))
|
|
{
|
|
/* If there were two surrogates, combine them otherwise treat them normally */
|
|
ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE;
|
|
mySource++;
|
|
}
|
|
}
|
|
else if (!args->flush)
|
|
{
|
|
args->converter->fromUnicodeStatus = ch;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (ch < 0x10000)
|
|
{
|
|
indexToWrite = 2;
|
|
temp[2] = (char) ((ch >> 12) | 0xe0);
|
|
}
|
|
else
|
|
{
|
|
indexToWrite = 3;
|
|
temp[3] = (char) ((ch >> 18) | 0xf0);
|
|
temp[2] = (char) ((ch >> 12) & 0x3f | 0x80);
|
|
}
|
|
temp[1] = (char) ((ch >> 6) & 0x3f | 0x80);
|
|
temp[0] = (char) (ch & 0x3f | 0x80);
|
|
|
|
for (; indexToWrite >= 0; indexToWrite--)
|
|
{
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myTarget++) = temp[indexToWrite];
|
|
}
|
|
else
|
|
{
|
|
args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite];
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
|
|
{
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
|
|
args->target = (char *) myTarget;
|
|
args->source = mySource;
|
|
}
|
|
|
|
U_CFUNC void T_UConverter_fromUnicode_UTF8_OFFSETS_LOGIC (UConverterFromUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const UChar *mySource = args->source;
|
|
unsigned char *myTarget = (unsigned char *) args->target;
|
|
int32_t *myOffsets = args->offsets;
|
|
const UChar *sourceLimit = args->sourceLimit;
|
|
const unsigned char *targetLimit = (unsigned char *) args->targetLimit;
|
|
uint32_t ch, ch2;
|
|
int32_t offsetNum = 0;
|
|
int16_t indexToWrite;
|
|
char temp[4];
|
|
|
|
if (args->converter->fromUnicodeStatus && myTarget < targetLimit)
|
|
{
|
|
ch = args->converter->fromUnicodeStatus;
|
|
args->converter->fromUnicodeStatus = 0;
|
|
goto lowsurogate;
|
|
}
|
|
|
|
while (mySource < sourceLimit && myTarget < targetLimit)
|
|
{
|
|
ch = *(mySource++);
|
|
|
|
if (ch < 0x80) /* Single byte */
|
|
{
|
|
*(myOffsets++) = offsetNum++;
|
|
*(myTarget++) = (char) ch;
|
|
}
|
|
else if (ch < 0x800) /* Double byte */
|
|
{
|
|
*(myOffsets++) = offsetNum;
|
|
*(myTarget++) = (char) ((ch >> 6) | 0xc0);
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myOffsets++) = offsetNum++;
|
|
*(myTarget++) = (char) ((ch & 0x3f) | 0x80);
|
|
}
|
|
else
|
|
{
|
|
args->converter->charErrorBuffer[0] = (char) ((ch & 0x3f) | 0x80);
|
|
args->converter->charErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
}
|
|
else
|
|
/* Check for surogates */
|
|
{
|
|
if ((ch >= SURROGATE_HIGH_START) && (ch <= SURROGATE_HIGH_END))
|
|
{
|
|
lowsurogate:
|
|
if (mySource < sourceLimit)
|
|
{
|
|
ch2 = *mySource;
|
|
if ((ch2 >= SURROGATE_LOW_START) && (ch2 <= SURROGATE_LOW_END))
|
|
{
|
|
/* If there were two surrogates, combine them otherwise treat them normally */
|
|
ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE;
|
|
mySource++;
|
|
}
|
|
}
|
|
else if (!args->flush)
|
|
{
|
|
args->converter->fromUnicodeStatus = ch;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (ch < 0x10000)
|
|
{
|
|
indexToWrite = 2;
|
|
temp[2] = (char) ((ch >> 12) | 0xe0);
|
|
}
|
|
else
|
|
{
|
|
indexToWrite = 3;
|
|
temp[3] = (char) ((ch >> 18) | 0xf0);
|
|
temp[2] = (char) ((ch >> 12) & 0x3f | 0x80);
|
|
}
|
|
temp[1] = (char) ((ch >> 6) & 0x3f | 0x80);
|
|
temp[0] = (char) (ch & 0x3f | 0x80);
|
|
|
|
for (; indexToWrite >= 0; indexToWrite--)
|
|
{
|
|
if (myTarget < targetLimit)
|
|
{
|
|
*(myOffsets++) = offsetNum;
|
|
*(myTarget++) = temp[indexToWrite];
|
|
}
|
|
else
|
|
{
|
|
args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite];
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
}
|
|
offsetNum += (ch >= 0x10000) + 1;
|
|
}
|
|
}
|
|
|
|
if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
|
|
{
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
|
|
args->target = (char *) myTarget;
|
|
args->source = mySource;
|
|
}
|
|
|
|
U_CFUNC UChar32 T_UConverter_getNextUChar_UTF8(UConverterToUnicodeArgs *args,
|
|
UErrorCode* err)
|
|
{
|
|
/*safe keeps a ptr to the beginning in case we need to step back*/
|
|
char const *sourceInitial = args->source;
|
|
uint16_t extraBytesToWrite;
|
|
uint8_t myByte;
|
|
UChar32 ch;
|
|
int8_t isLegalSequence = 1;
|
|
|
|
/*Input boundary check*/
|
|
if (args->source >= args->sourceLimit)
|
|
{
|
|
*err = U_INDEX_OUTOFBOUNDS_ERROR;
|
|
return 0xffff;
|
|
}
|
|
|
|
myByte = (uint8_t)*(args->source++);
|
|
if (myByte < 0x80)
|
|
{
|
|
return (UChar32)myByte;
|
|
}
|
|
extraBytesToWrite = (uint16_t)bytesFromUTF8[myByte];
|
|
if (extraBytesToWrite == 0) {
|
|
goto CALL_ERROR_FUNCTION;
|
|
}
|
|
|
|
/*The byte sequence is longer than the buffer area passed*/
|
|
|
|
if ((args->source + extraBytesToWrite - 1) > args->sourceLimit)
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
return 0xffff;
|
|
}
|
|
else
|
|
{
|
|
ch = myByte << 6;
|
|
switch(extraBytesToWrite)
|
|
{
|
|
/* note: code falls through cases! (sic)*/
|
|
case 6:
|
|
ch += (myByte = (uint8_t)*(args->source++));
|
|
ch <<= 6;
|
|
if ((myByte & 0xC0) != 0x80)
|
|
{
|
|
isLegalSequence = 0;
|
|
break;
|
|
}
|
|
case 5:
|
|
ch += (myByte = *(args->source++));
|
|
ch <<= 6;
|
|
if ((myByte & 0xC0) != 0x80)
|
|
{
|
|
isLegalSequence = 0;
|
|
break;
|
|
}
|
|
case 4:
|
|
ch += (myByte = *(args->source++));
|
|
ch <<= 6;
|
|
if ((myByte & 0xC0) != 0x80)
|
|
{
|
|
isLegalSequence = 0;
|
|
break;
|
|
}
|
|
case 3:
|
|
ch += (myByte = *(args->source++));
|
|
ch <<= 6;
|
|
if ((myByte & 0xC0) != 0x80)
|
|
{
|
|
isLegalSequence = 0;
|
|
break;
|
|
}
|
|
case 2:
|
|
ch += (myByte = *(args->source++));
|
|
if ((myByte & 0xC0) != 0x80)
|
|
{
|
|
isLegalSequence = 0;
|
|
}
|
|
};
|
|
}
|
|
ch -= offsetsFromUTF8[extraBytesToWrite];
|
|
|
|
if (isLegalSequence)
|
|
return ch; /* return the code point */
|
|
|
|
CALL_ERROR_FUNCTION:
|
|
{
|
|
UChar myUChar = (UChar)0xffff; /* ### TODO: this is a hack until we prepare the callbacks for code points */
|
|
UChar* myUCharPtr = &myUChar;
|
|
|
|
*err = U_ILLEGAL_CHAR_FOUND;
|
|
|
|
/*It is very likely that the ErrorFunctor will write to the
|
|
*internal buffers */
|
|
args->target = myUCharPtr;
|
|
args->targetLimit = myUCharPtr + 1;
|
|
args->converter->fromCharErrorBehaviour(args->converter->toUContext,
|
|
args,
|
|
sourceInitial,
|
|
args->source-sourceInitial,
|
|
UCNV_ILLEGAL,
|
|
err);
|
|
|
|
/*makes the internal caching transparent to the user*/
|
|
if (*err == U_BUFFER_OVERFLOW_ERROR)
|
|
*err = U_ZERO_ERROR;
|
|
|
|
return (UChar32)myUChar;
|
|
}
|
|
}
|
|
|
|
static const UConverterImpl _UTF8Impl={
|
|
UCNV_UTF8,
|
|
|
|
NULL,
|
|
NULL,
|
|
|
|
NULL,
|
|
NULL,
|
|
NULL,
|
|
|
|
T_UConverter_toUnicode_UTF8,
|
|
T_UConverter_toUnicode_UTF8_OFFSETS_LOGIC,
|
|
T_UConverter_fromUnicode_UTF8,
|
|
T_UConverter_fromUnicode_UTF8_OFFSETS_LOGIC,
|
|
T_UConverter_getNextUChar_UTF8,
|
|
|
|
NULL
|
|
};
|
|
|
|
/* Todo: verify that UTF-8 == (ccsid (ibm-codepage) 1208) for unicode version 2.0 and 3.0 */
|
|
const UConverterStaticData _UTF8StaticData={
|
|
sizeof(UConverterStaticData),
|
|
"UTF8",
|
|
1208, UCNV_IBM, UCNV_UTF8, 1, 4,
|
|
3, { 0xef, 0xbf, 0xbd, 0 }
|
|
};
|
|
|
|
|
|
const UConverterSharedData _UTF8Data={
|
|
sizeof(UConverterSharedData), ~((uint32_t) 0),
|
|
NULL, NULL, &_UTF8StaticData, FALSE, &_UTF8Impl,
|
|
0
|
|
};
|
|
|
|
/* UTF-16BE ----------------------------------------------------------------- */
|
|
|
|
U_CFUNC void T_UConverter_toUnicode_UTF16_BE (UConverterToUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const unsigned char *mySource = (unsigned char *) args->source;
|
|
UChar *myTarget = args->target;
|
|
int32_t mySourceIndex = 0;
|
|
int32_t myTargetIndex = 0;
|
|
int32_t targetLength = args->targetLimit - myTarget;
|
|
int32_t sourceLength = args->sourceLimit - (char *) mySource;
|
|
UChar mySourceChar = 0x0000;
|
|
UChar oldmySourceChar = 0x0000;
|
|
|
|
while (mySourceIndex < sourceLength)
|
|
{
|
|
if (myTargetIndex < targetLength)
|
|
{
|
|
/*gets the corresponding UChar */
|
|
mySourceChar = (unsigned char) mySource[mySourceIndex++];
|
|
oldmySourceChar = mySourceChar;
|
|
if (args->converter->toUnicodeStatus == 0)
|
|
{
|
|
args->converter->toUnicodeStatus =
|
|
(unsigned char) mySourceChar == 0 ? 0xFFFF : mySourceChar;
|
|
}
|
|
else
|
|
{
|
|
if (args->converter->toUnicodeStatus != 0xFFFF)
|
|
mySourceChar = (UChar) ((args->converter->toUnicodeStatus << 8) | mySourceChar);
|
|
args->converter->toUnicodeStatus = 0;
|
|
|
|
myTarget[myTargetIndex++] = mySourceChar;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (U_SUCCESS(*err) && args->flush
|
|
&& (mySourceIndex == sourceLength)
|
|
&& (args->converter->toUnicodeStatus != 0x00))
|
|
{
|
|
if (U_SUCCESS(*err))
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
args->converter->toUnicodeStatus = 0x00;
|
|
}
|
|
}
|
|
|
|
args->target += myTargetIndex;
|
|
args->source += mySourceIndex;
|
|
}
|
|
|
|
U_CFUNC void T_UConverter_fromUnicode_UTF16_BE (UConverterFromUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const UChar *mySource = args->source;
|
|
unsigned char *myTarget = (unsigned char *) args->target;
|
|
int32_t mySourceIndex = 0;
|
|
int32_t myTargetIndex = 0;
|
|
int32_t targetLength = args->targetLimit - (char *) myTarget;
|
|
int32_t sourceLength = args->sourceLimit - mySource;
|
|
UChar mySourceChar;
|
|
|
|
/*writing the char to the output stream */
|
|
while (mySourceIndex < sourceLength)
|
|
{
|
|
if (myTargetIndex < targetLength)
|
|
{
|
|
mySourceChar = (UChar) mySource[mySourceIndex++];
|
|
myTarget[myTargetIndex++] = (char) (mySourceChar >> 8);
|
|
if (myTargetIndex < targetLength)
|
|
{
|
|
myTarget[myTargetIndex++] = (char) mySourceChar;
|
|
}
|
|
else
|
|
{
|
|
args->converter->charErrorBuffer[0] = (char) mySourceChar;
|
|
args->converter->charErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
|
|
args->target += myTargetIndex;
|
|
args->source += mySourceIndex;
|
|
}
|
|
|
|
U_CFUNC UChar32 T_UConverter_getNextUChar_UTF16_BE(UConverterToUnicodeArgs* args,
|
|
UErrorCode* err)
|
|
{
|
|
UChar32 myUChar;
|
|
uint16_t first;
|
|
/*Checks boundaries and set appropriate error codes*/
|
|
if (args->source+2 > args->sourceLimit)
|
|
{
|
|
if (args->source >= args->sourceLimit)
|
|
{
|
|
/*Either caller has reached the end of the byte stream*/
|
|
*err = U_INDEX_OUTOFBOUNDS_ERROR;
|
|
}
|
|
else
|
|
{
|
|
/* a character was cut in half*/
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
}
|
|
return 0xffff;
|
|
}
|
|
|
|
/*Gets the corresponding codepoint*/
|
|
first = (uint16_t)(((uint16_t)(*(args->source)) << 8) |((uint8_t)*((args->source)+1)));
|
|
myUChar = first;
|
|
args->source += 2;
|
|
|
|
if(UTF_IS_FIRST_SURROGATE(first)) {
|
|
uint16_t second;
|
|
|
|
if (args->source+2 > args->sourceLimit) {
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
return 0xffff;
|
|
}
|
|
|
|
/* get the second surrogate and assemble the code point */
|
|
second = (uint16_t)(((uint16_t)(*(args->source)) << 8) |((uint8_t)*(args->source+1)));
|
|
|
|
/* ignore unmatched surrogates and just deliver the first one in such a case */
|
|
if(UTF_IS_SECOND_SURROGATE(second)) {
|
|
/* matched pair, get pair value */
|
|
myUChar = UTF16_GET_PAIR_VALUE(first, second);
|
|
args->source += 2;
|
|
}
|
|
}
|
|
|
|
return myUChar;
|
|
}
|
|
|
|
static const UConverterImpl _UTF16BEImpl={
|
|
UCNV_UTF16_BigEndian,
|
|
|
|
NULL,
|
|
NULL,
|
|
|
|
NULL,
|
|
NULL,
|
|
NULL,
|
|
|
|
T_UConverter_toUnicode_UTF16_BE,
|
|
NULL,
|
|
T_UConverter_fromUnicode_UTF16_BE,
|
|
NULL,
|
|
T_UConverter_getNextUChar_UTF16_BE,
|
|
|
|
NULL
|
|
};
|
|
|
|
/* Todo: verify that UTF-16BE == (ccsid (ibm-codepage) 1200) for unicode version 2.0 and 3.0 */
|
|
const UConverterStaticData _UTF16BEStaticData={
|
|
sizeof(UConverterStaticData),
|
|
"UTF16_BigEndian",
|
|
1200, UCNV_IBM, UCNV_UTF16_BigEndian, 2, 2,
|
|
2, { 0xff, 0xfd, 0, 0 }
|
|
};
|
|
|
|
|
|
const UConverterSharedData _UTF16BEData={
|
|
sizeof(UConverterSharedData), ~((uint32_t) 0),
|
|
NULL, NULL, &_UTF16BEStaticData, FALSE, &_UTF16BEImpl,
|
|
0
|
|
};
|
|
|
|
/* UTF-16LE ----------------------------------------------------------------- */
|
|
|
|
U_CFUNC void T_UConverter_toUnicode_UTF16_LE (UConverterToUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const unsigned char *mySource = (unsigned char *) args->source;
|
|
UChar *myTarget = args->target;
|
|
int32_t mySourceIndex = 0;
|
|
int32_t myTargetIndex = 0;
|
|
int32_t targetLength = args->targetLimit - myTarget;
|
|
int32_t sourceLength = args->sourceLimit - (char *) mySource;
|
|
UChar mySourceChar = 0x0000;
|
|
|
|
while (mySourceIndex < sourceLength)
|
|
{
|
|
if (myTargetIndex < targetLength)
|
|
{
|
|
/*gets the corresponding UniChar */
|
|
mySourceChar = (unsigned char) mySource[mySourceIndex++];
|
|
|
|
if (args->converter->toUnicodeStatus == 0x00)
|
|
{
|
|
args->converter->toUnicodeStatus = (unsigned char) mySourceChar == 0x00 ? 0xFFFF : mySourceChar;
|
|
}
|
|
else
|
|
{
|
|
if (args->converter->toUnicodeStatus == 0xFFFF) {
|
|
mySourceChar = (UChar) (mySourceChar << 8);
|
|
}
|
|
else
|
|
{
|
|
mySourceChar <<= 8;
|
|
mySourceChar |= (UChar) (args->converter->toUnicodeStatus);
|
|
}
|
|
args->converter->toUnicodeStatus = 0x00;
|
|
myTarget[myTargetIndex++] = mySourceChar;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
if (U_SUCCESS(*err) && args->flush
|
|
&& (mySourceIndex == sourceLength)
|
|
&& (args->converter->toUnicodeStatus != 0x00))
|
|
{
|
|
if (U_SUCCESS(*err))
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
args->converter->toUnicodeStatus = 0x00;
|
|
}
|
|
}
|
|
|
|
args->target += myTargetIndex;
|
|
args->source += mySourceIndex;
|
|
}
|
|
|
|
U_CFUNC void T_UConverter_fromUnicode_UTF16_LE (UConverterFromUnicodeArgs * args,
|
|
UErrorCode * err)
|
|
{
|
|
const UChar *mySource = args->source;
|
|
unsigned char *myTarget = (unsigned char *) args->target;
|
|
int32_t mySourceIndex = 0;
|
|
int32_t myTargetIndex = 0;
|
|
int32_t targetLength = args->targetLimit - (char *) myTarget;
|
|
int32_t sourceLength = args->sourceLimit - mySource;
|
|
UChar mySourceChar;
|
|
|
|
/*writing the char to the output stream */
|
|
while (mySourceIndex < sourceLength)
|
|
{
|
|
if (myTargetIndex < targetLength)
|
|
{
|
|
mySourceChar = (UChar) mySource[mySourceIndex++];
|
|
myTarget[myTargetIndex++] = (char) mySourceChar;
|
|
if (myTargetIndex < targetLength)
|
|
{
|
|
myTarget[myTargetIndex++] = (char) (mySourceChar >> 8);
|
|
}
|
|
else
|
|
{
|
|
args->converter->charErrorBuffer[0] = (char) (mySourceChar >> 8);
|
|
args->converter->charErrorBufferLength = 1;
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
*err = U_BUFFER_OVERFLOW_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
|
|
args->target += myTargetIndex;
|
|
args->source += mySourceIndex;
|
|
}
|
|
|
|
U_CFUNC UChar32 T_UConverter_getNextUChar_UTF16_LE(UConverterToUnicodeArgs* args,
|
|
UErrorCode* err)
|
|
{
|
|
UChar32 myUChar;
|
|
uint16_t first;
|
|
/*Checks boundaries and set appropriate error codes*/
|
|
if (args->source+2 > args->sourceLimit)
|
|
{
|
|
if (args->source >= args->sourceLimit)
|
|
{
|
|
/*Either caller has reached the end of the byte stream*/
|
|
*err = U_INDEX_OUTOFBOUNDS_ERROR;
|
|
}
|
|
else
|
|
{
|
|
/* a character was cut in half*/
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
}
|
|
|
|
return 0xffff;
|
|
}
|
|
|
|
/*Gets the corresponding codepoint*/
|
|
first = (uint16_t)(((uint16_t)*((args->source)+1) << 8) | ((uint8_t)(*(args->source))));
|
|
myUChar=first;
|
|
/*updates the source*/
|
|
args->source += 2;
|
|
|
|
if (UTF_IS_FIRST_SURROGATE(first))
|
|
{
|
|
uint16_t second;
|
|
|
|
if (args->source+2 > args->sourceLimit)
|
|
{
|
|
*err = U_TRUNCATED_CHAR_FOUND;
|
|
return 0xffff;
|
|
}
|
|
|
|
/* get the second surrogate and assemble the code point */
|
|
second = (uint16_t)(((uint16_t)*(args->source+1) << 8) |((uint8_t)(*(args->source))));
|
|
|
|
/* ignore unmatched surrogates and just deliver the first one in such a case */
|
|
if(UTF_IS_SECOND_SURROGATE(second))
|
|
{
|
|
/* matched pair, get pair value */
|
|
myUChar = UTF16_GET_PAIR_VALUE(first, second);
|
|
args->source += 2;
|
|
}
|
|
}
|
|
|
|
return myUChar;
|
|
}
|
|
|
|
static const UConverterImpl _UTF16LEImpl={
|
|
UCNV_UTF16_LittleEndian,
|
|
|
|
NULL,
|
|
NULL,
|
|
|
|
NULL,
|
|
NULL,
|
|
NULL,
|
|
|
|
T_UConverter_toUnicode_UTF16_LE,
|
|
NULL,
|
|
T_UConverter_fromUnicode_UTF16_LE,
|
|
NULL,
|
|
T_UConverter_getNextUChar_UTF16_LE,
|
|
|
|
NULL
|
|
};
|
|
|
|
|
|
/* Todo: verify that UTF-16LE == (ccsid (ibm-codepage) 1200) for unicode version 2.0 and 3.0 */
|
|
const UConverterStaticData _UTF16LEStaticData={
|
|
sizeof(UConverterStaticData),
|
|
"UTF16_LittleEndian",
|
|
1200, UCNV_IBM, UCNV_UTF16_LittleEndian, 2, 2,
|
|
2, { 0xfd, 0xff, 0, 0 }
|
|
};
|
|
|
|
|
|
const UConverterSharedData _UTF16LEData={
|
|
sizeof(UConverterSharedData), ~((uint32_t) 0),
|
|
NULL, NULL, &_UTF16LEStaticData, FALSE, &_UTF16LEImpl,
|
|
0
|
|
};
|