153 lines
3.6 KiB
C++
153 lines
3.6 KiB
C++
/*
|
|
**********************************************************************
|
|
* Copyright (C) 2005-2012, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
**********************************************************************
|
|
*/
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_CONVERSION
|
|
|
|
#include "csrucode.h"
|
|
#include "csmatch.h"
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
CharsetRecog_Unicode::~CharsetRecog_Unicode()
|
|
{
|
|
// nothing to do
|
|
}
|
|
|
|
CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
|
|
{
|
|
// nothing to do
|
|
}
|
|
|
|
const char *CharsetRecog_UTF_16_BE::getName() const
|
|
{
|
|
return "UTF-16BE";
|
|
}
|
|
|
|
UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
|
|
{
|
|
const uint8_t *input = textIn->fRawInput;
|
|
int32_t confidence = 0;
|
|
|
|
if (input[0] == 0xFE && input[1] == 0xFF) {
|
|
confidence = 100;
|
|
}
|
|
|
|
// TODO: Do some statastics to check for unsigned UTF-16BE
|
|
results->set(textIn, this, confidence);
|
|
return (confidence > 0);
|
|
}
|
|
|
|
CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
|
|
{
|
|
// nothing to do
|
|
}
|
|
|
|
const char *CharsetRecog_UTF_16_LE::getName() const
|
|
{
|
|
return "UTF-16LE";
|
|
}
|
|
|
|
UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
|
|
{
|
|
const uint8_t *input = textIn->fRawInput;
|
|
int32_t confidence = 0;
|
|
|
|
if (input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) {
|
|
confidence = 100;
|
|
}
|
|
|
|
// TODO: Do some statastics to check for unsigned UTF-16LE
|
|
results->set(textIn, this, confidence);
|
|
return (confidence > 0);
|
|
}
|
|
|
|
CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
|
|
{
|
|
// nothing to do
|
|
}
|
|
|
|
UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
|
|
{
|
|
const uint8_t *input = textIn->fRawInput;
|
|
int32_t limit = (textIn->fRawLength / 4) * 4;
|
|
int32_t numValid = 0;
|
|
int32_t numInvalid = 0;
|
|
bool hasBOM = FALSE;
|
|
int32_t confidence = 0;
|
|
|
|
if (getChar(input, 0) == 0x0000FEFFUL) {
|
|
hasBOM = TRUE;
|
|
}
|
|
|
|
for(int32_t i = 0; i < limit; i += 4) {
|
|
int32_t ch = getChar(input, i);
|
|
|
|
if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
|
|
numInvalid += 1;
|
|
} else {
|
|
numValid += 1;
|
|
}
|
|
}
|
|
|
|
|
|
// Cook up some sort of confidence score, based on presense of a BOM
|
|
// and the existence of valid and/or invalid multi-byte sequences.
|
|
if (hasBOM && numInvalid==0) {
|
|
confidence = 100;
|
|
} else if (hasBOM && numValid > numInvalid*10) {
|
|
confidence = 80;
|
|
} else if (numValid > 3 && numInvalid == 0) {
|
|
confidence = 100;
|
|
} else if (numValid > 0 && numInvalid == 0) {
|
|
confidence = 80;
|
|
} else if (numValid > numInvalid*10) {
|
|
// Probably corruput UTF-32BE data. Valid sequences aren't likely by chance.
|
|
confidence = 25;
|
|
}
|
|
|
|
results->set(textIn, this, confidence);
|
|
return (confidence > 0);
|
|
}
|
|
|
|
CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
|
|
{
|
|
// nothing to do
|
|
}
|
|
|
|
const char *CharsetRecog_UTF_32_BE::getName() const
|
|
{
|
|
return "UTF-32BE";
|
|
}
|
|
|
|
int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
|
|
{
|
|
return input[index + 0] << 24 | input[index + 1] << 16 |
|
|
input[index + 2] << 8 | input[index + 3];
|
|
}
|
|
|
|
CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
|
|
{
|
|
// nothing to do
|
|
}
|
|
|
|
const char *CharsetRecog_UTF_32_LE::getName() const
|
|
{
|
|
return "UTF-32LE";
|
|
}
|
|
|
|
int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
|
|
{
|
|
return input[index + 3] << 24 | input[index + 2] << 16 |
|
|
input[index + 1] << 8 | input[index + 0];
|
|
}
|
|
|
|
U_NAMESPACE_END
|
|
#endif
|
|
|