2006-02-06 18:03:11 +00:00
|
|
|
/*
|
|
|
|
**********************************************************************
|
2015-02-20 19:31:33 +00:00
|
|
|
* Copyright (C) 2005-2015, International Business Machines
|
2006-02-06 18:03:11 +00:00
|
|
|
* Corporation and others. All Rights Reserved.
|
|
|
|
**********************************************************************
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef __CSRSBCS_H
|
|
|
|
#define __CSRSBCS_H
|
|
|
|
|
|
|
|
#include "unicode/uobject.h"
|
|
|
|
|
2006-05-09 18:06:10 +00:00
|
|
|
#if !UCONFIG_NO_CONVERSION
|
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
#include "csrecog.h"
|
|
|
|
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
|
|
|
|
class NGramParser : public UMemory
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
private:
|
2006-02-06 18:03:11 +00:00
|
|
|
int32_t ngram;
|
2013-06-11 12:39:56 +00:00
|
|
|
const int32_t *ngramList;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
int32_t ngramCount;
|
|
|
|
int32_t hitCount;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2013-06-11 12:39:56 +00:00
|
|
|
protected:
|
|
|
|
int32_t byteIndex;
|
|
|
|
const uint8_t *charMap;
|
|
|
|
|
|
|
|
void addByte(int32_t b);
|
|
|
|
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap);
|
2015-03-11 16:59:36 +00:00
|
|
|
virtual ~NGramParser();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
|
|
|
private:
|
2006-02-06 18:03:11 +00:00
|
|
|
/*
|
2006-08-10 19:08:46 +00:00
|
|
|
* Binary search for value in table, which must have exactly 64 entries.
|
|
|
|
*/
|
2006-02-06 18:03:11 +00:00
|
|
|
int32_t search(const int32_t *table, int32_t value);
|
|
|
|
|
2006-08-10 19:08:46 +00:00
|
|
|
void lookup(int32_t thisNgram);
|
2013-06-11 12:39:56 +00:00
|
|
|
|
|
|
|
virtual int32_t nextByte(InputText *det);
|
|
|
|
virtual void parseCharacters(InputText *det);
|
2006-08-10 19:08:46 +00:00
|
|
|
|
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
int32_t parse(InputText *det);
|
|
|
|
|
|
|
|
};
|
|
|
|
|
2015-02-20 19:31:33 +00:00
|
|
|
#if !UCONFIG_ONLY_HTML_CONVERSION
|
2013-06-11 12:39:56 +00:00
|
|
|
class NGramParser_IBM420 : public NGramParser
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap);
|
2015-09-22 21:51:03 +00:00
|
|
|
~NGramParser_IBM420();
|
|
|
|
|
|
|
|
private:
|
|
|
|
int32_t alef;
|
|
|
|
int32_t isLamAlef(int32_t b);
|
|
|
|
int32_t nextByte(InputText *det);
|
|
|
|
void parseCharacters(InputText *det);
|
2013-06-11 12:39:56 +00:00
|
|
|
};
|
2015-02-20 19:31:33 +00:00
|
|
|
#endif
|
2013-06-11 12:39:56 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
class CharsetRecog_sbcs : public CharsetRecognizer
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
CharsetRecog_sbcs();
|
|
|
|
virtual ~CharsetRecog_sbcs();
|
|
|
|
virtual const char *getName() const = 0;
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const = 0;
|
|
|
|
virtual int32_t match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t charMap[]) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_1 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_1();
|
|
|
|
const char *getName() const;
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_2 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_2();
|
|
|
|
const char *getName() const;
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_5 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_5();
|
|
|
|
const char *getName() const;
|
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_6 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_6();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getName() const;
|
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_7 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_7();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getName() const;
|
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_8 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_8();
|
|
|
|
|
2006-08-09 20:13:18 +00:00
|
|
|
virtual const char *getName() const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_9 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_9();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getName() const;
|
|
|
|
};
|
|
|
|
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
|
|
|
|
class CharsetRecog_8859_5_ru : public CharsetRecog_8859_5
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_5_ru();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_6_ar : public CharsetRecog_8859_6
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_6_ar();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_7_el : public CharsetRecog_8859_7
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_7_el();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_8_I_he : public CharsetRecog_8859_8
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_8_I_he();
|
|
|
|
|
2006-08-09 20:13:18 +00:00
|
|
|
const char *getName() const;
|
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_8_he : public CharsetRecog_8859_8
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_8_he ();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_8859_9_tr : public CharsetRecog_8859_9
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_8859_9_tr ();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_windows_1256 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_windows_1256();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getName() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_windows_1251 : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_windows_1251();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getName() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
class CharsetRecog_KOI8_R : public CharsetRecog_sbcs
|
|
|
|
{
|
2006-08-10 19:08:46 +00:00
|
|
|
public:
|
2006-02-06 18:03:11 +00:00
|
|
|
virtual ~CharsetRecog_KOI8_R();
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getName() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
const char *getLanguage() const;
|
2006-08-10 19:08:46 +00:00
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2006-02-06 18:03:11 +00:00
|
|
|
};
|
|
|
|
|
2015-02-20 19:31:33 +00:00
|
|
|
#if !UCONFIG_ONLY_HTML_CONVERSION
|
2009-03-31 15:39:00 +00:00
|
|
|
class CharsetRecog_IBM424_he : public CharsetRecog_sbcs
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
virtual ~CharsetRecog_IBM424_he();
|
|
|
|
|
|
|
|
const char *getLanguage() const;
|
2009-04-13 21:32:21 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_IBM424_he_rtl : public CharsetRecog_IBM424_he {
|
|
|
|
public:
|
|
|
|
virtual ~CharsetRecog_IBM424_he_rtl();
|
|
|
|
|
|
|
|
const char *getName() const;
|
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2009-04-13 21:32:21 +00:00
|
|
|
};
|
2009-03-31 15:39:00 +00:00
|
|
|
|
2009-04-13 21:32:21 +00:00
|
|
|
class CharsetRecog_IBM424_he_ltr : public CharsetRecog_IBM424_he {
|
|
|
|
virtual ~CharsetRecog_IBM424_he_ltr();
|
|
|
|
|
|
|
|
const char *getName() const;
|
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2009-03-31 15:39:00 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_IBM420_ar : public CharsetRecog_sbcs
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
virtual ~CharsetRecog_IBM420_ar();
|
|
|
|
|
|
|
|
const char *getLanguage() const;
|
2013-06-11 12:39:56 +00:00
|
|
|
int32_t match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t charMap[]) const;
|
2009-04-13 21:32:21 +00:00
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
class CharsetRecog_IBM420_ar_rtl : public CharsetRecog_IBM420_ar {
|
|
|
|
public:
|
|
|
|
virtual ~CharsetRecog_IBM420_ar_rtl();
|
|
|
|
|
|
|
|
const char *getName() const;
|
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2009-04-13 21:32:21 +00:00
|
|
|
};
|
2009-03-31 15:39:00 +00:00
|
|
|
|
2009-04-13 21:32:21 +00:00
|
|
|
class CharsetRecog_IBM420_ar_ltr : public CharsetRecog_IBM420_ar {
|
|
|
|
virtual ~CharsetRecog_IBM420_ar_ltr();
|
|
|
|
|
|
|
|
const char *getName() const;
|
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
virtual UBool match(InputText *det, CharsetMatch *results) const;
|
2009-03-31 15:39:00 +00:00
|
|
|
};
|
2015-02-20 19:31:33 +00:00
|
|
|
#endif
|
2009-03-31 15:39:00 +00:00
|
|
|
|
2006-02-06 18:03:11 +00:00
|
|
|
U_NAMESPACE_END
|
|
|
|
|
2012-06-01 20:40:48 +00:00
|
|
|
#endif /* !UCONFIG_NO_CONVERSION */
|
2006-02-06 18:03:11 +00:00
|
|
|
#endif /* __CSRSBCS_H */
|