a111dd26b1
Make clear why we don't need to assert against out-of-bounda accesses in the generated code, provided the code point is within its bound, (Using one table's early entries as indices into later in the same table at which to look up indices into another table made it a little hard to work out what was going on, especially as nothing told me about the early / late distinction. Record what I discovered, to save the next person to stumble into this some confusion.) Change-Id: I8e5771a7f3d70c1911aeae1b0cabe5c47bc7e9c7 Reviewed-by: Konstantin Ritt <ritt.ks@gmail.com> Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
3194 lines
110 KiB
C++
3194 lines
110 KiB
C++
/****************************************************************************
|
|
**
|
|
** Copyright (C) 2020 The Qt Company Ltd.
|
|
** Contact: https://www.qt.io/licensing/
|
|
**
|
|
** This file is part of the utils of the Qt Toolkit.
|
|
**
|
|
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
|
|
** Commercial License Usage
|
|
** Licensees holding valid commercial Qt licenses may use this file in
|
|
** accordance with the commercial license agreement provided with the
|
|
** Software or, alternatively, in accordance with the terms contained in
|
|
** a written agreement between you and The Qt Company. For licensing terms
|
|
** and conditions see https://www.qt.io/terms-conditions. For further
|
|
** information use the contact form at https://www.qt.io/contact-us.
|
|
**
|
|
** GNU General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU
|
|
** General Public License version 3 as published by the Free Software
|
|
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
|
** included in the packaging of this file. Please review the following
|
|
** information to ensure the GNU General Public License requirements will
|
|
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
|
**
|
|
** $QT_END_LICENSE$
|
|
**
|
|
****************************************************************************/
|
|
|
|
#include <qbytearray.h>
|
|
#include <qchar.h>
|
|
#include <qdebug.h>
|
|
#include <qfile.h>
|
|
#include <qhash.h>
|
|
#include <qlist.h>
|
|
#include <qstring.h>
|
|
#if 0
|
|
#include <private/qunicodetables_p.h>
|
|
#endif
|
|
|
|
#define DATA_VERSION_S "13.0"
|
|
#define DATA_VERSION_STR "QChar::Unicode_13_0"
|
|
|
|
|
|
static QHash<QByteArray, QChar::UnicodeVersion> age_map;
|
|
|
|
static void initAgeMap()
|
|
{
|
|
struct AgeMap {
|
|
const QChar::UnicodeVersion version;
|
|
const char *age;
|
|
} ageMap[] = {
|
|
{ QChar::Unicode_1_1, "1.1" },
|
|
{ QChar::Unicode_2_0, "2.0" },
|
|
{ QChar::Unicode_2_1_2, "2.1" },
|
|
{ QChar::Unicode_3_0, "3.0" },
|
|
{ QChar::Unicode_3_1, "3.1" },
|
|
{ QChar::Unicode_3_2, "3.2" },
|
|
{ QChar::Unicode_4_0, "4.0" },
|
|
{ QChar::Unicode_4_1, "4.1" },
|
|
{ QChar::Unicode_5_0, "5.0" },
|
|
{ QChar::Unicode_5_1, "5.1" },
|
|
{ QChar::Unicode_5_2, "5.2" },
|
|
{ QChar::Unicode_6_0, "6.0" },
|
|
{ QChar::Unicode_6_1, "6.1" },
|
|
{ QChar::Unicode_6_2, "6.2" },
|
|
{ QChar::Unicode_6_3, "6.3" },
|
|
{ QChar::Unicode_7_0, "7.0" },
|
|
{ QChar::Unicode_8_0, "8.0" },
|
|
{ QChar::Unicode_9_0, "9.0" },
|
|
{ QChar::Unicode_10_0, "10.0" },
|
|
{ QChar::Unicode_11_0, "11.0" },
|
|
{ QChar::Unicode_12_0, "12.0" },
|
|
{ QChar::Unicode_12_1, "12.1" }, // UCD Revision 24
|
|
{ QChar::Unicode_13_0, "13.0" }, // UCD Revision 26
|
|
{ QChar::Unicode_Unassigned, 0 }
|
|
};
|
|
AgeMap *d = ageMap;
|
|
while (d->age) {
|
|
age_map.insert(d->age, d->version);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
static QHash<QByteArray, QChar::Category> categoryMap;
|
|
|
|
static void initCategoryMap()
|
|
{
|
|
struct Cat {
|
|
QChar::Category cat;
|
|
const char *name;
|
|
} categories[] = {
|
|
{ QChar::Mark_NonSpacing, "Mn" },
|
|
{ QChar::Mark_SpacingCombining, "Mc" },
|
|
{ QChar::Mark_Enclosing, "Me" },
|
|
|
|
{ QChar::Number_DecimalDigit, "Nd" },
|
|
{ QChar::Number_Letter, "Nl" },
|
|
{ QChar::Number_Other, "No" },
|
|
|
|
{ QChar::Separator_Space, "Zs" },
|
|
{ QChar::Separator_Line, "Zl" },
|
|
{ QChar::Separator_Paragraph, "Zp" },
|
|
|
|
{ QChar::Other_Control, "Cc" },
|
|
{ QChar::Other_Format, "Cf" },
|
|
{ QChar::Other_Surrogate, "Cs" },
|
|
{ QChar::Other_PrivateUse, "Co" },
|
|
{ QChar::Other_NotAssigned, "Cn" },
|
|
|
|
{ QChar::Letter_Uppercase, "Lu" },
|
|
{ QChar::Letter_Lowercase, "Ll" },
|
|
{ QChar::Letter_Titlecase, "Lt" },
|
|
{ QChar::Letter_Modifier, "Lm" },
|
|
{ QChar::Letter_Other, "Lo" },
|
|
|
|
{ QChar::Punctuation_Connector, "Pc" },
|
|
{ QChar::Punctuation_Dash, "Pd" },
|
|
{ QChar::Punctuation_Open, "Ps" },
|
|
{ QChar::Punctuation_Close, "Pe" },
|
|
{ QChar::Punctuation_InitialQuote, "Pi" },
|
|
{ QChar::Punctuation_FinalQuote, "Pf" },
|
|
{ QChar::Punctuation_Other, "Po" },
|
|
|
|
{ QChar::Symbol_Math, "Sm" },
|
|
{ QChar::Symbol_Currency, "Sc" },
|
|
{ QChar::Symbol_Modifier, "Sk" },
|
|
{ QChar::Symbol_Other, "So" },
|
|
{ QChar::Other_NotAssigned, 0 }
|
|
};
|
|
Cat *c = categories;
|
|
while (c->name) {
|
|
categoryMap.insert(c->name, c->cat);
|
|
++c;
|
|
}
|
|
}
|
|
|
|
|
|
static QHash<QByteArray, QChar::Decomposition> decompositionMap;
|
|
|
|
static void initDecompositionMap()
|
|
{
|
|
struct Dec {
|
|
QChar::Decomposition dec;
|
|
const char *name;
|
|
} decompositions[] = {
|
|
{ QChar::Canonical, "<canonical>" },
|
|
{ QChar::Font, "<font>" },
|
|
{ QChar::NoBreak, "<noBreak>" },
|
|
{ QChar::Initial, "<initial>" },
|
|
{ QChar::Medial, "<medial>" },
|
|
{ QChar::Final, "<final>" },
|
|
{ QChar::Isolated, "<isolated>" },
|
|
{ QChar::Circle, "<circle>" },
|
|
{ QChar::Super, "<super>" },
|
|
{ QChar::Sub, "<sub>" },
|
|
{ QChar::Vertical, "<vertical>" },
|
|
{ QChar::Wide, "<wide>" },
|
|
{ QChar::Narrow, "<narrow>" },
|
|
{ QChar::Small, "<small>" },
|
|
{ QChar::Square, "<square>" },
|
|
{ QChar::Compat, "<compat>" },
|
|
{ QChar::Fraction, "<fraction>" },
|
|
{ QChar::NoDecomposition, 0 }
|
|
};
|
|
Dec *d = decompositions;
|
|
while (d->name) {
|
|
decompositionMap.insert(d->name, d->dec);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
|
|
enum Direction {
|
|
DirL = QChar::DirL,
|
|
DirR = QChar::DirR,
|
|
DirEN = QChar::DirEN,
|
|
DirES = QChar::DirES,
|
|
DirET = QChar::DirET,
|
|
DirAN = QChar::DirAN,
|
|
DirCS = QChar::DirCS,
|
|
DirB = QChar::DirB,
|
|
DirS = QChar::DirS,
|
|
DirWS = QChar::DirWS,
|
|
DirON = QChar::DirON,
|
|
DirLRE = QChar::DirLRE,
|
|
DirLRO = QChar::DirLRO,
|
|
DirAL = QChar::DirAL,
|
|
DirRLE = QChar::DirRLE,
|
|
DirRLO = QChar::DirRLO,
|
|
DirPDF = QChar::DirPDF,
|
|
DirNSM = QChar::DirNSM,
|
|
DirBN = QChar::DirBN,
|
|
DirLRI = QChar::DirLRI,
|
|
DirRLI = QChar::DirRLI,
|
|
DirFSI = QChar::DirFSI,
|
|
DirPDI = QChar::DirPDI,
|
|
|
|
Dir_Unassigned
|
|
};
|
|
|
|
static QHash<QByteArray, Direction> directionMap;
|
|
|
|
static void initDirectionMap()
|
|
{
|
|
struct Dir {
|
|
Direction dir;
|
|
const char *name;
|
|
} directions[] = {
|
|
{ DirL, "L" },
|
|
{ DirR, "R" },
|
|
{ DirEN, "EN" },
|
|
{ DirES, "ES" },
|
|
{ DirET, "ET" },
|
|
{ DirAN, "AN" },
|
|
{ DirCS, "CS" },
|
|
{ DirB, "B" },
|
|
{ DirS, "S" },
|
|
{ DirWS, "WS" },
|
|
{ DirON, "ON" },
|
|
{ DirLRE, "LRE" },
|
|
{ DirLRO, "LRO" },
|
|
{ DirAL, "AL" },
|
|
{ DirRLE, "RLE" },
|
|
{ DirRLO, "RLO" },
|
|
{ DirPDF, "PDF" },
|
|
{ DirNSM, "NSM" },
|
|
{ DirBN, "BN" },
|
|
{ DirLRI, "LRI" },
|
|
{ DirRLI, "RLI" },
|
|
{ DirFSI, "FSI" },
|
|
{ DirPDI, "PDI" },
|
|
{ Dir_Unassigned, 0 }
|
|
};
|
|
Dir *d = directions;
|
|
while (d->name) {
|
|
directionMap.insert(d->name, d->dir);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
|
|
enum JoiningType {
|
|
Joining_None,
|
|
Joining_Causing,
|
|
Joining_Dual,
|
|
Joining_Right,
|
|
Joining_Left,
|
|
Joining_Transparent,
|
|
|
|
Joining_Unassigned
|
|
};
|
|
|
|
static QHash<QByteArray, JoiningType> joining_map;
|
|
|
|
static void initJoiningMap()
|
|
{
|
|
struct JoiningList {
|
|
JoiningType joining;
|
|
const char *name;
|
|
} joinings[] = {
|
|
{ Joining_None, "U" },
|
|
{ Joining_Causing, "C" },
|
|
{ Joining_Dual, "D" },
|
|
{ Joining_Right, "R" },
|
|
{ Joining_Left, "L" },
|
|
{ Joining_Transparent, "T" },
|
|
{ Joining_Unassigned, 0 }
|
|
};
|
|
JoiningList *d = joinings;
|
|
while (d->name) {
|
|
joining_map.insert(d->name, d->joining);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
|
|
static const char *grapheme_break_class_string =
|
|
"enum GraphemeBreakClass {\n"
|
|
" GraphemeBreak_Any,\n"
|
|
" GraphemeBreak_CR,\n"
|
|
" GraphemeBreak_LF,\n"
|
|
" GraphemeBreak_Control,\n"
|
|
" GraphemeBreak_Extend,\n"
|
|
" GraphemeBreak_ZWJ,\n"
|
|
" GraphemeBreak_RegionalIndicator,\n"
|
|
" GraphemeBreak_Prepend,\n"
|
|
" GraphemeBreak_SpacingMark,\n"
|
|
" GraphemeBreak_L,\n"
|
|
" GraphemeBreak_V,\n"
|
|
" GraphemeBreak_T,\n"
|
|
" GraphemeBreak_LV,\n"
|
|
" GraphemeBreak_LVT,\n"
|
|
" Graphemebreak_E_Base,\n"
|
|
" Graphemebreak_E_Modifier,\n"
|
|
" Graphemebreak_Glue_After_Zwj,\n"
|
|
" Graphemebreak_E_Base_GAZ,\n"
|
|
"\n"
|
|
" NumGraphemeBreakClasses\n"
|
|
"};\n\n";
|
|
|
|
enum GraphemeBreakClass {
|
|
GraphemeBreak_Any,
|
|
GraphemeBreak_CR,
|
|
GraphemeBreak_LF,
|
|
GraphemeBreak_Control,
|
|
GraphemeBreak_Extend,
|
|
GraphemeBreak_ZWJ,
|
|
GraphemeBreak_RegionalIndicator,
|
|
GraphemeBreak_Prepend,
|
|
GraphemeBreak_SpacingMark,
|
|
GraphemeBreak_L,
|
|
GraphemeBreak_V,
|
|
GraphemeBreak_T,
|
|
GraphemeBreak_LV,
|
|
GraphemeBreak_LVT,
|
|
Graphemebreak_E_Base,
|
|
Graphemebreak_E_Modifier,
|
|
Graphemebreak_Glue_After_Zwj,
|
|
Graphemebreak_E_Base_GAZ,
|
|
|
|
GraphemeBreak_Unassigned
|
|
};
|
|
|
|
static QHash<QByteArray, GraphemeBreakClass> grapheme_break_map;
|
|
|
|
static void initGraphemeBreak()
|
|
{
|
|
struct GraphemeBreakList {
|
|
GraphemeBreakClass brk;
|
|
const char *name;
|
|
} breaks[] = {
|
|
{ GraphemeBreak_Any, "Any" },
|
|
{ GraphemeBreak_CR, "CR" },
|
|
{ GraphemeBreak_LF, "LF" },
|
|
{ GraphemeBreak_Control, "Control" },
|
|
{ GraphemeBreak_Extend, "Extend" },
|
|
{ GraphemeBreak_ZWJ, "ZWJ" },
|
|
{ GraphemeBreak_RegionalIndicator, "Regional_Indicator" },
|
|
{ GraphemeBreak_Prepend, "Prepend" },
|
|
{ GraphemeBreak_SpacingMark, "SpacingMark" },
|
|
{ GraphemeBreak_L, "L" },
|
|
{ GraphemeBreak_V, "V" },
|
|
{ GraphemeBreak_T, "T" },
|
|
{ GraphemeBreak_LV, "LV" },
|
|
{ GraphemeBreak_LVT, "LVT" },
|
|
{ Graphemebreak_E_Base, "E_Base" },
|
|
{ Graphemebreak_E_Modifier, "E_Modifier" },
|
|
{ Graphemebreak_Glue_After_Zwj, "Glue_After_Zwj" },
|
|
{ Graphemebreak_E_Base_GAZ, "E_Base_GAZ" },
|
|
{ GraphemeBreak_Unassigned, 0 }
|
|
};
|
|
GraphemeBreakList *d = breaks;
|
|
while (d->name) {
|
|
grapheme_break_map.insert(d->name, d->brk);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
|
|
static const char *word_break_class_string =
|
|
"enum WordBreakClass {\n"
|
|
" WordBreak_Any,\n"
|
|
" WordBreak_CR,\n"
|
|
" WordBreak_LF,\n"
|
|
" WordBreak_Newline,\n"
|
|
" WordBreak_Extend,\n"
|
|
" WordBreak_ZWJ,\n"
|
|
" WordBreak_Format,\n"
|
|
" WordBreak_RegionalIndicator,\n"
|
|
" WordBreak_Katakana,\n"
|
|
" WordBreak_HebrewLetter,\n"
|
|
" WordBreak_ALetter,\n"
|
|
" WordBreak_SingleQuote,\n"
|
|
" WordBreak_DoubleQuote,\n"
|
|
" WordBreak_MidNumLet,\n"
|
|
" WordBreak_MidLetter,\n"
|
|
" WordBreak_MidNum,\n"
|
|
" WordBreak_Numeric,\n"
|
|
" WordBreak_ExtendNumLet,\n"
|
|
" WordBreak_E_Base,\n"
|
|
" WordBreak_E_Modifier,\n"
|
|
" WordBreak_Glue_After_Zwj,\n"
|
|
" WordBreak_E_Base_GAZ,\n"
|
|
" WordBreak_WSegSpace,\n"
|
|
"\n"
|
|
" NumWordBreakClasses\n"
|
|
"};\n\n";
|
|
|
|
enum WordBreakClass {
|
|
WordBreak_Any,
|
|
WordBreak_CR,
|
|
WordBreak_LF,
|
|
WordBreak_Newline,
|
|
WordBreak_Extend,
|
|
WordBreak_ZWJ,
|
|
WordBreak_Format,
|
|
WordBreak_RegionalIndicator,
|
|
WordBreak_Katakana,
|
|
WordBreak_HebrewLetter,
|
|
WordBreak_ALetter,
|
|
WordBreak_SingleQuote,
|
|
WordBreak_DoubleQuote,
|
|
WordBreak_MidNumLet,
|
|
WordBreak_MidLetter,
|
|
WordBreak_MidNum,
|
|
WordBreak_Numeric,
|
|
WordBreak_ExtendNumLet,
|
|
WordBreak_E_Base,
|
|
WordBreak_E_Modifier,
|
|
WordBreak_Glue_After_Zwj,
|
|
WordBreak_E_Base_GAZ,
|
|
WordBreak_WSegSpace,
|
|
|
|
WordBreak_Unassigned
|
|
};
|
|
|
|
static QHash<QByteArray, WordBreakClass> word_break_map;
|
|
|
|
static void initWordBreak()
|
|
{
|
|
struct WordBreakList {
|
|
WordBreakClass brk;
|
|
const char *name;
|
|
} breaks[] = {
|
|
{ WordBreak_Any, "Any" },
|
|
{ WordBreak_CR, "CR" },
|
|
{ WordBreak_LF, "LF" },
|
|
{ WordBreak_Newline, "Newline" },
|
|
{ WordBreak_Extend, "Extend" },
|
|
{ WordBreak_ZWJ, "ZWJ" },
|
|
{ WordBreak_Format, "Format" },
|
|
{ WordBreak_RegionalIndicator, "Regional_Indicator" },
|
|
{ WordBreak_Katakana, "Katakana" },
|
|
{ WordBreak_HebrewLetter, "Hebrew_Letter" },
|
|
{ WordBreak_ALetter, "ALetter" },
|
|
{ WordBreak_SingleQuote, "Single_Quote" },
|
|
{ WordBreak_DoubleQuote, "Double_Quote" },
|
|
{ WordBreak_MidNumLet, "MidNumLet" },
|
|
{ WordBreak_MidLetter, "MidLetter" },
|
|
{ WordBreak_MidNum, "MidNum" },
|
|
{ WordBreak_Numeric, "Numeric" },
|
|
{ WordBreak_ExtendNumLet, "ExtendNumLet" },
|
|
{ WordBreak_E_Base, "E_Base" },
|
|
{ WordBreak_E_Modifier, "E_Modifier" },
|
|
{ WordBreak_Glue_After_Zwj, "Glue_After_Zwj" },
|
|
{ WordBreak_E_Base_GAZ, "E_Base_GAZ" },
|
|
{ WordBreak_WSegSpace, "WSegSpace" },
|
|
{ WordBreak_Unassigned, 0 }
|
|
};
|
|
WordBreakList *d = breaks;
|
|
while (d->name) {
|
|
word_break_map.insert(d->name, d->brk);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
|
|
static const char *sentence_break_class_string =
|
|
"enum SentenceBreakClass {\n"
|
|
" SentenceBreak_Any,\n"
|
|
" SentenceBreak_CR,\n"
|
|
" SentenceBreak_LF,\n"
|
|
" SentenceBreak_Sep,\n"
|
|
" SentenceBreak_Extend,\n"
|
|
" SentenceBreak_Sp,\n"
|
|
" SentenceBreak_Lower,\n"
|
|
" SentenceBreak_Upper,\n"
|
|
" SentenceBreak_OLetter,\n"
|
|
" SentenceBreak_Numeric,\n"
|
|
" SentenceBreak_ATerm,\n"
|
|
" SentenceBreak_SContinue,\n"
|
|
" SentenceBreak_STerm,\n"
|
|
" SentenceBreak_Close,\n"
|
|
"\n"
|
|
" NumSentenceBreakClasses\n"
|
|
"};\n\n";
|
|
|
|
enum SentenceBreakClass {
|
|
SentenceBreak_Any,
|
|
SentenceBreak_CR,
|
|
SentenceBreak_LF,
|
|
SentenceBreak_Sep,
|
|
SentenceBreak_Extend,
|
|
SentenceBreak_Sp,
|
|
SentenceBreak_Lower,
|
|
SentenceBreak_Upper,
|
|
SentenceBreak_OLetter,
|
|
SentenceBreak_Numeric,
|
|
SentenceBreak_ATerm,
|
|
SentenceBreak_SContinue,
|
|
SentenceBreak_STerm,
|
|
SentenceBreak_Close,
|
|
|
|
SentenceBreak_Unassigned
|
|
};
|
|
|
|
static QHash<QByteArray, SentenceBreakClass> sentence_break_map;
|
|
|
|
static void initSentenceBreak()
|
|
{
|
|
struct SentenceBreakList {
|
|
SentenceBreakClass brk;
|
|
const char *name;
|
|
} breaks[] = {
|
|
{ SentenceBreak_Any, "Any" },
|
|
{ SentenceBreak_CR, "CR" },
|
|
{ SentenceBreak_LF, "LF" },
|
|
{ SentenceBreak_Sep, "Sep" },
|
|
{ SentenceBreak_Extend, "Extend" },
|
|
{ SentenceBreak_Extend, "Format" },
|
|
{ SentenceBreak_Sp, "Sp" },
|
|
{ SentenceBreak_Lower, "Lower" },
|
|
{ SentenceBreak_Upper, "Upper" },
|
|
{ SentenceBreak_OLetter, "OLetter" },
|
|
{ SentenceBreak_Numeric, "Numeric" },
|
|
{ SentenceBreak_ATerm, "ATerm" },
|
|
{ SentenceBreak_SContinue, "SContinue" },
|
|
{ SentenceBreak_STerm, "STerm" },
|
|
{ SentenceBreak_Close, "Close" },
|
|
{ SentenceBreak_Unassigned, 0 }
|
|
};
|
|
SentenceBreakList *d = breaks;
|
|
while (d->name) {
|
|
sentence_break_map.insert(d->name, d->brk);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
|
|
static const char *line_break_class_string =
|
|
"// see http://www.unicode.org/reports/tr14/tr14-30.html\n"
|
|
"// we don't use the XX and AI classes and map them to AL instead.\n"
|
|
"enum LineBreakClass {\n"
|
|
" LineBreak_OP, LineBreak_CL, LineBreak_CP, LineBreak_QU, LineBreak_GL,\n"
|
|
" LineBreak_NS, LineBreak_EX, LineBreak_SY, LineBreak_IS, LineBreak_PR,\n"
|
|
" LineBreak_PO, LineBreak_NU, LineBreak_AL, LineBreak_HL, LineBreak_ID,\n"
|
|
" LineBreak_IN, LineBreak_HY, LineBreak_BA, LineBreak_BB, LineBreak_B2,\n"
|
|
" LineBreak_ZW, LineBreak_CM, LineBreak_WJ, LineBreak_H2, LineBreak_H3,\n"
|
|
" LineBreak_JL, LineBreak_JV, LineBreak_JT, LineBreak_RI, LineBreak_CB,\n"
|
|
" LineBreak_EB, LineBreak_EM, LineBreak_ZWJ,\n"
|
|
" LineBreak_SA, LineBreak_SG, LineBreak_SP,\n"
|
|
" LineBreak_CR, LineBreak_LF, LineBreak_BK,\n"
|
|
"\n"
|
|
" NumLineBreakClasses\n"
|
|
"};\n\n";
|
|
|
|
enum LineBreakClass {
|
|
LineBreak_OP, LineBreak_CL, LineBreak_CP, LineBreak_QU, LineBreak_GL,
|
|
LineBreak_NS, LineBreak_EX, LineBreak_SY, LineBreak_IS, LineBreak_PR,
|
|
LineBreak_PO, LineBreak_NU, LineBreak_AL, LineBreak_HL, LineBreak_ID,
|
|
LineBreak_IN, LineBreak_HY, LineBreak_BA, LineBreak_BB, LineBreak_B2,
|
|
LineBreak_ZW, LineBreak_CM, LineBreak_WJ, LineBreak_H2, LineBreak_H3,
|
|
LineBreak_JL, LineBreak_JV, LineBreak_JT, LineBreak_RI, LineBreak_CB,
|
|
LineBreak_EB, LineBreak_EM, LineBreak_ZWJ,
|
|
LineBreak_SA, LineBreak_SG, LineBreak_SP,
|
|
LineBreak_CR, LineBreak_LF, LineBreak_BK,
|
|
|
|
LineBreak_Unassigned
|
|
};
|
|
|
|
static QHash<QByteArray, LineBreakClass> line_break_map;
|
|
|
|
static void initLineBreak()
|
|
{
|
|
// ### Classes XX and AI are left out and mapped to AL for now.
|
|
// ### Class NL is mapped to BK.
|
|
// ### Treating characters of class CJ as class NS will give CSS strict line breaking;
|
|
// treating them as class ID will give CSS normal breaking.
|
|
struct LineBreakList {
|
|
LineBreakClass brk;
|
|
const char *name;
|
|
} breaks[] = {
|
|
{ LineBreak_BK, "BK" },
|
|
{ LineBreak_CR, "CR" },
|
|
{ LineBreak_LF, "LF" },
|
|
{ LineBreak_CM, "CM" },
|
|
{ LineBreak_BK, "NL" },
|
|
{ LineBreak_SG, "SG" },
|
|
{ LineBreak_WJ, "WJ" },
|
|
{ LineBreak_ZW, "ZW" },
|
|
{ LineBreak_GL, "GL" },
|
|
{ LineBreak_SP, "SP" },
|
|
{ LineBreak_B2, "B2" },
|
|
{ LineBreak_BA, "BA" },
|
|
{ LineBreak_BB, "BB" },
|
|
{ LineBreak_HY, "HY" },
|
|
{ LineBreak_CB, "CB" },
|
|
{ LineBreak_NS, "CJ" },
|
|
{ LineBreak_CL, "CL" },
|
|
{ LineBreak_CP, "CP" },
|
|
{ LineBreak_EX, "EX" },
|
|
{ LineBreak_IN, "IN" },
|
|
{ LineBreak_NS, "NS" },
|
|
{ LineBreak_OP, "OP" },
|
|
{ LineBreak_QU, "QU" },
|
|
{ LineBreak_IS, "IS" },
|
|
{ LineBreak_NU, "NU" },
|
|
{ LineBreak_PO, "PO" },
|
|
{ LineBreak_PR, "PR" },
|
|
{ LineBreak_SY, "SY" },
|
|
{ LineBreak_AL, "AI" },
|
|
{ LineBreak_AL, "AL" },
|
|
{ LineBreak_HL, "HL" },
|
|
{ LineBreak_H2, "H2" },
|
|
{ LineBreak_H3, "H3" },
|
|
{ LineBreak_ID, "ID" },
|
|
{ LineBreak_JL, "JL" },
|
|
{ LineBreak_JV, "JV" },
|
|
{ LineBreak_JT, "JT" },
|
|
{ LineBreak_RI, "RI" },
|
|
{ LineBreak_SA, "SA" },
|
|
{ LineBreak_AL, "XX" },
|
|
{ LineBreak_EB, "EB" },
|
|
{ LineBreak_EM, "EM" },
|
|
{ LineBreak_ZWJ, "ZWJ" },
|
|
{ LineBreak_Unassigned, 0 }
|
|
};
|
|
LineBreakList *d = breaks;
|
|
while (d->name) {
|
|
line_break_map.insert(d->name, d->brk);
|
|
++d;
|
|
}
|
|
}
|
|
|
|
|
|
static QHash<QByteArray, QChar::Script> scriptMap;
|
|
|
|
static void initScriptMap()
|
|
{
|
|
struct Scrpt {
|
|
QChar::Script script;
|
|
const char *name;
|
|
} scripts[] = {
|
|
// general
|
|
{ QChar::Script_Unknown, "Unknown" },
|
|
{ QChar::Script_Inherited, "Inherited" },
|
|
{ QChar::Script_Common, "Common" },
|
|
// pre-4.0
|
|
{ QChar::Script_Latin, "Latin" },
|
|
{ QChar::Script_Greek, "Greek" },
|
|
{ QChar::Script_Cyrillic, "Cyrillic" },
|
|
{ QChar::Script_Armenian, "Armenian" },
|
|
{ QChar::Script_Hebrew, "Hebrew" },
|
|
{ QChar::Script_Arabic, "Arabic" },
|
|
{ QChar::Script_Syriac, "Syriac" },
|
|
{ QChar::Script_Thaana, "Thaana" },
|
|
{ QChar::Script_Devanagari, "Devanagari" },
|
|
{ QChar::Script_Bengali, "Bengali" },
|
|
{ QChar::Script_Gurmukhi, "Gurmukhi" },
|
|
{ QChar::Script_Gujarati, "Gujarati" },
|
|
{ QChar::Script_Oriya, "Oriya" },
|
|
{ QChar::Script_Tamil, "Tamil" },
|
|
{ QChar::Script_Telugu, "Telugu" },
|
|
{ QChar::Script_Kannada, "Kannada" },
|
|
{ QChar::Script_Malayalam, "Malayalam" },
|
|
{ QChar::Script_Sinhala, "Sinhala" },
|
|
{ QChar::Script_Thai, "Thai" },
|
|
{ QChar::Script_Lao, "Lao" },
|
|
{ QChar::Script_Tibetan, "Tibetan" },
|
|
{ QChar::Script_Myanmar, "Myanmar" },
|
|
{ QChar::Script_Georgian, "Georgian" },
|
|
{ QChar::Script_Hangul, "Hangul" },
|
|
{ QChar::Script_Ethiopic, "Ethiopic" },
|
|
{ QChar::Script_Cherokee, "Cherokee" },
|
|
{ QChar::Script_CanadianAboriginal, "CanadianAboriginal" },
|
|
{ QChar::Script_Ogham, "Ogham" },
|
|
{ QChar::Script_Runic, "Runic" },
|
|
{ QChar::Script_Khmer, "Khmer" },
|
|
{ QChar::Script_Mongolian, "Mongolian" },
|
|
{ QChar::Script_Hiragana, "Hiragana" },
|
|
{ QChar::Script_Katakana, "Katakana" },
|
|
{ QChar::Script_Bopomofo, "Bopomofo" },
|
|
{ QChar::Script_Han, "Han" },
|
|
{ QChar::Script_Yi, "Yi" },
|
|
{ QChar::Script_OldItalic, "OldItalic" },
|
|
{ QChar::Script_Gothic, "Gothic" },
|
|
{ QChar::Script_Deseret, "Deseret" },
|
|
{ QChar::Script_Tagalog, "Tagalog" },
|
|
{ QChar::Script_Hanunoo, "Hanunoo" },
|
|
{ QChar::Script_Buhid, "Buhid" },
|
|
{ QChar::Script_Tagbanwa, "Tagbanwa" },
|
|
{ QChar::Script_Coptic, "Coptic" },
|
|
// 4.0
|
|
{ QChar::Script_Limbu, "Limbu" },
|
|
{ QChar::Script_TaiLe, "TaiLe" },
|
|
{ QChar::Script_LinearB, "LinearB" },
|
|
{ QChar::Script_Ugaritic, "Ugaritic" },
|
|
{ QChar::Script_Shavian, "Shavian" },
|
|
{ QChar::Script_Osmanya, "Osmanya" },
|
|
{ QChar::Script_Cypriot, "Cypriot" },
|
|
{ QChar::Script_Braille, "Braille" },
|
|
// 4.1
|
|
{ QChar::Script_Buginese, "Buginese" },
|
|
{ QChar::Script_NewTaiLue, "NewTaiLue" },
|
|
{ QChar::Script_Glagolitic, "Glagolitic" },
|
|
{ QChar::Script_Tifinagh, "Tifinagh" },
|
|
{ QChar::Script_SylotiNagri, "SylotiNagri" },
|
|
{ QChar::Script_OldPersian, "OldPersian" },
|
|
{ QChar::Script_Kharoshthi, "Kharoshthi" },
|
|
// 5.0
|
|
{ QChar::Script_Balinese, "Balinese" },
|
|
{ QChar::Script_Cuneiform, "Cuneiform" },
|
|
{ QChar::Script_Phoenician, "Phoenician" },
|
|
{ QChar::Script_PhagsPa, "PhagsPa" },
|
|
{ QChar::Script_Nko, "Nko" },
|
|
// 5.1
|
|
{ QChar::Script_Sundanese, "Sundanese" },
|
|
{ QChar::Script_Lepcha, "Lepcha" },
|
|
{ QChar::Script_OlChiki, "OlChiki" },
|
|
{ QChar::Script_Vai, "Vai" },
|
|
{ QChar::Script_Saurashtra, "Saurashtra" },
|
|
{ QChar::Script_KayahLi, "KayahLi" },
|
|
{ QChar::Script_Rejang, "Rejang" },
|
|
{ QChar::Script_Lycian, "Lycian" },
|
|
{ QChar::Script_Carian, "Carian" },
|
|
{ QChar::Script_Lydian, "Lydian" },
|
|
{ QChar::Script_Cham, "Cham" },
|
|
// 5.2
|
|
{ QChar::Script_TaiTham, "TaiTham" },
|
|
{ QChar::Script_TaiViet, "TaiViet" },
|
|
{ QChar::Script_Avestan, "Avestan" },
|
|
{ QChar::Script_EgyptianHieroglyphs, "EgyptianHieroglyphs" },
|
|
{ QChar::Script_Samaritan, "Samaritan" },
|
|
{ QChar::Script_Lisu, "Lisu" },
|
|
{ QChar::Script_Bamum, "Bamum" },
|
|
{ QChar::Script_Javanese, "Javanese" },
|
|
{ QChar::Script_MeeteiMayek, "MeeteiMayek" },
|
|
{ QChar::Script_ImperialAramaic, "ImperialAramaic" },
|
|
{ QChar::Script_OldSouthArabian, "OldSouthArabian" },
|
|
{ QChar::Script_InscriptionalParthian, "InscriptionalParthian" },
|
|
{ QChar::Script_InscriptionalPahlavi, "InscriptionalPahlavi" },
|
|
{ QChar::Script_OldTurkic, "OldTurkic" },
|
|
{ QChar::Script_Kaithi, "Kaithi" },
|
|
// 6.0
|
|
{ QChar::Script_Batak, "Batak" },
|
|
{ QChar::Script_Brahmi, "Brahmi" },
|
|
{ QChar::Script_Mandaic, "Mandaic" },
|
|
// 6.1
|
|
{ QChar::Script_Chakma, "Chakma" },
|
|
{ QChar::Script_MeroiticCursive, "MeroiticCursive" },
|
|
{ QChar::Script_MeroiticHieroglyphs, "MeroiticHieroglyphs" },
|
|
{ QChar::Script_Miao, "Miao" },
|
|
{ QChar::Script_Sharada, "Sharada" },
|
|
{ QChar::Script_SoraSompeng, "SoraSompeng" },
|
|
{ QChar::Script_Takri, "Takri" },
|
|
// 7.0
|
|
{ QChar::Script_CaucasianAlbanian, "CaucasianAlbanian" },
|
|
{ QChar::Script_BassaVah, "BassaVah" },
|
|
{ QChar::Script_Duployan, "Duployan" },
|
|
{ QChar::Script_Elbasan, "Elbasan" },
|
|
{ QChar::Script_Grantha, "Grantha" },
|
|
{ QChar::Script_PahawhHmong, "PahawhHmong" },
|
|
{ QChar::Script_Khojki, "Khojki" },
|
|
{ QChar::Script_LinearA, "LinearA" },
|
|
{ QChar::Script_Mahajani, "Mahajani" },
|
|
{ QChar::Script_Manichaean, "Manichaean" },
|
|
{ QChar::Script_MendeKikakui, "MendeKikakui" },
|
|
{ QChar::Script_Modi, "Modi" },
|
|
{ QChar::Script_Mro, "Mro" },
|
|
{ QChar::Script_OldNorthArabian, "OldNorthArabian" },
|
|
{ QChar::Script_Nabataean, "Nabataean" },
|
|
{ QChar::Script_Palmyrene, "Palmyrene" },
|
|
{ QChar::Script_PauCinHau, "PauCinHau" },
|
|
{ QChar::Script_OldPermic, "OldPermic" },
|
|
{ QChar::Script_PsalterPahlavi, "PsalterPahlavi" },
|
|
{ QChar::Script_Siddham, "Siddham" },
|
|
{ QChar::Script_Khudawadi, "Khudawadi" },
|
|
{ QChar::Script_Tirhuta, "Tirhuta" },
|
|
{ QChar::Script_WarangCiti, "WarangCiti" },
|
|
// 8.0
|
|
{ QChar::Script_Ahom, "Ahom" },
|
|
{ QChar::Script_AnatolianHieroglyphs, "AnatolianHieroglyphs" },
|
|
{ QChar::Script_Hatran, "Hatran" },
|
|
{ QChar::Script_Multani, "Multani" },
|
|
{ QChar::Script_OldHungarian, "OldHungarian" },
|
|
{ QChar::Script_SignWriting, "SignWriting" },
|
|
// 9.0
|
|
{ QChar::Script_Adlam, "Adlam" },
|
|
{ QChar::Script_Bhaiksuki, "Bhaiksuki" },
|
|
{ QChar::Script_Marchen, "Marchen" },
|
|
{ QChar::Script_Newa, "Newa" },
|
|
{ QChar::Script_Osage, "Osage" },
|
|
{ QChar::Script_Tangut, "Tangut" },
|
|
// 10.0
|
|
{ QChar::Script_MasaramGondi, "MasaramGondi" },
|
|
{ QChar::Script_Nushu, "Nushu" },
|
|
{ QChar::Script_Soyombo, "Soyombo" },
|
|
{ QChar::Script_ZanabazarSquare, "ZanabazarSquare" },
|
|
// 12.1
|
|
{ QChar::Script_Dogra, "Dogra" },
|
|
{ QChar::Script_GunjalaGondi, "GunjalaGondi" },
|
|
{ QChar::Script_HanifiRohingya, "HanifiRohingya" },
|
|
{ QChar::Script_Makasar, "Makasar" },
|
|
{ QChar::Script_Medefaidrin, "Medefaidrin" },
|
|
{ QChar::Script_OldSogdian, "OldSogdian" },
|
|
{ QChar::Script_Sogdian, "Sogdian" },
|
|
{ QChar::Script_Elymaic, "Elymaic" },
|
|
{ QChar::Script_Nandinagari, "Nandinagari" },
|
|
{ QChar::Script_NyiakengPuachueHmong, "NyiakengPuachueHmong" },
|
|
{ QChar::Script_Wancho, "Wancho" },
|
|
// 13.0
|
|
{ QChar::Script_Chorasmian, "Chorasmian" },
|
|
{ QChar::Script_DivesAkuru, "DivesAkuru" },
|
|
{ QChar::Script_KhitanSmallScript, "KhitanSmallScript" },
|
|
{ QChar::Script_Yezidi, "Yezidi" },
|
|
|
|
// unhandled
|
|
{ QChar::Script_Unknown, 0 }
|
|
};
|
|
Scrpt *p = scripts;
|
|
while (p->name) {
|
|
scriptMap.insert(p->name, p->script);
|
|
++p;
|
|
}
|
|
}
|
|
|
|
// Keep this one in sync with the code in createPropertyInfo
|
|
static const char *property_string =
|
|
"enum Case {\n"
|
|
" LowerCase,\n"
|
|
" UpperCase,\n"
|
|
" TitleCase,\n"
|
|
" CaseFold,\n"
|
|
"\n"
|
|
" NumCases\n"
|
|
"};\n"
|
|
"\n"
|
|
"struct Properties {\n"
|
|
" ushort category : 8; /* 5 used */\n"
|
|
" ushort direction : 8; /* 5 used */\n"
|
|
" ushort combiningClass : 8;\n"
|
|
" ushort joining : 3;\n"
|
|
" signed short digitValue : 5;\n"
|
|
" signed short mirrorDiff : 16;\n"
|
|
" ushort unicodeVersion : 8; /* 5 used */\n"
|
|
" ushort nfQuickCheck : 8;\n" // could be narrowed
|
|
"#ifdef Q_OS_WASM\n"
|
|
" unsigned char : 0; //wasm 64 packing trick\n"
|
|
"#endif\n"
|
|
" struct {\n"
|
|
" ushort special : 1;\n"
|
|
" signed short diff : 15;\n"
|
|
" } cases[NumCases];\n"
|
|
"#ifdef Q_OS_WASM\n"
|
|
" unsigned char : 0; //wasm 64 packing trick\n"
|
|
"#endif\n"
|
|
" ushort graphemeBreakClass : 5; /* 5 used */\n"
|
|
" ushort wordBreakClass : 5; /* 5 used */\n"
|
|
" ushort lineBreakClass : 6; /* 6 used */\n"
|
|
" ushort sentenceBreakClass : 8; /* 4 used */\n"
|
|
" ushort script : 8;\n"
|
|
"};\n\n"
|
|
"Q_CORE_EXPORT const Properties * QT_FASTCALL properties(char32_t ucs4) noexcept;\n"
|
|
"Q_CORE_EXPORT const Properties * QT_FASTCALL properties(char16_t ucs2) noexcept;\n"
|
|
"\n";
|
|
|
|
static const char *methods =
|
|
"Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(char32_t ucs4) noexcept;\n"
|
|
"inline GraphemeBreakClass graphemeBreakClass(QChar ch) noexcept\n"
|
|
"{ return graphemeBreakClass(ch.unicode()); }\n"
|
|
"\n"
|
|
"Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(char32_t ucs4) noexcept;\n"
|
|
"inline WordBreakClass wordBreakClass(QChar ch) noexcept\n"
|
|
"{ return wordBreakClass(ch.unicode()); }\n"
|
|
"\n"
|
|
"Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(char32_t ucs4) noexcept;\n"
|
|
"inline SentenceBreakClass sentenceBreakClass(QChar ch) noexcept\n"
|
|
"{ return sentenceBreakClass(ch.unicode()); }\n"
|
|
"\n"
|
|
"Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(char32_t ucs4) noexcept;\n"
|
|
"inline LineBreakClass lineBreakClass(QChar ch) noexcept\n"
|
|
"{ return lineBreakClass(ch.unicode()); }\n"
|
|
"\n";
|
|
|
|
static const int SizeOfPropertiesStruct = 20;
|
|
|
|
static const QByteArray sizeOfPropertiesStructCheck =
|
|
"static_assert(sizeof(Properties) == " + QByteArray::number(SizeOfPropertiesStruct) + ");\n\n";
|
|
|
|
struct PropertyFlags {
|
|
bool operator==(const PropertyFlags &o) const {
|
|
return (combiningClass == o.combiningClass
|
|
&& category == o.category
|
|
&& direction == o.direction
|
|
&& joining == o.joining
|
|
&& age == o.age
|
|
&& digitValue == o.digitValue
|
|
&& mirrorDiff == o.mirrorDiff
|
|
&& lowerCaseDiff == o.lowerCaseDiff
|
|
&& upperCaseDiff == o.upperCaseDiff
|
|
&& titleCaseDiff == o.titleCaseDiff
|
|
&& caseFoldDiff == o.caseFoldDiff
|
|
&& lowerCaseSpecial == o.lowerCaseSpecial
|
|
&& upperCaseSpecial == o.upperCaseSpecial
|
|
&& titleCaseSpecial == o.titleCaseSpecial
|
|
&& caseFoldSpecial == o.caseFoldSpecial
|
|
&& graphemeBreakClass == o.graphemeBreakClass
|
|
&& wordBreakClass == o.wordBreakClass
|
|
&& sentenceBreakClass == o.sentenceBreakClass
|
|
&& lineBreakClass == o.lineBreakClass
|
|
&& script == o.script
|
|
&& nfQuickCheck == o.nfQuickCheck
|
|
);
|
|
}
|
|
// from UnicodeData.txt
|
|
uchar combiningClass : 8;
|
|
QChar::Category category : 5;
|
|
QChar::Direction direction : 5;
|
|
// from ArabicShaping.txt
|
|
QChar::JoiningType joining : 3;
|
|
// from DerivedAge.txt
|
|
QChar::UnicodeVersion age : 5;
|
|
int digitValue;
|
|
|
|
int mirrorDiff : 16;
|
|
|
|
int lowerCaseDiff;
|
|
int upperCaseDiff;
|
|
int titleCaseDiff;
|
|
int caseFoldDiff;
|
|
bool lowerCaseSpecial;
|
|
bool upperCaseSpecial;
|
|
bool titleCaseSpecial;
|
|
bool caseFoldSpecial;
|
|
GraphemeBreakClass graphemeBreakClass;
|
|
WordBreakClass wordBreakClass;
|
|
SentenceBreakClass sentenceBreakClass;
|
|
LineBreakClass lineBreakClass;
|
|
int script;
|
|
// from DerivedNormalizationProps.txt
|
|
uchar nfQuickCheck;
|
|
};
|
|
|
|
|
|
static QList<int> specialCaseMap;
|
|
|
|
static int appendToSpecialCaseMap(const QList<int> &map)
|
|
{
|
|
QList<int> utf16map;
|
|
for (int i = 0; i < map.size(); ++i) {
|
|
uint codepoint = map.at(i);
|
|
// if the condition below doesn't hold anymore we need to modify our special case mapping code
|
|
Q_ASSERT(!QChar::requiresSurrogates(codepoint));
|
|
if (QChar::requiresSurrogates(codepoint)) {
|
|
utf16map << QChar::highSurrogate(codepoint);
|
|
utf16map << QChar::lowSurrogate(codepoint);
|
|
} else {
|
|
utf16map << codepoint;
|
|
}
|
|
}
|
|
int length = utf16map.size();
|
|
utf16map.prepend(length);
|
|
|
|
if (specialCaseMap.isEmpty())
|
|
specialCaseMap << 0; // placeholder
|
|
|
|
int i = 1;
|
|
while (i < specialCaseMap.size()) {
|
|
int n = specialCaseMap.at(i);
|
|
if (n == length) {
|
|
int j;
|
|
for (j = 1; j <= n; ++j) {
|
|
if (specialCaseMap.at(i+j) != utf16map.at(j))
|
|
break;
|
|
}
|
|
if (j > n)
|
|
return i;
|
|
}
|
|
i += n + 1;
|
|
}
|
|
|
|
int pos = specialCaseMap.size();
|
|
specialCaseMap << utf16map;
|
|
return pos;
|
|
}
|
|
|
|
// DerivedCoreProperties.txt
|
|
static inline bool isDefaultIgnorable(uint ucs4)
|
|
{
|
|
// Default_Ignorable_Code_Point:
|
|
// Generated from
|
|
// Other_Default_Ignorable_Code_Point + Cf + Variation_Selector
|
|
// - White_Space - FFF9..FFFB (Annotation Characters)
|
|
// - 0600..0604, 06DD, 070F, 110BD (exceptional Cf characters that should be visible)
|
|
if (ucs4 <= 0xff)
|
|
return ucs4 == 0xad;
|
|
|
|
return ucs4 == 0x034f
|
|
|| ucs4 == 0x061c
|
|
|| (ucs4 >= 0x115f && ucs4 <= 0x1160)
|
|
|| (ucs4 >= 0x17b4 && ucs4 <= 0x17b5)
|
|
|| (ucs4 >= 0x180b && ucs4 <= 0x180d)
|
|
|| ucs4 == 0x180e
|
|
|| (ucs4 >= 0x200b && ucs4 <= 0x200f)
|
|
|| (ucs4 >= 0x202a && ucs4 <= 0x202e)
|
|
|| (ucs4 >= 0x2060 && ucs4 <= 0x206f)
|
|
|| ucs4 == 0x3164
|
|
|| (ucs4 >= 0xfe00 && ucs4 <= 0xfe0f)
|
|
|| ucs4 == 0xfeff
|
|
|| ucs4 == 0xffa0
|
|
|| (ucs4 >= 0xfff0 && ucs4 <= 0xfff8)
|
|
|| (ucs4 >= 0x1bca0 && ucs4 <= 0x1bca3)
|
|
|| (ucs4 >= 0x1d173 && ucs4 <= 0x1d17a)
|
|
|| (ucs4 >= 0xe0000 && ucs4 <= 0xe0fff);
|
|
}
|
|
|
|
struct UnicodeData {
|
|
UnicodeData(int codepoint = 0) {
|
|
p.category = QChar::Other_NotAssigned; // Cn
|
|
p.combiningClass = 0;
|
|
|
|
p.direction = QChar::DirL;
|
|
// DerivedBidiClass.txt
|
|
// The unassigned code points that default to AL are in the ranges:
|
|
// [U+0600..U+07BF, U+08A0..U+08FF, U+FB50..U+FDCF, U+FDF0..U+FDFF, U+FE70..U+FEFF, U+1EE00..U+1EEFF]
|
|
if ((codepoint >= 0x0600 && codepoint <= 0x07BF)
|
|
|| (codepoint >= 0x08A0 && codepoint <= 0x08FF)
|
|
|| (codepoint >= 0xFB50 && codepoint <= 0xFDCF)
|
|
|| (codepoint >= 0xFDF0 && codepoint <= 0xFDFF)
|
|
|| (codepoint >= 0xFE70 && codepoint <= 0xFEFF)
|
|
|| (codepoint >= 0x1EE00 && codepoint <= 0x1EEFF)) {
|
|
p.direction = QChar::DirAL;
|
|
}
|
|
// The unassigned code points that default to R are in the ranges:
|
|
// [U+0590..U+05FF, U+07C0..U+089F, U+FB1D..U+FB4F, U+10800..U+10FFF, U+1E800..U+1EDFF, U+1EF00..U+1EFFF]
|
|
else if ((codepoint >= 0x0590 && codepoint <= 0x05FF)
|
|
|| (codepoint >= 0x07C0 && codepoint <= 0x089F)
|
|
|| (codepoint >= 0xFB1D && codepoint <= 0xFB4F)
|
|
|| (codepoint >= 0x10800 && codepoint <= 0x10FFF)
|
|
|| (codepoint >= 0x1E800 && codepoint <= 0x1EDFF)
|
|
|| (codepoint >= 0x1EF00 && codepoint <= 0x1EFFF)) {
|
|
p.direction = QChar::DirR;
|
|
}
|
|
// The unassigned code points that default to ET are in the range:
|
|
// [U+20A0..U+20CF]
|
|
else if (codepoint >= 0x20A0 && codepoint <= 0x20CF) {
|
|
p.direction = QChar::DirET;
|
|
}
|
|
// The unassigned code points that default to BN have one of the following properties:
|
|
// Default_Ignorable_Code_Point
|
|
// Noncharacter_Code_Point
|
|
else if (QChar::isNonCharacter(codepoint) || isDefaultIgnorable(codepoint)) {
|
|
p.direction = QChar::DirBN;
|
|
}
|
|
|
|
p.lineBreakClass = LineBreak_AL; // XX -> AL
|
|
// LineBreak.txt
|
|
// The unassigned code points that default to "ID" include ranges in the following blocks:
|
|
// [U+3400..U+4DBF, U+4E00..U+9FFF, U+F900..U+FAFF, U+20000..U+2A6DF, U+2A700..U+2B73F, U+2B740..U+2B81F, U+2B820..U+2CEAF, U+2F800..U+2FA1F]
|
|
// and any other reserved code points on
|
|
// [U+20000..U+2FFFD, U+30000..U+3FFFD]
|
|
if ((codepoint >= 0x3400 && codepoint <= 0x4DBF)
|
|
|| (codepoint >= 0x4E00 && codepoint <= 0x9FFF)
|
|
|| (codepoint >= 0xF900 && codepoint <= 0xFAFF)
|
|
|| (codepoint >= 0x20000 && codepoint <= 0x2A6DF)
|
|
|| (codepoint >= 0x2A700 && codepoint <= 0x2B73F)
|
|
|| (codepoint >= 0x2B740 && codepoint <= 0x2B81F)
|
|
|| (codepoint >= 0x2B820 && codepoint <= 0x2CEAF)
|
|
|| (codepoint >= 0x2F800 && codepoint <= 0x2FA1F)
|
|
|| (codepoint >= 0x20000 && codepoint <= 0x2FFFD)
|
|
|| (codepoint >= 0x30000 && codepoint <= 0x3FFFD)) {
|
|
p.lineBreakClass = LineBreak_ID;
|
|
}
|
|
// The unassigned code points that default to "PR" comprise a range in the following block:
|
|
// [U+20A0..U+20CF]
|
|
else if (codepoint >= 0x20A0 && codepoint <= 0x20CF) {
|
|
p.lineBreakClass = LineBreak_PR;
|
|
}
|
|
|
|
mirroredChar = 0;
|
|
decompositionType = QChar::NoDecomposition;
|
|
p.joining = QChar::Joining_None;
|
|
p.age = QChar::Unicode_Unassigned;
|
|
p.mirrorDiff = 0;
|
|
p.digitValue = -1;
|
|
p.lowerCaseDiff = 0;
|
|
p.upperCaseDiff = 0;
|
|
p.titleCaseDiff = 0;
|
|
p.caseFoldDiff = 0;
|
|
p.lowerCaseSpecial = 0;
|
|
p.upperCaseSpecial = 0;
|
|
p.titleCaseSpecial = 0;
|
|
p.caseFoldSpecial = 0;
|
|
p.graphemeBreakClass = GraphemeBreak_Any;
|
|
p.wordBreakClass = WordBreak_Any;
|
|
p.sentenceBreakClass = SentenceBreak_Any;
|
|
p.script = QChar::Script_Unknown;
|
|
p.nfQuickCheck = 0;
|
|
propertyIndex = -1;
|
|
excludedComposition = false;
|
|
}
|
|
|
|
static UnicodeData &valueRef(int codepoint);
|
|
|
|
PropertyFlags p;
|
|
|
|
// from UnicodeData.txt
|
|
QChar::Decomposition decompositionType;
|
|
QList<int> decomposition;
|
|
|
|
QList<int> specialFolding;
|
|
|
|
// from BidiMirroring.txt
|
|
int mirroredChar;
|
|
|
|
// DerivedNormalizationProps.txt
|
|
bool excludedComposition;
|
|
|
|
// computed position of unicode property set
|
|
int propertyIndex;
|
|
};
|
|
|
|
static QList<UnicodeData> unicodeData;
|
|
|
|
UnicodeData &UnicodeData::valueRef(int codepoint)
|
|
{
|
|
static bool initialized = false;
|
|
if (!initialized) {
|
|
unicodeData.reserve(QChar::LastValidCodePoint + 1);
|
|
for (int uc = 0; uc <= QChar::LastValidCodePoint; ++uc)
|
|
unicodeData.append(UnicodeData(uc));
|
|
initialized = true;
|
|
}
|
|
|
|
Q_ASSERT(codepoint <= 0x10ffff);
|
|
return unicodeData[codepoint];
|
|
}
|
|
|
|
|
|
static QHash<int, int> decompositionLength;
|
|
static int highestComposedCharacter = 0;
|
|
static int numLigatures = 0;
|
|
static int highestLigature = 0;
|
|
|
|
struct Ligature {
|
|
int u1;
|
|
int u2;
|
|
int ligature;
|
|
};
|
|
// we need them sorted after the first component for fast lookup
|
|
bool operator < (const Ligature &l1, const Ligature &l2)
|
|
{ return l1.u1 < l2.u1; }
|
|
|
|
static QHash<int, QList<Ligature> > ligatureHashes;
|
|
|
|
static QHash<int, int> combiningClassUsage;
|
|
|
|
static int maxLowerCaseDiff = 0;
|
|
static int maxUpperCaseDiff = 0;
|
|
static int maxTitleCaseDiff = 0;
|
|
|
|
static void readUnicodeData()
|
|
{
|
|
qDebug("Reading UnicodeData.txt");
|
|
|
|
enum UniDataFields {
|
|
UD_Value,
|
|
UD_Name,
|
|
UD_Category,
|
|
UD_CombiningClass,
|
|
UD_BidiCategory,
|
|
UD_Decomposition,
|
|
UD_DecimalDigitValue,
|
|
UD_DigitValue,
|
|
UD_NumericValue,
|
|
UD_Mirrored,
|
|
UD_OldName,
|
|
UD_Comment,
|
|
UD_UpperCase,
|
|
UD_LowerCase,
|
|
UD_TitleCase
|
|
};
|
|
|
|
QFile f("data/UnicodeData.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find UnicodeData.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.truncate(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> properties = line.split(';');
|
|
bool ok;
|
|
int codepoint = properties[UD_Value].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
Q_ASSERT(codepoint <= QChar::LastValidCodePoint);
|
|
int lastCodepoint = codepoint;
|
|
|
|
QByteArray name = properties[UD_Name];
|
|
if (name.startsWith('<') && name.contains("First")) {
|
|
QByteArray nextLine;
|
|
nextLine.resize(1024);
|
|
f.readLine(nextLine.data(), 1024);
|
|
QList<QByteArray> properties = nextLine.split(';');
|
|
Q_ASSERT(properties[UD_Name].startsWith('<') && properties[UD_Name].contains("Last"));
|
|
lastCodepoint = properties[UD_Value].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
Q_ASSERT(lastCodepoint <= QChar::LastValidCodePoint);
|
|
}
|
|
|
|
UnicodeData &data = UnicodeData::valueRef(codepoint);
|
|
data.p.category = categoryMap.value(properties[UD_Category], QChar::Other_NotAssigned);
|
|
data.p.combiningClass = properties[UD_CombiningClass].toInt();
|
|
if (!combiningClassUsage.contains(data.p.combiningClass))
|
|
combiningClassUsage[data.p.combiningClass] = 1;
|
|
else
|
|
++combiningClassUsage[data.p.combiningClass];
|
|
|
|
Direction dir = directionMap.value(properties[UD_BidiCategory], Dir_Unassigned);
|
|
if (dir == Dir_Unassigned)
|
|
qFatal("unhandled direction value: %s", properties[UD_BidiCategory].constData());
|
|
data.p.direction = QChar::Direction(dir);
|
|
|
|
if (!properties[UD_UpperCase].isEmpty()) {
|
|
int upperCase = properties[UD_UpperCase].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int diff = upperCase - codepoint;
|
|
// if the conditions below doesn't hold anymore we need to modify our upper casing code
|
|
Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(upperCase));
|
|
if (QChar::requiresSurrogates(codepoint)) {
|
|
Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(upperCase));
|
|
Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(upperCase));
|
|
}
|
|
if (qAbs(diff) >= (1<<13)) {
|
|
data.p.upperCaseSpecial = true;
|
|
data.p.upperCaseDiff = appendToSpecialCaseMap(QList<int>() << upperCase);
|
|
} else {
|
|
data.p.upperCaseDiff = diff;
|
|
maxUpperCaseDiff = qMax(maxUpperCaseDiff, qAbs(diff));
|
|
}
|
|
}
|
|
if (!properties[UD_LowerCase].isEmpty()) {
|
|
int lowerCase = properties[UD_LowerCase].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int diff = lowerCase - codepoint;
|
|
// if the conditions below doesn't hold anymore we need to modify our lower casing code
|
|
Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(lowerCase));
|
|
if (QChar::requiresSurrogates(codepoint)) {
|
|
Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(lowerCase));
|
|
Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(lowerCase));
|
|
}
|
|
if (qAbs(diff) >= (1<<13)) {
|
|
data.p.lowerCaseSpecial = true;
|
|
data.p.lowerCaseDiff = appendToSpecialCaseMap(QList<int>() << lowerCase);
|
|
} else {
|
|
data.p.lowerCaseDiff = diff;
|
|
maxLowerCaseDiff = qMax(maxLowerCaseDiff, qAbs(diff));
|
|
}
|
|
}
|
|
// we want toTitleCase to map to ToUpper in case we don't have any titlecase.
|
|
if (properties[UD_TitleCase].isEmpty())
|
|
properties[UD_TitleCase] = properties[UD_UpperCase];
|
|
if (!properties[UD_TitleCase].isEmpty()) {
|
|
int titleCase = properties[UD_TitleCase].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int diff = titleCase - codepoint;
|
|
// if the conditions below doesn't hold anymore we need to modify our title casing code
|
|
Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(titleCase));
|
|
if (QChar::requiresSurrogates(codepoint)) {
|
|
Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(titleCase));
|
|
Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(titleCase));
|
|
}
|
|
if (qAbs(diff) >= (1<<13)) {
|
|
data.p.titleCaseSpecial = true;
|
|
data.p.titleCaseDiff = appendToSpecialCaseMap(QList<int>() << titleCase);
|
|
} else {
|
|
data.p.titleCaseDiff = diff;
|
|
maxTitleCaseDiff = qMax(maxTitleCaseDiff, qAbs(diff));
|
|
}
|
|
}
|
|
|
|
if (!properties[UD_DigitValue].isEmpty())
|
|
data.p.digitValue = properties[UD_DigitValue].toInt();
|
|
|
|
// decompositition
|
|
QByteArray decomposition = properties[UD_Decomposition];
|
|
if (!decomposition.isEmpty()) {
|
|
highestComposedCharacter = qMax(highestComposedCharacter, codepoint);
|
|
QList<QByteArray> d = decomposition.split(' ');
|
|
if (d[0].contains('<')) {
|
|
data.decompositionType = decompositionMap.value(d[0], QChar::NoDecomposition);
|
|
if (data.decompositionType == QChar::NoDecomposition)
|
|
qFatal("unhandled decomposition type: %s", d[0].constData());
|
|
d.takeFirst();
|
|
} else {
|
|
data.decompositionType = QChar::Canonical;
|
|
}
|
|
for (int i = 0; i < d.size(); ++i) {
|
|
data.decomposition.append(d[i].toInt(&ok, 16));
|
|
Q_ASSERT(ok);
|
|
}
|
|
++decompositionLength[data.decomposition.size()];
|
|
}
|
|
|
|
for (int i = codepoint; i <= lastCodepoint; ++i)
|
|
unicodeData[i] = data;
|
|
}
|
|
}
|
|
|
|
static int maxMirroredDiff = 0;
|
|
|
|
static void readBidiMirroring()
|
|
{
|
|
qDebug("Reading BidiMirroring.txt");
|
|
|
|
QFile f("data/BidiMirroring.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find BidiMirroring.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
line = line.replace(" ", "");
|
|
|
|
QList<QByteArray> pair = line.split(';');
|
|
Q_ASSERT(pair.size() == 2);
|
|
|
|
bool ok;
|
|
int codepoint = pair[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int mirror = pair[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
d.mirroredChar = mirror;
|
|
d.p.mirrorDiff = d.mirroredChar - codepoint;
|
|
maxMirroredDiff = qMax(maxMirroredDiff, qAbs(d.p.mirrorDiff));
|
|
}
|
|
}
|
|
|
|
static void readArabicShaping()
|
|
{
|
|
qDebug("Reading ArabicShaping.txt");
|
|
|
|
// Initialize defaults:
|
|
// Code points that are not explicitly listed in ArabicShaping.txt are either of joining type T or U:
|
|
// - Those that not explicitly listed that are of General Category Mn, Me, or Cf have joining type T.
|
|
// - All others not explicitly listed have joining type U.
|
|
for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
if (d.p.joining == QChar::Joining_None) {
|
|
if (d.p.category == QChar::Mark_NonSpacing || d.p.category == QChar::Mark_Enclosing || d.p.category == QChar::Other_Format)
|
|
d.p.joining = QChar::Joining_Transparent;
|
|
}
|
|
}
|
|
|
|
QFile f("data/ArabicShaping.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find ArabicShaping.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
line = line.trimmed();
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
Q_ASSERT(l.size() == 4);
|
|
|
|
bool ok;
|
|
int codepoint = l[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
JoiningType joining = joining_map.value(l[2].trimmed(), Joining_Unassigned);
|
|
switch (joining) {
|
|
case Joining_Unassigned:
|
|
qFatal("%x: unassigned or unhandled joining type: %s", codepoint, l[2].constData());
|
|
break;
|
|
case Joining_Transparent:
|
|
switch (d.p.category) {
|
|
case QChar::Mark_Enclosing:
|
|
case QChar::Mark_NonSpacing:
|
|
case QChar::Letter_Modifier:
|
|
case QChar::Other_Format:
|
|
break;
|
|
default:
|
|
qFatal("%x: joining type '%s' was met (category: %d); "
|
|
"the current implementation needs to be revised!",
|
|
codepoint, l[2].constData(), d.p.category);
|
|
}
|
|
Q_FALLTHROUGH();
|
|
default:
|
|
d.p.joining = QChar::JoiningType(joining);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void readDerivedAge()
|
|
{
|
|
qDebug("Reading DerivedAge.txt");
|
|
|
|
QFile f("data/DerivedAge.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find DerivedAge.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
line.replace(" ", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
Q_ASSERT(l.size() == 2);
|
|
|
|
QByteArray codes = l[0];
|
|
codes.replace("..", ".");
|
|
QList<QByteArray> cl = codes.split('.');
|
|
|
|
bool ok;
|
|
int from = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int to = from;
|
|
if (cl.size() == 2) {
|
|
to = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
QChar::UnicodeVersion age = age_map.value(l[1].trimmed(), QChar::Unicode_Unassigned);
|
|
//qDebug() << Qt::hex << from << ".." << to << ba << age;
|
|
if (age == QChar::Unicode_Unassigned)
|
|
qFatal("unassigned or unhandled age value: %s", l[1].constData());
|
|
|
|
for (int codepoint = from; codepoint <= to; ++codepoint) {
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
d.p.age = age;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void readDerivedNormalizationProps()
|
|
{
|
|
qDebug("Reading DerivedNormalizationProps.txt");
|
|
|
|
QFile f("data/DerivedNormalizationProps.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find DerivedNormalizationProps.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
|
|
if (line.trimmed().isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
Q_ASSERT(l.size() >= 2);
|
|
|
|
QByteArray propName = l[1].trimmed();
|
|
if (propName != "Full_Composition_Exclusion" &&
|
|
propName != "NFD_QC" && propName != "NFC_QC" &&
|
|
propName != "NFKD_QC" && propName != "NFKC_QC") {
|
|
// ###
|
|
continue;
|
|
}
|
|
|
|
QByteArray codes = l[0].trimmed();
|
|
codes.replace("..", ".");
|
|
QList<QByteArray> cl = codes.split('.');
|
|
|
|
bool ok;
|
|
int from = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int to = from;
|
|
if (cl.size() == 2) {
|
|
to = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
for (int codepoint = from; codepoint <= to; ++codepoint) {
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
if (propName == "Full_Composition_Exclusion") {
|
|
d.excludedComposition = true;
|
|
} else {
|
|
static_assert(QString::NormalizationForm_D == 0);
|
|
static_assert(QString::NormalizationForm_C == 1);
|
|
static_assert(QString::NormalizationForm_KD == 2);
|
|
static_assert(QString::NormalizationForm_KC == 3);
|
|
|
|
QString::NormalizationForm form;
|
|
if (propName == "NFD_QC")
|
|
form = QString::NormalizationForm_D;
|
|
else if (propName == "NFC_QC")
|
|
form = QString::NormalizationForm_C;
|
|
else if (propName == "NFKD_QC")
|
|
form = QString::NormalizationForm_KD;
|
|
else// if (propName == "NFKC_QC")
|
|
form = QString::NormalizationForm_KC;
|
|
|
|
Q_ASSERT(l.size() == 3);
|
|
l[2] = l[2].trimmed();
|
|
|
|
enum { NFQC_YES = 0, NFQC_NO = 1, NFQC_MAYBE = 3 };
|
|
uchar ynm = (l[2] == "N" ? NFQC_NO : l[2] == "M" ? NFQC_MAYBE : NFQC_YES);
|
|
if (ynm == NFQC_MAYBE) {
|
|
// if this changes, we need to revise the normalizationQuickCheckHelper() implementation
|
|
Q_ASSERT(form == QString::NormalizationForm_C || form == QString::NormalizationForm_KC);
|
|
}
|
|
d.p.nfQuickCheck |= (ynm << (form << 1)); // 2 bits per NF
|
|
}
|
|
}
|
|
}
|
|
|
|
for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
if (!d.excludedComposition
|
|
&& d.decompositionType == QChar::Canonical
|
|
&& d.decomposition.size() > 1) {
|
|
Q_ASSERT(d.decomposition.size() == 2);
|
|
|
|
int part1 = d.decomposition.at(0);
|
|
int part2 = d.decomposition.at(1);
|
|
|
|
// all non-starters are listed in DerivedNormalizationProps.txt
|
|
// and already excluded from composition
|
|
Q_ASSERT(UnicodeData::valueRef(part1).p.combiningClass == 0);
|
|
|
|
++numLigatures;
|
|
highestLigature = qMax(highestLigature, part1);
|
|
Ligature l = { part1, part2, codepoint };
|
|
ligatureHashes[part2].append(l);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
struct NormalizationCorrection {
|
|
uint codepoint;
|
|
uint mapped;
|
|
int version;
|
|
};
|
|
|
|
static QByteArray createNormalizationCorrections()
|
|
{
|
|
qDebug("Reading NormalizationCorrections.txt");
|
|
|
|
QFile f("data/NormalizationCorrections.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find NormalizationCorrections.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
QByteArray out
|
|
= "struct NormalizationCorrection {\n"
|
|
" uint ucs4;\n"
|
|
" uint old_mapping;\n"
|
|
" int version;\n"
|
|
"};\n\n"
|
|
|
|
"static const NormalizationCorrection uc_normalization_corrections[] = {\n";
|
|
|
|
int maxVersion = 0;
|
|
int numCorrections = 0;
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
line.replace(" ", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
Q_ASSERT(!line.contains(".."));
|
|
|
|
QList<QByteArray> fields = line.split(';');
|
|
Q_ASSERT(fields.size() == 4);
|
|
|
|
NormalizationCorrection c = { 0, 0, 0 };
|
|
bool ok;
|
|
c.codepoint = fields.at(0).toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
c.mapped = fields.at(1).toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
if (fields.at(3) == "3.2.0")
|
|
c.version = QChar::Unicode_3_2;
|
|
else if (fields.at(3) == "4.0.0")
|
|
c.version = QChar::Unicode_4_0;
|
|
else
|
|
qFatal("unknown unicode version in NormalizationCorrection.txt");
|
|
|
|
out += " { 0x" + QByteArray::number(c.codepoint, 16) + ", 0x"
|
|
+ QByteArray::number(c.mapped, 16) + ", "
|
|
+ QByteArray::number(c.version) + " },\n";
|
|
++numCorrections;
|
|
maxVersion = qMax(c.version, maxVersion);
|
|
}
|
|
if (out.endsWith(",\n"))
|
|
out.chop(2);
|
|
|
|
out += "\n};\n\n"
|
|
|
|
"enum { NumNormalizationCorrections = " + QByteArray::number(numCorrections) + " };\n"
|
|
"enum { NormalizationCorrectionsVersionMax = " + QByteArray::number(maxVersion) + " };\n\n";
|
|
|
|
return out;
|
|
}
|
|
|
|
static void readLineBreak()
|
|
{
|
|
qDebug("Reading LineBreak.txt");
|
|
|
|
QFile f("data/LineBreak.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find LineBreak.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
line.replace(" ", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
Q_ASSERT(l.size() == 2);
|
|
|
|
QByteArray codes = l[0];
|
|
codes.replace("..", ".");
|
|
QList<QByteArray> cl = codes.split('.');
|
|
|
|
bool ok;
|
|
int from = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int to = from;
|
|
if (cl.size() == 2) {
|
|
to = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
LineBreakClass lb = line_break_map.value(l[1], LineBreak_Unassigned);
|
|
if (lb == LineBreak_Unassigned)
|
|
qFatal("unassigned line break class: %s", l[1].constData());
|
|
|
|
for (int codepoint = from; codepoint <= to; ++codepoint) {
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
d.p.lineBreakClass = lb;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void readSpecialCasing()
|
|
{
|
|
qDebug("Reading SpecialCasing.txt");
|
|
|
|
QFile f("data/SpecialCasing.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find SpecialCasing.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
|
|
QByteArray condition = l.size() < 5 ? QByteArray() : l[4].trimmed();
|
|
if (!condition.isEmpty())
|
|
// #####
|
|
continue;
|
|
|
|
bool ok;
|
|
int codepoint = l[0].trimmed().toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
|
|
// if the condition below doesn't hold anymore we need to modify our
|
|
// lower/upper/title casing code and case folding code
|
|
Q_ASSERT(!QChar::requiresSurrogates(codepoint));
|
|
|
|
// qDebug() << "codepoint" << Qt::hex << codepoint;
|
|
// qDebug() << line;
|
|
|
|
QList<QByteArray> lower = l[1].trimmed().split(' ');
|
|
QList<int> lowerMap;
|
|
for (int i = 0; i < lower.size(); ++i) {
|
|
bool ok;
|
|
lowerMap.append(lower.at(i).toInt(&ok, 16));
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
QList<QByteArray> title = l[2].trimmed().split(' ');
|
|
QList<int> titleMap;
|
|
for (int i = 0; i < title.size(); ++i) {
|
|
bool ok;
|
|
titleMap.append(title.at(i).toInt(&ok, 16));
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
QList<QByteArray> upper = l[3].trimmed().split(' ');
|
|
QList<int> upperMap;
|
|
for (int i = 0; i < upper.size(); ++i) {
|
|
bool ok;
|
|
upperMap.append(upper.at(i).toInt(&ok, 16));
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
|
|
UnicodeData &ud = UnicodeData::valueRef(codepoint);
|
|
Q_ASSERT(lowerMap.size() > 1 || lowerMap.at(0) == codepoint + ud.p.lowerCaseDiff);
|
|
Q_ASSERT(titleMap.size() > 1 || titleMap.at(0) == codepoint + ud.p.titleCaseDiff);
|
|
Q_ASSERT(upperMap.size() > 1 || upperMap.at(0) == codepoint + ud.p.upperCaseDiff);
|
|
|
|
if (lowerMap.size() > 1) {
|
|
ud.p.lowerCaseSpecial = true;
|
|
ud.p.lowerCaseDiff = appendToSpecialCaseMap(lowerMap);
|
|
}
|
|
if (titleMap.size() > 1) {
|
|
ud.p.titleCaseSpecial = true;
|
|
ud.p.titleCaseDiff = appendToSpecialCaseMap(titleMap);
|
|
}
|
|
if (upperMap.size() > 1) {
|
|
ud.p.upperCaseSpecial = true;
|
|
ud.p.upperCaseDiff = appendToSpecialCaseMap(upperMap);
|
|
}
|
|
}
|
|
}
|
|
|
|
static int maxCaseFoldDiff = 0;
|
|
|
|
static void readCaseFolding()
|
|
{
|
|
qDebug("Reading CaseFolding.txt");
|
|
|
|
QFile f("data/CaseFolding.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find CaseFolding.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
|
|
bool ok;
|
|
int codepoint = l[0].trimmed().toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
|
|
|
|
l[1] = l[1].trimmed();
|
|
if (l[1] == "F" || l[1] == "T")
|
|
continue;
|
|
|
|
// qDebug() << "codepoint" << Qt::hex << codepoint;
|
|
// qDebug() << line;
|
|
QList<QByteArray> fold = l[2].trimmed().split(' ');
|
|
QList<int> foldMap;
|
|
for (int i = 0; i < fold.size(); ++i) {
|
|
bool ok;
|
|
foldMap.append(fold.at(i).toInt(&ok, 16));
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
UnicodeData &ud = UnicodeData::valueRef(codepoint);
|
|
if (foldMap.size() == 1) {
|
|
int caseFolded = foldMap.at(0);
|
|
int diff = caseFolded - codepoint;
|
|
// if the conditions below doesn't hold anymore we need to modify our case folding code
|
|
Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(caseFolded));
|
|
if (QChar::requiresSurrogates(codepoint)) {
|
|
Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(caseFolded));
|
|
Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(caseFolded));
|
|
}
|
|
if (qAbs(diff) >= (1<<13)) {
|
|
ud.p.caseFoldSpecial = true;
|
|
ud.p.caseFoldDiff = appendToSpecialCaseMap(foldMap);
|
|
} else {
|
|
ud.p.caseFoldDiff = diff;
|
|
maxCaseFoldDiff = qMax(maxCaseFoldDiff, qAbs(diff));
|
|
}
|
|
} else {
|
|
qFatal("we currently don't support full case foldings");
|
|
// qDebug() << "special" << Qt::hex << foldMap;
|
|
ud.p.caseFoldSpecial = true;
|
|
ud.p.caseFoldDiff = appendToSpecialCaseMap(foldMap);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void readGraphemeBreak()
|
|
{
|
|
qDebug("Reading GraphemeBreakProperty.txt");
|
|
|
|
QFile f("data/GraphemeBreakProperty.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find GraphemeBreakProperty.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
line.replace(" ", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
Q_ASSERT(l.size() == 2);
|
|
|
|
QByteArray codes = l[0];
|
|
codes.replace("..", ".");
|
|
QList<QByteArray> cl = codes.split('.');
|
|
|
|
bool ok;
|
|
int from = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int to = from;
|
|
if (cl.size() == 2) {
|
|
to = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
GraphemeBreakClass brk = grapheme_break_map.value(l[1], GraphemeBreak_Unassigned);
|
|
if (brk == GraphemeBreak_Unassigned)
|
|
qFatal("unassigned grapheme break class: %s", l[1].constData());
|
|
|
|
for (int codepoint = from; codepoint <= to; ++codepoint) {
|
|
UnicodeData &ud = UnicodeData::valueRef(codepoint);
|
|
ud.p.graphemeBreakClass = brk;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void readWordBreak()
|
|
{
|
|
qDebug("Reading WordBreakProperty.txt");
|
|
|
|
QFile f("data/WordBreakProperty.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find WordBreakProperty.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
line.replace(" ", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
Q_ASSERT(l.size() == 2);
|
|
|
|
QByteArray codes = l[0];
|
|
codes.replace("..", ".");
|
|
QList<QByteArray> cl = codes.split('.');
|
|
|
|
bool ok;
|
|
int from = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int to = from;
|
|
if (cl.size() == 2) {
|
|
to = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
WordBreakClass brk = word_break_map.value(l[1], WordBreak_Unassigned);
|
|
if (brk == WordBreak_Unassigned)
|
|
qFatal("unassigned word break class: %s", l[1].constData());
|
|
|
|
for (int codepoint = from; codepoint <= to; ++codepoint) {
|
|
// ### [
|
|
// as of Unicode 5.1, some punctuation marks were mapped to MidLetter and MidNumLet
|
|
// which caused "hi.there" to be treated like if it were just a single word;
|
|
// until we have a tailoring mechanism, retain the old behavior by remapping those characters here.
|
|
if (codepoint == 0x002E) // FULL STOP
|
|
brk = WordBreak_MidNum;
|
|
else if (codepoint == 0x003A) // COLON
|
|
brk = WordBreak_Any;
|
|
// ] ###
|
|
UnicodeData &ud = UnicodeData::valueRef(codepoint);
|
|
ud.p.wordBreakClass = brk;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void readSentenceBreak()
|
|
{
|
|
qDebug("Reading SentenceBreakProperty.txt");
|
|
|
|
QFile f("data/SentenceBreakProperty.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find SentenceBreakProperty.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line;
|
|
line.resize(1024);
|
|
int len = f.readLine(line.data(), 1024);
|
|
line.resize(len-1);
|
|
|
|
int comment = line.indexOf('#');
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
line.replace(" ", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
QList<QByteArray> l = line.split(';');
|
|
Q_ASSERT(l.size() == 2);
|
|
|
|
QByteArray codes = l[0];
|
|
codes.replace("..", ".");
|
|
QList<QByteArray> cl = codes.split('.');
|
|
|
|
bool ok;
|
|
int from = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int to = from;
|
|
if (cl.size() == 2) {
|
|
to = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
SentenceBreakClass brk = sentence_break_map.value(l[1], SentenceBreak_Unassigned);
|
|
if (brk == SentenceBreak_Unassigned)
|
|
qFatal("unassigned sentence break class: %s", l[1].constData());
|
|
|
|
for (int codepoint = from; codepoint <= to; ++codepoint) {
|
|
UnicodeData &ud = UnicodeData::valueRef(codepoint);
|
|
ud.p.sentenceBreakClass = brk;
|
|
}
|
|
}
|
|
}
|
|
|
|
#if 0
|
|
// this piece of code does full case folding and comparison. We currently
|
|
// don't use it, since this gives lots of issues with things as case insensitive
|
|
// search and replace.
|
|
static inline void foldCase(uint ch, ushort *out)
|
|
{
|
|
const QUnicodeTables::Properties *p = qGetProp(ch);
|
|
if (!p->caseFoldSpecial) {
|
|
*(out++) = ch + p->caseFoldDiff;
|
|
} else {
|
|
const ushort *folded = specialCaseMap + p->caseFoldDiff;
|
|
ushort length = *folded++;
|
|
while (length--)
|
|
*out++ = *folded++;
|
|
}
|
|
*out = 0;
|
|
}
|
|
|
|
static int ucstricmp(const ushort *a, const ushort *ae, const ushort *b, const ushort *be)
|
|
{
|
|
if (a == b)
|
|
return 0;
|
|
if (a == 0)
|
|
return 1;
|
|
if (b == 0)
|
|
return -1;
|
|
|
|
while (a != ae && b != be) {
|
|
const QUnicodeTables::Properties *pa = qGetProp(*a);
|
|
const QUnicodeTables::Properties *pb = qGetProp(*b);
|
|
if (pa->caseFoldSpecial | pb->caseFoldSpecial)
|
|
goto special;
|
|
int diff = (int)(*a + pa->caseFoldDiff) - (int)(*b + pb->caseFoldDiff);
|
|
if ((diff))
|
|
return diff;
|
|
++a;
|
|
++b;
|
|
}
|
|
}
|
|
if (a == ae) {
|
|
if (b == be)
|
|
return 0;
|
|
return -1;
|
|
}
|
|
return 1;
|
|
special:
|
|
ushort abuf[SPECIAL_CASE_MAX_LEN + 1];
|
|
ushort bbuf[SPECIAL_CASE_MAX_LEN + 1];
|
|
abuf[0] = bbuf[0] = 0;
|
|
ushort *ap = abuf;
|
|
ushort *bp = bbuf;
|
|
while (1) {
|
|
if (!*ap) {
|
|
if (a == ae) {
|
|
if (!*bp && b == be)
|
|
return 0;
|
|
return -1;
|
|
}
|
|
foldCase(*(a++), abuf);
|
|
ap = abuf;
|
|
}
|
|
if (!*bp) {
|
|
if (b == be)
|
|
return 1;
|
|
foldCase(*(b++), bbuf);
|
|
bp = bbuf;
|
|
}
|
|
if (*ap != *bp)
|
|
return (int)*ap - (int)*bp;
|
|
++ap;
|
|
++bp;
|
|
}
|
|
}
|
|
|
|
|
|
static int ucstricmp(const ushort *a, const ushort *ae, const uchar *b)
|
|
{
|
|
if (a == 0)
|
|
return 1;
|
|
if (b == 0)
|
|
return -1;
|
|
|
|
while (a != ae && *b) {
|
|
const QUnicodeTables::Properties *pa = qGetProp(*a);
|
|
const QUnicodeTables::Properties *pb = qGetProp((ushort)*b);
|
|
if (pa->caseFoldSpecial | pb->caseFoldSpecial)
|
|
goto special;
|
|
int diff = (int)(*a + pa->caseFoldDiff) - (int)(*b + pb->caseFoldDiff);
|
|
if ((diff))
|
|
return diff;
|
|
++a;
|
|
++b;
|
|
}
|
|
if (a == ae) {
|
|
if (!*b)
|
|
return 0;
|
|
return -1;
|
|
}
|
|
return 1;
|
|
|
|
special:
|
|
ushort abuf[SPECIAL_CASE_MAX_LEN + 1];
|
|
ushort bbuf[SPECIAL_CASE_MAX_LEN + 1];
|
|
abuf[0] = bbuf[0] = 0;
|
|
ushort *ap = abuf;
|
|
ushort *bp = bbuf;
|
|
while (1) {
|
|
if (!*ap) {
|
|
if (a == ae) {
|
|
if (!*bp && !*b)
|
|
return 0;
|
|
return -1;
|
|
}
|
|
foldCase(*(a++), abuf);
|
|
ap = abuf;
|
|
}
|
|
if (!*bp) {
|
|
if (!*b)
|
|
return 1;
|
|
foldCase(*(b++), bbuf);
|
|
bp = bbuf;
|
|
}
|
|
if (*ap != *bp)
|
|
return (int)*ap - (int)*bp;
|
|
++ap;
|
|
++bp;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
#if 0
|
|
static QList<QByteArray> blockNames;
|
|
struct BlockInfo
|
|
{
|
|
int blockIndex;
|
|
int firstCodePoint;
|
|
int lastCodePoint;
|
|
};
|
|
static QList<BlockInfo> blockInfoList;
|
|
|
|
static void readBlocks()
|
|
{
|
|
qDebug("Reading Blocks.txt");
|
|
|
|
QFile f("data/Blocks.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find Blocks.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line = f.readLine();
|
|
line.resize(line.size() - 1);
|
|
|
|
int comment = line.indexOf("#");
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
|
|
line.replace(" ", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
int semicolon = line.indexOf(';');
|
|
Q_ASSERT(semicolon >= 0);
|
|
QByteArray codePoints = line.left(semicolon);
|
|
QByteArray blockName = line.mid(semicolon + 1);
|
|
|
|
int blockIndex = blockNames.indexOf(blockName);
|
|
if (blockIndex == -1) {
|
|
blockIndex = blockNames.size();
|
|
blockNames.append(blockName);
|
|
}
|
|
|
|
codePoints.replace("..", ".");
|
|
QList<QByteArray> cl = codePoints.split('.');
|
|
|
|
bool ok;
|
|
int first = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int last = first;
|
|
if (cl.size() == 2) {
|
|
last = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
BlockInfo blockInfo = { blockIndex, first, last };
|
|
blockInfoList.append(blockInfo);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
static void readScripts()
|
|
{
|
|
qDebug("Reading Scripts.txt");
|
|
|
|
QFile f("data/Scripts.txt");
|
|
if (!f.exists())
|
|
qFatal("Couldn't find Scripts.txt");
|
|
|
|
f.open(QFile::ReadOnly);
|
|
|
|
while (!f.atEnd()) {
|
|
QByteArray line = f.readLine();
|
|
line.resize(line.size() - 1);
|
|
|
|
int comment = line.indexOf("#");
|
|
if (comment >= 0)
|
|
line = line.left(comment);
|
|
|
|
line.replace(" ", "");
|
|
line.replace("_", "");
|
|
|
|
if (line.isEmpty())
|
|
continue;
|
|
|
|
int semicolon = line.indexOf(';');
|
|
Q_ASSERT(semicolon >= 0);
|
|
QByteArray codePoints = line.left(semicolon);
|
|
QByteArray scriptName = line.mid(semicolon + 1);
|
|
|
|
codePoints.replace("..", ".");
|
|
QList<QByteArray> cl = codePoints.split('.');
|
|
|
|
bool ok;
|
|
int first = cl[0].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
int last = first;
|
|
if (cl.size() == 2) {
|
|
last = cl[1].toInt(&ok, 16);
|
|
Q_ASSERT(ok);
|
|
}
|
|
|
|
if (!scriptMap.contains(scriptName))
|
|
qFatal("Unhandled script property value: %s", scriptName.constData());
|
|
QChar::Script script = scriptMap.value(scriptName, QChar::Script_Unknown);
|
|
|
|
for (int codepoint = first; codepoint <= last; ++codepoint) {
|
|
UnicodeData &ud = UnicodeData::valueRef(codepoint);
|
|
ud.p.script = script;
|
|
}
|
|
}
|
|
}
|
|
|
|
#if 0
|
|
static void dump(int from, int to)
|
|
{
|
|
for (int i = from; i <= to; ++i) {
|
|
UnicodeData &d = UnicodeData::valueRef(i);
|
|
qDebug("0x%04x: cat=%d combining=%d dir=%d case=%x mirror=%x joining=%d age=%d",
|
|
i, d.p.category, d.p.combiningClass, d.p.direction, d.otherCase, d.mirroredChar, d.p.joining, d.p.age);
|
|
if (d.decompositionType != QChar::NoDecomposition) {
|
|
qDebug(" decomposition: type=%d, length=%d, first=%x", d.decompositionType, d.decomposition.size(),
|
|
d.decomposition[0]);
|
|
}
|
|
}
|
|
qDebug(" ");
|
|
}
|
|
#endif
|
|
|
|
static QList<PropertyFlags> uniqueProperties;
|
|
|
|
static void computeUniqueProperties()
|
|
{
|
|
qDebug("computeUniqueProperties:");
|
|
for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
|
|
UnicodeData &d = UnicodeData::valueRef(codepoint);
|
|
int index = uniqueProperties.indexOf(d.p);
|
|
if (index == -1) {
|
|
index = uniqueProperties.size();
|
|
uniqueProperties.append(d.p);
|
|
}
|
|
d.propertyIndex = index;
|
|
}
|
|
qDebug(" %zd unique unicode properties found", ssize_t(uniqueProperties.size()));
|
|
}
|
|
|
|
struct UniqueBlock {
|
|
inline UniqueBlock() : index(-1) {}
|
|
|
|
inline bool operator==(const UniqueBlock &other) const
|
|
{ return values == other.values; }
|
|
|
|
int index;
|
|
QList<int> values;
|
|
};
|
|
|
|
static QByteArray createPropertyInfo()
|
|
{
|
|
qDebug("createPropertyInfo:");
|
|
|
|
// we reserve one bit more than in the assert below for the sign
|
|
Q_ASSERT(maxMirroredDiff < (1<<12));
|
|
Q_ASSERT(maxLowerCaseDiff < (1<<13));
|
|
Q_ASSERT(maxUpperCaseDiff < (1<<13));
|
|
Q_ASSERT(maxTitleCaseDiff < (1<<13));
|
|
Q_ASSERT(maxCaseFoldDiff < (1<<13));
|
|
|
|
const int BMP_BLOCKSIZE = 32;
|
|
const int BMP_SHIFT = 5;
|
|
const int BMP_END = 0x11000;
|
|
const int SMP_END = 0x110000;
|
|
const int SMP_BLOCKSIZE = 256;
|
|
const int SMP_SHIFT = 8;
|
|
|
|
QList<UniqueBlock> uniqueBlocks;
|
|
QList<int> blockMap;
|
|
int used = 0;
|
|
|
|
// Group BMP data into blocks indexed by their 12 most significant bits
|
|
// (blockId = ucs >> 5):
|
|
for (int block = 0; block < BMP_END/BMP_BLOCKSIZE; ++block) {
|
|
UniqueBlock b;
|
|
b.values.reserve(BMP_BLOCKSIZE);
|
|
for (int i = 0; i < BMP_BLOCKSIZE; ++i) {
|
|
int uc = block*BMP_BLOCKSIZE + i;
|
|
UnicodeData &d = UnicodeData::valueRef(uc);
|
|
b.values.append(d.propertyIndex);
|
|
}
|
|
int index = uniqueBlocks.indexOf(b);
|
|
if (index == -1) {
|
|
index = uniqueBlocks.size();
|
|
b.index = used;
|
|
used += BMP_BLOCKSIZE;
|
|
uniqueBlocks.append(b);
|
|
}
|
|
blockMap.append(uniqueBlocks.at(index).index);
|
|
}
|
|
int bmp_blocks = uniqueBlocks.size();
|
|
|
|
// Group SMP data into blocks indexed by their 9 most significant bits, plus
|
|
// an offset to put them after the BMP blocks (blockId = (ucs >> 8) + 0x880):
|
|
for (int block = BMP_END/SMP_BLOCKSIZE; block < SMP_END/SMP_BLOCKSIZE; ++block) {
|
|
UniqueBlock b;
|
|
b.values.reserve(SMP_BLOCKSIZE);
|
|
for (int i = 0; i < SMP_BLOCKSIZE; ++i) {
|
|
int uc = block*SMP_BLOCKSIZE + i;
|
|
UnicodeData &d = UnicodeData::valueRef(uc);
|
|
b.values.append(d.propertyIndex);
|
|
}
|
|
int index = uniqueBlocks.indexOf(b);
|
|
if (index == -1) {
|
|
index = uniqueBlocks.size();
|
|
b.index = used;
|
|
used += SMP_BLOCKSIZE;
|
|
uniqueBlocks.append(b);
|
|
}
|
|
blockMap.append(uniqueBlocks.at(index).index);
|
|
}
|
|
int smp_blocks = uniqueBlocks.size() - bmp_blocks;
|
|
|
|
int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int bmp_trie = BMP_END/BMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int bmp_mem = bmp_block_data + bmp_trie;
|
|
qDebug(" %d unique blocks in BMP.", bmp_blocks);
|
|
qDebug(" block data uses: %d bytes", bmp_block_data);
|
|
qDebug(" trie data uses : %d bytes", bmp_trie);
|
|
|
|
int smp_block_data = smp_blocks*SMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int smp_mem = smp_block_data + smp_trie;
|
|
qDebug(" %d unique blocks in SMP.", smp_blocks);
|
|
qDebug(" block data uses: %d bytes", smp_block_data);
|
|
qDebug(" trie data uses : %d bytes", smp_trie);
|
|
|
|
int prop_data = uniqueProperties.size() * SizeOfPropertiesStruct;
|
|
qDebug("\n properties data uses : %d bytes", prop_data);
|
|
qDebug(" memory usage: %d bytes", bmp_mem + smp_mem + prop_data);
|
|
|
|
Q_ASSERT(blockMap.size() == BMP_END/BMP_BLOCKSIZE +(SMP_END-BMP_END)/SMP_BLOCKSIZE); // 0x1870
|
|
Q_ASSERT(blockMap.last() + blockMap.size() < (1<<(sizeof(unsigned short)*8)));
|
|
|
|
QByteArray out = "static const unsigned short uc_property_trie[] = {\n";
|
|
// First write the map from blockId to indices of unique blocks:
|
|
out += " // [0x0..0x" + QByteArray::number(BMP_END, 16) + ")";
|
|
for (int i = 0; i < BMP_END/BMP_BLOCKSIZE; ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
if (!((i*BMP_BLOCKSIZE) % 0x1000))
|
|
out += "\n";
|
|
out += "\n ";
|
|
}
|
|
out += QByteArray::number(blockMap.at(i) + blockMap.size());
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n\n // [0x" + QByteArray::number(BMP_END, 16) + "..0x" + QByteArray::number(SMP_END, 16) + ")\n";
|
|
for (int i = BMP_END/BMP_BLOCKSIZE; i < blockMap.size(); ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
if (!(i % (0x10000/SMP_BLOCKSIZE)))
|
|
out += "\n";
|
|
out += "\n ";
|
|
}
|
|
out += QByteArray::number(blockMap.at(i) + blockMap.size());
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n";
|
|
// Then write the contents of the unique blocks, at the anticipated indices.
|
|
// Each unique block is a list of UnicodeData::propertyIndex values, whch
|
|
// are indices into the uc_properties table.
|
|
for (int i = 0; i < uniqueBlocks.size(); ++i) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n";
|
|
const UniqueBlock &b = uniqueBlocks.at(i);
|
|
for (int j = 0; j < b.values.size(); ++j) {
|
|
if (!(j % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n ";
|
|
}
|
|
out += QByteArray::number(b.values.at(j));
|
|
out += ", ";
|
|
}
|
|
}
|
|
if (out.endsWith(", "))
|
|
out.chop(2);
|
|
out += "\n};\n\n";
|
|
|
|
out += "static const Properties uc_properties[] = {";
|
|
// keep in sync with the property declaration
|
|
for (int i = 0; i < uniqueProperties.size(); ++i) {
|
|
const PropertyFlags &p = uniqueProperties.at(i);
|
|
out += "\n { ";
|
|
// " ushort category : 8; /* 5 used */\n"
|
|
out += QByteArray::number( p.category );
|
|
out += ", ";
|
|
// " ushort direction : 8; /* 5 used */\n"
|
|
out += QByteArray::number( p.direction );
|
|
out += ", ";
|
|
// " ushort combiningClass : 8;\n"
|
|
out += QByteArray::number( p.combiningClass );
|
|
out += ", ";
|
|
// " ushort joining : 3;\n"
|
|
out += QByteArray::number( p.joining );
|
|
out += ", ";
|
|
// " signed short digitValue : 5;\n"
|
|
out += QByteArray::number( p.digitValue );
|
|
out += ", ";
|
|
// " signed short mirrorDiff : 16;\n"
|
|
out += QByteArray::number( p.mirrorDiff );
|
|
out += ", ";
|
|
// " ushort unicodeVersion : 8; /* 5 used */\n"
|
|
out += QByteArray::number( p.age );
|
|
out += ", ";
|
|
// " ushort nfQuickCheck : 8;\n"
|
|
out += QByteArray::number( p.nfQuickCheck );
|
|
out += ", ";
|
|
// " struct {\n"
|
|
// " ushort special : 1;\n"
|
|
// " signed short diff : 15;\n"
|
|
// " } cases[NumCases];\n"
|
|
out += " { {";
|
|
out += QByteArray::number( p.lowerCaseSpecial );
|
|
out += ", ";
|
|
out += QByteArray::number( p.lowerCaseDiff );
|
|
out += "}, {";
|
|
out += QByteArray::number( p.upperCaseSpecial );
|
|
out += ", ";
|
|
out += QByteArray::number( p.upperCaseDiff );
|
|
out += "}, {";
|
|
out += QByteArray::number( p.titleCaseSpecial );
|
|
out += ", ";
|
|
out += QByteArray::number( p.titleCaseDiff );
|
|
out += "}, {";
|
|
out += QByteArray::number( p.caseFoldSpecial );
|
|
out += ", ";
|
|
out += QByteArray::number( p.caseFoldDiff );
|
|
out += "} }, ";
|
|
// " ushort graphemeBreakClass : 5; /* 5 used */\n"
|
|
// " ushort wordBreakClass : 5; /* 5 used */\n"
|
|
// " ushort lineBreakClass : 6; /* 6 used */\n"
|
|
out += QByteArray::number( p.graphemeBreakClass );
|
|
out += ", ";
|
|
out += QByteArray::number( p.wordBreakClass );
|
|
out += ", ";
|
|
out += QByteArray::number( p.lineBreakClass );
|
|
out += ", ";
|
|
// " ushort sentenceBreakClass : 8; /* 4 used */\n"
|
|
out += QByteArray::number( p.sentenceBreakClass );
|
|
out += ", ";
|
|
// " ushort script : 8;\n"
|
|
out += QByteArray::number( p.script );
|
|
out += " },";
|
|
}
|
|
if (out.endsWith(','))
|
|
out.chop(1);
|
|
out += "\n};\n\n";
|
|
|
|
out += "Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(char32_t ucs4) noexcept\n"
|
|
"{\n"
|
|
" if (ucs4 < 0x" + QByteArray::number(BMP_END, 16) + ")\n"
|
|
" return uc_properties + uc_property_trie[uc_property_trie[ucs4 >> "
|
|
+ QByteArray::number(BMP_SHIFT) + "] + (ucs4 & 0x"
|
|
+ QByteArray::number(BMP_BLOCKSIZE - 1, 16)+ ")];\n"
|
|
"\n"
|
|
" return uc_properties\n"
|
|
" + uc_property_trie[uc_property_trie[((ucs4 - 0x"
|
|
+ QByteArray::number(BMP_END, 16) + ") >> "
|
|
+ QByteArray::number(SMP_SHIFT) + ") + 0x"
|
|
+ QByteArray::number(BMP_END / BMP_BLOCKSIZE, 16) + "] + (ucs4 & 0x"
|
|
+ QByteArray::number(SMP_BLOCKSIZE - 1, 16) + ")];\n"
|
|
"}\n"
|
|
"\n"
|
|
"Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(char16_t ucs2) noexcept\n"
|
|
"{\n"
|
|
" return uc_properties + uc_property_trie[uc_property_trie[ucs2 >> "
|
|
+ QByteArray::number(BMP_SHIFT) + "] + (ucs2 & 0x"
|
|
+ QByteArray::number(BMP_BLOCKSIZE - 1, 16) + ")];\n"
|
|
"}\n"
|
|
"\n"
|
|
"Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(char32_t ucs4) noexcept\n"
|
|
"{\n"
|
|
" return qGetProp(ucs4);\n"
|
|
"}\n"
|
|
"\n"
|
|
"Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(char16_t ucs2) noexcept\n"
|
|
"{\n"
|
|
" return qGetProp(ucs2);\n"
|
|
"}\n\n";
|
|
|
|
out += "Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(char32_t ucs4) noexcept\n"
|
|
"{\n"
|
|
" return static_cast<GraphemeBreakClass>(qGetProp(ucs4)->graphemeBreakClass);\n"
|
|
"}\n"
|
|
"\n"
|
|
"Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(char32_t ucs4) noexcept\n"
|
|
"{\n"
|
|
" return static_cast<WordBreakClass>(qGetProp(ucs4)->wordBreakClass);\n"
|
|
"}\n"
|
|
"\n"
|
|
"Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(char32_t ucs4) noexcept\n"
|
|
"{\n"
|
|
" return static_cast<SentenceBreakClass>(qGetProp(ucs4)->sentenceBreakClass);\n"
|
|
"}\n"
|
|
"\n"
|
|
"Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(char32_t ucs4) noexcept\n"
|
|
"{\n"
|
|
" return static_cast<LineBreakClass>(qGetProp(ucs4)->lineBreakClass);\n"
|
|
"}\n"
|
|
"\n";
|
|
|
|
return out;
|
|
}
|
|
|
|
static QByteArray createSpecialCaseMap()
|
|
{
|
|
qDebug("createSpecialCaseMap:");
|
|
|
|
QByteArray out
|
|
= "static const unsigned short specialCaseMap[] = {\n"
|
|
" 0x0, // placeholder";
|
|
|
|
int i = 1;
|
|
int maxN = 0;
|
|
while (i < specialCaseMap.size()) {
|
|
out += "\n ";
|
|
int n = specialCaseMap.at(i);
|
|
for (int j = 0; j <= n; ++j) {
|
|
out += QByteArray(" 0x") + QByteArray::number(specialCaseMap.at(i+j), 16);
|
|
out += ",";
|
|
}
|
|
i += n + 1;
|
|
maxN = std::max(maxN, n);
|
|
}
|
|
out.chop(1);
|
|
out += "\n};\n\nconst unsigned int MaxSpecialCaseLength = ";
|
|
out += QByteArray::number(maxN);
|
|
out += ";\n\n";
|
|
|
|
qDebug(" memory usage: %zd bytes", ssize_t(specialCaseMap.size() * sizeof(unsigned short)));
|
|
|
|
return out;
|
|
}
|
|
|
|
|
|
static QByteArray createCompositionInfo()
|
|
{
|
|
qDebug("createCompositionInfo: highestComposedCharacter=0x%x", highestComposedCharacter);
|
|
|
|
const int BMP_BLOCKSIZE = 16;
|
|
const int BMP_SHIFT = 4;
|
|
const int BMP_END = 0x3400; // start of Han
|
|
const int SMP_END = 0x30000;
|
|
const int SMP_BLOCKSIZE = 256;
|
|
const int SMP_SHIFT = 8;
|
|
|
|
if (SMP_END <= highestComposedCharacter)
|
|
qFatal("end of table smaller than highest composed character 0x%x", highestComposedCharacter);
|
|
|
|
QList<unsigned short> decompositions;
|
|
int tableIndex = 0;
|
|
|
|
QList<UniqueBlock> uniqueBlocks;
|
|
QList<int> blockMap;
|
|
int used = 0;
|
|
|
|
for (int block = 0; block < BMP_END/BMP_BLOCKSIZE; ++block) {
|
|
UniqueBlock b;
|
|
b.values.reserve(BMP_BLOCKSIZE);
|
|
for (int i = 0; i < BMP_BLOCKSIZE; ++i) {
|
|
int uc = block*BMP_BLOCKSIZE + i;
|
|
UnicodeData &d = UnicodeData::valueRef(uc);
|
|
if (!d.decomposition.isEmpty()) {
|
|
int utf16Length = 0;
|
|
decompositions.append(0);
|
|
for (int j = 0; j < d.decomposition.size(); ++j) {
|
|
int code = d.decomposition.at(j);
|
|
if (QChar::requiresSurrogates(code)) {
|
|
// save as surrogate pair
|
|
decompositions.append(QChar::highSurrogate(code));
|
|
decompositions.append(QChar::lowSurrogate(code));
|
|
utf16Length += 2;
|
|
} else {
|
|
decompositions.append(code);
|
|
utf16Length++;
|
|
}
|
|
}
|
|
decompositions[tableIndex] = d.decompositionType + (utf16Length<<8);
|
|
b.values.append(tableIndex);
|
|
tableIndex += utf16Length + 1;
|
|
} else {
|
|
b.values.append(0xffff);
|
|
}
|
|
}
|
|
int index = uniqueBlocks.indexOf(b);
|
|
if (index == -1) {
|
|
index = uniqueBlocks.size();
|
|
b.index = used;
|
|
used += BMP_BLOCKSIZE;
|
|
uniqueBlocks.append(b);
|
|
}
|
|
blockMap.append(uniqueBlocks.at(index).index);
|
|
}
|
|
int bmp_blocks = uniqueBlocks.size();
|
|
|
|
for (int block = BMP_END/SMP_BLOCKSIZE; block < SMP_END/SMP_BLOCKSIZE; ++block) {
|
|
UniqueBlock b;
|
|
b.values.reserve(SMP_BLOCKSIZE);
|
|
for (int i = 0; i < SMP_BLOCKSIZE; ++i) {
|
|
int uc = block*SMP_BLOCKSIZE + i;
|
|
UnicodeData &d = UnicodeData::valueRef(uc);
|
|
if (!d.decomposition.isEmpty()) {
|
|
int utf16Length = 0;
|
|
decompositions.append(0);
|
|
for (int j = 0; j < d.decomposition.size(); ++j) {
|
|
int code = d.decomposition.at(j);
|
|
if (QChar::requiresSurrogates(code)) {
|
|
// save as surrogate pair
|
|
decompositions.append(QChar::highSurrogate(code));
|
|
decompositions.append(QChar::lowSurrogate(code));
|
|
utf16Length += 2;
|
|
} else {
|
|
decompositions.append(code);
|
|
utf16Length++;
|
|
}
|
|
}
|
|
decompositions[tableIndex] = d.decompositionType + (utf16Length<<8);
|
|
b.values.append(tableIndex);
|
|
tableIndex += utf16Length + 1;
|
|
} else {
|
|
b.values.append(0xffff);
|
|
}
|
|
}
|
|
int index = uniqueBlocks.indexOf(b);
|
|
if (index == -1) {
|
|
index = uniqueBlocks.size();
|
|
b.index = used;
|
|
used += SMP_BLOCKSIZE;
|
|
uniqueBlocks.append(b);
|
|
}
|
|
blockMap.append(uniqueBlocks.at(index).index);
|
|
}
|
|
int smp_blocks = uniqueBlocks.size() - bmp_blocks;
|
|
|
|
// if the condition below doesn't hold anymore we need to modify our decomposition code
|
|
Q_ASSERT(tableIndex < 0xffff);
|
|
|
|
int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int bmp_trie = BMP_END/BMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int bmp_mem = bmp_block_data + bmp_trie;
|
|
qDebug(" %d unique blocks in BMP.", bmp_blocks);
|
|
qDebug(" block data uses: %d bytes", bmp_block_data);
|
|
qDebug(" trie data uses : %d bytes", bmp_trie);
|
|
|
|
int smp_block_data = smp_blocks*SMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int smp_mem = smp_block_data + smp_trie;
|
|
qDebug(" %d unique blocks in SMP.", smp_blocks);
|
|
qDebug(" block data uses: %d bytes", smp_block_data);
|
|
qDebug(" trie data uses : %d bytes", smp_trie);
|
|
|
|
int decomposition_data = decompositions.size() * 2;
|
|
qDebug("\n decomposition data uses : %d bytes", decomposition_data);
|
|
qDebug(" memory usage: %d bytes", bmp_mem + smp_mem + decomposition_data);
|
|
|
|
Q_ASSERT(blockMap.last() + blockMap.size() < (1<<(sizeof(unsigned short)*8)));
|
|
|
|
QByteArray out = "static const unsigned short uc_decomposition_trie[] = {\n";
|
|
// first write the map
|
|
out += " // 0 - 0x" + QByteArray::number(BMP_END, 16);
|
|
for (int i = 0; i < BMP_END/BMP_BLOCKSIZE; ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
if (!((i*BMP_BLOCKSIZE) % 0x1000))
|
|
out += "\n";
|
|
out += "\n ";
|
|
}
|
|
out += QByteArray::number(blockMap.at(i) + blockMap.size());
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n\n // 0x" + QByteArray::number(BMP_END, 16) + " - 0x" + QByteArray::number(SMP_END, 16) + "\n";
|
|
for (int i = BMP_END/BMP_BLOCKSIZE; i < blockMap.size(); ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
if (!(i % (0x10000/SMP_BLOCKSIZE)))
|
|
out += "\n";
|
|
out += "\n ";
|
|
}
|
|
out += QByteArray::number(blockMap.at(i) + blockMap.size());
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n";
|
|
// write the data
|
|
for (int i = 0; i < uniqueBlocks.size(); ++i) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n";
|
|
const UniqueBlock &b = uniqueBlocks.at(i);
|
|
for (int j = 0; j < b.values.size(); ++j) {
|
|
if (!(j % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n ";
|
|
}
|
|
out += "0x" + QByteArray::number(b.values.at(j), 16);
|
|
out += ", ";
|
|
}
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(2);
|
|
out += "\n};\n\n";
|
|
|
|
out += "#define GET_DECOMPOSITION_INDEX(ucs4) \\\n"
|
|
" (ucs4 < 0x" + QByteArray::number(BMP_END, 16) + " \\\n"
|
|
" ? (uc_decomposition_trie[uc_decomposition_trie[ucs4 >> "
|
|
+ QByteArray::number(BMP_SHIFT) + "] + (ucs4 & 0x"
|
|
+ QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")]) \\\n"
|
|
" : ucs4 < 0x" + QByteArray::number(SMP_END, 16) + " \\\n"
|
|
" ? uc_decomposition_trie[uc_decomposition_trie[((ucs4 - 0x"
|
|
+ QByteArray::number(BMP_END, 16) + ") >> "
|
|
+ QByteArray::number(SMP_SHIFT) + ") + 0x"
|
|
+ QByteArray::number(BMP_END/BMP_BLOCKSIZE, 16) + "] + (ucs4 & 0x"
|
|
+ QByteArray::number(SMP_BLOCKSIZE-1, 16) + ")] \\\n"
|
|
" : 0xffff)\n\n";
|
|
|
|
out += "static const unsigned short uc_decomposition_map[] = {";
|
|
for (int i = 0; i < decompositions.size(); ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n ";
|
|
}
|
|
out += "0x" + QByteArray::number(decompositions.at(i), 16);
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(2);
|
|
out += "\n};\n\n";
|
|
|
|
return out;
|
|
}
|
|
|
|
static QByteArray createLigatureInfo()
|
|
{
|
|
qDebug("createLigatureInfo: numLigatures=%d, highestLigature=0x%x", numLigatures, highestLigature);
|
|
|
|
for (int i = 0; i < ligatureHashes.size(); ++i) {
|
|
const QList<Ligature> &l = ligatureHashes.value(i);
|
|
for (int j = 0; j < l.size(); ++j) {
|
|
// if the condition below doesn't hold anymore we need to modify our ligatureHelper code
|
|
Q_ASSERT(QChar::requiresSurrogates(l.at(j).u2) == QChar::requiresSurrogates(l.at(j).ligature) &&
|
|
QChar::requiresSurrogates(l.at(j).u2) == QChar::requiresSurrogates(l.at(j).u1));
|
|
}
|
|
}
|
|
|
|
const int BMP_BLOCKSIZE = 32;
|
|
const int BMP_SHIFT = 5;
|
|
const int BMP_END = 0x3100;
|
|
const int SMP_END = 0x12000;
|
|
const int SMP_BLOCKSIZE = 256;
|
|
const int SMP_SHIFT = 8;
|
|
|
|
if (SMP_END <= highestLigature)
|
|
qFatal("end of table smaller than highest ligature character 0x%x", highestLigature);
|
|
|
|
QList<unsigned short> ligatures;
|
|
int tableIndex = 0;
|
|
|
|
QList<UniqueBlock> uniqueBlocks;
|
|
QList<int> blockMap;
|
|
int used = 0;
|
|
|
|
for (int block = 0; block < BMP_END/BMP_BLOCKSIZE; ++block) {
|
|
UniqueBlock b;
|
|
b.values.reserve(BMP_BLOCKSIZE);
|
|
for (int i = 0; i < BMP_BLOCKSIZE; ++i) {
|
|
int uc = block*BMP_BLOCKSIZE + i;
|
|
QList<Ligature> l = ligatureHashes.value(uc);
|
|
if (!l.isEmpty()) {
|
|
Q_ASSERT(!QChar::requiresSurrogates(uc));
|
|
std::sort(l.begin(), l.end()); // needed for bsearch in ligatureHelper code
|
|
|
|
ligatures.append(l.size());
|
|
for (int j = 0; j < l.size(); ++j) {
|
|
ligatures.append(l.at(j).u1);
|
|
ligatures.append(l.at(j).ligature);
|
|
}
|
|
b.values.append(tableIndex);
|
|
tableIndex += 2*l.size() + 1;
|
|
} else {
|
|
b.values.append(0xffff);
|
|
}
|
|
}
|
|
int index = uniqueBlocks.indexOf(b);
|
|
if (index == -1) {
|
|
index = uniqueBlocks.size();
|
|
b.index = used;
|
|
used += BMP_BLOCKSIZE;
|
|
uniqueBlocks.append(b);
|
|
}
|
|
blockMap.append(uniqueBlocks.at(index).index);
|
|
}
|
|
int bmp_blocks = uniqueBlocks.size();
|
|
|
|
for (int block = BMP_END/SMP_BLOCKSIZE; block < SMP_END/SMP_BLOCKSIZE; ++block) {
|
|
UniqueBlock b;
|
|
b.values.reserve(SMP_BLOCKSIZE);
|
|
for (int i = 0; i < SMP_BLOCKSIZE; ++i) {
|
|
int uc = block*SMP_BLOCKSIZE + i;
|
|
QList<Ligature> l = ligatureHashes.value(uc);
|
|
if (!l.isEmpty()) {
|
|
Q_ASSERT(QChar::requiresSurrogates(uc));
|
|
std::sort(l.begin(), l.end()); // needed for bsearch in ligatureHelper code
|
|
|
|
ligatures.append(l.size());
|
|
for (int j = 0; j < l.size(); ++j) {
|
|
ligatures.append(QChar::highSurrogate(l.at(j).u1));
|
|
ligatures.append(QChar::lowSurrogate(l.at(j).u1));
|
|
ligatures.append(QChar::highSurrogate(l.at(j).ligature));
|
|
ligatures.append(QChar::lowSurrogate(l.at(j).ligature));
|
|
}
|
|
b.values.append(tableIndex);
|
|
tableIndex += 4*l.size() + 1;
|
|
} else {
|
|
b.values.append(0xffff);
|
|
}
|
|
}
|
|
int index = uniqueBlocks.indexOf(b);
|
|
if (index == -1) {
|
|
index = uniqueBlocks.size();
|
|
b.index = used;
|
|
used += SMP_BLOCKSIZE;
|
|
uniqueBlocks.append(b);
|
|
}
|
|
blockMap.append(uniqueBlocks.at(index).index);
|
|
}
|
|
int smp_blocks = uniqueBlocks.size() - bmp_blocks;
|
|
|
|
// if the condition below doesn't hold anymore we need to modify our composition code
|
|
Q_ASSERT(tableIndex < 0xffff);
|
|
|
|
int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int bmp_trie = BMP_END/BMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int bmp_mem = bmp_block_data + bmp_trie;
|
|
qDebug(" %d unique blocks in BMP.", bmp_blocks);
|
|
qDebug(" block data uses: %d bytes", bmp_block_data);
|
|
qDebug(" trie data uses : %d bytes", bmp_trie);
|
|
|
|
int smp_block_data = smp_blocks*SMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*sizeof(unsigned short);
|
|
int smp_mem = smp_block_data + smp_trie;
|
|
qDebug(" %d unique blocks in SMP.", smp_blocks);
|
|
qDebug(" block data uses: %d bytes", smp_block_data);
|
|
qDebug(" trie data uses : %d bytes", smp_trie);
|
|
|
|
int ligature_data = ligatures.size() * 2;
|
|
qDebug("\n ligature data uses : %d bytes", ligature_data);
|
|
qDebug(" memory usage: %d bytes", bmp_mem + smp_mem + ligature_data);
|
|
|
|
Q_ASSERT(blockMap.last() + blockMap.size() < (1<<(sizeof(unsigned short)*8)));
|
|
|
|
QByteArray out = "static const unsigned short uc_ligature_trie[] = {\n";
|
|
// first write the map
|
|
out += " // 0 - 0x" + QByteArray::number(BMP_END, 16);
|
|
for (int i = 0; i < BMP_END/BMP_BLOCKSIZE; ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
if (!((i*BMP_BLOCKSIZE) % 0x1000))
|
|
out += "\n";
|
|
out += "\n ";
|
|
}
|
|
out += QByteArray::number(blockMap.at(i) + blockMap.size());
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n\n // 0x" + QByteArray::number(BMP_END, 16) + " - 0x" + QByteArray::number(SMP_END, 16) + "\n";
|
|
for (int i = BMP_END/BMP_BLOCKSIZE; i < blockMap.size(); ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
if (!(i % (0x10000/SMP_BLOCKSIZE)))
|
|
out += "\n";
|
|
out += "\n ";
|
|
}
|
|
out += QByteArray::number(blockMap.at(i) + blockMap.size());
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n";
|
|
// write the data
|
|
for (int i = 0; i < uniqueBlocks.size(); ++i) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n";
|
|
const UniqueBlock &b = uniqueBlocks.at(i);
|
|
for (int j = 0; j < b.values.size(); ++j) {
|
|
if (!(j % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n ";
|
|
}
|
|
out += "0x" + QByteArray::number(b.values.at(j), 16);
|
|
out += ", ";
|
|
}
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(2);
|
|
out += "\n};\n\n";
|
|
|
|
out += "#define GET_LIGATURE_INDEX(ucs4) \\\n"
|
|
" (ucs4 < 0x" + QByteArray::number(BMP_END, 16) + " \\\n"
|
|
" ? (uc_ligature_trie[uc_ligature_trie[ucs4 >> "
|
|
+ QByteArray::number(BMP_SHIFT) + "] + (ucs4 & 0x"
|
|
+ QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")]) \\\n"
|
|
" : ucs4 < 0x" + QByteArray::number(SMP_END, 16) + " \\\n"
|
|
" ? uc_ligature_trie[uc_ligature_trie[((ucs4 - 0x"
|
|
+ QByteArray::number(BMP_END, 16) + ") >> "
|
|
+ QByteArray::number(SMP_SHIFT) + ") + 0x"
|
|
+ QByteArray::number(BMP_END/BMP_BLOCKSIZE, 16) + "]" " + (ucs4 & 0x"
|
|
+ QByteArray::number(SMP_BLOCKSIZE-1, 16) + ")] \\\n"
|
|
" : 0xffff)\n\n";
|
|
|
|
out += "static const unsigned short uc_ligature_map[] = {";
|
|
for (int i = 0; i < ligatures.size(); ++i) {
|
|
if (!(i % 8)) {
|
|
if (out.endsWith(' '))
|
|
out.chop(1);
|
|
out += "\n ";
|
|
}
|
|
out += "0x" + QByteArray::number(ligatures.at(i), 16);
|
|
out += ", ";
|
|
}
|
|
if (out.endsWith(' '))
|
|
out.chop(2);
|
|
out += "\n};\n";
|
|
|
|
return out;
|
|
}
|
|
|
|
QByteArray createCasingInfo()
|
|
{
|
|
QByteArray out
|
|
= "struct CasingInfo {\n"
|
|
" uint codePoint : 16;\n"
|
|
" uint flags : 8;\n"
|
|
" uint offset : 8;\n"
|
|
"};\n\n";
|
|
|
|
return out;
|
|
}
|
|
|
|
|
|
int main(int, char **)
|
|
{
|
|
initAgeMap();
|
|
initCategoryMap();
|
|
initDecompositionMap();
|
|
initDirectionMap();
|
|
initJoiningMap();
|
|
initGraphemeBreak();
|
|
initWordBreak();
|
|
initSentenceBreak();
|
|
initLineBreak();
|
|
initScriptMap();
|
|
|
|
readUnicodeData();
|
|
readBidiMirroring();
|
|
readArabicShaping();
|
|
readDerivedAge();
|
|
readDerivedNormalizationProps();
|
|
readSpecialCasing();
|
|
readCaseFolding();
|
|
// readBlocks();
|
|
readScripts();
|
|
readGraphemeBreak();
|
|
readWordBreak();
|
|
readSentenceBreak();
|
|
readLineBreak();
|
|
|
|
computeUniqueProperties();
|
|
QByteArray properties = createPropertyInfo();
|
|
QByteArray specialCases = createSpecialCaseMap();
|
|
QByteArray compositions = createCompositionInfo();
|
|
QByteArray ligatures = createLigatureInfo();
|
|
QByteArray normalizationCorrections = createNormalizationCorrections();
|
|
|
|
QByteArray header =
|
|
"/****************************************************************************\n"
|
|
"**\n"
|
|
"** Copyright (C) 2020 The Qt Company Ltd.\n"
|
|
"** Contact: https://www.qt.io/licensing/\n"
|
|
"**\n"
|
|
"** This file is part of the QtCore module of the Qt Toolkit.\n"
|
|
"**\n"
|
|
"** $QT_BEGIN_LICENSE:LGPL$\n"
|
|
"** Commercial License Usage\n"
|
|
"** Licensees holding valid commercial Qt licenses may use this file in\n"
|
|
"** accordance with the commercial license agreement provided with the\n"
|
|
"** Software or, alternatively, in accordance with the terms contained in\n"
|
|
"** a written agreement between you and The Qt Company. For licensing terms\n"
|
|
"** and conditions see https://www.qt.io/terms-conditions. For further\n"
|
|
"** information use the contact form at https://www.qt.io/contact-us.\n"
|
|
"**\n"
|
|
"** GNU Lesser General Public License Usage\n"
|
|
"** Alternatively, this file may be used under the terms of the GNU Lesser\n"
|
|
"** General Public License version 3 as published by the Free Software\n"
|
|
"** Foundation and appearing in the file LICENSE.LGPL3 included in the\n"
|
|
"** packaging of this file. Please review the following information to\n"
|
|
"** ensure the GNU Lesser General Public License version 3 requirements\n"
|
|
"** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.\n"
|
|
"**\n"
|
|
"** GNU General Public License Usage\n"
|
|
"** Alternatively, this file may be used under the terms of the GNU\n"
|
|
"** General Public License version 2.0 or (at your option) the GNU General\n"
|
|
"** Public license version 3 or any later version approved by the KDE Free\n"
|
|
"** Qt Foundation. The licenses are as published by the Free Software\n"
|
|
"** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3\n"
|
|
"** included in the packaging of this file. Please review the following\n"
|
|
"** information to ensure the GNU General Public License requirements will\n"
|
|
"** be met: https://www.gnu.org/licenses/gpl-2.0.html and\n"
|
|
"** https://www.gnu.org/licenses/gpl-3.0.html.\n"
|
|
"**\n"
|
|
"** $QT_END_LICENSE$\n"
|
|
"**\n"
|
|
"****************************************************************************/\n\n";
|
|
|
|
QByteArray note =
|
|
"/* This file is autogenerated from the Unicode " DATA_VERSION_S " database. Do not edit */\n\n";
|
|
|
|
QByteArray warning =
|
|
"//\n"
|
|
"// W A R N I N G\n"
|
|
"// -------------\n"
|
|
"//\n"
|
|
"// This file is not part of the Qt API. It exists for the convenience\n"
|
|
"// of internal files. This header file may change from version to version\n"
|
|
"// without notice, or even be removed.\n"
|
|
"//\n"
|
|
"// We mean it.\n"
|
|
"//\n\n";
|
|
|
|
QFile f("../../src/corelib/text/qunicodetables.cpp");
|
|
f.open(QFile::WriteOnly|QFile::Truncate);
|
|
f.write(header);
|
|
f.write(note);
|
|
f.write("#include \"qunicodetables_p.h\"\n\n");
|
|
f.write("QT_BEGIN_NAMESPACE\n\n");
|
|
f.write("namespace QUnicodeTables {\n\n");
|
|
f.write(properties);
|
|
f.write(specialCases);
|
|
f.write(compositions);
|
|
f.write(ligatures);
|
|
f.write("\n");
|
|
f.write(normalizationCorrections);
|
|
f.write("} // namespace QUnicodeTables\n\n");
|
|
f.write("using namespace QUnicodeTables;\n\n");
|
|
f.write("QT_END_NAMESPACE\n");
|
|
f.close();
|
|
|
|
f.setFileName("../../src/corelib/text/qunicodetables_p.h");
|
|
f.open(QFile::WriteOnly | QFile::Truncate);
|
|
f.write(header);
|
|
f.write(note);
|
|
f.write(warning);
|
|
f.write("#ifndef QUNICODETABLES_P_H\n"
|
|
"#define QUNICODETABLES_P_H\n\n"
|
|
"#include <QtCore/private/qglobal_p.h>\n\n"
|
|
"#include <QtCore/qchar.h>\n\n"
|
|
"QT_BEGIN_NAMESPACE\n\n");
|
|
f.write("#define UNICODE_DATA_VERSION " DATA_VERSION_STR "\n\n");
|
|
f.write("namespace QUnicodeTables {\n\n");
|
|
f.write(property_string);
|
|
f.write(sizeOfPropertiesStructCheck);
|
|
f.write(grapheme_break_class_string);
|
|
f.write(word_break_class_string);
|
|
f.write(sentence_break_class_string);
|
|
f.write(line_break_class_string);
|
|
f.write(methods);
|
|
f.write("} // namespace QUnicodeTables\n\n"
|
|
"QT_END_NAMESPACE\n\n"
|
|
"#endif // QUNICODETABLES_P_H\n");
|
|
f.close();
|
|
|
|
qDebug() << "maxMirroredDiff = " << Qt::hex << maxMirroredDiff;
|
|
qDebug() << "maxLowerCaseDiff = " << Qt::hex << maxLowerCaseDiff;
|
|
qDebug() << "maxUpperCaseDiff = " << Qt::hex << maxUpperCaseDiff;
|
|
qDebug() << "maxTitleCaseDiff = " << Qt::hex << maxTitleCaseDiff;
|
|
qDebug() << "maxCaseFoldDiff = " << Qt::hex << maxCaseFoldDiff;
|
|
#if 0
|
|
// dump(0, 0x7f);
|
|
// dump(0x620, 0x640);
|
|
// dump(0x10000, 0x10020);
|
|
// dump(0x10800, 0x10820);
|
|
|
|
qDebug("decompositionLength used:");
|
|
int totalcompositions = 0;
|
|
int sum = 0;
|
|
for (int i = 1; i < 20; ++i) {
|
|
qDebug(" length %d used %d times", i, decompositionLength.value(i, 0));
|
|
totalcompositions += i*decompositionLength.value(i, 0);
|
|
sum += decompositionLength.value(i, 0);
|
|
}
|
|
qDebug(" len decomposition map %d, average length %f, num composed chars %d",
|
|
totalcompositions, (float)totalcompositions/(float)sum, sum);
|
|
qDebug("highest composed character %x", highestComposedCharacter);
|
|
qDebug("num ligatures = %d highest=%x, maxLength=%d", numLigatures, highestLigature, longestLigature);
|
|
|
|
qBubbleSort(ligatures);
|
|
for (int i = 0; i < ligatures.size(); ++i)
|
|
qDebug("%s", ligatures.at(i).data());
|
|
|
|
// qDebug("combiningClass usage:");
|
|
// int numClasses = 0;
|
|
// for (int i = 0; i < 255; ++i) {
|
|
// int num = combiningClassUsage.value(i, 0);
|
|
// if (num) {
|
|
// ++numClasses;
|
|
// qDebug(" combiningClass %d used %d times", i, num);
|
|
// }
|
|
// }
|
|
// qDebug("total of %d combining classes used", numClasses);
|
|
|
|
#endif
|
|
}
|