536 lines
18 KiB
C++
536 lines
18 KiB
C++
/*
|
|
*******************************************************************************
|
|
*
|
|
* Copyright (C) 2004-2012, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
*
|
|
*******************************************************************************
|
|
* file name: bidipropsbuilder.cpp (was genbidi/store.c)
|
|
* encoding: US-ASCII
|
|
* tab size: 8 (not used)
|
|
* indentation:4
|
|
*
|
|
* created on: 2004dec30
|
|
* created by: Markus W. Scherer
|
|
*
|
|
* Store Unicode bidi/shaping properties efficiently for
|
|
* random access.
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include "unicode/utypes.h"
|
|
#include "unicode/uchar.h"
|
|
#include "unicode/uniset.h"
|
|
#include "cmemory.h"
|
|
#include "cstring.h"
|
|
#include "ppucd.h"
|
|
#include "uarrsort.h"
|
|
#include "unicode/udata.h"
|
|
#include "unewdata.h"
|
|
#include "utrie2.h"
|
|
#include "writesrc.h"
|
|
#include "ubidi_props.h"
|
|
#include "genprops.h"
|
|
|
|
/* Unicode bidi/shaping properties file format ---------------------------------
|
|
|
|
The file format prepared and written here contains several data
|
|
structures that store indexes or data.
|
|
|
|
Before the data contents described below, there are the headers required by
|
|
the udata API for loading ICU data. Especially, a UDataInfo structure
|
|
precedes the actual data. It contains platform properties values and the
|
|
file format version.
|
|
|
|
The following is a description of format version 2.0 .
|
|
|
|
The file contains the following structures:
|
|
|
|
const int32_t indexes[i0] with values i0, i1, ...:
|
|
(see UBIDI_IX_... constants for names of indexes)
|
|
|
|
i0 indexLength; -- length of indexes[] (UBIDI_IX_TOP)
|
|
i1 dataLength; -- length in bytes of the post-header data (incl. indexes[])
|
|
i2 trieSize; -- size in bytes of the bidi/shaping properties trie
|
|
i3 mirrorLength; -- length in uint32_t of the bidi mirroring array
|
|
|
|
i4 jgStart; -- first code point with Joining_Group data
|
|
i5 jgLimit; -- limit code point for Joining_Group data
|
|
|
|
i6..i14 reservedIndexes; -- reserved values; 0 for now
|
|
|
|
i15 maxValues; -- maximum code values for enumerated properties
|
|
bits 23..16 contain the max value for Joining_Group,
|
|
otherwise the bits are used like enum fields in the trie word
|
|
|
|
Serialized trie, see utrie2.h;
|
|
|
|
const uint32_t mirrors[mirrorLength];
|
|
|
|
const uint8_t jgArray[i5-i4]; -- (i5-i4) is always a multiple of 4
|
|
|
|
Trie data word:
|
|
Bits
|
|
15..13 signed delta to bidi mirroring code point
|
|
(add delta to input code point)
|
|
0 no such code point (source maps to itself)
|
|
-3..-1, 1..3 delta
|
|
-4 look in mirrors table
|
|
12 is mirrored
|
|
11 Bidi_Control
|
|
10 Join_Control
|
|
9.. 8 reserved (set to 0)
|
|
7.. 5 Joining_Type
|
|
4.. 0 BiDi category
|
|
|
|
|
|
Mirrors:
|
|
Stores some of the bidi mirroring data, where each code point maps to
|
|
at most one other.
|
|
Most code points do not have a mirroring code point; most that do have a signed
|
|
delta stored in the trie data value. Only those where the delta does not fit
|
|
into the trie data are stored in this table.
|
|
|
|
Logically, this is a two-column table with source and mirror code points.
|
|
|
|
Physically, the table is compressed by taking advantage of the fact that each
|
|
mirror code point is also a source code point
|
|
(each of them is a mirror of the other).
|
|
Therefore, both logical columns contain the same set of code points, which needs
|
|
to be stored only once.
|
|
|
|
The table stores source code points, and also for each the index of its mirror
|
|
code point in the same table, in a simple array of uint32_t.
|
|
Bits
|
|
31..21 index to mirror code point (unsigned)
|
|
20.. 0 source code point
|
|
|
|
The table is sorted by source code points.
|
|
|
|
|
|
Joining_Group array:
|
|
The Joining_Group values do not fit into the 16-bit trie, but the data is also
|
|
limited to a small range of code points (Arabic and Syriac) and not
|
|
well compressible.
|
|
|
|
The start and limit code points for the range are stored in the indexes[]
|
|
array, and the jgArray[] stores a byte for each of these code points,
|
|
containing the Joining_Group value.
|
|
|
|
All code points outside of this range have No_Joining_Group (0).
|
|
|
|
--- Changes in format version 2 ---
|
|
|
|
Change from UTrie to UTrie2.
|
|
|
|
----------------------------------------------------------------------------- */
|
|
|
|
U_NAMESPACE_USE
|
|
|
|
/* UDataInfo cf. udata.h */
|
|
static UDataInfo dataInfo={
|
|
sizeof(UDataInfo),
|
|
0,
|
|
|
|
U_IS_BIG_ENDIAN,
|
|
U_CHARSET_FAMILY,
|
|
U_SIZEOF_UCHAR,
|
|
0,
|
|
|
|
/* dataFormat="BiDi" */
|
|
{ UBIDI_FMT_0, UBIDI_FMT_1, UBIDI_FMT_2, UBIDI_FMT_3 },
|
|
{ 2, 0, 0, 0 }, /* formatVersion */
|
|
{ 6, 0, 0, 0 } /* dataVersion */
|
|
};
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
class BiDiPropsBuilder : public PropsBuilder {
|
|
public:
|
|
BiDiPropsBuilder(UErrorCode &errorCode);
|
|
virtual ~BiDiPropsBuilder();
|
|
|
|
virtual void setUnicodeVersion(const UVersionInfo version);
|
|
virtual void setProps(const UniProps &, const UnicodeSet &newValues, UErrorCode &errorCode);
|
|
virtual void build(UErrorCode &errorCode);
|
|
virtual void writeCSourceFile(const char *path, UErrorCode &errorCode);
|
|
virtual void writeBinaryData(const char *path, UBool withCopyright, UErrorCode &errorCode);
|
|
|
|
private:
|
|
int32_t encodeBidiMirroringGlyph(UChar32 src, UChar32 end, UChar32 mirror, UErrorCode &errorCode);
|
|
void makeMirror(UErrorCode &errorCode);
|
|
|
|
UnicodeSet relevantProps;
|
|
UTrie2 *pTrie;
|
|
uint8_t jgArray[0x300]; /* at most for U+0600..U+08FF */
|
|
UChar32 jgStart; // First code point with a Joining_Group.
|
|
UChar32 jgLimit; // One past the last one.
|
|
uint32_t mirrors[UBIDI_MAX_MIRROR_INDEX+1][2];
|
|
int32_t mirrorTop;
|
|
};
|
|
|
|
BiDiPropsBuilder::BiDiPropsBuilder(UErrorCode &errorCode)
|
|
: pTrie(NULL),
|
|
jgStart(0), jgLimit(0),
|
|
mirrorTop(0) {
|
|
// This builder encodes the following properties.
|
|
relevantProps.
|
|
add(UCHAR_BIDI_CONTROL).
|
|
add(UCHAR_BIDI_MIRRORED).
|
|
add(UCHAR_BIDI_CLASS).
|
|
add(UCHAR_BIDI_MIRRORING_GLYPH).
|
|
add(UCHAR_JOIN_CONTROL).
|
|
add(UCHAR_JOINING_GROUP).
|
|
add(UCHAR_JOINING_TYPE);
|
|
pTrie=utrie2_open(0, 0, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "genprops error: bidipropsbuilder utrie2_open() failed - %s\n",
|
|
u_errorName(errorCode));
|
|
}
|
|
uprv_memset(jgArray, U_JG_NO_JOINING_GROUP, sizeof(jgArray));
|
|
}
|
|
|
|
BiDiPropsBuilder::~BiDiPropsBuilder() {
|
|
utrie2_close(pTrie);
|
|
}
|
|
|
|
void
|
|
BiDiPropsBuilder::setUnicodeVersion(const UVersionInfo version) {
|
|
uprv_memcpy(dataInfo.dataVersion, version, 4);
|
|
}
|
|
|
|
/* bidi mirroring table ----------------------------------------------------- */
|
|
|
|
int32_t
|
|
BiDiPropsBuilder::encodeBidiMirroringGlyph(UChar32 src, UChar32 end, UChar32 mirror,
|
|
UErrorCode &errorCode) {
|
|
if(U_FAILURE(errorCode) || mirror<0) {
|
|
return 0;
|
|
}
|
|
if(src!=end) {
|
|
fprintf(stderr,
|
|
"genprops error: range U+%04lX..U+%04lX all with the same "
|
|
"Bidi_Mirroring_Glyph U+%04lX\n",
|
|
(long)src, (long)end, (long)mirror);
|
|
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
|
|
return 0;
|
|
}
|
|
int32_t delta=mirror-src;
|
|
if(delta==0) {
|
|
return 0; /* mapping to self=no mapping */
|
|
}
|
|
|
|
if(delta<UBIDI_MIN_MIRROR_DELTA || UBIDI_MAX_MIRROR_DELTA<delta) {
|
|
/* delta does not fit into the trie properties value, store in the mirrors[] table */
|
|
if(mirrorTop==LENGTHOF(mirrors)) {
|
|
fprintf(stderr,
|
|
"genprops error: too many long-distance Bidi_Mirroring_Glyph mappings; "
|
|
"UBIDI_MAX_MIRROR_INDEX can only be increased by "
|
|
"redesigning the ubidi.icu data bit fields\n");
|
|
errorCode=U_BUFFER_OVERFLOW_ERROR;
|
|
return 0;
|
|
}
|
|
|
|
/* possible: search the table so far and see if src is already listed */
|
|
|
|
mirrors[mirrorTop][0]=(uint32_t)src;
|
|
mirrors[mirrorTop][1]=(uint32_t)mirror;
|
|
++mirrorTop;
|
|
|
|
/* set an escape marker in src's properties */
|
|
delta=UBIDI_ESC_MIRROR_DELTA;
|
|
}
|
|
return delta;
|
|
}
|
|
|
|
void
|
|
BiDiPropsBuilder::setProps(const UniProps &props, const UnicodeSet &newValues,
|
|
UErrorCode &errorCode) {
|
|
if(U_FAILURE(errorCode) || newValues.containsNone(relevantProps)) { return; }
|
|
|
|
UChar32 start=props.start;
|
|
UChar32 end=props.end;
|
|
|
|
int32_t delta=encodeBidiMirroringGlyph(start, end, props.bmg, errorCode);
|
|
uint32_t value=(uint32_t)delta<<UBIDI_MIRROR_DELTA_SHIFT;
|
|
if(props.binProps[UCHAR_BIDI_MIRRORED]) {
|
|
value|=U_MASK(UBIDI_IS_MIRRORED_SHIFT);
|
|
}
|
|
if(props.binProps[UCHAR_BIDI_CONTROL]) {
|
|
value|=U_MASK(UBIDI_BIDI_CONTROL_SHIFT);
|
|
}
|
|
if(props.binProps[UCHAR_JOIN_CONTROL]) {
|
|
value|=U_MASK(UBIDI_JOIN_CONTROL_SHIFT);
|
|
}
|
|
value|=(uint32_t)props.getIntProp(UCHAR_JOINING_TYPE)<<UBIDI_JT_SHIFT;
|
|
value|=(uint32_t)props.getIntProp(UCHAR_BIDI_CLASS);
|
|
utrie2_setRange32(pTrie, start, end, value, TRUE, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "genprops error: BiDiPropsBuilder utrie2_setRange32() failed - %s\n",
|
|
u_errorName(errorCode));
|
|
return;
|
|
}
|
|
|
|
/* store Joining_Group values from vector column 1 in a simple byte array */
|
|
int32_t jg=props.getIntProp(UCHAR_JOINING_GROUP);
|
|
if(jg!=U_JG_NO_JOINING_GROUP) {
|
|
if(start<0x600 || 0x8ff<end) {
|
|
fprintf(stderr, "genprops error: Joining_Group for out-of-range code points U+%04lx..U+%04lx\n",
|
|
(long)start, (long)end);
|
|
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
|
|
return;
|
|
}
|
|
if(start<jgStart || jgStart==0) { jgStart=start; }
|
|
if(end>=jgLimit || jgLimit==0) { jgLimit=end+1; }
|
|
}
|
|
// On the off-chance that a block is defined with a Joining_Group
|
|
// that is then overridden by No_Joining_Group,
|
|
// we set that too, but only inside U+0600..U+08FF.
|
|
if(end>=0x600 && start<=0x8ff) {
|
|
if(start<0x600) { start=0x600; }
|
|
if(end>0x8ff) { end=0x8ff; }
|
|
|
|
/* set Joining_Group value for start..end */
|
|
for(UChar32 c=start; c<=end; ++c) {
|
|
jgArray[c-0x600]=(uint8_t)jg;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* generate output data ----------------------------------------------------- */
|
|
|
|
static int32_t U_CALLCONV
|
|
compareMirror(const void *context, const void *left, const void *right) {
|
|
UChar32 l, r;
|
|
|
|
l=UBIDI_GET_MIRROR_CODE_POINT(((const uint32_t *)left)[0]);
|
|
r=UBIDI_GET_MIRROR_CODE_POINT(((const uint32_t *)right)[0]);
|
|
return l-r;
|
|
}
|
|
|
|
void
|
|
BiDiPropsBuilder::makeMirror(UErrorCode &errorCode) {
|
|
/* sort the mirroring table by source code points */
|
|
uprv_sortArray(mirrors, mirrorTop, 8,
|
|
compareMirror, NULL, FALSE, &errorCode);
|
|
if(U_FAILURE(errorCode)) { return; }
|
|
|
|
/*
|
|
* reduce the 2-column table to a single column
|
|
* by putting the index to the mirror entry into the source entry
|
|
*
|
|
* first:
|
|
* find each mirror code point in the source column and set each other's indexes
|
|
*
|
|
* second:
|
|
* reduce the table, combine the source code points with their indexes
|
|
* and store as a simple array of uint32_t
|
|
*/
|
|
for(int32_t i=0; i<mirrorTop; ++i) {
|
|
uint32_t c=mirrors[i][1]; /* mirror code point */
|
|
if(c>0x1fffff) {
|
|
continue; /* this entry already has an index */
|
|
}
|
|
|
|
/* search for the mirror code point in the source column */
|
|
int32_t start, limit, step;
|
|
if(c<mirrors[i][0]) {
|
|
/* search before i */
|
|
start=i-1;
|
|
limit=-1;
|
|
step=-1;
|
|
} else {
|
|
start=i+1;
|
|
limit=mirrorTop;
|
|
step=1;
|
|
}
|
|
|
|
for(int32_t j=start;; j+=step) {
|
|
if(j==limit) {
|
|
fprintf(stderr,
|
|
"genprops error: bidi mirror does not roundtrip - %04lx->%04lx->?\n",
|
|
(long)mirrors[i][0], (long)mirrors[i][1]);
|
|
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
|
|
}
|
|
if(c==mirrors[j][0]) {
|
|
/*
|
|
* found the mirror code point c in the source column,
|
|
* set both entries' indexes to each other
|
|
*/
|
|
if(UBIDI_GET_MIRROR_CODE_POINT(mirrors[i][0])!=UBIDI_GET_MIRROR_CODE_POINT(mirrors[j][1])) {
|
|
/* roundtrip check fails */
|
|
fprintf(stderr,
|
|
"genprops error: bidi mirrors do not roundtrip - %04lx->%04lx->%04lx\n",
|
|
(long)mirrors[i][0], (long)mirrors[i][1], (long)mirrors[j][1]);
|
|
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
|
|
} else {
|
|
mirrors[i][1]|=(uint32_t)j<<UBIDI_MIRROR_INDEX_SHIFT;
|
|
mirrors[j][1]|=(uint32_t)i<<UBIDI_MIRROR_INDEX_SHIFT;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* now the second step, the actual reduction of the table */
|
|
uint32_t *reducedMirror=mirrors[0];
|
|
for(int32_t i=0; i<mirrorTop; ++i) {
|
|
reducedMirror[i]=mirrors[i][0]|(mirrors[i][1]&~0x1fffff);
|
|
}
|
|
}
|
|
|
|
static int32_t indexes[UBIDI_IX_TOP]={
|
|
UBIDI_IX_TOP, 0, 0, 0,
|
|
0, 0, 0, 0,
|
|
0, 0, 0, 0,
|
|
0, 0, 0, 0
|
|
};
|
|
|
|
static uint8_t trieBlock[40000];
|
|
static int32_t trieSize;
|
|
|
|
void
|
|
BiDiPropsBuilder::build(UErrorCode &errorCode) {
|
|
makeMirror(errorCode);
|
|
if(U_FAILURE(errorCode)) { return; }
|
|
|
|
utrie2_freeze(pTrie, UTRIE2_16_VALUE_BITS, &errorCode);
|
|
trieSize=utrie2_serialize(pTrie, trieBlock, sizeof(trieBlock), &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "genprops error: utrie2_freeze()+utrie2_serialize() failed: %s (length %ld)\n",
|
|
u_errorName(errorCode), (long)trieSize);
|
|
return;
|
|
}
|
|
|
|
/* finish jgArray, pad to multiple of 4 */
|
|
while((jgLimit-jgStart)&3) {
|
|
// Prefer rounding down jgStart before rounding up jgLimit
|
|
// so that we are guaranteed not to increase jgLimit beyond 0x900
|
|
// which (after the offset) is the end of the jgArray.
|
|
if(jgStart&3) { --jgStart; } else { ++jgLimit; }
|
|
}
|
|
indexes[UBIDI_IX_JG_START]=jgStart;
|
|
indexes[UBIDI_IX_JG_LIMIT]=jgLimit;
|
|
|
|
indexes[UBIDI_IX_TRIE_SIZE]=trieSize;
|
|
indexes[UBIDI_IX_MIRROR_LENGTH]=mirrorTop;
|
|
indexes[UBIDI_IX_LENGTH]=
|
|
(int32_t)sizeof(indexes)+
|
|
trieSize+
|
|
4*mirrorTop+
|
|
(jgLimit-jgStart);
|
|
|
|
if(!beQuiet) {
|
|
puts("* ubidi.icu stats *");
|
|
printf("trie size in bytes: %5d\n", (int)trieSize);
|
|
printf("size in bytes of mirroring table: %5d\n", (int)(4*mirrorTop));
|
|
printf("length of Joining_Group array: %5d (U+%04x..U+%04x)\n", (int)(jgLimit-jgStart), (int)jgStart, (int)(jgLimit-1));
|
|
printf("data size: %5d\n", (int)indexes[UBIDI_IX_LENGTH]);
|
|
}
|
|
|
|
indexes[UBIDI_MAX_VALUES_INDEX]=
|
|
((int32_t)U_CHAR_DIRECTION_COUNT-1)|
|
|
(((int32_t)U_JT_COUNT-1)<<UBIDI_JT_SHIFT)|
|
|
(((int32_t)U_JG_COUNT-1)<<UBIDI_MAX_JG_SHIFT);
|
|
}
|
|
|
|
void
|
|
BiDiPropsBuilder::writeCSourceFile(const char *path, UErrorCode &errorCode) {
|
|
if(U_FAILURE(errorCode)) { return; }
|
|
|
|
FILE *f=usrc_create(path, "ubidi_props_data.h",
|
|
"icu/tools/unicode/c/genprops/bidipropsbuilder.cpp");
|
|
if(f==NULL) {
|
|
errorCode=U_FILE_ACCESS_ERROR;
|
|
return;
|
|
}
|
|
fputs("#ifndef INCLUDED_FROM_UBIDI_PROPS_C\n"
|
|
"# error This file must be #included from ubidi_props.c only.\n"
|
|
"#endif\n\n", f);
|
|
usrc_writeArray(f,
|
|
"static const UVersionInfo ubidi_props_dataVersion={",
|
|
dataInfo.dataVersion, 8, 4,
|
|
"};\n\n");
|
|
usrc_writeArray(f,
|
|
"static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={",
|
|
indexes, 32, UBIDI_IX_TOP,
|
|
"};\n\n");
|
|
usrc_writeUTrie2Arrays(f,
|
|
"static const uint16_t ubidi_props_trieIndex[%ld]={\n", NULL,
|
|
pTrie,
|
|
"\n};\n\n");
|
|
usrc_writeArray(f,
|
|
"static const uint32_t ubidi_props_mirrors[%ld]={\n",
|
|
mirrors, 32, mirrorTop,
|
|
"\n};\n\n");
|
|
usrc_writeArray(f,
|
|
"static const uint8_t ubidi_props_jgArray[%ld]={\n",
|
|
jgArray+(jgStart-0x600), 8, jgLimit-jgStart,
|
|
"\n};\n\n");
|
|
fputs(
|
|
"static const UBiDiProps ubidi_props_singleton={\n"
|
|
" NULL,\n"
|
|
" ubidi_props_indexes,\n"
|
|
" ubidi_props_mirrors,\n"
|
|
" ubidi_props_jgArray,\n",
|
|
f);
|
|
usrc_writeUTrie2Struct(f,
|
|
" {\n",
|
|
pTrie, "ubidi_props_trieIndex", NULL,
|
|
" },\n");
|
|
usrc_writeArray(f, " { ", dataInfo.formatVersion, 8, 4, " }\n");
|
|
fputs("};\n", f);
|
|
fclose(f);
|
|
}
|
|
|
|
void
|
|
BiDiPropsBuilder::writeBinaryData(const char *path, UBool withCopyright, UErrorCode &errorCode) {
|
|
if(U_FAILURE(errorCode)) { return; }
|
|
|
|
UNewDataMemory *pData=udata_create(path, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, &dataInfo,
|
|
withCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "genprops: udata_create(%s, ubidi.icu) failed - %s\n",
|
|
path, u_errorName(errorCode));
|
|
return;
|
|
}
|
|
|
|
udata_writeBlock(pData, indexes, sizeof(indexes));
|
|
udata_writeBlock(pData, trieBlock, trieSize);
|
|
udata_writeBlock(pData, mirrors, 4*mirrorTop);
|
|
udata_writeBlock(pData, jgArray+(jgStart-0x600), jgLimit-jgStart);
|
|
|
|
long dataLength=udata_finish(pData, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "genprops error: bidipropsbuilder %d writing the output file\n", errorCode);
|
|
return;
|
|
}
|
|
|
|
if(dataLength!=indexes[UBIDI_IX_LENGTH]) {
|
|
fprintf(stderr,
|
|
"udata_finish(ubidi.icu) reports %ld bytes written but should be %ld\n",
|
|
dataLength, (long)indexes[UBIDI_IX_LENGTH]);
|
|
errorCode=U_INTERNAL_PROGRAM_ERROR;
|
|
}
|
|
}
|
|
|
|
PropsBuilder *
|
|
createBiDiPropsBuilder(UErrorCode &errorCode) {
|
|
if(U_FAILURE(errorCode)) { return NULL; }
|
|
PropsBuilder *pb=new BiDiPropsBuilder(errorCode);
|
|
if(pb==NULL) {
|
|
errorCode=U_MEMORY_ALLOCATION_ERROR;
|
|
}
|
|
return pb;
|
|
}
|
|
|
|
/*
|
|
* Hey, Emacs, please set the following:
|
|
*
|
|
* Local Variables:
|
|
* indent-tabs-mode: nil
|
|
* End:
|
|
*
|
|
*/
|