AuroraMiddleware/scuffed-code
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ICU-8972 merge genbidi into genprops

Browse Source 
X-SVN-Rev: 31172
This commit is contained in:
Markus Scherer 2011-12-25 01:25:50 +00:00
parent 162f137de9
commit 241e2a07fb
12 changed files with 546 additions and 1699 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.gitignore vendored
View File

@ -964,20 +964,6 @@ tools/trac/IcuCodeTools/0.11/icucodetools/*.pyc
tools/trac/IcuCodeTools/0.12/*.egg-info
tools/trac/IcuCodeTools/0.12/build
tools/trac/IcuCodeTools/0.12/icucodetools/*.pyc
tools/unicode/c/genbidi/*.d
tools/unicode/c/genbidi/*.o
tools/unicode/c/genbidi/*.pdb
tools/unicode/c/genbidi/*.plg
tools/unicode/c/genbidi/Debug
tools/unicode/c/genbidi/Makefile
tools/unicode/c/genbidi/Release
tools/unicode/c/genbidi/debug
tools/unicode/c/genbidi/genbidi
tools/unicode/c/genbidi/genbidi.[0-9]
tools/unicode/c/genbidi/genbidi.vcproj.*.*.user
tools/unicode/c/genbidi/release
tools/unicode/c/genbidi/x64
tools/unicode/c/genbidi/x86
tools/unicode/c/gencase/*.d
tools/unicode/c/gencase/*.ncb
tools/unicode/c/gencase/*.o

1
tools/unicode/c/CMakeLists.txt
View File

@ -17,7 +17,6 @@ include_directories(
${ICU_SRC_DIR}/source/i18n
${ICU_SRC_DIR}/source/tools/toolutil)
link_directories(${ICU_INST_DIR}/lib)
add_subdirectory(genbidi)
add_subdirectory(gencase)
add_subdirectory(gennames)
add_subdirectory(gennorm)

9
tools/unicode/c/genbidi/CMakeLists.txt
View File

@ -1,9 +0,0 @@
# Copyright (C) 2010, International Business Machines
# Corporation and others. All Rights Reserved.
#
# created on: 2010jun03
# created by: Markus W. Scherer
# edited on: 2010jul20
# edited by: Stuart G. Gill
add_executable(genbidi genbidi.c store.c)
target_link_libraries(genbidi icuuc icutu)

95
tools/unicode/c/genbidi/Makefile.in
View File

@ -1,95 +0,0 @@
## Makefile.in for ICU - tools/genbidi
## Copyright (c) 1999-2005, International Business Machines Corporation and
## others. All Rights Reserved.
## Steven R. Loomis
## Source directory information
srcdir = @srcdir@
top_srcdir = @top_srcdir@
top_builddir = ../..
include $(top_builddir)/icudefs.mk
## Build directory information
subdir = tools/genbidi
TARGET_STUB_NAME = genbidi
SECTION = 8
#MAN_FILES = $(TARGET_STUB_NAME).$(SECTION)
## Extra files to remove for 'make clean'
CLEANFILES = *~ $(DEPS) $(MAN_FILES)
## Target information
TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT)
ifneq ($(top_builddir),$(top_srcdir))
CPPFLAGS += -I$(top_builddir)/common
endif
CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil
LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M)
OBJECTS = genbidi.o store.o
DEPS = $(OBJECTS:.o=.d)
## List of phony targets
.PHONY : all all-local install install-local clean clean-local \
distclean distclean-local dist dist-local check check-local install-man
## Clear suffix list
.SUFFIXES :
## List of standard targets
all: all-local
install: install-local
clean: clean-local
distclean : distclean-local
dist: dist-local
check: all check-local
all-local: $(TARGET) $(MAN_FILES)
install-local: all-local install-man
install-man: $(MAN_FILES)
# $(MKINSTALLDIRS) $(DESTDIR)$(mandir)/man$(SECTION)
# $(INSTALL_DATA) $? $(DESTDIR)$(mandir)/man$(SECTION)
dist-local:
clean-local:
test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES)
$(RMV) $(TARGET) $(OBJECTS)
distclean-local: clean-local
$(RMV) Makefile
check-local: all-local
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status
$(TARGET) : $(OBJECTS)
$(LINK.cc) $(OUTOPT)$@ $^ $(LIBS)
$(POST_BUILD_STEP)
%.$(SECTION): $(srcdir)/%.$(SECTION).in
cd $(top_builddir) \
&& CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status
ifeq (,$(MAKECMDGOALS))
-include $(DEPS)
else
ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),)
-include $(DEPS)
endif
endif

643
tools/unicode/c/genbidi/genbidi.c
View File

@ -1,643 +0,0 @@
/*
*******************************************************************************
*
* Copyright (C) 2004-2011, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: genbidi.c
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2004dec30
* created by: Markus W. Scherer
*
* This program reads several of the Unicode character database text files,
* parses them, and extracts the bidi/shaping properties for each character.
* It then writes a binary file containing the properties
* that is designed to be used directly for random-access to
* the properties of each Unicode character.
*/
#include <stdio.h>
#include "unicode/utypes.h"
#include "unicode/uchar.h"
#include "unicode/putil.h"
#include "unicode/uclean.h"
#include "cmemory.h"
#include "cstring.h"
#include "uarrsort.h"
#include "unewdata.h"
#include "uoptions.h"
#include "uparse.h"
#include "propsvec.h"
#include "ubidi_props.h"
#include "genbidi.h"
#define LENGTHOF(array) (sizeof(array)/sizeof((array)[0]))
/* data --------------------------------------------------------------------- */
UPropsVectors *pv;
UBool beVerbose=FALSE, haveCopyright=TRUE;
/* prototypes --------------------------------------------------------------- */
static UBool
isToken(const char *token, const char *s);
static void
parseBidiMirroring(const char *filename, UErrorCode *pErrorCode);
static void
parseDB(const char *filename, UErrorCode *pErrorCode);
/* miscellaneous ------------------------------------------------------------ */
/* TODO: more common code, move functions to uparse.h|c */
static char *
trimTerminateField(char *s, char *limit) {
/* trim leading whitespace */
s=(char *)u_skipWhitespace(s);
/* trim trailing whitespace */
while(s<limit && (*(limit-1)==' ' || *(limit-1)=='\t')) {
--limit;
}
*limit=0;
return s;
}
static void
parseTwoFieldFile(char *filename, char *basename,
const char *ucdFile, const char *suffix,
UParseLineFn *lineFn,
UErrorCode *pErrorCode) {
char *fields[2][2];
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return;
}
writeUCDFilename(basename, ucdFile, suffix);
u_parseDelimitedFile(filename, ';', fields, 2, lineFn, NULL, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "error parsing %s.txt: %s\n", ucdFile, u_errorName(*pErrorCode));
}
}
static void U_CALLCONV
bidiClassLineFn(void *context,
char *fields[][2], int32_t fieldCount,
UErrorCode *pErrorCode);
/* parse files with single enumerated properties ---------------------------- */
/* TODO: more common code, move functions to uparse.h|c */
struct SingleEnum {
const char *ucdFile, *propName;
UProperty prop;
int32_t vecWord, vecShift;
uint32_t vecMask;
};
typedef struct SingleEnum SingleEnum;
static void
parseSingleEnumFile(char *filename, char *basename, const char *suffix,
const SingleEnum *sen,
UErrorCode *pErrorCode);
static const SingleEnum jtSingleEnum={
"DerivedJoiningType", "joining type",
UCHAR_JOINING_TYPE,
0, UBIDI_JT_SHIFT, UBIDI_JT_MASK
};
static const SingleEnum jgSingleEnum={
"DerivedJoiningGroup", "joining group",
UCHAR_JOINING_GROUP,
1, 0, 0xff /* column 1 bits 7..0 */
};
static void U_CALLCONV
singleEnumLineFn(void *context,
char *fields[][2], int32_t fieldCount,
UErrorCode *pErrorCode) {
const SingleEnum *sen;
char *s;
uint32_t start, end, uv;
int32_t value;
sen=(const SingleEnum *)context;
u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi: syntax error in %s.txt field 0 at %s\n", sen->ucdFile, fields[0][0]);
exit(*pErrorCode);
}
/* parse property alias */
s=trimTerminateField(fields[1][0], fields[1][1]);
value=u_getPropertyValueEnum(sen->prop, s);
if(value<0) {
if(sen->prop==UCHAR_BLOCK) {
if(isToken("Greek", s)) {
value=UBLOCK_GREEK; /* Unicode 3.2 renames this to "Greek and Coptic" */
} else if(isToken("Combining Marks for Symbols", s)) {
value=UBLOCK_COMBINING_MARKS_FOR_SYMBOLS; /* Unicode 3.2 renames this to "Combining Diacritical Marks for Symbols" */
} else if(isToken("Private Use", s)) {
value=UBLOCK_PRIVATE_USE; /* Unicode 3.2 renames this to "Private Use Area" */
}
}
}
if(value<0) {
fprintf(stderr, "genbidi error: unknown %s name in %s.txt field 1 at %s\n",
sen->propName, sen->ucdFile, s);
exit(U_PARSE_ERROR);
}
uv=(uint32_t)(value<<sen->vecShift);
if((uv&sen->vecMask)!=uv) {
fprintf(stderr, "genbidi error: %s value overflow (0x%x) at %s\n",
sen->propName, (int)uv, s);
exit(U_INTERNAL_PROGRAM_ERROR);
}
upvec_setValue(pv, start, end, sen->vecWord, uv, sen->vecMask, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi error: unable to set %s code: %s\n",
sen->propName, u_errorName(*pErrorCode));
exit(*pErrorCode);
}
}
static void
parseSingleEnumFile(char *filename, char *basename, const char *suffix,
const SingleEnum *sen,
UErrorCode *pErrorCode) {
char *fields[2][2];
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return;
}
writeUCDFilename(basename, sen->ucdFile, suffix);
u_parseDelimitedFile(filename, ';', fields, 2, singleEnumLineFn, (void *)sen, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "error parsing %s.txt: %s\n", sen->ucdFile, u_errorName(*pErrorCode));
}
}
/* parse files with multiple binary properties ------------------------------ */
/* TODO: more common code, move functions to uparse.h|c */
/* TODO: similar to genbidi/props2.c but not the same; same as in gencase/gencase.c */
struct Binary {
const char *propName;
int32_t vecWord;
uint32_t vecValue, vecMask;
};
typedef struct Binary Binary;
struct Binaries {
const char *ucdFile;
const Binary *binaries;
int32_t binariesCount;
};
typedef struct Binaries Binaries;
static const Binary
propListNames[]={
{ "Bidi_Control", 0, U_MASK(UBIDI_BIDI_CONTROL_SHIFT), U_MASK(UBIDI_BIDI_CONTROL_SHIFT) },
{ "Join_Control", 0, U_MASK(UBIDI_JOIN_CONTROL_SHIFT), U_MASK(UBIDI_JOIN_CONTROL_SHIFT) }
};
static const Binaries
propListBinaries={
"PropList", propListNames, LENGTHOF(propListNames)
};
static void U_CALLCONV
binariesLineFn(void *context,
char *fields[][2], int32_t fieldCount,
UErrorCode *pErrorCode) {
const Binaries *bin;
char *s;
uint32_t start, end;
int32_t i;
bin=(const Binaries *)context;
u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi: syntax error in %s.txt field 0 at %s\n", bin->ucdFile, fields[0][0]);
exit(*pErrorCode);
}
/* parse binary property name */
s=(char *)u_skipWhitespace(fields[1][0]);
for(i=0;; ++i) {
if(i==bin->binariesCount) {
/* ignore unrecognized properties */
return;
}
if(isToken(bin->binaries[i].propName, s)) {
break;
}
}
if(bin->binaries[i].vecMask==0) {
fprintf(stderr, "genbidi error: mask value %d==0 for %s %s\n",
(int)bin->binaries[i].vecMask, bin->ucdFile, bin->binaries[i].propName);
exit(U_INTERNAL_PROGRAM_ERROR);
}
upvec_setValue(pv, start, end, bin->binaries[i].vecWord, bin->binaries[i].vecValue, bin->binaries[i].vecMask, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi error: unable to set %s, code: %s\n",
bin->binaries[i].propName, u_errorName(*pErrorCode));
exit(*pErrorCode);
}
}
static void
parseBinariesFile(char *filename, char *basename, const char *suffix,
const Binaries *bin,
UErrorCode *pErrorCode) {
char *fields[2][2];
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return;
}
writeUCDFilename(basename, bin->ucdFile, suffix);
u_parseDelimitedFile(filename, ';', fields, 2, binariesLineFn, (void *)bin, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "error parsing %s.txt: %s\n", bin->ucdFile, u_errorName(*pErrorCode));
}
}
/* -------------------------------------------------------------------------- */
enum {
HELP_H,
HELP_QUESTION_MARK,
VERBOSE,
COPYRIGHT,
DESTDIR,
SOURCEDIR,
UNICODE_VERSION,
ICUDATADIR,
CSOURCE
};
/* Keep these values in sync with the above enums */
static UOption options[]={
UOPTION_HELP_H,
UOPTION_HELP_QUESTION_MARK,
UOPTION_VERBOSE,
UOPTION_COPYRIGHT,
UOPTION_DESTDIR,
UOPTION_SOURCEDIR,
UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG),
UOPTION_ICUDATADIR,
UOPTION_DEF("csource", 'C', UOPT_NO_ARG)
};
extern int
main(int argc, char* argv[]) {
char filename[300];
const char *srcDir=NULL, *destDir=NULL, *suffix=NULL;
char *basename=NULL;
UErrorCode errorCode=U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
options[DESTDIR].value=u_getDataDirectory();
options[SOURCEDIR].value="";
options[UNICODE_VERSION].value="";
options[ICUDATADIR].value=u_getDataDirectory();
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc<0) {
fprintf(stderr,
"error in command line argument \"%s\"\n",
argv[-argc]);
}
if(argc<0 || options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
/*
* Broken into chucks because the C89 standard says the minimum
* required supported string length is 509 bytes.
*/
fprintf(stderr,
"Usage: %s [-options] [suffix]\n"
"\n"
"read the UnicodeData.txt file and other Unicode properties files and\n"
"create a binary file " UBIDI_DATA_NAME "." UBIDI_DATA_TYPE " with the bidi/shaping properties\n"
"\n",
argv[0]);
fprintf(stderr,
"Options:\n"
"\t-h or -? or --help this usage text\n"
"\t-v or --verbose verbose output\n"
"\t-c or --copyright include a copyright notice\n"
"\t-u or --unicode Unicode version, followed by the version like 3.0.0\n"
"\t-C or --csource generate a .c source file rather than the .icu binary\n");
fprintf(stderr,
"\t-d or --destdir destination directory, followed by the path\n"
"\t-s or --sourcedir source directory, followed by the path\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
"\t followed by path, defaults to %s\n"
"\tsuffix suffix that is to be appended with a '-'\n"
"\t to the source file basenames before opening;\n"
"\t 'genbidi new' will read UnicodeData-new.txt etc.\n",
u_getDataDirectory());
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
}
/* get the options values */
beVerbose=options[VERBOSE].doesOccur;
haveCopyright=options[COPYRIGHT].doesOccur;
srcDir=options[SOURCEDIR].value;
destDir=options[DESTDIR].value;
if(argc>=2) {
suffix=argv[1];
} else {
suffix=NULL;
}
if(options[UNICODE_VERSION].doesOccur) {
setUnicodeVersion(options[UNICODE_VERSION].value);
}
/* else use the default dataVersion in store.c */
if (options[ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ICUDATADIR].value);
}
/* prepare the filename beginning with the source dir */
uprv_strcpy(filename, srcDir);
basename=filename+uprv_strlen(filename);
if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
*basename++=U_FILE_SEP_CHAR;
}
/* initialize */
pv=upvec_open(2, &errorCode);
/* process BidiMirroring.txt */
writeUCDFilename(basename, "BidiMirroring", suffix);
parseBidiMirroring(filename, &errorCode);
/* process additional properties files */
*basename=0;
parseBinariesFile(filename, basename, suffix, &propListBinaries, &errorCode);
parseSingleEnumFile(filename, basename, suffix, &jtSingleEnum, &errorCode);
parseSingleEnumFile(filename, basename, suffix, &jgSingleEnum, &errorCode);
/* process UnicodeData.txt */
writeUCDFilename(basename, "UnicodeData", suffix);
parseDB(filename, &errorCode);
/* set proper bidi class for unassigned code points (Cn) */
parseTwoFieldFile(filename, basename, "DerivedBidiClass", suffix, bidiClassLineFn, &errorCode);
/* process parsed data */
if(U_SUCCESS(errorCode)) {
/* write the properties data file */
generateData(destDir, options[CSOURCE].doesOccur);
}
u_cleanup();
return errorCode;
}
U_CFUNC void
writeUCDFilename(char *basename, const char *filename, const char *suffix) {
int32_t length=(int32_t)uprv_strlen(filename);
uprv_strcpy(basename, filename);
if(suffix!=NULL) {
basename[length++]='-';
uprv_strcpy(basename+length, suffix);
length+=(int32_t)uprv_strlen(suffix);
}
uprv_strcpy(basename+length, ".txt");
}
/* TODO: move to toolutil */
static UBool
isToken(const char *token, const char *s) {
const char *z;
int32_t j;
s=u_skipWhitespace(s);
for(j=0;; ++j) {
if(token[j]!=0) {
if(s[j]!=token[j]) {
break;
}
} else {
z=u_skipWhitespace(s+j);
if(*z==';' || *z==0) {
return TRUE;
} else {
break;
}
}
}
return FALSE;
}
/* parser for BidiMirroring.txt --------------------------------------------- */
static void U_CALLCONV
mirrorLineFn(void *context,
char *fields[][2], int32_t fieldCount,
UErrorCode *pErrorCode) {
const char *s;
char *end;
UChar32 src, mirror;
/* ignore "<code point>" which is on the @missing line */
s=u_skipWhitespace(fields[1][0]);
if(0==uprv_strncmp(s, "<code point>", 12)) {
return;
}
src=(UChar32)uprv_strtoul(fields[0][0], &end, 16);
if(end<=fields[0][0] || end!=fields[0][1]) {
fprintf(stderr, "genbidi: syntax error in BidiMirroring.txt field 0 at %s\n", fields[0][0]);
*pErrorCode=U_PARSE_ERROR;
exit(U_PARSE_ERROR);
}
mirror=(UChar32)uprv_strtoul(s, &end, 16);
if(end<=s || end!=fields[1][1]) {
fprintf(stderr, "genbidi: syntax error in BidiMirroring.txt field 1 at %s\n", fields[1][0]);
*pErrorCode=U_PARSE_ERROR;
exit(U_PARSE_ERROR);
}
addMirror(src, mirror);
}
static void
parseBidiMirroring(const char *filename, UErrorCode *pErrorCode) {
char *fields[2][2];
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return;
}
u_parseDelimitedFile(filename, ';', fields, 2, mirrorLineFn, NULL, pErrorCode);
}
/* parser for UnicodeData.txt ----------------------------------------------- */
static void U_CALLCONV
unicodeDataLineFn(void *context,
char *fields[][2], int32_t fieldCount,
UErrorCode *pErrorCode) {
char *end;
UErrorCode errorCode;
UChar32 c;
errorCode=U_ZERO_ERROR;
/* get the character code, field 0 */
c=(UChar32)uprv_strtoul(fields[0][0], &end, 16);
if(end<=fields[0][0] || end!=fields[0][1]) {
fprintf(stderr, "genbidi: syntax error in field 0 at %s\n", fields[0][0]);
*pErrorCode=U_PARSE_ERROR;
exit(U_PARSE_ERROR);
}
/* get Mirrored flag, field 9 */
if(*fields[9][0]=='Y') {
upvec_setValue(pv, c, c, 0, U_MASK(UBIDI_IS_MIRRORED_SHIFT), U_MASK(UBIDI_IS_MIRRORED_SHIFT), &errorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi error: unable to set 'is mirrored' for U+%04lx, code: %s\n",
(long)c, u_errorName(errorCode));
exit(errorCode);
}
} else if(fields[9][1]-fields[9][0]!=1 || *fields[9][0]!='N') {
fprintf(stderr, "genbidi: syntax error in field 9 at U+%04lx\n",
(long)c);
*pErrorCode=U_PARSE_ERROR;
exit(U_PARSE_ERROR);
}
}
static void
parseDB(const char *filename, UErrorCode *pErrorCode) {
/* default Bidi classes for unassigned code points */
static const UChar32 defaultBidi[][3]={ /* { start, end, class } */
/* R: U+0590..U+05FF, U+07C0..U+08FF, U+FB1D..U+FB4F, U+10800..U+10FFF */
{ 0x0590, 0x05FF, U_RIGHT_TO_LEFT },
{ 0x07C0, 0x08FF, U_RIGHT_TO_LEFT },
{ 0xFB1D, 0xFB4F, U_RIGHT_TO_LEFT },
{ 0x10800, 0x10FFF, U_RIGHT_TO_LEFT },
/* AL: U+0600..U+07BF, U+FB50..U+FDCF, U+FDF0..U+FDFF, U+FE70..U+FEFE */
{ 0x0600, 0x07BF, U_RIGHT_TO_LEFT_ARABIC },
{ 0xFB50, 0xFDCF, U_RIGHT_TO_LEFT_ARABIC },
{ 0xFDF0, 0xFDFF, U_RIGHT_TO_LEFT_ARABIC },
{ 0xFE70, 0xFEFE, U_RIGHT_TO_LEFT_ARABIC }
/* L otherwise */
};
char *fields[15][2];
UChar32 start, end;
int32_t i;
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
return;
}
/*
* Set default Bidi classes for unassigned code points.
* See the documentation for Bidi_Class in UCD.html in the Unicode data.
* http://www.unicode.org/Public/
*
* Starting with Unicode 5.0, DerivedBidiClass.txt should (re)set
* the Bidi_Class values for all code points including unassigned ones
* and including L values for these.
* This code becomes unnecesary but harmless. Leave it for now in case
* someone uses genbidi on pre-Unicode 5.0 data.
*/
for(i=0; i<LENGTHOF(defaultBidi); ++i) {
start=defaultBidi[i][0];
end=defaultBidi[i][1];
upvec_setValue(pv, start, end, 0, (uint32_t)defaultBidi[i][2], UBIDI_CLASS_MASK, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi error: unable to set default bidi class for U+%04lx..U+%04lx, code: %s\n",
(long)start, (long)end, u_errorName(*pErrorCode));
exit(*pErrorCode);
}
}
u_parseDelimitedFile(filename, ';', fields, 15, unicodeDataLineFn, NULL, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
return;
}
}
/* DerivedBidiClass.txt ----------------------------------------------------- */
static void U_CALLCONV
bidiClassLineFn(void *context,
char *fields[][2], int32_t fieldCount,
UErrorCode *pErrorCode) {
char *s;
uint32_t start, end, value;
/* get the code point range */
u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi: syntax error in DerivedBidiClass.txt field 0 at %s\n", fields[0][0]);
exit(*pErrorCode);
}
/* parse bidi class */
s=trimTerminateField(fields[1][0], fields[1][1]);
value=u_getPropertyValueEnum(UCHAR_BIDI_CLASS, s);
if((int32_t)value<0) {
fprintf(stderr, "genbidi error: unknown bidi class in DerivedBidiClass.txt field 1 at %s\n", s);
exit(U_PARSE_ERROR);
}
upvec_setValue(pv, start, end, 0, value, UBIDI_CLASS_MASK, pErrorCode);
if(U_FAILURE(*pErrorCode)) {
fprintf(stderr, "genbidi error: unable to set derived bidi class for U+%04x..U+%04x - %s\n",
(int)start, (int)end, u_errorName(*pErrorCode));
exit(*pErrorCode);
}
}
/*
* Hey, Emacs, please set the following:
*
* Local Variables:
* indent-tabs-mode: nil
* End:
*
*/

48
tools/unicode/c/genbidi/genbidi.h
View File

@ -1,48 +0,0 @@
/*
*******************************************************************************
*
* Copyright (C) 2004-2008, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: genbidi.h
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2004dec30
* created by: Markus W. Scherer
*/
#ifndef __GENBIDI_H__
#define __GENBIDI_H__
#include "unicode/utypes.h"
#include "propsvec.h"
U_CDECL_BEGIN
/* genbidi ------------------------------------------------------------------ */
/* global flags */
extern UBool beVerbose, haveCopyright;
/* properties vectors in genbidi.c */
extern UPropsVectors *pv;
/* prototypes */
U_CFUNC void
writeUCDFilename(char *basename, const char *filename, const char *suffix);
extern void
setUnicodeVersion(const char *v);
extern void
addMirror(UChar32 src, UChar32 mirror);
extern void
generateData(const char *dataDir, UBool csource);
U_CDECL_END
#endif

422
tools/unicode/c/genbidi/genbidi.vcproj
View File

@ -1,422 +0,0 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="9.00"
Name="genbidi"
ProjectGUID="{DB312A49-12A9-4E07-9E96-451DC2D8FF62}"
TargetFrameworkVersion="131072"
>
<Platforms>
<Platform
Name="Win32"
/>
<Platform
Name="x64"
/>
</Platforms>
<ToolFiles>
</ToolFiles>
<Configurations>
<Configuration
Name="Release|Win32"
OutputDirectory=".\x86\Release"
IntermediateDirectory=".\x86\Release"
ConfigurationType="1"
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
UseOfMFC="0"
ATLMinimizesCRunTimeLibraryUsage="false"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
CommandLine="copy &quot;$(TargetPath)&quot; ..\..\..\bin&#x0D;&#x0A;"
Outputs="..\..\..\bin\$(TargetFileName)"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TypeLibraryName=".\x86\Release/genbidi.tlb"
/>
<Tool
Name="VCCLCompilerTool"
AdditionalIncludeDirectories="..\..\common;..\toolutil"
PreprocessorDefinitions="WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE"
StringPooling="true"
RuntimeLibrary="2"
EnableFunctionLevelLinking="true"
DisableLanguageExtensions="true"
TreatWChar_tAsBuiltInType="true"
PrecompiledHeaderFile=".\x86\Release/genbidi.pch"
AssemblerListingLocation=".\x86\Release/"
ObjectFile=".\x86\Release/"
ProgramDataBaseFileName=".\x86\Release/"
WarningLevel="3"
SuppressStartupBanner="true"
CompileAs="0"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
PreprocessorDefinitions="NDEBUG"
Culture="1033"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
OutputFile=".\x86\Release/genbidi.exe"
LinkIncremental="1"
SuppressStartupBanner="true"
ProgramDatabaseFile=".\x86\Release/genbidi.pdb"
SubSystem="1"
RandomizedBaseAddress="1"
DataExecutionPrevention="0"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Debug|Win32"
OutputDirectory=".\x86\Debug"
IntermediateDirectory=".\x86\Debug"
ConfigurationType="1"
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
UseOfMFC="0"
ATLMinimizesCRunTimeLibraryUsage="false"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
CommandLine="copy &quot;$(TargetPath)&quot; ..\..\..\bin&#x0D;&#x0A;"
Outputs="..\..\..\bin\$(TargetFileName)"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TypeLibraryName=".\x86\Debug/genbidi.tlb"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="..\..\common;..\toolutil"
PreprocessorDefinitions="WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
BufferSecurityCheck="true"
DisableLanguageExtensions="true"
TreatWChar_tAsBuiltInType="true"
PrecompiledHeaderFile=".\x86\Debug/genbidi.pch"
AssemblerListingLocation=".\x86\Debug/"
ObjectFile=".\x86\Debug/"
ProgramDataBaseFileName=".\x86\Debug/"
BrowseInformation="1"
WarningLevel="3"
SuppressStartupBanner="true"
DebugInformationFormat="4"
CompileAs="0"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
PreprocessorDefinitions="_DEBUG"
Culture="1033"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
OutputFile=".\x86\Debug/genbidi.exe"
LinkIncremental="2"
SuppressStartupBanner="true"
GenerateDebugInformation="true"
ProgramDatabaseFile=".\x86\Debug/genbidi.pdb"
SubSystem="1"
RandomizedBaseAddress="1"
DataExecutionPrevention="0"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
UseFAT32Workaround="true"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|x64"
OutputDirectory=".\x64\Release"
IntermediateDirectory=".\x64\Release"
ConfigurationType="1"
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
UseOfMFC="0"
ATLMinimizesCRunTimeLibraryUsage="false"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
CommandLine="copy &quot;$(TargetPath)&quot; ..\..\..\bin64&#x0D;&#x0A;"
Outputs="..\..\..\bin64\$(TargetFileName)"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
TypeLibraryName=".\x64\Release/genbidi.tlb"
/>
<Tool
Name="VCCLCompilerTool"
AdditionalIncludeDirectories="..\..\common;..\toolutil"
PreprocessorDefinitions="WIN64;WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE"
StringPooling="true"
RuntimeLibrary="2"
EnableFunctionLevelLinking="true"
DisableLanguageExtensions="true"
TreatWChar_tAsBuiltInType="true"
PrecompiledHeaderFile=".\x64\Release/genbidi.pch"
AssemblerListingLocation=".\x64\Release/"
ObjectFile=".\x64\Release/"
ProgramDataBaseFileName=".\x64\Release/"
WarningLevel="3"
SuppressStartupBanner="true"
CompileAs="0"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
PreprocessorDefinitions="NDEBUG"
Culture="1033"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
OutputFile=".\x64\Release/genbidi.exe"
LinkIncremental="1"
SuppressStartupBanner="true"
ProgramDatabaseFile=".\x64\Release/genbidi.pdb"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCWebDeploymentTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory=".\x64\Debug"
IntermediateDirectory=".\x64\Debug"
ConfigurationType="1"
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
UseOfMFC="0"
ATLMinimizesCRunTimeLibraryUsage="false"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
CommandLine="copy &quot;$(TargetPath)&quot; ..\..\..\bin64&#x0D;&#x0A;"
Outputs="..\..\..\bin64\$(TargetFileName)"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
TypeLibraryName=".\x64\Debug/genbidi.tlb"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="..\..\common;..\toolutil"
PreprocessorDefinitions="WIN64;WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
BufferSecurityCheck="true"
DisableLanguageExtensions="true"
TreatWChar_tAsBuiltInType="true"
PrecompiledHeaderFile=".\x64\Debug/genbidi.pch"
AssemblerListingLocation=".\x64\Debug/"
ObjectFile=".\x64\Debug/"
ProgramDataBaseFileName=".\x64\Debug/"
BrowseInformation="1"
WarningLevel="3"
SuppressStartupBanner="true"
DebugInformationFormat="3"
CompileAs="0"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
PreprocessorDefinitions="_DEBUG"
Culture="1033"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
OutputFile=".\x64\Debug/genbidi.exe"
LinkIncremental="2"
SuppressStartupBanner="true"
GenerateDebugInformation="true"
ProgramDatabaseFile=".\x64\Debug/genbidi.pdb"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
UseFAT32Workaround="true"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCWebDeploymentTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="c;cpp;rc"
>
<File
RelativePath=".\genbidi.c"
>
</File>
<File
RelativePath=".\store.c"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h"
>
<File
RelativePath=".\genbidi.h"
>
</File>
</Filter>
<Filter
Name="Resource Files"
Filter="ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe"
>
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

466
tools/unicode/c/genbidi/store.c
View File

@ -1,466 +0,0 @@
/*
*******************************************************************************
*
* Copyright (C) 2004-2011, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: store.c
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2004dec30
* created by: Markus W. Scherer
*
* Store Unicode bidi/shaping properties efficiently for
* random access.
*/
#include <stdio.h>
#include <stdlib.h>
#include "unicode/utypes.h"
#include "unicode/uchar.h"
#include "cmemory.h"
#include "cstring.h"
#include "utrie2.h"
#include "uarrsort.h"
#include "unicode/udata.h"
#include "unewdata.h"
#include "propsvec.h"
#include "writesrc.h"
#include "ubidi_props.h"
#include "genbidi.h"
#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
/* Unicode bidi/shaping properties file format ---------------------------------
The file format prepared and written here contains several data
structures that store indexes or data.
Before the data contents described below, there are the headers required by
the udata API for loading ICU data. Especially, a UDataInfo structure
precedes the actual data. It contains platform properties values and the
file format version.
The following is a description of format version 2.0 .
The file contains the following structures:
const int32_t indexes[i0] with values i0, i1, ...:
(see UBIDI_IX_... constants for names of indexes)
i0 indexLength; -- length of indexes[] (UBIDI_IX_TOP)
i1 dataLength; -- length in bytes of the post-header data (incl. indexes[])
i2 trieSize; -- size in bytes of the bidi/shaping properties trie
i3 mirrorLength; -- length in uint32_t of the bidi mirroring array
i4 jgStart; -- first code point with Joining_Group data
i5 jgLimit; -- limit code point for Joining_Group data
i6..i14 reservedIndexes; -- reserved values; 0 for now
i15 maxValues; -- maximum code values for enumerated properties
bits 23..16 contain the max value for Joining_Group,
otherwise the bits are used like enum fields in the trie word
Serialized trie, see utrie2.h;
const uint32_t mirrors[mirrorLength];
const uint8_t jgArray[i5-i4]; -- (i5-i4) is always a multiple of 4
Trie data word:
Bits
15..13 signed delta to bidi mirroring code point
(add delta to input code point)
0 no such code point (source maps to itself)
-3..-1, 1..3 delta
-4 look in mirrors table
12 is mirrored
11 Bidi_Control
10 Join_Control
9.. 8 reserved (set to 0)
7.. 5 Joining_Type
4.. 0 BiDi category
Mirrors:
Stores some of the bidi mirroring data, where each code point maps to
at most one other.
Most code points do not have a mirroring code point; most that do have a signed
delta stored in the trie data value. Only those where the delta does not fit
into the trie data are stored in this table.
Logically, this is a two-column table with source and mirror code points.
Physically, the table is compressed by taking advantage of the fact that each
mirror code point is also a source code point
(each of them is a mirror of the other).
Therefore, both logical columns contain the same set of code points, which needs
to be stored only once.
The table stores source code points, and also for each the index of its mirror
code point in the same table, in a simple array of uint32_t.
Bits
31..21 index to mirror code point (unsigned)
20.. 0 source code point
The table is sorted by source code points.
Joining_Group array:
The Joining_Group values do not fit into the 16-bit trie, but the data is also
limited to a small range of code points (Arabic and Syriac) and not
well compressible.
The start and limit code points for the range are stored in the indexes[]
array, and the jgArray[] stores a byte for each of these code points,
containing the Joining_Group value.
All code points outside of this range have No_Joining_Group (0).
--- Changes in format version 2 ---
Change from UTrie to UTrie2.
----------------------------------------------------------------------------- */
/* UDataInfo cf. udata.h */
static UDataInfo dataInfo={
sizeof(UDataInfo),
0,
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
0,
/* dataFormat="BiDi" */
{ UBIDI_FMT_0, UBIDI_FMT_1, UBIDI_FMT_2, UBIDI_FMT_3 },
{ 2, 0, 0, 0 }, /* formatVersion */
{ 6, 0, 0, 0 } /* dataVersion */
};
/* exceptions values */
static uint32_t mirrors[UBIDI_MAX_MIRROR_INDEX+1][2];
static uint16_t mirrorTop=0;
/* -------------------------------------------------------------------------- */
extern void
setUnicodeVersion(const char *v) {
UVersionInfo version;
u_versionFromString(version, v);
uprv_memcpy(dataInfo.dataVersion, version, 4);
}
/* bidi mirroring table ----------------------------------------------------- */
extern void
addMirror(UChar32 src, UChar32 mirror) {
UErrorCode errorCode;
int32_t delta;
delta=mirror-src;
if(delta==0) {
return; /* mapping to self=no mapping */
}
if(delta<UBIDI_MIN_MIRROR_DELTA || UBIDI_MAX_MIRROR_DELTA<delta) {
/* delta does not fit into the trie properties value, store in the mirrors[] table */
if(mirrorTop==LENGTHOF(mirrors)) {
fprintf(stderr, "genbidi error: too many long-distance mirroring mappings\n");
exit(U_BUFFER_OVERFLOW_ERROR);
}
/* possible: search the table so far and see if src is already listed */
mirrors[mirrorTop][0]=(uint32_t)src;
mirrors[mirrorTop][1]=(uint32_t)mirror;
++mirrorTop;
/* set an escape marker in src's properties */
delta=UBIDI_ESC_MIRROR_DELTA;
}
errorCode=U_ZERO_ERROR;
upvec_setValue(
pv, src, src, 0,
(uint32_t)delta<<UBIDI_MIRROR_DELTA_SHIFT, (uint32_t)(-1)<<UBIDI_MIRROR_DELTA_SHIFT,
&errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi error: unable to set mirroring delta, code: %s\n",
u_errorName(errorCode));
exit(errorCode);
}
}
static int32_t U_CALLCONV
compareMirror(const void *context, const void *left, const void *right) {
UChar32 l, r;
l=UBIDI_GET_MIRROR_CODE_POINT(((const uint32_t *)left)[0]);
r=UBIDI_GET_MIRROR_CODE_POINT(((const uint32_t *)right)[0]);
return l-r;
}
static void
makeMirror() {
uint32_t *reducedMirror;
UErrorCode errorCode;
int32_t i, j, start, limit, step;
uint32_t c;
/* sort the mirroring table by source code points */
errorCode=U_ZERO_ERROR;
uprv_sortArray(mirrors, mirrorTop, 8,
compareMirror, NULL, FALSE, &errorCode);
/*
* reduce the 2-column table to a single column
* by putting the index to the mirror entry into the source entry
*
* first:
* find each mirror code point in the source column and set each other's indexes
*
* second:
* reduce the table, combine the source code points with their indexes
* and store as a simple array of uint32_t
*/
for(i=0; i<mirrorTop; ++i) {
c=mirrors[i][1]; /* mirror code point */
if(c>0x1fffff) {
continue; /* this entry already has an index */
}
/* search for the mirror code point in the source column */
if(c<mirrors[i][0]) {
/* search before i */
start=i-1;
limit=-1;
step=-1;
} else {
start=i+1;
limit=mirrorTop;
step=1;
}
for(j=start;; j+=step) {
if(j==limit) {
fprintf(stderr,
"genbidi error: bidi mirror does not roundtrip - %04lx->%04lx->?\n",
(long)mirrors[i][0], (long)mirrors[i][1]);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
}
if(c==mirrors[j][0]) {
/*
* found the mirror code point c in the source column,
* set both entries' indexes to each other
*/
if(UBIDI_GET_MIRROR_CODE_POINT(mirrors[i][0])!=UBIDI_GET_MIRROR_CODE_POINT(mirrors[j][1])) {
/* roundtrip check fails */
fprintf(stderr,
"genbidi error: bidi mirrors do not roundtrip - %04lx->%04lx->%04lx\n",
(long)mirrors[i][0], (long)mirrors[i][1], (long)mirrors[j][1]);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
} else {
mirrors[i][1]|=(uint32_t)j<<UBIDI_MIRROR_INDEX_SHIFT;
mirrors[j][1]|=(uint32_t)i<<UBIDI_MIRROR_INDEX_SHIFT;
}
break;
}
}
}
/* now the second step, the actual reduction of the table */
reducedMirror=mirrors[0];
for(i=0; i<mirrorTop; ++i) {
reducedMirror[i]=mirrors[i][0]|(mirrors[i][1]&~0x1fffff);
}
if(U_FAILURE(errorCode)) {
exit(errorCode);
}
}
/* generate output data ----------------------------------------------------- */
extern void
generateData(const char *dataDir, UBool csource) {
static int32_t indexes[UBIDI_IX_TOP]={
UBIDI_IX_TOP
};
static uint8_t trieBlock[40000];
static uint8_t jgArray[0x300]; /* at most for U+0600..U+08FF */
const uint32_t *row;
UChar32 start, end, prev, jgStart;
int32_t i;
UNewDataMemory *pData;
UTrie2 *pTrie;
UErrorCode errorCode=U_ZERO_ERROR;
int32_t trieSize;
long dataLength;
makeMirror();
pTrie=utrie2_open(0, 0, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi error: utrie2_open() failed - %s\n", u_errorName(errorCode));
exit(errorCode);
}
prev=jgStart=0;
for(i=0; (row=upvec_getRow(pv, i, &start, &end))!=NULL && start<UPVEC_FIRST_SPECIAL_CP; ++i) {
/* store most values from vector column 0 in the trie */
utrie2_setRange32(pTrie, start, end, *row, TRUE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi error: utrie2_setRange32() failed - %s\n", u_errorName(errorCode));
exit(errorCode);
}
/* store Joining_Group values from vector column 1 in a simple byte array */
if(row[1]!=0) {
if(start<0x600 || 0x8ff<end) {
fprintf(stderr, "genbidi error: Joining_Group for out-of-range code points U+%04lx..U+%04lx\n",
(long)start, (long)end);
exit(U_ILLEGAL_ARGUMENT_ERROR);
}
if(prev==0) {
/* first code point with any value */
prev=jgStart=start;
} else {
/* add No_Joining_Group for code points between prev and start */
while(prev<start) {
jgArray[prev++ -jgStart]=0;
}
}
/* set Joining_Group value for start..end */
while(prev<=end) {
jgArray[prev++ -jgStart]=(uint8_t)row[1];
}
}
}
/* finish jgArray, pad to multiple of 4 */
while((prev-jgStart)&3) {
jgArray[prev++ -jgStart]=0;
}
indexes[UBIDI_IX_JG_START]=jgStart;
indexes[UBIDI_IX_JG_LIMIT]=prev;
utrie2_freeze(pTrie, UTRIE2_16_VALUE_BITS, &errorCode);
trieSize=utrie2_serialize(pTrie, trieBlock, sizeof(trieBlock), &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi error: utrie2_freeze()+utrie2_serialize() failed: %s (length %ld)\n",
u_errorName(errorCode), (long)trieSize);
exit(errorCode);
}
indexes[UBIDI_IX_TRIE_SIZE]=trieSize;
indexes[UBIDI_IX_MIRROR_LENGTH]=mirrorTop;
indexes[UBIDI_IX_LENGTH]=
(int32_t)sizeof(indexes)+
trieSize+
4*mirrorTop+
(prev-jgStart);
if(beVerbose) {
printf("trie size in bytes: %5d\n", (int)trieSize);
printf("size in bytes of mirroring table: %5d\n", (int)(4*mirrorTop));
printf("length of Joining_Group array: %5d (U+%04x..U+%04x)\n", (int)(prev-jgStart), (int)jgStart, (int)(prev-1));
printf("data size: %5d\n", (int)indexes[UBIDI_IX_LENGTH]);
}
indexes[UBIDI_MAX_VALUES_INDEX]=
((int32_t)U_CHAR_DIRECTION_COUNT-1)|
(((int32_t)U_JT_COUNT-1)<<UBIDI_JT_SHIFT)|
(((int32_t)U_JG_COUNT-1)<<UBIDI_MAX_JG_SHIFT);
if(csource) {
/* write .c file for hardcoded data */
FILE *f=usrc_create(dataDir, "ubidi_props_data.h");
if(f!=NULL) {
fputs("#ifndef INCLUDED_FROM_UBIDI_PROPS_C\n"
"# error This file must be #included from ubidi_props.c only.\n"
"#endif\n\n", f);
usrc_writeArray(f,
"static const UVersionInfo ubidi_props_dataVersion={",
dataInfo.dataVersion, 8, 4,
"};\n\n");
usrc_writeArray(f,
"static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={",
indexes, 32, UBIDI_IX_TOP,
"};\n\n");
usrc_writeUTrie2Arrays(f,
"static const uint16_t ubidi_props_trieIndex[%ld]={\n", NULL,
pTrie,
"\n};\n\n");
usrc_writeArray(f,
"static const uint32_t ubidi_props_mirrors[%ld]={\n",
mirrors, 32, mirrorTop,
"\n};\n\n");
usrc_writeArray(f,
"static const uint8_t ubidi_props_jgArray[%ld]={\n",
jgArray, 8, prev-jgStart,
"\n};\n\n");
fputs(
"static const UBiDiProps ubidi_props_singleton={\n"
" NULL,\n"
" ubidi_props_indexes,\n"
" ubidi_props_mirrors,\n"
" ubidi_props_jgArray,\n",
f);
usrc_writeUTrie2Struct(f,
" {\n",
pTrie, "ubidi_props_trieIndex", NULL,
" },\n");
usrc_writeArray(f, " { ", dataInfo.formatVersion, 8, 4, " }\n");
fputs("};\n", f);
fclose(f);
}
} else {
/* write the data */
pData=udata_create(dataDir, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, &dataInfo,
haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi: unable to create data memory, %s\n", u_errorName(errorCode));
exit(errorCode);
}
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, trieBlock, trieSize);
udata_writeBlock(pData, mirrors, 4*mirrorTop);
udata_writeBlock(pData, jgArray, prev-jgStart);
/* finish up */
dataLength=udata_finish(pData, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genbidi: error %d writing the output file\n", errorCode);
exit(errorCode);
}
if(dataLength!=indexes[UBIDI_IX_LENGTH]) {
fprintf(stderr, "genbidi: data length %ld != calculated size %d\n",
dataLength, (int)indexes[UBIDI_IX_LENGTH]);
exit(U_INTERNAL_PROGRAM_ERROR);
}
}
utrie2_close(pTrie);
upvec_close(pv);
}
/*
* Hey, Emacs, please set the following:
*
* Local Variables:
* indent-tabs-mode: nil
* End:
*
*/

4
tools/unicode/c/genprops/CMakeLists.txt
View File

@ -5,5 +5,7 @@
# created by: Markus W. Scherer
# edited on: 2010jul20
# edited by: Stuart G. Gill
add_executable(genprops genprops.cpp corepropsbuilder.cpp pnamesbuilder.cpp)
add_executable(genprops genprops.cpp
pnamesbuilder.cpp corepropsbuilder.cpp
bidipropsbuilder.cpp)
target_link_libraries(genprops icuuc icutu)

536
tools/unicode/c/genprops/bidipropsbuilder.cpp Normal file
View File

@ -0,0 +1,536 @@
/*
*******************************************************************************
*
* Copyright (C) 2004-2011, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: bidipropsbuilder.cpp (was genbidi/store.c)
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2004dec30
* created by: Markus W. Scherer
*
* Store Unicode bidi/shaping properties efficiently for
* random access.
*/
#include <stdio.h>
#include <stdlib.h>
#include "unicode/utypes.h"
#include "unicode/uchar.h"
#include "unicode/uniset.h"
#include "cmemory.h"
#include "cstring.h"
#include "ppucd.h"
#include "uarrsort.h"
#include "unicode/udata.h"
#include "unewdata.h"
#include "utrie2.h"
#include "writesrc.h"
#include "ubidi_props.h"
#include "genprops.h"
#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
/* Unicode bidi/shaping properties file format ---------------------------------
The file format prepared and written here contains several data
structures that store indexes or data.
Before the data contents described below, there are the headers required by
the udata API for loading ICU data. Especially, a UDataInfo structure
precedes the actual data. It contains platform properties values and the
file format version.
The following is a description of format version 2.0 .
The file contains the following structures:
const int32_t indexes[i0] with values i0, i1, ...:
(see UBIDI_IX_... constants for names of indexes)
i0 indexLength; -- length of indexes[] (UBIDI_IX_TOP)
i1 dataLength; -- length in bytes of the post-header data (incl. indexes[])
i2 trieSize; -- size in bytes of the bidi/shaping properties trie
i3 mirrorLength; -- length in uint32_t of the bidi mirroring array
i4 jgStart; -- first code point with Joining_Group data
i5 jgLimit; -- limit code point for Joining_Group data
i6..i14 reservedIndexes; -- reserved values; 0 for now
i15 maxValues; -- maximum code values for enumerated properties
bits 23..16 contain the max value for Joining_Group,
otherwise the bits are used like enum fields in the trie word
Serialized trie, see utrie2.h;
const uint32_t mirrors[mirrorLength];
const uint8_t jgArray[i5-i4]; -- (i5-i4) is always a multiple of 4
Trie data word:
Bits
15..13 signed delta to bidi mirroring code point
(add delta to input code point)
0 no such code point (source maps to itself)
-3..-1, 1..3 delta
-4 look in mirrors table
12 is mirrored
11 Bidi_Control
10 Join_Control
9.. 8 reserved (set to 0)
7.. 5 Joining_Type
4.. 0 BiDi category
Mirrors:
Stores some of the bidi mirroring data, where each code point maps to
at most one other.
Most code points do not have a mirroring code point; most that do have a signed
delta stored in the trie data value. Only those where the delta does not fit
into the trie data are stored in this table.
Logically, this is a two-column table with source and mirror code points.
Physically, the table is compressed by taking advantage of the fact that each
mirror code point is also a source code point
(each of them is a mirror of the other).
Therefore, both logical columns contain the same set of code points, which needs
to be stored only once.
The table stores source code points, and also for each the index of its mirror
code point in the same table, in a simple array of uint32_t.
Bits
31..21 index to mirror code point (unsigned)
20.. 0 source code point
The table is sorted by source code points.
Joining_Group array:
The Joining_Group values do not fit into the 16-bit trie, but the data is also
limited to a small range of code points (Arabic and Syriac) and not
well compressible.
The start and limit code points for the range are stored in the indexes[]
array, and the jgArray[] stores a byte for each of these code points,
containing the Joining_Group value.
All code points outside of this range have No_Joining_Group (0).
--- Changes in format version 2 ---
Change from UTrie to UTrie2.
----------------------------------------------------------------------------- */
U_NAMESPACE_USE
/* UDataInfo cf. udata.h */
static UDataInfo dataInfo={
sizeof(UDataInfo),
0,
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
0,
/* dataFormat="BiDi" */
{ UBIDI_FMT_0, UBIDI_FMT_1, UBIDI_FMT_2, UBIDI_FMT_3 },
{ 2, 0, 0, 0 }, /* formatVersion */
{ 6, 0, 0, 0 } /* dataVersion */
};
/* -------------------------------------------------------------------------- */
class BiDiPropsBuilder : public PropsBuilder {
public:
BiDiPropsBuilder(UErrorCode &errorCode);
virtual ~BiDiPropsBuilder();
virtual void setUnicodeVersion(const UVersionInfo version);
virtual void setProps(const UniProps &, const UnicodeSet &newValues, UErrorCode &errorCode);
virtual void build(UErrorCode &errorCode);
virtual void writeCSourceFile(const char *path, UErrorCode &errorCode);
virtual void writeBinaryData(const char *path, UBool withCopyright, UErrorCode &errorCode);
private:
int32_t encodeBidiMirroringGlyph(UChar32 src, UChar32 end, UChar32 mirror, UErrorCode &errorCode);
void makeMirror(UErrorCode &errorCode);
UnicodeSet relevantProps;
UTrie2 *pTrie;
uint8_t jgArray[0x300]; /* at most for U+0600..U+08FF */
UChar32 jgStart; // First code point with a Joining_Group.
UChar32 jgLimit; // One past the last one.
uint32_t mirrors[UBIDI_MAX_MIRROR_INDEX+1][2];
int32_t mirrorTop;
};
BiDiPropsBuilder::BiDiPropsBuilder(UErrorCode &errorCode)
: pTrie(NULL),
jgStart(0), jgLimit(0),
mirrorTop(0) {
// This builder encodes the following properties.
relevantProps.
add(UCHAR_BIDI_CONTROL).
add(UCHAR_BIDI_MIRRORED).
add(UCHAR_BIDI_CLASS).
add(UCHAR_BIDI_MIRRORING_GLYPH).
add(UCHAR_JOIN_CONTROL).
add(UCHAR_JOINING_GROUP).
add(UCHAR_JOINING_TYPE);
pTrie=utrie2_open(0, 0, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops error: bidipropsbuilder utrie2_open() failed - %s\n",
u_errorName(errorCode));
}
uprv_memset(jgArray, U_JG_NO_JOINING_GROUP, sizeof(jgArray));
}
BiDiPropsBuilder::~BiDiPropsBuilder() {
utrie2_close(pTrie);
}
void
BiDiPropsBuilder::setUnicodeVersion(const UVersionInfo version) {
uprv_memcpy(dataInfo.dataVersion, version, 4);
}
/* bidi mirroring table ----------------------------------------------------- */
int32_t
BiDiPropsBuilder::encodeBidiMirroringGlyph(UChar32 src, UChar32 end, UChar32 mirror,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode) || mirror<0) {
return 0;
}
if(src!=end) {
fprintf(stderr,
"genprops error: range U+%04lX..U+%04lX all with the same "
"Bidi_Mirroring_Glyph U+%04lX\n",
(long)src, (long)end, (long)mirror);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
int32_t delta=mirror-src;
if(delta==0) {
return 0; /* mapping to self=no mapping */
}
if(delta<UBIDI_MIN_MIRROR_DELTA || UBIDI_MAX_MIRROR_DELTA<delta) {
/* delta does not fit into the trie properties value, store in the mirrors[] table */
if(mirrorTop==LENGTHOF(mirrors)) {
fprintf(stderr,
"genprops error: too many long-distance Bidi_Mirroring_Glyph mappings; "
"UBIDI_MAX_MIRROR_INDEX can only be increased by "
"redesigning the ubidi.icu data bit fields\n");
errorCode=U_BUFFER_OVERFLOW_ERROR;
return 0;
}
/* possible: search the table so far and see if src is already listed */
mirrors[mirrorTop][0]=(uint32_t)src;
mirrors[mirrorTop][1]=(uint32_t)mirror;
++mirrorTop;
/* set an escape marker in src's properties */
delta=UBIDI_ESC_MIRROR_DELTA;
}
return delta;
}
void
BiDiPropsBuilder::setProps(const UniProps &props, const UnicodeSet &newValues,
UErrorCode &errorCode) {
if(U_FAILURE(errorCode) || newValues.containsNone(relevantProps)) { return; }
UChar32 start=props.start;
UChar32 end=props.end;
int32_t delta=encodeBidiMirroringGlyph(start, end, props.bmg, errorCode);
uint32_t value=(uint32_t)delta<<UBIDI_MIRROR_DELTA_SHIFT;
if(props.binProps[UCHAR_BIDI_MIRRORED]) {
value|=U_MASK(UBIDI_IS_MIRRORED_SHIFT);
}
if(props.binProps[UCHAR_BIDI_CONTROL]) {
value|=U_MASK(UBIDI_BIDI_CONTROL_SHIFT);
}
if(props.binProps[UCHAR_JOIN_CONTROL]) {
value|=U_MASK(UBIDI_JOIN_CONTROL_SHIFT);
}
value|=(uint32_t)props.getIntProp(UCHAR_JOINING_TYPE)<<UBIDI_JT_SHIFT;
value|=(uint32_t)props.getIntProp(UCHAR_BIDI_CLASS);
utrie2_setRange32(pTrie, start, end, value, TRUE, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops error: BiDiPropsBuilder utrie2_setRange32() failed - %s\n",
u_errorName(errorCode));
return;
}
/* store Joining_Group values from vector column 1 in a simple byte array */
int32_t jg=props.getIntProp(UCHAR_JOINING_GROUP);
if(jg!=U_JG_NO_JOINING_GROUP) {
if(start<0x600 || 0x8ff<end) {
fprintf(stderr, "genprops error: Joining_Group for out-of-range code points U+%04lx..U+%04lx\n",
(long)start, (long)end);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
return;
}
if(start<jgStart || jgStart==0) { jgStart=start; }
if(end>=jgLimit || jgLimit==0) { jgLimit=end+1; }
}
// On the off-chance that a block is defined with a Joining_Group
// that is then overridden by No_Joining_Group,
// we set that too, but only inside U+0600..U+08FF.
if(end>=0x600 && start<=0x8ff) {
if(start<0x600) { start=0x600; }
if(end>0x8ff) { end=0x8ff; }
/* set Joining_Group value for start..end */
for(UChar32 c=start; c<=end; ++c) {
jgArray[c-0x600]=(uint8_t)jg;
}
}
}
/* generate output data ----------------------------------------------------- */
static int32_t U_CALLCONV
compareMirror(const void *context, const void *left, const void *right) {
UChar32 l, r;
l=UBIDI_GET_MIRROR_CODE_POINT(((const uint32_t *)left)[0]);
r=UBIDI_GET_MIRROR_CODE_POINT(((const uint32_t *)right)[0]);
return l-r;
}
void
BiDiPropsBuilder::makeMirror(UErrorCode &errorCode) {
/* sort the mirroring table by source code points */
uprv_sortArray(mirrors, mirrorTop, 8,
compareMirror, NULL, FALSE, &errorCode);
if(U_FAILURE(errorCode)) { return; }
/*
* reduce the 2-column table to a single column
* by putting the index to the mirror entry into the source entry
*
* first:
* find each mirror code point in the source column and set each other's indexes
*
* second:
* reduce the table, combine the source code points with their indexes
* and store as a simple array of uint32_t
*/
for(int32_t i=0; i<mirrorTop; ++i) {
uint32_t c=mirrors[i][1]; /* mirror code point */
if(c>0x1fffff) {
continue; /* this entry already has an index */
}
/* search for the mirror code point in the source column */
int32_t start, limit, step;
if(c<mirrors[i][0]) {
/* search before i */
start=i-1;
limit=-1;
step=-1;
} else {
start=i+1;
limit=mirrorTop;
step=1;
}
for(int32_t j=start;; j+=step) {
if(j==limit) {
fprintf(stderr,
"genprops error: bidi mirror does not roundtrip - %04lx->%04lx->?\n",
(long)mirrors[i][0], (long)mirrors[i][1]);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
}
if(c==mirrors[j][0]) {
/*
* found the mirror code point c in the source column,
* set both entries' indexes to each other
*/
if(UBIDI_GET_MIRROR_CODE_POINT(mirrors[i][0])!=UBIDI_GET_MIRROR_CODE_POINT(mirrors[j][1])) {
/* roundtrip check fails */
fprintf(stderr,
"genprops error: bidi mirrors do not roundtrip - %04lx->%04lx->%04lx\n",
(long)mirrors[i][0], (long)mirrors[i][1], (long)mirrors[j][1]);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
} else {
mirrors[i][1]|=(uint32_t)j<<UBIDI_MIRROR_INDEX_SHIFT;
mirrors[j][1]|=(uint32_t)i<<UBIDI_MIRROR_INDEX_SHIFT;
}
break;
}
}
}
/* now the second step, the actual reduction of the table */
uint32_t *reducedMirror=mirrors[0];
for(int32_t i=0; i<mirrorTop; ++i) {
reducedMirror[i]=mirrors[i][0]|(mirrors[i][1]&~0x1fffff);
}
}
static int32_t indexes[UBIDI_IX_TOP]={
UBIDI_IX_TOP, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0
};
static uint8_t trieBlock[40000];
static int32_t trieSize;
void
BiDiPropsBuilder::build(UErrorCode &errorCode) {
makeMirror(errorCode);
if(U_FAILURE(errorCode)) { return; }
utrie2_freeze(pTrie, UTRIE2_16_VALUE_BITS, &errorCode);
trieSize=utrie2_serialize(pTrie, trieBlock, sizeof(trieBlock), &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops error: utrie2_freeze()+utrie2_serialize() failed: %s (length %ld)\n",
u_errorName(errorCode), (long)trieSize);
return;
}
/* finish jgArray, pad to multiple of 4 */
while((jgLimit-jgStart)&3) {
// Prefer rounding down jgStart before rounding up jgLimit
// so that we are guaranteed not to increase jgLimit beyond 0x900
// which (after the offset) is the end of the jgArray.
if(jgStart&3) { --jgStart; } else { ++jgLimit; }
}
indexes[UBIDI_IX_JG_START]=jgStart;
indexes[UBIDI_IX_JG_LIMIT]=jgLimit;
indexes[UBIDI_IX_TRIE_SIZE]=trieSize;
indexes[UBIDI_IX_MIRROR_LENGTH]=mirrorTop;
indexes[UBIDI_IX_LENGTH]=
(int32_t)sizeof(indexes)+
trieSize+
4*mirrorTop+
(jgLimit-jgStart);
if(beVerbose) {
puts("* ubidi.icu stats *");
printf("trie size in bytes: %5d\n", (int)trieSize);
printf("size in bytes of mirroring table: %5d\n", (int)(4*mirrorTop));
printf("length of Joining_Group array: %5d (U+%04x..U+%04x)\n", (int)(jgLimit-jgStart), (int)jgStart, (int)(jgLimit-1));
printf("data size: %5d\n", (int)indexes[UBIDI_IX_LENGTH]);
}
indexes[UBIDI_MAX_VALUES_INDEX]=
((int32_t)U_CHAR_DIRECTION_COUNT-1)|
(((int32_t)U_JT_COUNT-1)<<UBIDI_JT_SHIFT)|
(((int32_t)U_JG_COUNT-1)<<UBIDI_MAX_JG_SHIFT);
}
void
BiDiPropsBuilder::writeCSourceFile(const char *path, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
FILE *f=usrc_createFromGenerator(path, "ubidi_props_data.h",
"icu/tools/unicode/c/genprops/bidipropsbuilder.cpp");
if(f==NULL) {
errorCode=U_FILE_ACCESS_ERROR;
return;
}
fputs("#ifndef INCLUDED_FROM_UBIDI_PROPS_C\n"
"# error This file must be #included from ubidi_props.c only.\n"
"#endif\n\n", f);
usrc_writeArray(f,
"static const UVersionInfo ubidi_props_dataVersion={",
dataInfo.dataVersion, 8, 4,
"};\n\n");
usrc_writeArray(f,
"static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={",
indexes, 32, UBIDI_IX_TOP,
"};\n\n");
usrc_writeUTrie2Arrays(f,
"static const uint16_t ubidi_props_trieIndex[%ld]={\n", NULL,
pTrie,
"\n};\n\n");
usrc_writeArray(f,
"static const uint32_t ubidi_props_mirrors[%ld]={\n",
mirrors, 32, mirrorTop,
"\n};\n\n");
usrc_writeArray(f,
"static const uint8_t ubidi_props_jgArray[%ld]={\n",
jgArray+(jgStart-0x600), 8, jgLimit-jgStart,
"\n};\n\n");
fputs(
"static const UBiDiProps ubidi_props_singleton={\n"
" NULL,\n"
" ubidi_props_indexes,\n"
" ubidi_props_mirrors,\n"
" ubidi_props_jgArray,\n",
f);
usrc_writeUTrie2Struct(f,
" {\n",
pTrie, "ubidi_props_trieIndex", NULL,
" },\n");
usrc_writeArray(f, " { ", dataInfo.formatVersion, 8, 4, " }\n");
fputs("};\n", f);
fclose(f);
}
void
BiDiPropsBuilder::writeBinaryData(const char *path, UBool withCopyright, UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return; }
UNewDataMemory *pData=udata_create(path, UBIDI_DATA_TYPE, UBIDI_DATA_NAME, &dataInfo,
withCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops: udata_create(%s, ubidi.icu) failed - %s\n",
path, u_errorName(errorCode));
return;
}
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, trieBlock, trieSize);
udata_writeBlock(pData, mirrors, 4*mirrorTop);
udata_writeBlock(pData, jgArray+(jgStart-0x600), jgLimit-jgStart);
long dataLength=udata_finish(pData, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "genprops: error %d writing the output file\n", errorCode);
return;
}
if(dataLength!=indexes[UBIDI_IX_LENGTH]) {
fprintf(stderr, "genprops: data length %ld != calculated size %d\n",
dataLength, (int)indexes[UBIDI_IX_LENGTH]);
errorCode=U_INTERNAL_PROGRAM_ERROR;
}
}
PropsBuilder *
createBiDiPropsBuilder(UErrorCode &errorCode) {
if(U_FAILURE(errorCode)) { return NULL; }
PropsBuilder *pw=new BiDiPropsBuilder(errorCode);
if(pw==NULL) {
errorCode=U_MEMORY_ALLOCATION_ERROR;
}
return pw;
}
/*
* Hey, Emacs, please set the following:
*
* Local Variables:
* indent-tabs-mode: nil
* End:
*
*/

6
tools/unicode/c/genprops/genprops.cpp
View File

@ -97,6 +97,7 @@ main(int argc, char* argv[]) {
IcuToolErrorCode errorCode("genprops");
LocalPointer<PNamesBuilder> pnamesBuilder(createPNamesBuilder(errorCode));
LocalPointer<PropsBuilder> corePropsBuilder(createCorePropsBuilder(errorCode));
LocalPointer<PropsBuilder> bidiPropsBuilder(createBiDiPropsBuilder(errorCode));
if(errorCode.isFailure()) {
fprintf(stderr, "genprops: unable to create PropsBuilders - %s\n", errorCode.errorName());
return errorCode.reset();
@ -136,9 +137,11 @@ main(int argc, char* argv[]) {
if(ppucd.lineHasPropertyValues()) {
const UniProps *props=ppucd.getProps(newValues, errorCode);
corePropsBuilder->setProps(*props, newValues, errorCode);
bidiPropsBuilder->setProps(*props, newValues, errorCode);
} else if(lineType==PreparsedUCD::UNICODE_VERSION_LINE) {
const UVersionInfo &version=ppucd.getUnicodeVersion();
corePropsBuilder->setUnicodeVersion(version);
bidiPropsBuilder->setUnicodeVersion(version);
}
if(errorCode.isFailure()) {
fprintf(stderr,
@ -149,6 +152,7 @@ main(int argc, char* argv[]) {
}
corePropsBuilder->build(errorCode);
bidiPropsBuilder->build(errorCode);
if(errorCode.isFailure()) {
fprintf(stderr, "genprops error: failure finalizing the data - %s\n",
errorCode.errorName());
@ -168,6 +172,8 @@ main(int argc, char* argv[]) {
pnamesBuilder->writeBinaryData(sourceDataIn.data(), withCopyright, errorCode);
corePropsBuilder->writeCSourceFile(sourceCommon.data(), errorCode);
corePropsBuilder->writeBinaryData(sourceDataIn.data(), withCopyright, errorCode);
bidiPropsBuilder->writeCSourceFile(sourceCommon.data(), errorCode);
bidiPropsBuilder->writeBinaryData(sourceDataIn.data(), withCopyright, errorCode);
return errorCode;
}

1
tools/unicode/c/genprops/genprops.h
View File

@ -41,6 +41,7 @@ public:
PNamesBuilder *createPNamesBuilder(UErrorCode &errorCode);
PropsBuilder *createCorePropsBuilder(UErrorCode &errorCode);
PropsBuilder *createBiDiPropsBuilder(UErrorCode &errorCode);
/* global flags */
U_CFUNC UBool beVerbose;