AuroraMiddleware/scuffed-code
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ICU-983 Added collation attributes and normalization options.

Browse Source 
X-SVN-Rev: 5023
This commit is contained in:
Syn Wee Quek 2001-06-19 23:03:34 +00:00
parent b6476563b2
commit 26cb25d8a2
1 changed files with 435 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

528
icu4c/source/tools/dumpce/dumpce.cpp
View File

@ -18,12 +18,7 @@
* This program outputs the collation elements used for a requested tailoring. * This program outputs the collation elements used for a requested tailoring.
* *
* Usage: * Usage:
* dumpce options... * dumpce options... please check main function.
* -locale name ICU locale to use. Default is en_US
* -serialize Serializes the collation elements in -locale
* and outputs them into -outputfile
* -destdir dir_name Path for outputing the serialized collation
* elements. Default standard output.
*/ */
#include <unicode/utypes.h> #include <unicode/utypes.h>
#include <unicode/ucol.h> #include <unicode/ucol.h>
@ -32,6 +27,7 @@
#include <unicode/uchar.h> #include <unicode/uchar.h>
#include <unicode/utf16.h> #include <unicode/utf16.h>
#include <unicode/putil.h> #include <unicode/putil.h>
#include <unicode/ustring.h>
#include <stdio.h> #include <stdio.h>
#include "ucol_tok.h" #include "ucol_tok.h"
#include "cstring.h" #include "cstring.h"
@ -46,9 +42,13 @@
static UOption options[]={ static UOption options[]={
UOPTION_HELP_H, UOPTION_HELP_H,
UOPTION_HELP_QUESTION_MARK, UOPTION_HELP_QUESTION_MARK,
{"locale", NULL, NULL, NULL, 'l', UOPT_REQUIRES_ARG, 0}, {"locale", NULL, NULL, NULL, 'l', UOPT_REQUIRES_ARG, 0},
{"serialize", NULL, NULL, NULL, 's', UOPT_NO_ARG, 0}, {"serialize", NULL, NULL, NULL, 'z', UOPT_NO_ARG, 0},
UOPTION_DESTDIR, UOPTION_DESTDIR,
UOPTION_SOURCEDIR,
{"attribute", NULL, NULL, NULL, 'a', UOPT_REQUIRES_ARG, 0},
{"rule", NULL, NULL, NULL, 'r', UOPT_REQUIRES_ARG, 0},
{"normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0},
UOPTION_VERBOSE UOPTION_VERBOSE
}; };
@ -61,6 +61,61 @@ static UCollator *COLLATOR_;
*/ */
static FILE *OUTPUT_; static FILE *OUTPUT_;
static UColAttributeValue ATTRIBUTE_[UCOL_ATTRIBUTE_COUNT] = {
UCOL_DEFAULT, UCOL_DEFAULT, UCOL_DEFAULT, UCOL_DEFAULT, UCOL_DEFAULT,
UCOL_DEFAULT
};
static UNormalizationMode NORMALIZATION_ = UCOL_DEFAULT_NORMALIZATION;
static const char *ATTRIBUTE_NAME_[UCOL_ATTRIBUTE_COUNT] = {
"UCOL_FRENCH_COLLATION",
"UCOL_ALTERNATE_HANDLING",
"UCOL_CASE_FIRST",
"UCOL_CASE_LEVEL",
"UCOL_NORMALIZATION_MODE|UCOL_DECOMPOSITION_MODE",
"UCOL_STRENGTH"};
static const char *ATTRIBUTE_VALUE_[UCOL_ATTRIBUTE_VALUE_COUNT] = {
"UCOL_PRIMARY",
"UCOL_SECONDARY",
"UCOL_TERTIARY|UCOL_DEFAULT_STRENGTH",
"UCOL_QUATERNARY",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"UCOL_IDENTICAL",
"UCOL_OFF",
"UCOL_ON",
"",
"",
"UCOL_SHIFTED",
"UCOL_NON_IGNORABLE",
"",
"",
"UCOL_LOWER_FIRST",
"UCOL_UPPER_FIRST",
"",
"",
"UCOL_ON_WITHOUT_HANGUL"};
static const char *NORMALIZATION_VALUE_[UNORM_MODE_COUNT] = {
"",
"UCOL_NO_NORMALIZATION|UNORM_NONE",
"UCOL_DECOMP_CAN|UNORM_NFD",
"UCOL_DECOMP_COMPAT|UNORM_NFKD|UCOL_DEFAULT_NORMALIZATION",
"UCOL_DECOMP_CAN_COMP_COMPAT|UNORM_DEFAULT|UNORM_NFC",
"UCOL_DECOMP_COMPAT_COMP_CAN|UNORM_NFKC"
};
/** /**
* Writes the hexadecimal of a null-terminated array of codepoints into a * Writes the hexadecimal of a null-terminated array of codepoints into a
* file * file
@ -122,7 +177,7 @@ void serialize(FILE *f, UCollationElements *iter) {
sortkeylength = ucol_getSortKey(iter->iteratordata_.coll, codepoint, sortkeylength = ucol_getSortKey(iter->iteratordata_.coll, codepoint,
-1, sortkey, 64); -1, sortkey, 64);
} }
if (options[5].doesOccur) { if (options[9].doesOccur) {
serialize(stdout, codepoint); serialize(stdout, codepoint);
fprintf(stdout, "\n"); fprintf(stdout, "\n");
} }
@ -135,21 +190,32 @@ void serialize(FILE *f, UCollationElements *iter) {
fprintf(f, "Error retrieving collation elements\n"); fprintf(f, "Error retrieving collation elements\n");
return; return;
} }
fprintf(f, "[");
while (TRUE) { while (TRUE) {
fprintf(f, "%08x", ce); fprintf(f, "[");
if (UCOL_PRIMARYORDER(ce) != 0) {
fprintf(f, "%04x", UCOL_PRIMARYORDER(ce));
}
fprintf(f, ",");
if (UCOL_SECONDARYORDER(ce) != 0) {
fprintf(f, " %02x", UCOL_SECONDARYORDER(ce));
}
fprintf(f, ",");
if (UCOL_TERTIARYORDER(ce) != 0) {
fprintf(f, " %02x", UCOL_TERTIARYORDER(ce));
}
fprintf(f, "] ");
ce = ucol_next(iter, &error); ce = ucol_next(iter, &error);
if (ce == UCOL_NULLORDER) { if (ce == UCOL_NULLORDER) {
break; break;
} }
fprintf(f, " ");
if (U_FAILURE(error)) { if (U_FAILURE(error)) {
fprintf(stdout, "Error retrieving collation elements"); fprintf(stdout, "Error retrieving collation elements");
return; return;
} }
} }
fprintf(f, "] ");
if (sortkeylength > 64) { if (sortkeylength > 64) {
fprintf(f, "Sortkey exceeds pre-allocated size"); fprintf(f, "Sortkey exceeds pre-allocated size");
} }
@ -213,23 +279,63 @@ void serialize(FILE *f, UChar *rule, int rlen, UBool contractiononly,
} }
} }
/**
* Prints the attribute values in the argument collator into the output stream
* @param collator
*/
void outputAttribute(UCollator *collator, UErrorCode *error)
{
UColAttribute attribute = UCOL_FRENCH_COLLATION;
while (attribute < UCOL_ATTRIBUTE_COUNT) {
fprintf(OUTPUT_, "%s = %s\n", ATTRIBUTE_NAME_[attribute],
ATTRIBUTE_VALUE_[ucol_getAttribute(collator, attribute, error)]);
if (U_FAILURE(*error)) {
fprintf(stdout, "Failure in reading collator attribute\n");
return;
}
attribute = (UColAttribute)(attribute + 1);
}
}
/**
* Prints the normalization mode in the argument collator into the output stream
* @param collator
*/
void outputNormalization(UCollator *collator)
{
fprintf(OUTPUT_, "NORMALIZATION MODE = %s\n",
NORMALIZATION_VALUE_[ucol_getNormalization(collator)]);
}
/** /**
* Output the collation element belonging to the locale into a file * Output the collation element belonging to the locale into a file
* @param locale string
* @param fullrules flag to indicate if only tailored collation elements are to * @param fullrules flag to indicate if only tailored collation elements are to
* be output or all collation elements * be output or all collation elements
*/ */
void serialize(UBool tailoredonly) { void serialize(const char *locale, UBool tailoredonly) {
UErrorCode error = U_ZERO_ERROR; UErrorCode error = U_ZERO_ERROR;
UChar str[128]; UChar str[128];
int strlen = 0; int strlen = 0;
fprintf(OUTPUT_, "# This file contains the serialized collation elements\n"); fprintf(OUTPUT_, "# This file contains the serialized collation elements\n");
fprintf(OUTPUT_, "# as of the collation version indicated below.\n"); fprintf(OUTPUT_, "# as of the collation version indicated below.\n");
fprintf(OUTPUT_, "# Data format: xxxx xxxx..; [yyyyyyyy yyyyyy..] [zz zz..\n"); fprintf(OUTPUT_, "# Data format: xxxx xxxx..; [yyyy, yyyy, yyyyyy] [yyyy, yyyy, yyyyyy] ... [yyyy, yyyy, yyyyyy] [zz zz..\n");
fprintf(OUTPUT_, "# where xxxx are codepoints in hexadecimals,\n"); fprintf(OUTPUT_, "# where xxxx are codepoints in hexadecimals,\n");
fprintf(OUTPUT_, "# yyyyyyyy are the corresponding\n"); fprintf(OUTPUT_, "# yyyyyyyy are the corresponding\n");
fprintf(OUTPUT_, "# collation elements in hexadecimals\n"); fprintf(OUTPUT_, "# collation elements in hexadecimals\n");
fprintf(OUTPUT_, "# and zz are the sortkey values in hexadecimals\n"); fprintf(OUTPUT_, "# and zz are the sortkey values in hexadecimals\n");
fprintf(OUTPUT_, "\n# Collator information\n");
fprintf(OUTPUT_, "\nLocale: %s\n", locale);
fprintf(stdout, "Locale: %s\n", locale);
UVersionInfo version;
ucol_getVersion(COLLATOR_, version);
fprintf(OUTPUT_, "Version number: %d.%d.%d.%d\n",
version[0], version[1], version[2], version[3]);
outputAttribute(COLLATOR_, &error);
outputNormalization(COLLATOR_);
UCollationElements *iter = ucol_openElements(COLLATOR_, str, strlen, UCollationElements *iter = ucol_openElements(COLLATOR_, str, strlen,
&error); &error);
@ -296,105 +402,320 @@ void serialize(UBool tailoredonly) {
ucol_closeElements(iter); ucol_closeElements(iter);
} }
/**
* Sets the collator with the attribute values
* @param collator
* @param error status
*/
void setAttributes(UCollator *collator, UErrorCode *error)
{
int count = 0;
while (count < UCOL_ATTRIBUTE_COUNT) {
if (ATTRIBUTE_[count] != UCOL_DEFAULT) {
ucol_setAttribute(collator, (UColAttribute)count,
ATTRIBUTE_[count], error);
if (U_FAILURE(*error)) {
return;
}
}
count ++;
}
}
/**
* Sets the collator with the normalization mode
* @param collator
*/
void setNormalization(UCollator *collator)
{
if (NORMALIZATION_ != UCOL_DEFAULT_NORMALIZATION) {
ucol_setNormalization(collator, NORMALIZATION_);
}
}
/**
* Appends directory path with an ending seperator if necessary.
* @param path with enough space to append one seperator
* @return new directory path length
*/
int appendDirSeparator(char *dir)
{
int dirlength = strlen(dir);
char dirending = dir[dirlength - 1];
if (dirending != U_FILE_SEP_CHAR) {
dir[dirlength] = U_FILE_SEP_CHAR;
dir[dirlength + 1] = 0;
return dirlength + 1;
}
return dirlength;
}
/** /**
* Output the collation element into a file * Output the collation element into a file
*/ */
void serialize() { void serialize() {
const char *locale = (char *)options[2].value;
int32_t localeindex = 0;
char filename[128]; char filename[128];
int dirlength = 0; int dirlength = 0;
if (options[4].doesOccur) { if (options[4].doesOccur) {
const char *dir = options[4].value; strcpy(filename, options[4].value);
strcpy(filename, dir); dirlength = appendDirSeparator(filename);
dirlength = strlen(dir);
char dirending = dir[dirlength - 1];
if (dirending != U_FILE_SEP_CHAR) {
filename[dirlength] = U_FILE_SEP_CHAR;
filename[dirlength + 1] = 0;
}
dirlength ++;
}
if (strcmp(locale, "all") == 0) {
if (options[4].doesOccur) {
strcat(filename, "UCA.txt");
OUTPUT_ = fopen(filename, "w");
if (OUTPUT_ == NULL) {
fprintf(stdout, "Cannot open file:%s\n", filename);
return;
}
}
fprintf(OUTPUT_, "# UCA collation elements\n");
fprintf(stdout, "UCA\n");
UErrorCode error = U_ZERO_ERROR;
COLLATOR_ = ucol_open("en_US", &error);
if (U_FAILURE(error)) {
fprintf(stdout, "Collator creation failed:");
fprintf(stdout, u_errorName(error));
return;
}
UVersionInfo version;
ucol_getVersion(COLLATOR_, version);
fprintf(OUTPUT_, "# Collation version number: %d.%d.%d.%d\n",
version[0], version[1], version[2], version[3]);
serialize(FALSE);
if (options[4].doesOccur) {
filename[dirlength] = 0;
fclose(OUTPUT_);
}
ucol_close(COLLATOR_);
localeindex = ucol_countAvailable() - 1;
fprintf(stdout, "Number of locales: %d\n", localeindex + 1);
locale = ucol_getAvailable(localeindex);
} }
while (TRUE) { if (options[2].doesOccur) {
UErrorCode error = U_ZERO_ERROR; const char *locale = (char *)options[2].value;
COLLATOR_ = ucol_open(locale, &error); int32_t localeindex = 0;
if (U_FAILURE(error)) {
fprintf(stdout, "Collator creation failed:"); if (strcmp(locale, "all") == 0) {
fprintf(stdout, u_errorName(error)); if (options[4].doesOccur) {
return; strcat(filename, "UCA.txt");
} OUTPUT_ = fopen(filename, "w");
if (OUTPUT_ == NULL) {
if (options[4].doesOccur) { fprintf(stdout, "Cannot open file:%s\n", filename);
strcat(filename, locale); return;
strcat(filename, ".txt"); }
OUTPUT_ = fopen(filename, "w"); }
if (OUTPUT_ == NULL) { fprintf(stdout, "UCA\n");
fprintf(stdout, "Cannot open file:%s\n", filename); UErrorCode error = U_ZERO_ERROR;
COLLATOR_ = ucol_open("en_US", &error);
if (U_FAILURE(error)) {
fprintf(stdout, "Collator creation failed:");
fprintf(stdout, u_errorName(error));
goto CLOSEUCA;
return; return;
} }
setAttributes(COLLATOR_, &error);
setNormalization(COLLATOR_);
if (U_FAILURE(error)) {
fprintf(stdout, "Collator attribute setting failed:");
fprintf(stdout, u_errorName(error));
goto CLOSEUCA;
return;
}
serialize("UCA", FALSE);
CLOSEUCA :
if (options[4].doesOccur) {
filename[dirlength] = 0;
fclose(OUTPUT_);
}
ucol_close(COLLATOR_);
localeindex = ucol_countAvailable() - 1;
fprintf(stdout, "Number of locales: %d\n", localeindex + 1);
locale = ucol_getAvailable(localeindex);
}
while (TRUE) {
UErrorCode error = U_ZERO_ERROR;
COLLATOR_ = ucol_open(locale, &error);
if (U_FAILURE(error)) {
fprintf(stdout, "Collator creation failed:");
fprintf(stdout, u_errorName(error));
goto CLOSETAILOR;
return;
}
setAttributes(COLLATOR_, &error);
setNormalization(COLLATOR_);
if (U_FAILURE(error)) {
fprintf(stdout, "Collator attribute setting failed:");
fprintf(stdout, u_errorName(error));
goto CLOSETAILOR;
return;
}
if (options[4].doesOccur) {
strcat(filename, locale);
strcat(filename, ".txt");
OUTPUT_ = fopen(filename, "w");
if (OUTPUT_ == NULL) {
fprintf(stdout, "Cannot open file:%s\n", filename);
return;
}
}
if (options[3].doesOccur) {
serialize(locale, TRUE);
}
ucol_close(COLLATOR_);
CLOSETAILOR :
if (options[4].doesOccur) {
filename[dirlength] = 0;
fclose(OUTPUT_);
}
localeindex --;
if (localeindex < 0) {
break;
}
locale = ucol_getAvailable(localeindex);
}
}
if (options[7].doesOccur) {
char inputfilename[128];
// rules are to be used
if (options[5].doesOccur) {
strcpy(inputfilename, options[5].value);
appendDirSeparator(inputfilename);
}
strcat(inputfilename, options[7].value);
FILE *input = fopen(inputfilename, "r");
if (input == NULL) {
fprintf(stdout, "Cannot open file:%s\n", filename);
return;
} }
fprintf(OUTPUT_, "# Locale: %s\n", locale); char s[1024];
fprintf(stdout, "Locale: %s\n", locale); UChar rule[1024];
UVersionInfo version; UChar *prule = rule;
ucol_getVersion(COLLATOR_, version); int size = 1024;
fprintf(OUTPUT_, "# Collation version number: %d.%d.%d.%d\n", // synwee TODO: make this part dynamic
version[0], version[1], version[2], version[3]); while (fscanf(input, "%[^\n]s", s) != EOF) {
size -= u_unescape(s, prule, size);
prule = prule + u_strlen(prule);
}
fclose(input);
if (options[3].doesOccur) { if (options[4].doesOccur) {
serialize(TRUE); strcat(filename, "Rules.txt");
OUTPUT_ = fopen(filename, "w");
if (OUTPUT_ == NULL) {
fprintf(stdout, "Cannot open file:%s\n", filename);
return;
}
} }
fprintf(stdout, "Rules\n");
UErrorCode error = U_ZERO_ERROR;
COLLATOR_ = ucol_openRules(rule, u_strlen(rule), NORMALIZATION_,
UCOL_DEFAULT_STRENGTH, &error);
if (U_FAILURE(error)) {
fprintf(stdout, "Collator creation failed:");
fprintf(stdout, u_errorName(error));
goto CLOSERULES;
return;
}
setAttributes(COLLATOR_, &error);
if (U_FAILURE(error)) {
fprintf(stdout, "Collator attribute setting failed:");
fprintf(stdout, u_errorName(error));
goto CLOSERULES;
return;
}
serialize("Rule-based", TRUE);
ucol_close(COLLATOR_); ucol_close(COLLATOR_);
CLOSERULES :
if (options[4].doesOccur) { if (options[4].doesOccur) {
filename[dirlength] = 0; filename[dirlength] = 0;
fclose(OUTPUT_); fclose(OUTPUT_);
} }
localeindex --;
if (localeindex < 0) {
break;
}
locale = ucol_getAvailable(localeindex);
} }
} }
/**
* Parse for enum values.
* Note this only works for positive enum values.
* @param enumarray array containing names of the enum values in string at their
* corresponding value index. use "" at indexes with no corresponding
* declared enum value.
* @param enumlength length of enum array
* @param str string to be parsed
* @return corresponding integer enum value or -1 if value is not found.
*/
int parseEnums(const char *enumarray[], int enumlength, const char *str)
{
const char *enumvalue = enumarray[0];
int result = atoi(str);
if (result == 0 && str[0] != '0') {
while (enumvalue[0] == 0 || strcmp(enumvalue, str) != 0) {
// checking for multiple enum names sharing the same values
enumvalue = strstr(enumvalue, str);
if (enumvalue != NULL) {
int size = strchr(enumvalue, '|') - enumvalue;
if (size < 0) {
size = strlen(enumvalue);
}
if (size == (int)strlen(str)) {
return result;
}
}
result ++;
if (result == enumlength) {
return -1;
}
enumvalue = enumarray[result];
}
}
if (result >= enumlength) {
return -1;
}
return result;
}
/**
* Parser for attribute name value pair
*/
void parseAttributes() {
char str[32];
const char *pname = options[6].value;
const char *pend = options[6].value + strlen(options[6].value);
const char *pvalue;
while (pname < pend) {
pvalue = strchr(pname, '=');
if (pvalue == NULL) {
fprintf(stdout,
"No matching value found for attribute argument %s\n",
pname);
return;
}
int count = pvalue - pname;
strncpy(str, pname, count);
str[count] = 0;
int name = parseEnums(ATTRIBUTE_NAME_, UCOL_ATTRIBUTE_COUNT, str);
if (name == -1) {
fprintf(stdout, "Attribute name not found: %s\n", str);
return;
}
pvalue ++;
// getting corresponding enum value
pname = strchr(pvalue, ',');
if (pname == NULL) {
pname = pend;
}
count = pname - pvalue;
strncpy(str, pvalue, count);
str[count] = 0;
int value = parseEnums(ATTRIBUTE_VALUE_, UCOL_ATTRIBUTE_VALUE_COUNT,
str);
if (value == -1) {
fprintf(stdout, "Attribute value not found: %s\n", str);
return;
}
ATTRIBUTE_[name] = (UColAttributeValue)value;
pname ++;
}
}
/**
* Parser for normalization mode
*/
void parseNormalization() {
const char *str = options[6].value;
int norm = parseEnums(NORMALIZATION_VALUE_, UNORM_MODE_COUNT, str);
if (norm == -1) {
fprintf(stdout, "Normalization mode not found: %s\n", str);
return;
}
NORMALIZATION_ = (UNormalizationMode)norm;
}
/** /**
* Main -- process command line, read in and pre-process the test file, * Main -- process command line, read in and pre-process the test file,
* call other functions to do the actual tests. * call other functions to do the actual tests.
@ -412,14 +733,35 @@ int main(int argc, char *argv[]) {
} }
if (argc < 0 || options[0].doesOccur || options[1].doesOccur) { if (argc < 0 || options[0].doesOccur || options[1].doesOccur) {
fprintf(stdout, "Usage: strperf options...\n" fprintf(stdout, "Usage: strperf options...\n"
"--help Display this message.\n" "--help\n"
"--locale [name|all] ICU locale to use. Default is en_US\n" " Display this message.\n"
"--serialize Serializes the collation elements in -locale or all locales available and outputs them into --outputdir/locale_ce.txt\n" "--locale name|all\n"
"--destdir dir_name Path for outputing the serialized collation elements. Defaults to stdout if no defined\n"); " ICU locale to use. Default is en_US\n"
"--serialize\n"
" Serializes the collation elements in -locale or all locales available and outputs them into --outputdir/locale_ce.txt\n"
"--destdir dir_name\n"
" Path for outputing the serialized collation elements. Defaults to stdout if no defined\n"
"--sourcedir dir_name\n"
" Path for the input rule file for collation\n"
"--attribute name=value,name=value...\n"
" Pairs of attribute names and values for setting\n"
"--rule filename\n"
" Name of file containing the collation rules.\n"
"--normalizaton mode\n"
" UNormalizationMode mode to be used.\n");
fprintf(stdout, "Example: dumpce --serialize --locale af --destdir /temp --attribute UCOL_STRENGTH=UCOL_DEFAULT_STRENGTH,4=17");
return argc < 0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; return argc < 0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
} }
OUTPUT_ = stdout; OUTPUT_ = stdout;
if (options[6].doesOccur) {
fprintf(stdout, "attributes %s\n", options[6].value);
parseAttributes();
}
if (options[8].doesOccur) {
fprintf(stdout, "normalization mode %s\n", options[8].value);
parseNormalization();
}
if (options[3].doesOccur) { if (options[3].doesOccur) {
serialize(); serialize();
} }