ICU-983 Added collation attributes and normalization options.

X-SVN-Rev: 5023

icu4c/source/tools/dumpce/dumpce.cpp

@@ -18,12 +18,7 @@
 * This program outputs the collation elements used for a requested tailoring.
 *
 * Usage:
-*     dumpce options...
+*     dumpce options... please check main function.
-*         -locale name          ICU locale to use.  Default is en_US
-*         -serialize            Serializes the collation elements in -locale 
-*                               and outputs them into -outputfile
-*         -destdir     dir_name Path for outputing the serialized collation 
-*                               elements. Default standard output.
 */
 #include <unicode/utypes.h>
 #include <unicode/ucol.h>
@@ -32,6 +27,7 @@
 #include <unicode/uchar.h>
 #include <unicode/utf16.h>
 #include <unicode/putil.h>
+#include <unicode/ustring.h>
 #include <stdio.h>
 #include "ucol_tok.h"
 #include "cstring.h"
@@ -47,8 +43,12 @@ static UOption options[]={
     UOPTION_HELP_H,
     UOPTION_HELP_QUESTION_MARK,
     {"locale",        NULL, NULL, NULL, 'l', UOPT_REQUIRES_ARG, 0},
-    {"serialize",   NULL, NULL, NULL, 's', UOPT_NO_ARG, 0},
+    {"serialize",     NULL, NULL, NULL, 'z', UOPT_NO_ARG, 0},
 	UOPTION_DESTDIR,
+    UOPTION_SOURCEDIR,
+    {"attribute",     NULL, NULL, NULL, 'a', UOPT_REQUIRES_ARG, 0},
+    {"rule",          NULL, NULL, NULL, 'r', UOPT_REQUIRES_ARG, 0},
+    {"normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0},
 	UOPTION_VERBOSE
 };
 
@@ -61,6 +61,61 @@ static UCollator *COLLATOR_;
 */
 static FILE *OUTPUT_;
 
+static UColAttributeValue ATTRIBUTE_[UCOL_ATTRIBUTE_COUNT] = {
+    UCOL_DEFAULT, UCOL_DEFAULT, UCOL_DEFAULT, UCOL_DEFAULT, UCOL_DEFAULT, 
+    UCOL_DEFAULT
+};
+
+static UNormalizationMode NORMALIZATION_ = UCOL_DEFAULT_NORMALIZATION;
+
+static const char *ATTRIBUTE_NAME_[UCOL_ATTRIBUTE_COUNT] = {
+                        "UCOL_FRENCH_COLLATION", 
+                        "UCOL_ALTERNATE_HANDLING", 
+                        "UCOL_CASE_FIRST", 
+                        "UCOL_CASE_LEVEL", 
+                        "UCOL_NORMALIZATION_MODE|UCOL_DECOMPOSITION_MODE",
+                        "UCOL_STRENGTH"};
+     
+static const char *ATTRIBUTE_VALUE_[UCOL_ATTRIBUTE_VALUE_COUNT] = {
+                                        "UCOL_PRIMARY",
+                                        "UCOL_SECONDARY",
+                                        "UCOL_TERTIARY|UCOL_DEFAULT_STRENGTH",
+                                        "UCOL_QUATERNARY",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "",
+                                        "UCOL_IDENTICAL",
+                                        "UCOL_OFF",
+                                        "UCOL_ON",
+                                        "",
+                                        "",
+                                        "UCOL_SHIFTED",
+                                        "UCOL_NON_IGNORABLE",
+                                        "",
+                                        "",
+                                        "UCOL_LOWER_FIRST",
+                                        "UCOL_UPPER_FIRST",
+                                        "",
+                                        "",
+                                        "UCOL_ON_WITHOUT_HANGUL"};
+
+static const char *NORMALIZATION_VALUE_[UNORM_MODE_COUNT] = {
+    "",
+    "UCOL_NO_NORMALIZATION|UNORM_NONE",
+    "UCOL_DECOMP_CAN|UNORM_NFD",
+    "UCOL_DECOMP_COMPAT|UNORM_NFKD|UCOL_DEFAULT_NORMALIZATION",
+    "UCOL_DECOMP_CAN_COMP_COMPAT|UNORM_DEFAULT|UNORM_NFC",
+    "UCOL_DECOMP_COMPAT_COMP_CAN|UNORM_NFKC"
+};
+
 /**
 * Writes the hexadecimal of a null-terminated array of codepoints into a 
 * file
@@ -122,7 +177,7 @@ void serialize(FILE *f, UCollationElements *iter) {
         sortkeylength = ucol_getSortKey(iter->iteratordata_.coll, codepoint, 
                                         -1, sortkey, 64);
     }
-    if (options[5].doesOccur) {
+    if (options[9].doesOccur) {
         serialize(stdout, codepoint);
         fprintf(stdout, "\n");
     }
@@ -135,20 +190,31 @@ void serialize(FILE *f, UCollationElements *iter) {
         fprintf(f, "Error retrieving collation elements\n");
         return;
     }
-    fprintf(f, "[");
+    
     while (TRUE) {
-        fprintf(f, "%08x", ce);
+        fprintf(f, "[");
+        if (UCOL_PRIMARYORDER(ce) != 0) {
+            fprintf(f, "%04x", UCOL_PRIMARYORDER(ce));
+        }
+        fprintf(f, ",");
+        if (UCOL_SECONDARYORDER(ce) != 0) {
+            fprintf(f, " %02x", UCOL_SECONDARYORDER(ce));
+        }
+        fprintf(f, ",");
+        if (UCOL_TERTIARYORDER(ce) != 0) {
+            fprintf(f, " %02x", UCOL_TERTIARYORDER(ce));
+        }
+        fprintf(f, "] ");
+
         ce = ucol_next(iter, &error);
         if (ce == UCOL_NULLORDER) {
             break;
         }
-        fprintf(f, " ");
         if (U_FAILURE(error)) {
             fprintf(stdout, "Error retrieving collation elements");
             return;
         }
     }
-    fprintf(f, "] ");
     
     if (sortkeylength > 64) {
         fprintf(f, "Sortkey exceeds pre-allocated size");
@@ -213,24 +279,64 @@ void serialize(FILE *f, UChar *rule, int rlen, UBool contractiononly,
     }
 }
 
+/**
+* Prints the attribute values in the argument collator into the output stream
+* @param collator
+*/
+void outputAttribute(UCollator *collator, UErrorCode *error) 
+{
+    UColAttribute attribute = UCOL_FRENCH_COLLATION;
+    while (attribute < UCOL_ATTRIBUTE_COUNT) {
+        fprintf(OUTPUT_, "%s = %s\n", ATTRIBUTE_NAME_[attribute],
+            ATTRIBUTE_VALUE_[ucol_getAttribute(collator, attribute, error)]);
+        if (U_FAILURE(*error)) {
+            fprintf(stdout, "Failure in reading collator attribute\n");
+            return;
+        }
+        attribute = (UColAttribute)(attribute + 1);
+    }
+}
+
+/**
+* Prints the normalization mode in the argument collator into the output stream
+* @param collator
+*/
+void outputNormalization(UCollator *collator) 
+{
+    fprintf(OUTPUT_, "NORMALIZATION MODE = %s\n", 
+            NORMALIZATION_VALUE_[ucol_getNormalization(collator)]);
+}
+
 /**
 * Output the collation element belonging to the locale into a file
+* @param locale string
 * @param fullrules flag to indicate if only tailored collation elements are to
 *        be output or all collation elements
 */
-void serialize(UBool tailoredonly) {
+void serialize(const char *locale, UBool tailoredonly) {
     UErrorCode  error              = U_ZERO_ERROR;
     UChar       str[128];
     int         strlen = 0;
 
     fprintf(OUTPUT_, "# This file contains the serialized collation elements\n");
     fprintf(OUTPUT_, "# as of the collation version indicated below.\n");
-    fprintf(OUTPUT_, "# Data format: xxxx xxxx..; [yyyyyyyy yyyyyy..] [zz zz..\n");
+    fprintf(OUTPUT_, "# Data format: xxxx xxxx..; [yyyy, yyyy, yyyyyy] [yyyy, yyyy, yyyyyy] ... [yyyy, yyyy, yyyyyy] [zz zz..\n");
     fprintf(OUTPUT_, "#              where xxxx are codepoints in hexadecimals,\n");
     fprintf(OUTPUT_, "#              yyyyyyyy are the corresponding\n");
     fprintf(OUTPUT_, "#              collation elements in hexadecimals\n");
     fprintf(OUTPUT_, "#              and zz are the sortkey values in hexadecimals\n");
 
+    fprintf(OUTPUT_, "\n# Collator information\n");
+
+    fprintf(OUTPUT_, "\nLocale: %s\n", locale);
+    fprintf(stdout, "Locale: %s\n", locale);
+    UVersionInfo version;
+    ucol_getVersion(COLLATOR_, version);
+    fprintf(OUTPUT_, "Version number: %d.%d.%d.%d\n", 
+                      version[0], version[1], version[2], version[3]);
+    outputAttribute(COLLATOR_, &error);
+    outputNormalization(COLLATOR_);
+    
     UCollationElements *iter = ucol_openElements(COLLATOR_, str, strlen, 
                                                  &error);
     if (U_FAILURE(error)) {
@@ -296,26 +402,70 @@ void serialize(UBool tailoredonly) {
     ucol_closeElements(iter);
 }
 
+/**
+* Sets the collator with the attribute values
+* @param collator
+* @param error status
+*/
+void setAttributes(UCollator *collator, UErrorCode *error) 
+{
+    int count = 0;
+    while (count < UCOL_ATTRIBUTE_COUNT) {
+        if (ATTRIBUTE_[count] != UCOL_DEFAULT) {
+            ucol_setAttribute(collator, (UColAttribute)count, 
+                              ATTRIBUTE_[count], error);
+            if (U_FAILURE(*error)) {
+                return;
+            }
+        }
+        count ++;
+    }
+}
+
+/**
+* Sets the collator with the normalization mode
+* @param collator
+*/
+void setNormalization(UCollator *collator) 
+{
+    if (NORMALIZATION_ != UCOL_DEFAULT_NORMALIZATION) {
+        ucol_setNormalization(collator, NORMALIZATION_);
+    }
+}
+
+/**
+* Appends directory path with an ending seperator if necessary.
+* @param path with enough space to append one seperator
+* @return new directory path length
+*/
+int appendDirSeparator(char *dir) 
+{
+    int dirlength = strlen(dir);
+    char dirending = dir[dirlength - 1];
+    if (dirending != U_FILE_SEP_CHAR) {
+        dir[dirlength] = U_FILE_SEP_CHAR;
+        dir[dirlength + 1] = 0;
+        return dirlength + 1;
+    }
+    return dirlength;
+}
+
 /**
 * Output the collation element into a file
 */
 void serialize() {
-    const char    *locale      = (char *)options[2].value;
-          int32_t  localeindex = 0;
     char filename[128];
     int  dirlength = 0;
 
     if (options[4].doesOccur) {
-        const char *dir      = options[4].value;
+        strcpy(filename, options[4].value);
-        strcpy(filename, dir);
+        dirlength = appendDirSeparator(filename);
-        dirlength = strlen(dir);
-        char dirending = dir[dirlength - 1];
-        if (dirending != U_FILE_SEP_CHAR) {
-            filename[dirlength] = U_FILE_SEP_CHAR;
-            filename[dirlength + 1] = 0;
-        }
-        dirlength ++;
     }
+
+    if (options[2].doesOccur) {
+        const char    *locale      = (char *)options[2].value;
+              int32_t  localeindex = 0;
+        
         if (strcmp(locale, "all") == 0) {
             if (options[4].doesOccur) {
                 strcat(filename, "UCA.txt");
@@ -325,21 +475,26 @@ void serialize() {
                     return;
                 }
             }
-        fprintf(OUTPUT_, "# UCA collation elements\n");
             fprintf(stdout, "UCA\n");
             UErrorCode error = U_ZERO_ERROR;
             COLLATOR_ = ucol_open("en_US", &error);
             if (U_FAILURE(error)) {
                 fprintf(stdout, "Collator creation failed:");
                 fprintf(stdout, u_errorName(error));
+                goto CLOSEUCA;
+                return;
+            }
+            setAttributes(COLLATOR_, &error);
+            setNormalization(COLLATOR_);
+            if (U_FAILURE(error)) {
+                fprintf(stdout, "Collator attribute setting failed:");
+                fprintf(stdout, u_errorName(error));
+                goto CLOSEUCA;
                 return;
             }
-        UVersionInfo version;
-        ucol_getVersion(COLLATOR_, version);
-        fprintf(OUTPUT_, "# Collation version number: %d.%d.%d.%d\n", 
-                         version[0], version[1], version[2], version[3]);
         
-        serialize(FALSE);
+            serialize("UCA", FALSE);
+CLOSEUCA :  
             if (options[4].doesOccur) {
                 filename[dirlength] = 0;
                 fclose(OUTPUT_);
@@ -356,6 +511,15 @@ void serialize() {
             if (U_FAILURE(error)) {
                 fprintf(stdout, "Collator creation failed:");
                 fprintf(stdout, u_errorName(error));
+                goto CLOSETAILOR;
+                return;
+            }
+            setAttributes(COLLATOR_, &error);
+            setNormalization(COLLATOR_);
+            if (U_FAILURE(error)) {
+                fprintf(stdout, "Collator attribute setting failed:");
+                fprintf(stdout, u_errorName(error));
+                goto CLOSETAILOR;
                 return;
             }
 
@@ -369,19 +533,13 @@ void serialize() {
                 }
             }
 
-        fprintf(OUTPUT_, "# Locale: %s\n", locale);
-        fprintf(stdout, "Locale: %s\n", locale);
-        UVersionInfo version;
-        ucol_getVersion(COLLATOR_, version);
-        fprintf(OUTPUT_, "# Collation version number: %d.%d.%d.%d\n", 
-                         version[0], version[1], version[2], version[3]);
-
             if (options[3].doesOccur) {
-            serialize(TRUE);
+                serialize(locale, TRUE);
             }
 
             ucol_close(COLLATOR_);
 
+CLOSETAILOR : 
             if (options[4].doesOccur) {
                 filename[dirlength] = 0;
                 fclose(OUTPUT_);
@@ -393,6 +551,169 @@ void serialize() {
             }
             locale = ucol_getAvailable(localeindex);
         }
+    }
+
+    if (options[7].doesOccur) {
+        char inputfilename[128];
+        // rules are to be used
+        if (options[5].doesOccur) {
+            strcpy(inputfilename, options[5].value);
+            appendDirSeparator(inputfilename);
+        }
+        strcat(inputfilename, options[7].value);
+        FILE *input = fopen(inputfilename, "r");
+        if (input == NULL) {
+            fprintf(stdout, "Cannot open file:%s\n", filename);
+            return;
+        }
+        
+        char   s[1024];
+        UChar  rule[1024];
+        UChar *prule = rule;
+        int    size = 1024;
+        // synwee TODO: make this part dynamic
+        while (fscanf(input, "%[^\n]s", s) != EOF) {
+            size -= u_unescape(s, prule, size);
+            prule = prule + u_strlen(prule);
+        }
+        fclose(input);
+
+        if (options[4].doesOccur) {
+            strcat(filename, "Rules.txt");
+            OUTPUT_ = fopen(filename, "w");
+            if (OUTPUT_ == NULL) {
+                fprintf(stdout, "Cannot open file:%s\n", filename);
+                return;
+            }
+        }
+
+        fprintf(stdout, "Rules\n");
+        UErrorCode error = U_ZERO_ERROR;
+        COLLATOR_ = ucol_openRules(rule, u_strlen(rule), NORMALIZATION_, 
+                                   UCOL_DEFAULT_STRENGTH, &error);
+        if (U_FAILURE(error)) {
+            fprintf(stdout, "Collator creation failed:");
+            fprintf(stdout, u_errorName(error));
+            goto CLOSERULES;
+            return;
+        }
+        setAttributes(COLLATOR_, &error);
+        if (U_FAILURE(error)) {
+            fprintf(stdout, "Collator attribute setting failed:");
+            fprintf(stdout, u_errorName(error));
+            goto CLOSERULES;
+            return;
+        }
+        
+        serialize("Rule-based", TRUE);
+        ucol_close(COLLATOR_);
+
+CLOSERULES :
+        if (options[4].doesOccur) {
+            filename[dirlength] = 0;
+            fclose(OUTPUT_);
+        }
+    }
+}
+
+/**
+* Parse for enum values.
+* Note this only works for positive enum values.
+* @param enumarray array containing names of the enum values in string at their
+*        corresponding value index. use "" at indexes with no corresponding 
+*        declared enum value.
+* @param enumlength length of enum array
+* @param str string to be parsed
+* @return corresponding integer enum value or -1 if value is not found.
+*/
+int parseEnums(const char *enumarray[], int enumlength, const char *str) 
+{
+    const char *enumvalue = enumarray[0];
+    int result = atoi(str);
+    if (result == 0 && str[0] != '0') {
+        while (enumvalue[0] == 0 || strcmp(enumvalue, str) != 0) {
+            // checking for multiple enum names sharing the same values
+            enumvalue = strstr(enumvalue, str);
+            if (enumvalue != NULL) {
+                int size = strchr(enumvalue, '|') - enumvalue;
+                if (size < 0) {
+                    size = strlen(enumvalue);
+                }
+                if (size == (int)strlen(str)) {
+                    return result;
+                }
+            }
+            result ++;
+            if (result == enumlength) {
+                return -1;
+            }
+            enumvalue = enumarray[result];
+        }
+    }
+    if (result >= enumlength) {
+        return -1;
+    }
+    return result;
+}
+
+/**
+* Parser for attribute name value pair
+*/
+void parseAttributes() {
+    char str[32];
+    const char *pname = options[6].value;
+    const char *pend  = options[6].value + strlen(options[6].value);
+    const char *pvalue;
+    
+    while (pname < pend) {
+        pvalue = strchr(pname, '=');
+        if (pvalue == NULL) {
+            fprintf(stdout, 
+                    "No matching value found for attribute argument %s\n", 
+                    pname);        
+            return;
+        }
+        int count = pvalue - pname;
+        strncpy(str, pname, count);
+        str[count] = 0;
+
+        int name = parseEnums(ATTRIBUTE_NAME_, UCOL_ATTRIBUTE_COUNT, str);
+        if (name == -1) {
+            fprintf(stdout, "Attribute name not found: %s\n", str);
+            return;
+        }
+        
+        pvalue ++;
+        // getting corresponding enum value
+        pname = strchr(pvalue, ',');
+        if (pname == NULL) {
+            pname = pend;
+        }
+        count = pname - pvalue;
+        strncpy(str, pvalue, count);
+        str[count] = 0;
+        int value = parseEnums(ATTRIBUTE_VALUE_, UCOL_ATTRIBUTE_VALUE_COUNT, 
+                               str);
+        if (value == -1) {
+            fprintf(stdout, "Attribute value not found: %s\n", str);
+            return;
+        }
+        ATTRIBUTE_[name] = (UColAttributeValue)value;
+        pname ++;
+    }
+}
+
+/**
+* Parser for normalization mode
+*/
+void parseNormalization() {
+    const char *str = options[6].value;
+    int norm = parseEnums(NORMALIZATION_VALUE_, UNORM_MODE_COUNT, str);
+    if (norm == -1) {
+        fprintf(stdout, "Normalization mode not found: %s\n", str);
+        return;
+    }
+    NORMALIZATION_ = (UNormalizationMode)norm;
 }
 
 /** 
@@ -412,14 +733,35 @@ int main(int argc, char *argv[]) {
     }
     if (argc < 0 || options[0].doesOccur || options[1].doesOccur) {
         fprintf(stdout, "Usage: strperf options...\n"
-                        "--help                 Display this message.\n"
+                        "--help\n"
-                        "--locale    [name|all] ICU locale to use. Default is en_US\n"
+                        "    Display this message.\n"
-                        "--serialize            Serializes the collation elements in -locale or all locales available and outputs them into --outputdir/locale_ce.txt\n"
+                        "--locale name|all\n"
-                        "--destdir   dir_name   Path for outputing the serialized collation elements. Defaults to stdout if no defined\n");
+                        "    ICU locale to use. Default is en_US\n"
+                        "--serialize\n"
+                        "    Serializes the collation elements in -locale or all locales available and outputs them into --outputdir/locale_ce.txt\n"
+                        "--destdir dir_name\n"
+                        "    Path for outputing the serialized collation elements. Defaults to stdout if no defined\n"
+                        "--sourcedir dir_name\n"
+                        "    Path for the input rule file for collation\n"
+                        "--attribute name=value,name=value...\n" 
+                        "    Pairs of attribute names and values for setting\n"
+                        "--rule filename\n" 
+                        "    Name of file containing the collation rules.\n"
+                        "--normalizaton mode\n" 
+                        "    UNormalizationMode mode to be used.\n");
+        fprintf(stdout, "Example: dumpce --serialize --locale af --destdir /temp --attribute UCOL_STRENGTH=UCOL_DEFAULT_STRENGTH,4=17");
         return argc < 0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
     }
 
     OUTPUT_ = stdout;
+    if (options[6].doesOccur) {
+        fprintf(stdout, "attributes %s\n", options[6].value);
+        parseAttributes();
+    }
+    if (options[8].doesOccur) {
+        fprintf(stdout, "normalization mode %s\n", options[8].value);
+        parseNormalization();
+    }
     if (options[3].doesOccur) {
         serialize();
     }