529 lines
16 KiB
C
529 lines
16 KiB
C
/*
|
|
*******************************************************************************
|
|
*
|
|
* Copyright (C) 2001-2002, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
*
|
|
*******************************************************************************
|
|
* file name: genidn.c
|
|
* encoding: US-ASCII
|
|
* tab size: 8 (not used)
|
|
* indentation:4
|
|
*
|
|
* created on: 2003-02-06
|
|
* created by: Ram Viswanadha
|
|
*
|
|
* This program reads the rfc3454_*.txt files,
|
|
* parses them, and extracts the data for Nameprep conformance.
|
|
* It then preprocesses it and writes a binary file for efficient use
|
|
* in various IDNA conversion processes.
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include "unicode/utypes.h"
|
|
#include "unicode/uchar.h"
|
|
#include "unicode/putil.h"
|
|
#include "cmemory.h"
|
|
#include "cstring.h"
|
|
#include "unicode/udata.h"
|
|
#include "unewdata.h"
|
|
#include "uoptions.h"
|
|
#include "uparse.h"
|
|
#include "unicode/uset.h"
|
|
#include "uprops.h"
|
|
|
|
U_CDECL_BEGIN
|
|
#include "genidna.h"
|
|
U_CDECL_END
|
|
|
|
#ifdef WIN32
|
|
# pragma warning(disable: 4100)
|
|
#endif
|
|
|
|
UBool beVerbose=FALSE, haveCopyright=TRUE, printRules = FALSE;
|
|
|
|
/* prototypes --------------------------------------------------------------- */
|
|
|
|
static void
|
|
parseMappings(const char *filename, UBool withNorm, UBool reportError, UErrorCode *pErrorCode);
|
|
|
|
static void
|
|
parseTable(const char *filename, UBool isUnassigned, UErrorCode *pErrorCode);
|
|
|
|
static void
|
|
parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode);
|
|
|
|
static void
|
|
setLDHValues(UErrorCode* pErrorCode);
|
|
|
|
static void
|
|
setLabelSeperators(UErrorCode* pErrorCode);
|
|
|
|
static void
|
|
printMapping(UChar32 cp,UChar32* mapping, int32_t mappingLength);
|
|
|
|
static const char* fileNames[] = {
|
|
"rfc3454_A_1.txt", /* contains unassigned code points */
|
|
"rfc3454_C_X.txt", /* contains code points that are prohibited */
|
|
"rfc3454_B_1.txt", /* contains case mappings when normalization is turned off */
|
|
"rfc3454_B_2.txt", /* contains case mappings when normalization it turned on */
|
|
"NormalizationCorrections.txt",/* normalization corrections */
|
|
};
|
|
static const char *UNIDATA_DIR = "unidata";
|
|
static const char *MISC_DIR = "misc";
|
|
|
|
/* -------------------------------------------------------------------------- */
|
|
|
|
static UOption options[]={
|
|
UOPTION_HELP_H,
|
|
UOPTION_HELP_QUESTION_MARK,
|
|
UOPTION_VERBOSE,
|
|
UOPTION_COPYRIGHT,
|
|
UOPTION_DESTDIR,
|
|
UOPTION_SOURCEDIR,
|
|
{ "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 },
|
|
{ "generate-rules", NULL, NULL, NULL, 'g', UOPT_NO_ARG, 0 }
|
|
};
|
|
|
|
extern int
|
|
main(int argc, char* argv[]) {
|
|
char filename[300];
|
|
const char *srcDir=NULL, *destDir=NULL, *suffix=NULL;
|
|
char *basename=NULL;
|
|
char *saveBasename = NULL;
|
|
UErrorCode errorCode=U_ZERO_ERROR;
|
|
|
|
U_MAIN_INIT_ARGS(argc, argv);
|
|
|
|
/* preset then read command line options */
|
|
options[4].value=u_getDataDirectory();
|
|
options[5].value="";
|
|
options[6].value="3.0.0";
|
|
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
|
|
|
|
/* error handling, printing usage message */
|
|
if(argc<0) {
|
|
fprintf(stderr,
|
|
"error in command line argument \"%s\"\n",
|
|
argv[-argc]);
|
|
}
|
|
if(argc<0 || options[0].doesOccur || options[1].doesOccur) {
|
|
/*
|
|
* Broken into chucks because the C89 standard says the minimum
|
|
* required supported string length is 509 bytes.
|
|
*/
|
|
fprintf(stderr,
|
|
"Usage: %s [-options] [suffix]\n"
|
|
"\n"
|
|
"Read the rfc3454_*.txt files and\n"
|
|
"create a binary file " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE " with the normalization data\n"
|
|
"\n",
|
|
argv[0]);
|
|
fprintf(stderr,
|
|
"Options:\n"
|
|
"\t-h or -? or --help this usage text\n"
|
|
"\t-v or --verbose verbose output\n"
|
|
"\t-c or --copyright include a copyright notice\n");
|
|
fprintf(stderr,
|
|
"\t-d or --destdir destination directory, followed by the path\n"
|
|
"\t-s or --sourcedir source directory of ICU data, followed by the path\n"
|
|
"\t-g or --generate-rules generate IDN rules for testing. Will print out rules to STDOUT\n"
|
|
);
|
|
return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
|
|
}
|
|
|
|
/* get the options values */
|
|
beVerbose=options[2].doesOccur;
|
|
haveCopyright=options[3].doesOccur;
|
|
srcDir=options[5].value;
|
|
destDir=options[4].value;
|
|
printRules = options[7].doesOccur;
|
|
|
|
if(argc>=2) {
|
|
suffix=argv[1];
|
|
} else {
|
|
suffix=NULL;
|
|
}
|
|
|
|
setUnicodeVersion(options[6].value);
|
|
|
|
/* prepare the filename beginning with the source dir */
|
|
if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL){
|
|
filename[0] = 0x2E;
|
|
filename[1] = U_FILE_SEP_CHAR;
|
|
uprv_strcpy(filename+2,srcDir);
|
|
}else{
|
|
uprv_strcpy(filename, srcDir);
|
|
}
|
|
basename=filename+uprv_strlen(filename);
|
|
if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
|
|
*basename++=U_FILE_SEP_CHAR;
|
|
}
|
|
|
|
/* initialize */
|
|
init();
|
|
if(printRules){
|
|
printf("// Copyright (C) 2003, International Business Machines\n\n");
|
|
printf("// WARNING: This file is machine generated by %s tool. Please DO NOT edit.\n\n",argv[0]);
|
|
|
|
printf("idn_rules{\n");
|
|
}
|
|
|
|
/* first copy misc directory */
|
|
saveBasename = basename;
|
|
uprv_strcpy(basename,MISC_DIR);
|
|
basename = basename + uprv_strlen(MISC_DIR);
|
|
*basename++=U_FILE_SEP_CHAR;
|
|
|
|
/* process unassigned */
|
|
uprv_strcpy(basename,fileNames[0]);
|
|
parseTable(filename,TRUE, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "Could not open file %s for reading \n", filename);
|
|
return errorCode;
|
|
}
|
|
/* process prohibited */
|
|
uprv_strcpy(basename,fileNames[1]);
|
|
parseTable(filename,FALSE, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "Could not open file %s for reading \n", filename);
|
|
return errorCode;
|
|
}
|
|
|
|
/* setLDHValues(&errorCode); */
|
|
setLabelSeperators(&errorCode);
|
|
|
|
/* process mappings */
|
|
if(printRules){
|
|
printf("\n\tMapNoNormalization{\n");
|
|
}
|
|
uprv_strcpy(basename,fileNames[2]);
|
|
parseMappings(filename, FALSE, FALSE, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "Could not open file %s for reading \n", filename);
|
|
return errorCode;
|
|
}
|
|
if(printRules){
|
|
printf("\n\t}\n");
|
|
}
|
|
|
|
if(printRules){
|
|
printf("\n\tMapNFKC{\n");
|
|
}
|
|
uprv_strcpy(basename,fileNames[3]);
|
|
parseMappings(filename, TRUE, FALSE, &errorCode);
|
|
if(U_FAILURE(errorCode)) {
|
|
fprintf(stderr, "Could not open file %s for reading \n", filename);
|
|
return errorCode;
|
|
}
|
|
/* set up directory for NormalizationCorrections.txt */
|
|
basename = saveBasename;
|
|
uprv_strcpy(basename,UNIDATA_DIR);
|
|
basename = basename + uprv_strlen(UNIDATA_DIR);
|
|
*basename++=U_FILE_SEP_CHAR;
|
|
uprv_strcpy(basename,fileNames[4]);
|
|
|
|
parseNormalizationCorrections(filename,&errorCode);
|
|
if(U_FAILURE(errorCode)){
|
|
fprintf(stderr,"Could not open file %s for reading \n", filename);
|
|
return errorCode;
|
|
}
|
|
|
|
/* process parsed data */
|
|
if(U_SUCCESS(errorCode)) {
|
|
/* write the data file */
|
|
generateData(destDir);
|
|
|
|
cleanUpData();
|
|
}
|
|
if(printRules){
|
|
printf("\t\t\"::[:AGE=3.2:]NFKC;\"\n\t}\n}");
|
|
}
|
|
return errorCode;
|
|
}
|
|
|
|
static void U_CALLCONV
|
|
normalizationCorrectionsLineFn(void *context,
|
|
char *fields[][2], int32_t fieldCount,
|
|
UErrorCode *pErrorCode) {
|
|
uint32_t mapping[40];
|
|
char *end, *s;
|
|
uint32_t code;
|
|
int32_t length;
|
|
UVersionInfo version;
|
|
UVersionInfo thisVersion;
|
|
|
|
/* get the character code, field 0 */
|
|
code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
|
|
if(U_FAILURE(*pErrorCode)) {
|
|
fprintf(stderr, "genidn: error parsing FCNFKC_3_2_0.txt mapping at %s\n", fields[0][0]);
|
|
exit(*pErrorCode);
|
|
}
|
|
/* Original (erroneous) decomposition */
|
|
s = fields[1][0];
|
|
|
|
/* parse the mapping string */
|
|
length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);
|
|
|
|
/* ignore corrected decomposition */
|
|
|
|
u_versionFromString(version,fields[3][0] );
|
|
u_versionFromString(thisVersion, "3.2.0");
|
|
|
|
|
|
|
|
if(U_FAILURE(*pErrorCode)) {
|
|
fprintf(stderr, "genidn error parsing NormalizationCorrection of U+%04lx - %s\n",
|
|
(long)code, u_errorName(*pErrorCode));
|
|
exit(*pErrorCode);
|
|
}
|
|
|
|
/* store the mapping */
|
|
if( version[0] > thisVersion[0] ||
|
|
((version[0]==thisVersion[0]) && (version[1] > thisVersion[1]))
|
|
){
|
|
storeMapping(code,mapping, length, TRUE, pErrorCode);
|
|
if(printRules){
|
|
printMapping(code,(UChar32*)mapping,length);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) {
|
|
char *fields[4][2];
|
|
|
|
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
|
|
return;
|
|
}
|
|
|
|
u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode);
|
|
|
|
/* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */
|
|
|
|
if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) {
|
|
fprintf(stderr, "genidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
|
|
exit(*pErrorCode);
|
|
}
|
|
}
|
|
|
|
static void U_CALLCONV
|
|
caseMapLineFn(void *context,
|
|
char *fields[][2], int32_t fieldCount,
|
|
UErrorCode *pErrorCode) {
|
|
uint32_t mapping[40];
|
|
char *end, *s;
|
|
uint32_t code;
|
|
int32_t length;
|
|
UBool* mapWithNorm = (UBool*) context;
|
|
|
|
|
|
/* get the character code, field 0 */
|
|
code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
|
|
if(end<=fields[0][0] || end!=fields[0][1]) {
|
|
fprintf(stderr, "genidn: syntax error in field 0 at %s\n", fields[0][0]);
|
|
*pErrorCode=U_PARSE_ERROR;
|
|
exit(U_PARSE_ERROR);
|
|
}
|
|
|
|
s = fields[1][0];
|
|
/* parse the mapping string */
|
|
length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);
|
|
|
|
if(U_FAILURE(*pErrorCode)) {
|
|
fprintf(stderr, "genidn error parsing UnicodeData.txt decomposition of U+%04lx - %s\n",
|
|
(long)code, u_errorName(*pErrorCode));
|
|
exit(*pErrorCode);
|
|
}
|
|
|
|
/* store the mapping */
|
|
|
|
storeMapping(code,mapping, length, *mapWithNorm, pErrorCode);
|
|
if(printRules){
|
|
printMapping(code,(UChar32*)mapping,length);
|
|
}
|
|
}
|
|
|
|
static void
|
|
parseMappings(const char *filename,UBool withNorm, UBool reportError, UErrorCode *pErrorCode) {
|
|
char *fields[3][2];
|
|
|
|
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
|
|
return;
|
|
}
|
|
|
|
u_parseDelimitedFile(filename, ';', fields, 3, caseMapLineFn, &withNorm, pErrorCode);
|
|
|
|
/*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/
|
|
|
|
if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
|
|
fprintf(stderr, "genidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
|
|
exit(*pErrorCode);
|
|
}
|
|
}
|
|
|
|
/* parser for UnicodeData.txt ----------------------------------------------- */
|
|
static int32_t printedCharCount = 0;
|
|
|
|
static void printEscaped(UChar32 ch){
|
|
if(ch > 0xFFFF){
|
|
printf("\\\\U%08X",ch);
|
|
printedCharCount+=11;
|
|
}else{
|
|
if(uprv_isRuleWhiteSpace(ch)){
|
|
/* double escape the rule white space */
|
|
printf("\\\\u%04X", ch);
|
|
printedCharCount+=7;
|
|
}else if(0x20< ch && ch <0x7f){
|
|
if(ch == 0x2E){
|
|
/* double escape dot */
|
|
printf("\\\\%c",(char)ch);
|
|
printedCharCount+=3;
|
|
}else{
|
|
printf("%c",(char)ch);
|
|
printedCharCount++;
|
|
}
|
|
}else{
|
|
printf("\\\\u%04X",ch);
|
|
printedCharCount+=7;
|
|
}
|
|
}
|
|
}
|
|
static void printEscapedRange(UChar32 rangeStart, UChar32 rangeEnd){
|
|
if(rangeStart != rangeEnd){
|
|
printEscaped(rangeStart);
|
|
printf("-");
|
|
printedCharCount++;
|
|
printEscaped(rangeEnd);
|
|
printf(" ");
|
|
}else{
|
|
printEscaped(rangeStart);
|
|
printf(" ");
|
|
}
|
|
if(printedCharCount > 70){
|
|
printf("\"\n\t\t\t\"");
|
|
printedCharCount =0 ;
|
|
}
|
|
}
|
|
static void printMapping( UChar32 cp, UChar32* mapping, int32_t mappingLength){
|
|
|
|
int32_t i;
|
|
printf("\t\t\"");
|
|
printEscaped(cp);
|
|
printf(" > ");
|
|
for(i=0;i<mappingLength;i++){
|
|
printEscaped(mapping[i]);
|
|
}
|
|
printf(";\"\n");
|
|
|
|
printedCharCount=0;
|
|
}
|
|
static void U_CALLCONV
|
|
unicodeDataLineFn(void *context,
|
|
char *fields[][2], int32_t fieldCount,
|
|
UErrorCode *pErrorCode) {
|
|
uint32_t rangeStart=0,rangeEnd =0;
|
|
UBool* isUnassigned = (UBool*) context;
|
|
|
|
|
|
u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
|
|
|
|
if(U_FAILURE(*pErrorCode)){
|
|
fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
|
|
return;
|
|
}
|
|
|
|
if(*isUnassigned == TRUE){
|
|
storeRange(rangeStart,rangeEnd,UIDNA_UNASSIGNED, pErrorCode);
|
|
}else{
|
|
storeRange(rangeStart,rangeEnd,UIDNA_PROHIBITED, pErrorCode);
|
|
}
|
|
/*TODO: comment out the printer */
|
|
if(printRules){
|
|
printEscapedRange(rangeStart,rangeEnd);
|
|
}
|
|
}
|
|
|
|
static void
|
|
parseTable(const char *filename,UBool isUnassigned, UErrorCode *pErrorCode) {
|
|
char *fields[1][2];
|
|
int32_t len=0;
|
|
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
|
|
return;
|
|
}
|
|
/*TODO: comment out the printer */
|
|
if(printRules){
|
|
printedCharCount = 0;
|
|
if(isUnassigned){
|
|
printf("\n\tUnassignedSet{\"[ ");
|
|
}else{
|
|
printf("\n\tProhibitedSet{\"[ ");
|
|
}
|
|
}
|
|
u_parseDelimitedFile(filename, ';', fields, 1, unicodeDataLineFn, &isUnassigned, pErrorCode);
|
|
|
|
|
|
if(U_FAILURE(*pErrorCode)) {
|
|
fprintf(stderr, "genidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
|
|
exit(*pErrorCode);
|
|
}
|
|
if(printRules){
|
|
printf("]\"}\n");
|
|
}
|
|
}
|
|
|
|
/*
|
|
static void
|
|
setLDHValues(UErrorCode* pErrorCode){
|
|
USet* set = uset_openPattern(LDH_PATTERN, LDH_PATTERN_LEN, pErrorCode);
|
|
int32_t itemCount;
|
|
int32_t index = 0;
|
|
UChar32 start,end;
|
|
|
|
if(U_FAILURE(*pErrorCode)){
|
|
fprintf(stderr,"Could not open USet. Error :%s \n",u_errorName(*pErrorCode));
|
|
exit(*pErrorCode);
|
|
}
|
|
|
|
itemCount = uset_getItemCount(set);
|
|
|
|
for(;index < itemCount; index++){
|
|
uset_getItem(set,index, &start, &end, NULL, 0, pErrorCode);
|
|
storeRange(start,end,UIDNA_LDH_OR_MAP_NFKC, pErrorCode);
|
|
}
|
|
if(printRules){
|
|
printf(PAT);
|
|
}
|
|
|
|
}
|
|
*/
|
|
static void
|
|
setLabelSeperators(UErrorCode *pErrorCode){
|
|
/* U+002E, U+3002, U+FF0E, U+FF61 */
|
|
storeRange(0x002E, 0x002E, UIDNA_LABEL_SEPARATOR, pErrorCode);
|
|
storeRange(0x3002, 0x3002, UIDNA_LABEL_SEPARATOR, pErrorCode);
|
|
storeRange(0xFF0E, 0xFF0E, UIDNA_LABEL_SEPARATOR, pErrorCode);
|
|
storeRange(0xFF61, 0xFF61, UIDNA_LABEL_SEPARATOR, pErrorCode);
|
|
if(U_FAILURE(*pErrorCode)){
|
|
fprintf(stderr, "Could not store values for label separators\n");
|
|
}
|
|
if(printRules){
|
|
printf("\tLabelSeparatorSet{\"[ ");
|
|
printEscaped(0x002E);
|
|
printEscaped(0x3002);
|
|
printEscaped(0xFF0E);
|
|
printEscaped(0xFF61);
|
|
printf(" ]\"}\n\n");
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Hey, Emacs, please set the following:
|
|
*
|
|
* Local Variables:
|
|
* indent-tabs-mode: nil
|
|
* End:
|
|
*
|
|
*/
|