2001-03-17 00:46:46 +00:00
|
|
|
/*
|
|
|
|
*******************************************************************************
|
|
|
|
*
|
|
|
|
* Copyright (C) 2001, International Business Machines
|
|
|
|
* Corporation and others. All Rights Reserved.
|
|
|
|
*
|
|
|
|
*******************************************************************************
|
|
|
|
* file name: ucol_tok.cpp
|
|
|
|
* encoding: US-ASCII
|
|
|
|
* tab size: 8 (not used)
|
|
|
|
* indentation:4
|
|
|
|
*
|
|
|
|
* created 02/22/2001
|
|
|
|
* created by: Vladimir Weinstein
|
|
|
|
*
|
|
|
|
* This module reads a tailoring rule string and produces a list of
|
|
|
|
* tokens that will be turned into collation elements
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2001-01-31 07:20:56 +00:00
|
|
|
#include "unicode/ustring.h"
|
2001-03-01 00:14:29 +00:00
|
|
|
#include "unicode/uchar.h"
|
2001-02-28 19:01:23 +00:00
|
|
|
|
2001-01-31 07:20:56 +00:00
|
|
|
#include "cmemory.h"
|
2001-03-08 17:40:42 +00:00
|
|
|
#include "ucol_tok.h"
|
2001-01-29 22:09:24 +00:00
|
|
|
#include "uhash.h"
|
2001-01-31 07:20:56 +00:00
|
|
|
#include "ucmp32.h"
|
2001-01-29 22:09:24 +00:00
|
|
|
|
2001-03-01 00:14:29 +00:00
|
|
|
static UHashtable *uchars2tokens = 0;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
|
|
|
static const UChar *rulesToParse = 0;
|
|
|
|
|
|
|
|
/* will use a small structure, tokHash */
|
|
|
|
|
2001-02-28 19:01:23 +00:00
|
|
|
int32_t
|
2001-01-31 07:20:56 +00:00
|
|
|
uhash_hashTokens(const void *k) {
|
|
|
|
int32_t hash = 0;
|
|
|
|
if (k != NULL) {
|
|
|
|
const UColToken *key = (const UColToken *)k;
|
|
|
|
int32_t len = (key->source & 0xFF000000)>>24;
|
|
|
|
int32_t inc = ((len - 32) / 32) + 1;
|
|
|
|
|
|
|
|
const UChar *p = (key->source & 0x00FFFFFF) + rulesToParse;
|
|
|
|
const UChar *limit = p + len;
|
|
|
|
|
|
|
|
while (p<limit) {
|
|
|
|
hash = (hash * 37) + *p;
|
|
|
|
p += inc;
|
|
|
|
}
|
|
|
|
|
|
|
|
if((len = ((key->expansion & 0xFF000000)>>24)) != 0) {
|
|
|
|
p = (key->expansion & 0x00FFFFFF) + rulesToParse;
|
|
|
|
limit = p + len;
|
|
|
|
while (p<limit) {
|
|
|
|
hash = (hash * 37) + *p;
|
|
|
|
p += inc;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return hash;
|
|
|
|
}
|
|
|
|
|
2001-02-28 19:01:23 +00:00
|
|
|
UBool uhash_compareTokens(const void *key1, const void *key2) {
|
2001-01-31 07:20:56 +00:00
|
|
|
const UColToken *p1 = (const UColToken*) key1;
|
|
|
|
const UColToken *p2 = (const UColToken*) key2;
|
|
|
|
const UChar *s1 = (p1->source & 0x00FFFFFF) + rulesToParse;
|
|
|
|
const UChar *s2 = (p2->source & 0x00FFFFFF) + rulesToParse;
|
|
|
|
uint32_t s1L = ((p1->source & 0xFF000000) >> 24);
|
|
|
|
uint32_t s2L = ((p2->source & 0xFF000000) >> 24);
|
|
|
|
|
|
|
|
if (p1 == p2) {
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
if (p1 == NULL || p2 == NULL) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
if(p1->source == p2->source && p1->expansion == p2->expansion) {
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
if(s1L != s2L) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
while(s1 < s1+s1L-1 && *s1 == *s2) {
|
|
|
|
++s1;
|
|
|
|
++s2;
|
|
|
|
}
|
|
|
|
if(*s1 == *s2) {
|
|
|
|
s1 = (p1->expansion & 0x00FFFFFF) + rulesToParse;
|
|
|
|
s2 = (p2->expansion & 0x00FFFFFF) + rulesToParse;
|
|
|
|
s1L = ((p1->expansion & 0xFF000000) >> 24);
|
|
|
|
s2L = ((p2->expansion & 0xFF000000) >> 24);
|
|
|
|
if(s1L != s2L) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
if(s1L != 0) {
|
|
|
|
while(s1 < s1+s1L-1 && *s1 == *s2) {
|
|
|
|
++s1;
|
|
|
|
++s2;
|
|
|
|
}
|
|
|
|
return (UBool)(*s1 == *s2);
|
|
|
|
} else {
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
}
|
2001-01-29 22:09:24 +00:00
|
|
|
|
2001-02-28 21:50:23 +00:00
|
|
|
void deleteToken(void *token) {
|
|
|
|
UColToken *tok = (UColToken *)token;
|
|
|
|
uprv_free(tok);
|
2001-01-29 22:09:24 +00:00
|
|
|
}
|
|
|
|
|
2001-01-31 07:20:56 +00:00
|
|
|
void ucol_tok_initTokenList(UColTokenParser *src, UErrorCode *status) {
|
2001-01-29 22:09:24 +00:00
|
|
|
if(U_FAILURE(*status)) {
|
|
|
|
return;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
rulesToParse = src->source;
|
2001-02-28 21:50:23 +00:00
|
|
|
src->lh = (UColTokListHeader *)uprv_malloc(256*sizeof(UColTokListHeader));
|
|
|
|
src->resultLen = 0;
|
2001-01-31 07:20:56 +00:00
|
|
|
uchars2tokens = uhash_open(uhash_hashTokens, uhash_compareTokens, status);
|
2001-02-28 21:50:23 +00:00
|
|
|
uhash_setValueDeleter(uchars2tokens, deleteToken);
|
2001-01-29 22:09:24 +00:00
|
|
|
}
|
|
|
|
|
2001-03-06 07:44:37 +00:00
|
|
|
/* -1 off, 1 on, 0 neither */
|
|
|
|
int32_t ucol_uprv_tok_isOnorOf(const UChar* onoff) {
|
|
|
|
if(onoff) {
|
|
|
|
if(u_tolower(*onoff) == 0x006F /*'o'*/) {
|
|
|
|
onoff++;
|
|
|
|
if(u_tolower(*onoff) == 0x006e /*'n'*/) {
|
|
|
|
return 1;
|
|
|
|
} else if(u_tolower(*onoff) == 0x0066 /*'f'*/) {
|
|
|
|
onoff++;
|
|
|
|
if(u_tolower(*onoff) == 0x0066 /*'f'*/) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2001-03-06 00:57:48 +00:00
|
|
|
|
2001-03-30 00:23:46 +00:00
|
|
|
void ucol_uprv_tok_setOptionInImage(UColOptionSet *opts, UColAttribute attrib, UColAttributeValue value) {
|
2001-03-07 19:43:06 +00:00
|
|
|
switch(attrib) {
|
|
|
|
case UCOL_FRENCH_COLLATION:
|
2001-03-30 00:23:46 +00:00
|
|
|
opts->frenchCollation = value;
|
2001-03-07 19:43:06 +00:00
|
|
|
break;
|
|
|
|
case UCOL_ALTERNATE_HANDLING:
|
2001-03-30 00:23:46 +00:00
|
|
|
opts->alternateHandling = value;
|
2001-03-07 19:43:06 +00:00
|
|
|
break;
|
|
|
|
case UCOL_CASE_FIRST:
|
2001-03-30 00:23:46 +00:00
|
|
|
opts->caseFirst = value;
|
2001-03-07 19:43:06 +00:00
|
|
|
break;
|
|
|
|
case UCOL_CASE_LEVEL:
|
2001-03-30 00:23:46 +00:00
|
|
|
opts->caseLevel = value;
|
2001-03-07 19:43:06 +00:00
|
|
|
break;
|
|
|
|
case UCOL_NORMALIZATION_MODE:
|
2001-03-30 00:23:46 +00:00
|
|
|
opts->normalizationMode = value;
|
2001-03-07 19:43:06 +00:00
|
|
|
break;
|
|
|
|
case UCOL_STRENGTH:
|
2001-03-30 00:23:46 +00:00
|
|
|
opts->strength = value;
|
2001-03-07 19:43:06 +00:00
|
|
|
break;
|
|
|
|
case UCOL_ATTRIBUTE_COUNT:
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2001-05-02 05:05:06 +00:00
|
|
|
#define UTOK_OPTION_COUNT 13
|
2001-03-07 19:43:06 +00:00
|
|
|
|
2001-03-22 21:16:20 +00:00
|
|
|
static UBool didInit = FALSE;
|
2001-03-07 19:43:06 +00:00
|
|
|
/* we can be strict, or we can be lenient */
|
|
|
|
/* I'd surely be lenient with the option arguments */
|
|
|
|
/* maybe even with options */
|
|
|
|
U_STRING_DECL(suboption_00, "non-ignorable", 13);
|
|
|
|
U_STRING_DECL(suboption_01, "shifted", 7);
|
|
|
|
|
|
|
|
U_STRING_DECL(suboption_02, "lower", 5);
|
|
|
|
U_STRING_DECL(suboption_03, "upper", 5);
|
|
|
|
U_STRING_DECL(suboption_04, "off", 3);
|
|
|
|
U_STRING_DECL(suboption_05, "on", 2);
|
2001-05-02 05:05:06 +00:00
|
|
|
U_STRING_DECL(suboption_06, "1", 1);
|
|
|
|
U_STRING_DECL(suboption_07, "2", 1);
|
|
|
|
U_STRING_DECL(suboption_08, "3", 1);
|
2001-03-07 19:43:06 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
U_STRING_DECL(option_00, "undefined", 9);
|
|
|
|
U_STRING_DECL(option_01, "rearrange", 9);
|
|
|
|
U_STRING_DECL(option_02, "alternate", 9);
|
|
|
|
U_STRING_DECL(option_03, "backwards", 9);
|
|
|
|
U_STRING_DECL(option_04, "variable top", 12);
|
|
|
|
U_STRING_DECL(option_05, "top", 3);
|
|
|
|
U_STRING_DECL(option_06, "normalization", 13);
|
|
|
|
U_STRING_DECL(option_07, "caseLevel", 9);
|
|
|
|
U_STRING_DECL(option_08, "caseFirst", 9);
|
|
|
|
U_STRING_DECL(option_09, "scriptOrder", 11);
|
|
|
|
U_STRING_DECL(option_10, "charsetname", 11);
|
|
|
|
U_STRING_DECL(option_11, "charset", 7);
|
2001-05-02 05:05:06 +00:00
|
|
|
U_STRING_DECL(option_12, "before", 6);
|
|
|
|
|
2001-03-07 19:43:06 +00:00
|
|
|
|
|
|
|
ucolTokSuboption alternateSub[2] = {
|
|
|
|
{suboption_00, 13, UCOL_NON_IGNORABLE},
|
|
|
|
{suboption_01, 7, UCOL_SHIFTED}
|
|
|
|
};
|
|
|
|
|
|
|
|
ucolTokSuboption caseFirstSub[3] = {
|
2001-04-25 22:28:30 +00:00
|
|
|
{suboption_02, 5, UCOL_LOWER_FIRST},
|
|
|
|
{suboption_03, 5, UCOL_UPPER_FIRST},
|
|
|
|
{suboption_04, 3, UCOL_OFF},
|
2001-03-07 19:43:06 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
ucolTokSuboption onOffSub[2] = {
|
|
|
|
{suboption_04, 3, UCOL_OFF},
|
|
|
|
{suboption_05, 2, UCOL_ON}
|
|
|
|
};
|
|
|
|
|
|
|
|
ucolTokSuboption frenchSub[1] = {
|
2001-05-02 05:05:06 +00:00
|
|
|
{suboption_07, 1, UCOL_ON}
|
2001-03-07 19:43:06 +00:00
|
|
|
};
|
|
|
|
|
2001-05-02 05:05:06 +00:00
|
|
|
ucolTokSuboption beforeSub[3] = {
|
|
|
|
{suboption_06, 1, UCOL_PRIMARY},
|
|
|
|
{suboption_07, 1, UCOL_SECONDARY},
|
|
|
|
{suboption_08, 1, UCOL_TERTIARY}
|
|
|
|
};
|
2001-03-07 19:43:06 +00:00
|
|
|
|
|
|
|
|
|
|
|
ucolTokOption rulesOptions[UTOK_OPTION_COUNT] = {
|
|
|
|
{option_02, 9, alternateSub, 2, UCOL_ALTERNATE_HANDLING}, /*"alternate" */
|
|
|
|
{option_03, 9, frenchSub, 1, UCOL_FRENCH_COLLATION}, /*"backwards" */
|
|
|
|
{option_07, 9, onOffSub, 2, UCOL_CASE_LEVEL}, /*"caseLevel" */
|
|
|
|
{option_08, 9, caseFirstSub, 3, UCOL_CASE_FIRST}, /*"caseFirst" */
|
|
|
|
{option_06, 13, onOffSub, 2, UCOL_NORMALIZATION_MODE}, /*"normalization" */
|
|
|
|
{option_04, 12, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"variable top" */
|
|
|
|
{option_01, 9, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"rearrange" */
|
|
|
|
{option_05, 3, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"top" */
|
2001-05-02 05:05:06 +00:00
|
|
|
{option_12, 6, beforeSub, 3, UCOL_ATTRIBUTE_COUNT}, /*"before" */
|
2001-03-07 19:43:06 +00:00
|
|
|
{option_00, 9, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"undefined" */
|
|
|
|
{option_09, 11, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"scriptOrder" */
|
|
|
|
{option_10, 11, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"charsetname" */
|
|
|
|
{option_11, 7, NULL, 0, UCOL_ATTRIBUTE_COUNT} /*"charset" */
|
|
|
|
};
|
|
|
|
|
2001-04-19 17:08:07 +00:00
|
|
|
int32_t
|
|
|
|
u_strncmpNoCase(const UChar *s1,
|
|
|
|
const UChar *s2,
|
|
|
|
int32_t n)
|
|
|
|
{
|
|
|
|
if(n > 0) {
|
|
|
|
int32_t rc;
|
|
|
|
for(;;) {
|
|
|
|
rc = (int32_t)u_tolower(*s1) - (int32_t)u_tolower(*s2);
|
|
|
|
if(rc != 0 || *s1 == 0 || --n == 0) {
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
++s1;
|
|
|
|
++s2;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2001-05-02 05:05:06 +00:00
|
|
|
uint8_t ucol_uprv_tok_readAndSetOption(UColOptionSet *opts, const UChar* start, const UChar *end, UErrorCode *status) {
|
2001-03-06 07:44:37 +00:00
|
|
|
uint32_t i = 0;
|
2001-03-07 19:43:06 +00:00
|
|
|
int32_t j=0;
|
2001-03-06 07:44:37 +00:00
|
|
|
UBool foundOption = FALSE;
|
|
|
|
const UChar *optionArg = NULL;
|
2001-03-06 00:57:48 +00:00
|
|
|
if(!didInit) {
|
2001-03-07 19:43:06 +00:00
|
|
|
U_STRING_INIT(suboption_00, "non-ignorable", 13);
|
|
|
|
U_STRING_INIT(suboption_01, "shifted", 7);
|
|
|
|
|
|
|
|
U_STRING_INIT(suboption_02, "lower", 5);
|
|
|
|
U_STRING_INIT(suboption_03, "upper", 5);
|
|
|
|
U_STRING_INIT(suboption_04, "off", 3);
|
|
|
|
U_STRING_INIT(suboption_05, "on", 2);
|
|
|
|
|
2001-05-02 05:05:06 +00:00
|
|
|
U_STRING_INIT(suboption_06, "1", 1);
|
|
|
|
U_STRING_INIT(suboption_07, "2", 1);
|
|
|
|
U_STRING_INIT(suboption_08, "3", 1);
|
2001-03-07 19:43:06 +00:00
|
|
|
|
|
|
|
|
2001-03-06 07:44:37 +00:00
|
|
|
U_STRING_INIT(option_00, "undefined", 9);
|
|
|
|
U_STRING_INIT(option_01, "rearrange", 9);
|
|
|
|
U_STRING_INIT(option_02, "alternate", 9);
|
|
|
|
U_STRING_INIT(option_03, "backwards", 9);
|
|
|
|
U_STRING_INIT(option_04, "variable top", 12);
|
|
|
|
U_STRING_INIT(option_05, "top", 3);
|
|
|
|
U_STRING_INIT(option_06, "normalization", 13);
|
|
|
|
U_STRING_INIT(option_07, "caseLevel", 9);
|
|
|
|
U_STRING_INIT(option_08, "caseFirst", 9);
|
|
|
|
U_STRING_INIT(option_09, "scriptOrder", 11);
|
|
|
|
U_STRING_INIT(option_10, "charsetname", 11);
|
|
|
|
U_STRING_INIT(option_11, "charset", 7);
|
2001-05-02 05:05:06 +00:00
|
|
|
U_STRING_INIT(option_12, "before", 6);
|
2001-03-06 07:44:37 +00:00
|
|
|
}
|
|
|
|
start++; /*skip opening '['*/
|
|
|
|
while(i < UTOK_OPTION_COUNT) {
|
2001-04-19 17:08:07 +00:00
|
|
|
if(u_strncmpNoCase(start, rulesOptions[i].optionName, rulesOptions[i].optionLen) == 0) {
|
2001-03-06 07:44:37 +00:00
|
|
|
foundOption = TRUE;
|
|
|
|
if(end - start > rulesOptions[i].optionLen) {
|
|
|
|
optionArg = start+rulesOptions[i].optionLen+1; /* start of the options, skip space */
|
|
|
|
while(u_isWhitespace(*optionArg)) { /* eat whitespace */
|
|
|
|
optionArg++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(!foundOption) {
|
|
|
|
*status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
|
|
return FALSE;
|
2001-03-06 00:57:48 +00:00
|
|
|
}
|
2001-03-06 07:44:37 +00:00
|
|
|
|
2001-03-07 19:43:06 +00:00
|
|
|
if(i<5) {
|
2001-03-06 07:44:37 +00:00
|
|
|
if(optionArg) {
|
2001-03-07 19:43:06 +00:00
|
|
|
for(j = 0; j<rulesOptions[i].subSize; j++) {
|
2001-04-25 22:28:30 +00:00
|
|
|
if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
|
2001-03-30 00:23:46 +00:00
|
|
|
ucol_uprv_tok_setOptionInImage(opts, rulesOptions[i].attr, rulesOptions[i].subopts[j].attrVal);
|
2001-05-02 05:05:06 +00:00
|
|
|
return UCOL_TOK_SUCCESS;
|
2001-03-07 19:43:06 +00:00
|
|
|
}
|
|
|
|
}
|
2001-03-06 07:44:37 +00:00
|
|
|
}
|
2001-03-07 19:43:06 +00:00
|
|
|
*status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
|
|
return FALSE;
|
|
|
|
} else if(i == 5) { /* variable top */
|
2001-05-02 05:05:06 +00:00
|
|
|
return UCOL_TOK_SUCCESS | UCOL_TOK_VARIABLE_TOP;
|
2001-03-07 19:43:06 +00:00
|
|
|
} else if(i == 6) { /*rearange */
|
2001-05-02 05:05:06 +00:00
|
|
|
return UCOL_TOK_SUCCESS;
|
2001-03-07 19:43:06 +00:00
|
|
|
} else if(i == 7) { /*top */
|
2001-05-02 05:05:06 +00:00
|
|
|
return UCOL_TOK_SUCCESS | UCOL_TOK_TOP;
|
|
|
|
} else if(i == 8) { /*before*/
|
|
|
|
if(optionArg) {
|
|
|
|
for(j = 0; j<rulesOptions[i].subSize; j++) {
|
|
|
|
if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
|
|
|
|
return UCOL_TOK_SUCCESS | rulesOptions[i].subopts[j].attrVal + 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
|
|
return 0;
|
2001-03-07 19:43:06 +00:00
|
|
|
} else {
|
|
|
|
*status = U_UNSUPPORTED_ERROR;
|
2001-05-02 05:05:06 +00:00
|
|
|
return 0;
|
2001-03-06 07:44:37 +00:00
|
|
|
}
|
2001-03-06 00:57:48 +00:00
|
|
|
}
|
2001-02-27 21:01:11 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
const UChar *ucol_tok_parseNextToken(UColTokenParser *src,
|
|
|
|
uint32_t *strength,
|
|
|
|
uint32_t *chOffset, uint32_t *chLen,
|
|
|
|
uint32_t *exOffset, uint32_t *exLen,
|
2001-05-02 05:05:06 +00:00
|
|
|
uint8_t *specs,
|
2001-03-20 00:56:37 +00:00
|
|
|
UBool startOfRules,
|
|
|
|
UErrorCode *status) {
|
|
|
|
/* parsing part */
|
2001-01-29 22:09:24 +00:00
|
|
|
|
2001-03-06 00:57:48 +00:00
|
|
|
UBool variableTop = FALSE;
|
2001-03-07 19:43:06 +00:00
|
|
|
UBool top = FALSE;
|
2001-03-20 00:56:37 +00:00
|
|
|
UBool inChars = TRUE;
|
|
|
|
UBool inQuote = FALSE;
|
|
|
|
UBool wasInQuote = FALSE;
|
|
|
|
UChar *optionEnd = NULL;
|
2001-05-02 05:05:06 +00:00
|
|
|
uint8_t before = 0;
|
2001-01-31 20:17:12 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
uint32_t newCharsLen = 0, newExtensionLen = 0;
|
|
|
|
uint32_t charsOffset = 0, extensionOffset = 0;
|
2001-01-31 20:17:12 +00:00
|
|
|
uint32_t newStrength = UCOL_TOK_UNSET;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
while (src->current < src->end) {
|
|
|
|
UChar ch = *(src->current);
|
2001-02-28 21:50:23 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
if (inQuote) {
|
|
|
|
if (ch == 0x0027/*'\''*/) {
|
|
|
|
inQuote = FALSE;
|
|
|
|
} else {
|
|
|
|
if ((newCharsLen == 0) || inChars) {
|
|
|
|
if(newCharsLen == 0) {
|
|
|
|
charsOffset = src->extraCurrent - src->source;
|
|
|
|
}
|
|
|
|
newCharsLen++;
|
|
|
|
} else {
|
|
|
|
if(newExtensionLen == 0) {
|
|
|
|
extensionOffset = src->extraCurrent - src->source;
|
|
|
|
}
|
|
|
|
newExtensionLen++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* Sets the strength for this entry */
|
|
|
|
switch (ch) {
|
|
|
|
case 0x003D/*'='*/ :
|
|
|
|
if (newStrength != UCOL_TOK_UNSET) {
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
/* if we start with strength, we'll reset to top */
|
|
|
|
if(startOfRules == TRUE) {
|
|
|
|
top = TRUE;
|
|
|
|
newStrength = UCOL_TOK_RESET;
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
|
|
|
newStrength = UCOL_IDENTICAL;
|
|
|
|
break;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
case 0x002C/*','*/:
|
|
|
|
if (newStrength != UCOL_TOK_UNSET) {
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
2001-01-31 21:10:55 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
/* if we start with strength, we'll reset to top */
|
|
|
|
if(startOfRules == TRUE) {
|
|
|
|
top = TRUE;
|
|
|
|
newStrength = UCOL_TOK_RESET;
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
|
|
|
newStrength = UCOL_TERTIARY;
|
|
|
|
break;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
case 0x003B/*';'*/:
|
|
|
|
if (newStrength != UCOL_TOK_UNSET) {
|
|
|
|
goto EndOfLoop;
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
/* if we start with strength, we'll reset to top */
|
|
|
|
if(startOfRules == TRUE) {
|
|
|
|
top = TRUE;
|
|
|
|
newStrength = UCOL_TOK_RESET;
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
|
|
|
newStrength = UCOL_SECONDARY;
|
|
|
|
break;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
case 0x003C/*'<'*/:
|
|
|
|
if (newStrength != UCOL_TOK_UNSET) {
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
/* if we start with strength, we'll reset to top */
|
|
|
|
if(startOfRules == TRUE) {
|
|
|
|
top = TRUE;
|
|
|
|
newStrength = UCOL_TOK_RESET;
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
|
|
|
/* before this, do a scan to verify whether this is */
|
|
|
|
/* another strength */
|
|
|
|
if(*(src->current+1) == 0x003C) {
|
|
|
|
src->current++;
|
|
|
|
if(*(src->current+1) == 0x003C) {
|
|
|
|
src->current++; /* three in a row! */
|
2001-01-31 07:20:56 +00:00
|
|
|
newStrength = UCOL_TERTIARY;
|
2001-03-20 00:56:37 +00:00
|
|
|
} else { /* two in a row */
|
2001-01-31 07:20:56 +00:00
|
|
|
newStrength = UCOL_SECONDARY;
|
2001-03-20 00:56:37 +00:00
|
|
|
}
|
|
|
|
} else { /* just one */
|
|
|
|
newStrength = UCOL_PRIMARY;
|
|
|
|
}
|
|
|
|
break;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
case 0x0026/*'&'*/:
|
|
|
|
if (newStrength != UCOL_TOK_UNSET) {
|
|
|
|
goto EndOfLoop;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
newStrength = UCOL_TOK_RESET; /* PatternEntry::RESET = 0 */
|
|
|
|
break;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
case 0x005b/*'['*/:
|
|
|
|
/* options - read an option, analyze it */
|
|
|
|
if((optionEnd = u_strchr(src->current, 0x005d /*']'*/)) != NULL) {
|
2001-05-02 05:05:06 +00:00
|
|
|
uint8_t result = ucol_uprv_tok_readAndSetOption(src->opts, src->current, optionEnd, status);
|
2001-03-20 00:56:37 +00:00
|
|
|
src->current = optionEnd;
|
2001-05-02 05:05:06 +00:00
|
|
|
if(U_SUCCESS(*status)) {
|
|
|
|
if(result & UCOL_TOK_TOP) {
|
|
|
|
if(newStrength == UCOL_TOK_RESET) {
|
|
|
|
top = TRUE;
|
|
|
|
src->current++;
|
|
|
|
goto EndOfLoop;
|
|
|
|
} else {
|
|
|
|
*status = U_INVALID_FORMAT_ERROR;
|
|
|
|
}
|
|
|
|
} else if(result & UCOL_TOK_VARIABLE_TOP) {
|
|
|
|
if(newStrength != UCOL_TOK_RESET && newStrength != UCOL_TOK_UNSET) {
|
|
|
|
variableTop = TRUE;
|
|
|
|
charsOffset = src->extraCurrent - src->source;
|
|
|
|
newCharsLen = 1;
|
|
|
|
*src->extraCurrent++ = 0xFFFF;
|
|
|
|
src->current++;
|
|
|
|
goto EndOfLoop;
|
|
|
|
} else {
|
|
|
|
*status = U_INVALID_FORMAT_ERROR;
|
|
|
|
}
|
|
|
|
} else if (result & UCOL_TOK_BEFORE){
|
|
|
|
if(newStrength == UCOL_TOK_RESET) {
|
|
|
|
before = result & UCOL_TOK_BEFORE;
|
|
|
|
} else {
|
|
|
|
*status = U_INVALID_FORMAT_ERROR;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
2001-03-20 00:56:37 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
/* Ignore the white spaces */
|
|
|
|
case 0x0009/*'\t'*/:
|
|
|
|
case 0x000C/*'\f'*/:
|
|
|
|
case 0x000D/*'\r'*/:
|
|
|
|
case 0x000A/*'\n'*/:
|
|
|
|
case 0x0020/*' '*/:
|
|
|
|
break; /* skip whitespace TODO use Unicode */
|
|
|
|
|
|
|
|
case 0x002F/*'/'*/:
|
|
|
|
/* This entry has an extension. */
|
|
|
|
inChars = FALSE;
|
|
|
|
break;
|
|
|
|
|
|
|
|
/* found a quote, we're gonna start copying */
|
|
|
|
case 0x0027/*'\''*/:
|
|
|
|
inQuote = TRUE;
|
|
|
|
wasInQuote = TRUE;
|
|
|
|
|
|
|
|
if (newCharsLen == 0) {
|
|
|
|
charsOffset = src->extraCurrent - src->source;
|
|
|
|
newCharsLen++;
|
|
|
|
} else if (inChars) { /* we're reading some chars */
|
|
|
|
charsOffset = src->extraCurrent - src->source;
|
|
|
|
if(newCharsLen != 0) {
|
|
|
|
uprv_memcpy(src->extraCurrent, src->current - newCharsLen, newCharsLen*sizeof(UChar));
|
|
|
|
src->extraCurrent += newCharsLen;
|
|
|
|
}
|
|
|
|
newCharsLen++;
|
|
|
|
} else {
|
|
|
|
if(newExtensionLen != 0) {
|
|
|
|
uprv_memcpy(src->extraCurrent, src->current - newExtensionLen, newExtensionLen*sizeof(UChar));
|
|
|
|
src->extraCurrent += newExtensionLen;
|
|
|
|
}
|
|
|
|
newExtensionLen++;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
ch = *(++(src->current)); /*pattern[++index]; */
|
2001-03-21 00:22:35 +00:00
|
|
|
if(ch == 0x0027) {
|
|
|
|
*src->extraCurrent++ = ch;
|
|
|
|
inQuote = FALSE;
|
|
|
|
}
|
2001-03-20 00:56:37 +00:00
|
|
|
break;
|
2001-02-26 10:28:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
/* '@' is french only if the strength is not currently set */
|
|
|
|
/* if it is, it's just a regular character in collation rules */
|
|
|
|
case 0x0040/*'@'*/:
|
|
|
|
if (newStrength == UCOL_TOK_UNSET) {
|
2001-03-30 00:23:46 +00:00
|
|
|
src->opts->frenchCollation = UCOL_ON;
|
2001-03-20 00:56:37 +00:00
|
|
|
break;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
default:
|
|
|
|
if (newStrength == UCOL_TOK_UNSET) {
|
|
|
|
*status = U_INVALID_FORMAT_ERROR;
|
|
|
|
return NULL;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
if (ucol_tok_isSpecialChar(ch) && (inQuote == FALSE)) {
|
|
|
|
*status = U_INVALID_FORMAT_ERROR;
|
|
|
|
return NULL;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-05-15 17:36:39 +00:00
|
|
|
if(ch == 0x0000 && src->current+1 == src->end) {
|
|
|
|
break;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
if (inChars) {
|
|
|
|
if(newCharsLen == 0) {
|
|
|
|
charsOffset = src->current - src->source;
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
2001-03-20 00:56:37 +00:00
|
|
|
newCharsLen++;
|
|
|
|
} else {
|
|
|
|
if(newExtensionLen == 0) {
|
|
|
|
extensionOffset = src->current - src->source;
|
|
|
|
}
|
|
|
|
newExtensionLen++;
|
2001-03-14 00:12:46 +00:00
|
|
|
}
|
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
break;
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
2001-03-20 00:56:37 +00:00
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
if(wasInQuote) {
|
2001-03-21 00:22:35 +00:00
|
|
|
if(ch != 0x27) {
|
2001-03-20 00:56:37 +00:00
|
|
|
*src->extraCurrent++ = ch;
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
2001-03-20 00:56:37 +00:00
|
|
|
if(src->extraCurrent == src->extraEnd) {
|
|
|
|
/* reallocate */
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
src->current++;
|
|
|
|
}
|
|
|
|
|
|
|
|
EndOfLoop:
|
|
|
|
wasInQuote = FALSE;
|
|
|
|
if (newStrength == UCOL_TOK_UNSET) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2001-04-19 17:08:07 +00:00
|
|
|
if (newCharsLen == 0 && top == FALSE) {
|
2001-03-20 00:56:37 +00:00
|
|
|
*status = U_INVALID_FORMAT_ERROR;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
*strength = newStrength;
|
|
|
|
|
|
|
|
*chOffset = charsOffset;
|
|
|
|
*chLen = newCharsLen;
|
|
|
|
*exOffset = extensionOffset;
|
|
|
|
*exLen = newExtensionLen;
|
2001-05-02 05:05:06 +00:00
|
|
|
*specs = (UCOL_TOK_VARIABLE_TOP * (variableTop?1:0)) | (UCOL_TOK_TOP * (top?1:0)) | before;
|
2001-03-20 00:56:37 +00:00
|
|
|
|
|
|
|
return src->current;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
Processing Description
|
|
|
|
1 Build a ListList. Each list has a header, which contains two lists (positive
|
|
|
|
and negative), a reset token, a baseCE, nextCE, and previousCE. The lists and
|
|
|
|
reset may be null.
|
|
|
|
2 As you process, you keep a LAST pointer that points to the last token you
|
|
|
|
handled.
|
|
|
|
*/
|
|
|
|
|
|
|
|
uint32_t ucol_uprv_tok_assembleTokenList(UColTokenParser *src, UErrorCode *status) {
|
|
|
|
UColToken *lastToken = NULL;
|
|
|
|
const UChar *parseEnd = NULL;
|
|
|
|
uint32_t expandNext = 0;
|
|
|
|
UBool variableTop = FALSE;
|
|
|
|
UBool top = FALSE;
|
2001-05-02 05:05:06 +00:00
|
|
|
uint8_t specs = 0;
|
2001-03-20 00:56:37 +00:00
|
|
|
|
|
|
|
UColTokListHeader *ListList = NULL;
|
|
|
|
|
|
|
|
uint32_t newCharsLen = 0, newExtensionsLen = 0;
|
|
|
|
uint32_t charsOffset = 0, extensionOffset = 0;
|
|
|
|
uint32_t newStrength = UCOL_TOK_UNSET;
|
|
|
|
|
|
|
|
ucol_tok_initTokenList(src, status);
|
|
|
|
|
|
|
|
ListList = src->lh;
|
|
|
|
|
|
|
|
while(src->current < src->end) {
|
|
|
|
|
|
|
|
parseEnd = ucol_tok_parseNextToken(src,
|
|
|
|
&newStrength,
|
|
|
|
&charsOffset, &newCharsLen,
|
|
|
|
&extensionOffset, &newExtensionsLen,
|
2001-05-02 05:05:06 +00:00
|
|
|
&specs,
|
2001-03-20 00:56:37 +00:00
|
|
|
(UBool)(lastToken == NULL),
|
|
|
|
status);
|
|
|
|
|
2001-05-02 05:05:06 +00:00
|
|
|
variableTop = ((specs & UCOL_TOK_VARIABLE_TOP) != 0);
|
|
|
|
top = ((specs & UCOL_TOK_TOP) != 0);
|
|
|
|
|
2001-03-20 00:56:37 +00:00
|
|
|
if(U_SUCCESS(*status) && parseEnd != NULL) {
|
2001-01-31 07:20:56 +00:00
|
|
|
UColToken *sourceToken = NULL;
|
|
|
|
UColToken key;
|
2001-05-18 23:48:16 +00:00
|
|
|
uint32_t lastStrength = UCOL_TOK_UNSET;
|
|
|
|
|
|
|
|
if(lastToken != NULL ) {
|
|
|
|
lastStrength = lastToken->strength;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
|
|
|
|
key.source = newCharsLen << 24 | charsOffset;
|
|
|
|
key.expansion = newExtensionsLen << 24 | extensionOffset;
|
|
|
|
|
|
|
|
/* 4 Lookup each [source, expansion] in the CharsToToken map, and find a sourceToken */
|
|
|
|
sourceToken = (UColToken *)uhash_get(uchars2tokens, &key);
|
2001-01-31 21:10:55 +00:00
|
|
|
|
2001-01-31 07:20:56 +00:00
|
|
|
if(newStrength != UCOL_TOK_RESET) {
|
2001-01-31 20:17:12 +00:00
|
|
|
if(lastToken == NULL) { /* this means that rules haven't started properly */
|
|
|
|
*status = U_INVALID_FORMAT_ERROR;
|
|
|
|
return 0;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
/* 6 Otherwise (when relation != reset) */
|
|
|
|
if(sourceToken == NULL) {
|
|
|
|
/* If sourceToken is null, create new one, */
|
|
|
|
sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
|
|
|
|
sourceToken->source = newCharsLen << 24 | charsOffset;
|
|
|
|
sourceToken->expansion = newExtensionsLen << 24 | extensionOffset;
|
2001-01-31 21:10:55 +00:00
|
|
|
|
|
|
|
sourceToken->debugSource = *(src->source + charsOffset);
|
2001-01-31 23:12:37 +00:00
|
|
|
if(newExtensionsLen > 0) {
|
|
|
|
sourceToken->debugExpansion = *(src->source + extensionOffset);
|
|
|
|
} else {
|
|
|
|
sourceToken->debugExpansion = 0;
|
|
|
|
}
|
|
|
|
|
2001-01-31 21:10:55 +00:00
|
|
|
|
2001-01-31 23:12:37 +00:00
|
|
|
sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
|
2001-01-31 07:20:56 +00:00
|
|
|
sourceToken->next = NULL;
|
|
|
|
sourceToken->previous = NULL;
|
2001-02-26 10:28:56 +00:00
|
|
|
sourceToken->noOfCEs = 0;
|
|
|
|
sourceToken->noOfExpCEs = 0;
|
2001-01-31 07:20:56 +00:00
|
|
|
uhash_put(uchars2tokens, sourceToken, sourceToken, status);
|
|
|
|
} else {
|
|
|
|
/* we could have fished out a reset here */
|
|
|
|
if(sourceToken->strength != UCOL_TOK_RESET) {
|
|
|
|
/* otherwise remove sourceToken from where it was. */
|
|
|
|
if(sourceToken->next != NULL) {
|
2001-05-18 23:48:16 +00:00
|
|
|
if(sourceToken->next->strength > sourceToken->strength) {
|
|
|
|
sourceToken->next->strength = sourceToken->strength;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
sourceToken->next->previous = sourceToken->previous;
|
|
|
|
} else {
|
|
|
|
sourceToken->listHeader->last[sourceToken->polarity] = sourceToken->previous;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(sourceToken->previous != NULL) {
|
|
|
|
sourceToken->previous->next = sourceToken->next;
|
|
|
|
} else {
|
|
|
|
sourceToken->listHeader->first[sourceToken->polarity] = sourceToken->next;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2001-05-18 23:48:16 +00:00
|
|
|
sourceToken->next = NULL;
|
|
|
|
sourceToken->previous = NULL;
|
2001-01-31 07:20:56 +00:00
|
|
|
sourceToken->strength = newStrength;
|
2001-01-31 23:12:37 +00:00
|
|
|
sourceToken->listHeader = lastToken->listHeader;
|
2001-04-19 17:08:07 +00:00
|
|
|
|
|
|
|
/* if we had a variable top, we're gonna put it in */
|
|
|
|
if(variableTop == TRUE && src->varTop == NULL) {
|
|
|
|
variableTop = FALSE;
|
|
|
|
src->varTop = sourceToken;
|
|
|
|
}
|
|
|
|
|
2001-01-31 23:12:37 +00:00
|
|
|
/*
|
|
|
|
1. Find the strongest strength in each list, and set strongestP and strongestN
|
|
|
|
accordingly in the headers.
|
|
|
|
*/
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-05-19 02:37:47 +00:00
|
|
|
if(lastStrength == UCOL_TOK_RESET
|
|
|
|
|| sourceToken->listHeader->first[sourceToken->polarity] == 0) {
|
2001-01-31 07:20:56 +00:00
|
|
|
/* If LAST is a reset
|
2001-05-18 23:48:16 +00:00
|
|
|
insert sourceToken in the list. */
|
|
|
|
|
2001-01-31 23:12:37 +00:00
|
|
|
if(sourceToken->listHeader->first[sourceToken->polarity] == 0) {
|
|
|
|
sourceToken->listHeader->first[sourceToken->polarity] = sourceToken;
|
|
|
|
sourceToken->listHeader->last[sourceToken->polarity] = sourceToken;
|
2001-05-18 23:48:16 +00:00
|
|
|
} else { /* we need to find a place for us */
|
|
|
|
/* and we'll get in front of the same strength */
|
|
|
|
if(sourceToken->listHeader->first[sourceToken->polarity]->strength <= sourceToken->strength) {
|
|
|
|
sourceToken->next = sourceToken->listHeader->first[sourceToken->polarity];
|
|
|
|
sourceToken->next->previous = sourceToken;
|
|
|
|
sourceToken->listHeader->first[sourceToken->polarity] = sourceToken;
|
|
|
|
sourceToken->previous = NULL;
|
|
|
|
} else {
|
|
|
|
lastToken = sourceToken->listHeader->first[sourceToken->polarity];
|
|
|
|
while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
|
|
|
|
lastToken = lastToken->next;
|
|
|
|
}
|
|
|
|
if(lastToken->next != NULL) {
|
|
|
|
lastToken->next->previous = sourceToken;
|
|
|
|
} else {
|
|
|
|
sourceToken->listHeader->last[sourceToken->polarity] = sourceToken;
|
|
|
|
}
|
|
|
|
sourceToken->previous = lastToken;
|
|
|
|
sourceToken->next = lastToken->next;
|
|
|
|
lastToken->next = sourceToken;
|
|
|
|
}
|
2001-01-31 23:12:37 +00:00
|
|
|
}
|
|
|
|
|
2001-01-31 20:17:12 +00:00
|
|
|
/*
|
|
|
|
If "xy" doesn't occur earlier in the list or in the UCA, convert &xy * c *
|
|
|
|
d * ... into &x * c/y * d * ...
|
|
|
|
*/
|
2001-01-31 23:12:37 +00:00
|
|
|
if(expandNext != 0 && sourceToken->expansion == 0) {
|
|
|
|
sourceToken->expansion = expandNext;
|
|
|
|
sourceToken->debugExpansion = *(src->source + (expandNext & 0xFFFFFF));
|
|
|
|
expandNext = 0;
|
2001-01-31 20:17:12 +00:00
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
} else {
|
|
|
|
/* Otherwise (when LAST is not a reset)
|
|
|
|
if polarity (LAST) == polarity(relation), insert sourceToken after LAST,
|
|
|
|
otherwise insert before.
|
|
|
|
when inserting after or before, search to the next position with the same
|
|
|
|
strength in that direction. (This is called postpone insertion). */
|
2001-05-18 23:48:16 +00:00
|
|
|
if(sourceToken != lastToken) {
|
|
|
|
if(lastToken->polarity == sourceToken->polarity) {
|
|
|
|
while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
|
|
|
|
lastToken = lastToken->next;
|
|
|
|
}
|
|
|
|
sourceToken->previous = lastToken;
|
|
|
|
if(lastToken->next != NULL) {
|
|
|
|
lastToken->next->previous = sourceToken;
|
|
|
|
} else {
|
|
|
|
sourceToken->listHeader->last[sourceToken->polarity] = sourceToken;
|
|
|
|
}
|
|
|
|
|
|
|
|
sourceToken->next = lastToken->next;
|
|
|
|
lastToken->next = sourceToken;
|
2001-01-31 07:20:56 +00:00
|
|
|
} else {
|
2001-05-18 23:48:16 +00:00
|
|
|
while(lastToken->previous != NULL && lastToken->previous->strength > sourceToken->strength) {
|
|
|
|
lastToken = lastToken->previous;
|
|
|
|
}
|
|
|
|
sourceToken->next = lastToken;
|
|
|
|
if(lastToken->previous != NULL) {
|
|
|
|
lastToken->previous->next = sourceToken;
|
|
|
|
} else {
|
|
|
|
sourceToken->listHeader->first[sourceToken->polarity] = sourceToken;
|
|
|
|
}
|
|
|
|
sourceToken->previous = lastToken->previous;
|
|
|
|
lastToken->previous = sourceToken;
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
2001-05-18 23:48:16 +00:00
|
|
|
} else { /* repeated one thing twice in rules, stay with the stronger strength */
|
|
|
|
if(lastStrength < sourceToken->strength) {
|
|
|
|
sourceToken->strength = lastStrength;
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
2001-01-31 21:10:55 +00:00
|
|
|
uint32_t CE = UCOL_NOT_FOUND, SecondCE = UCOL_NOT_FOUND;
|
|
|
|
collIterate s;
|
|
|
|
|
2001-05-18 23:48:16 +00:00
|
|
|
if((specs & UCOL_TOK_BEFORE) != 0) { /* we're doing before */
|
2001-05-02 05:05:06 +00:00
|
|
|
uint8_t strength = (specs & UCOL_TOK_BEFORE) - 1;
|
2001-05-18 23:48:16 +00:00
|
|
|
if(sourceToken != NULL && sourceToken->strength != UCOL_TOK_RESET) {
|
|
|
|
/* this is a before that is already ordered in the UCA - so we need to get the previous with good strength */
|
|
|
|
if(sourceToken->previous != NULL) {
|
|
|
|
sourceToken = sourceToken->previous;
|
|
|
|
while(sourceToken->previous != NULL && sourceToken->previous->strength > strength) {
|
|
|
|
sourceToken = sourceToken->previous;
|
|
|
|
}
|
|
|
|
if(sourceToken->previous == NULL && sourceToken->strength > strength) {
|
|
|
|
sourceToken = sourceToken->listHeader->reset;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
sourceToken = sourceToken->listHeader->reset;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* this is a virgin before - we need to fish the anchor from the UCA */
|
|
|
|
uint32_t baseCE = UCOL_NOT_FOUND, baseContCE = UCOL_NOT_FOUND;
|
|
|
|
uint32_t invPos;
|
|
|
|
init_collIterate(src->UCA, src->source+charsOffset, 1, &s);
|
|
|
|
|
|
|
|
baseCE = ucol_getNextCE(src->UCA, &s, status) & 0xFFFFFF3F;
|
|
|
|
baseContCE = ucol_getNextCE(src->UCA, &s, status);
|
|
|
|
if(baseContCE == UCOL_NO_MORE_CES) {
|
|
|
|
baseContCE = 0;
|
|
|
|
}
|
2001-05-02 05:05:06 +00:00
|
|
|
|
2001-05-18 23:48:16 +00:00
|
|
|
invPos = ucol_inv_getPrevCE(baseCE, baseContCE, &CE, &SecondCE, strength);
|
2001-05-02 05:05:06 +00:00
|
|
|
|
2001-05-18 23:48:16 +00:00
|
|
|
uint32_t *CETable = (uint32_t *)((uint8_t *)src->invUCA+src->invUCA->table);
|
|
|
|
uint32_t ch = CETable[3*invPos+2];
|
2001-05-02 05:05:06 +00:00
|
|
|
|
2001-05-18 23:48:16 +00:00
|
|
|
if((ch & UCOL_INV_SIZEMASK) != 0) {
|
|
|
|
uint32_t *conts = (uint32_t *)((uint8_t *)src->invUCA+src->invUCA->conts);
|
|
|
|
uint32_t offset = (ch & UCOL_INV_OFFSETMASK);
|
|
|
|
ch = conts[offset];
|
|
|
|
}
|
|
|
|
*src->extraCurrent++ = (UChar)ch;
|
|
|
|
charsOffset = src->extraCurrent - src->source - 1;
|
|
|
|
newCharsLen = 1;
|
|
|
|
}
|
2001-05-02 05:05:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2001-05-18 23:48:16 +00:00
|
|
|
if(lastToken != NULL && lastStrength == UCOL_TOK_RESET) {
|
2001-04-05 21:27:44 +00:00
|
|
|
/* if the previous token was also a reset, */
|
|
|
|
/*this means that we have two consecutive resets */
|
|
|
|
/* and we want to remove the previous one if empty*/
|
|
|
|
if(ListList[src->resultLen-1].first[UCOL_TOK_POLARITY_POSITIVE] == NULL) {
|
|
|
|
src->resultLen--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2001-01-31 23:12:37 +00:00
|
|
|
if(newCharsLen > 1) {
|
|
|
|
expandNext = ((newCharsLen-1)<<24) | (charsOffset + 1);
|
|
|
|
} else {
|
|
|
|
expandNext = 0;
|
|
|
|
}
|
|
|
|
|
2001-01-31 07:20:56 +00:00
|
|
|
/* 5 If the relation is a reset:
|
|
|
|
If sourceToken is null
|
|
|
|
Create new list, create new sourceToken, make the baseCE from source, put
|
|
|
|
the sourceToken in ListHeader of the new list */
|
|
|
|
if(sourceToken == NULL) {
|
2001-02-05 05:36:12 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
3. The rule for "& abcdefg < xyz" is a bit tricky. What it turns into is:
|
|
|
|
|
|
|
|
a. Find the longest sequence in "abcdefg" that is in UCA *OR* in the
|
|
|
|
tailoring so far. Suppose that is "abcd".
|
|
|
|
b. Then treat this rule as equivalent to:
|
|
|
|
"& abcd < xyz / efg"
|
|
|
|
*/
|
2001-01-31 21:10:55 +00:00
|
|
|
if(newCharsLen > 1) {
|
|
|
|
key.source = 0x01000000 | charsOffset;
|
|
|
|
sourceToken = (UColToken *)uhash_get(uchars2tokens, &key);
|
|
|
|
if(sourceToken != NULL) {
|
|
|
|
lastToken = sourceToken;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
/* do the reset thing */
|
|
|
|
sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
|
|
|
|
sourceToken->source = newCharsLen << 24 | charsOffset;
|
|
|
|
sourceToken->expansion = newExtensionsLen << 24 | extensionOffset;
|
2001-01-31 21:10:55 +00:00
|
|
|
|
|
|
|
sourceToken->debugSource = *(src->source + charsOffset);
|
|
|
|
sourceToken->debugExpansion = *(src->source + extensionOffset);
|
|
|
|
|
|
|
|
|
2001-01-31 23:12:37 +00:00
|
|
|
sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
|
2001-01-31 07:20:56 +00:00
|
|
|
sourceToken->strength = UCOL_TOK_RESET;
|
|
|
|
sourceToken->next = NULL;
|
|
|
|
sourceToken->previous = NULL;
|
2001-02-28 21:50:23 +00:00
|
|
|
sourceToken->listHeader = &ListList[src->resultLen];
|
2001-01-31 20:17:12 +00:00
|
|
|
/*
|
|
|
|
3 Consider each item: relation, source, and expansion: e.g. ...< x / y ...
|
|
|
|
First convert all expansions into normal form. Examples:
|
|
|
|
If "xy" doesn't occur earlier in the list or in the UCA, convert &xy * c *
|
|
|
|
d * ... into &x * c/y * d * ...
|
|
|
|
Note: reset values can never have expansions, although they can cause the
|
|
|
|
very next item to have one. They may be contractions, if they are found
|
|
|
|
earlier in the list.
|
|
|
|
*/
|
2001-03-07 19:43:06 +00:00
|
|
|
if(top == FALSE) {
|
|
|
|
if(newCharsLen > 1) {
|
|
|
|
sourceToken->source = 0x01000000 | charsOffset;
|
|
|
|
}
|
2001-01-31 20:17:12 +00:00
|
|
|
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-04-06 23:37:48 +00:00
|
|
|
init_collIterate(src->UCA, src->source+charsOffset, 1, &s); /* or newCharsLen instead of 1??? */
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-03-07 19:43:06 +00:00
|
|
|
CE = ucol_getNextCE(src->UCA, &s, status);
|
|
|
|
SecondCE = ucol_getNextCE(src->UCA, &s, status);
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-05-10 22:33:50 +00:00
|
|
|
ListList[src->resultLen].baseCE = CE & 0xFFFFFF3F;
|
2001-03-07 19:43:06 +00:00
|
|
|
if(isContinuation(SecondCE)) {
|
|
|
|
ListList[src->resultLen].baseContCE = SecondCE;
|
|
|
|
} else {
|
|
|
|
ListList[src->resultLen].baseContCE = 0;
|
|
|
|
}
|
|
|
|
} else { /* top == TRUE */
|
|
|
|
top = FALSE;
|
|
|
|
ListList[src->resultLen].baseCE = UCOL_RESET_TOP_VALUE;
|
2001-02-28 21:50:23 +00:00
|
|
|
ListList[src->resultLen].baseContCE = 0;
|
2001-02-08 01:04:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2001-02-28 21:50:23 +00:00
|
|
|
ListList[src->resultLen].first[UCOL_TOK_POLARITY_NEGATIVE] = NULL;
|
|
|
|
ListList[src->resultLen].last[UCOL_TOK_POLARITY_NEGATIVE] = NULL;
|
|
|
|
ListList[src->resultLen].first[UCOL_TOK_POLARITY_POSITIVE] = NULL;
|
|
|
|
ListList[src->resultLen].last[UCOL_TOK_POLARITY_POSITIVE] = NULL;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-02-28 21:50:23 +00:00
|
|
|
ListList[src->resultLen].reset = sourceToken;
|
2001-01-31 07:20:56 +00:00
|
|
|
|
2001-02-28 21:50:23 +00:00
|
|
|
src->resultLen++;
|
2001-01-31 07:20:56 +00:00
|
|
|
uhash_put(uchars2tokens, sourceToken, sourceToken, status);
|
|
|
|
} else { /* reset to something already in rules */
|
2001-03-14 00:12:46 +00:00
|
|
|
top = FALSE;
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
/* 7 After all this, set LAST to point to sourceToken, and goto step 3. */
|
|
|
|
lastToken = sourceToken;
|
2001-03-20 00:56:37 +00:00
|
|
|
} else {
|
|
|
|
return 0;
|
|
|
|
}
|
2001-01-31 07:20:56 +00:00
|
|
|
}
|
|
|
|
|
2001-04-05 22:48:23 +00:00
|
|
|
if(src->resultLen > 0 && ListList[src->resultLen-1].first[UCOL_TOK_POLARITY_POSITIVE] == NULL) {
|
2001-04-05 21:27:44 +00:00
|
|
|
src->resultLen--;
|
|
|
|
}
|
2001-02-28 21:50:23 +00:00
|
|
|
return src->resultLen;
|
2001-01-29 22:09:24 +00:00
|
|
|
}
|
|
|
|
|
2001-02-27 21:01:11 +00:00
|
|
|
uint32_t ucol_tok_assembleTokenList(UColTokenParser *src, UErrorCode *status) {
|
2001-02-28 16:12:45 +00:00
|
|
|
uint32_t res = ucol_uprv_tok_assembleTokenList(src, status);
|
|
|
|
return res;
|
2001-02-28 19:01:23 +00:00
|
|
|
}
|
|
|
|
|
2001-02-28 21:50:23 +00:00
|
|
|
void ucol_tok_closeTokenList(UColTokenParser *src) {
|
|
|
|
uhash_close(uchars2tokens);
|
|
|
|
uprv_free(src->lh);
|
2001-03-14 00:12:46 +00:00
|
|
|
uprv_free(src->source);
|
2001-02-28 21:50:23 +00:00
|
|
|
}
|
2001-03-06 00:57:48 +00:00
|
|
|
|