545 lines
20 KiB
C++
545 lines
20 KiB
C++
/*
|
|
**********************************************************************
|
|
* Copyright (C) 1999, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
**********************************************************************
|
|
* Date Name Description
|
|
* 11/17/99 aliu Creation.
|
|
**********************************************************************
|
|
*/
|
|
#include "rbt_pars.h"
|
|
#include "unicode/rbt.h"
|
|
#include "rbt_rule.h"
|
|
#include "unirange.h"
|
|
#include "rbt_data.h"
|
|
#include "unicode/uniset.h"
|
|
#include "cstring.h"
|
|
#include "unicode/parsepos.h"
|
|
#include "symtable.h"
|
|
#include "unicode/parseerr.h"
|
|
|
|
// Operators
|
|
const UChar TransliterationRuleParser::VARIABLE_DEF_OP = 0x003D/*=*/;
|
|
const UChar TransliterationRuleParser::FORWARD_RULE_OP = 0x003E/*>*/;
|
|
const UChar TransliterationRuleParser::REVERSE_RULE_OP = 0x003C/*<*/;
|
|
const UChar TransliterationRuleParser::FWDREV_RULE_OP = 0x007E/*~*/; // internal rep of <> op
|
|
const UnicodeString TransliterationRuleParser::OPERATORS = UNICODE_STRING("=><", 3);
|
|
|
|
// Other special characters
|
|
const UChar TransliterationRuleParser::QUOTE = 0x0027/*'*/;
|
|
const UChar TransliterationRuleParser::ESCAPE = 0x005C/*\*/;
|
|
const UChar TransliterationRuleParser::END_OF_RULE = 0x003B/*;*/;
|
|
const UChar TransliterationRuleParser::RULE_COMMENT_CHAR = 0x0023/*#*/;
|
|
|
|
const UChar TransliterationRuleParser::VARIABLE_REF_OPEN = 0x007B/*{*/;
|
|
const UChar TransliterationRuleParser::VARIABLE_REF_CLOSE = 0x007D/*}*/;
|
|
const UChar TransliterationRuleParser::CONTEXT_OPEN = 0x0028/*(*/;
|
|
const UChar TransliterationRuleParser::CONTEXT_CLOSE = 0x0029/*)*/;
|
|
const UChar TransliterationRuleParser::SET_OPEN = 0x005B/*[*/;
|
|
const UChar TransliterationRuleParser::SET_CLOSE = 0x005D/*]*/;
|
|
const UChar TransliterationRuleParser::CURSOR_POS = 0x007C/*|*/;
|
|
|
|
//----------------------------------------------------------------------
|
|
// BEGIN ParseData
|
|
//----------------------------------------------------------------------
|
|
|
|
/**
|
|
* This class implements the SymbolTable interface. It is used
|
|
* during parsing to give UnicodeSet access to variables that
|
|
* have been defined so far. Note that it uses setVariablesVector,
|
|
* _not_ data.setVariables.
|
|
*/
|
|
class ParseData : public SymbolTable {
|
|
public:
|
|
const TransliterationRuleData* data; // alias
|
|
|
|
const UVector* setVariablesVector; // alias
|
|
|
|
ParseData(const TransliterationRuleData* data = 0,
|
|
const UVector* setVariablesVector = 0);
|
|
|
|
/**
|
|
* Lookup the object associated with this string and return it.
|
|
* Return U_ILLEGAL_ARGUMENT_ERROR status if the name does not
|
|
* exist. Return a non-NULL set if the name is mapped to a set;
|
|
* otherwise return a NULL set.
|
|
*/
|
|
virtual void lookup(const UnicodeString& name, UChar& c, UnicodeSet*& set,
|
|
UErrorCode& status) const;
|
|
};
|
|
|
|
ParseData::ParseData(const TransliterationRuleData* d,
|
|
const UVector* sets) :
|
|
data(d), setVariablesVector(sets) {}
|
|
|
|
/**
|
|
* Implement SymbolTable API. Lookup a variable, returning
|
|
* either a Character, a UnicodeSet, or null.
|
|
*/
|
|
void ParseData::lookup(const UnicodeString& name, UChar& c, UnicodeSet*& set,
|
|
UErrorCode& status) const {
|
|
c = data->lookupVariable(name, status);
|
|
if (U_SUCCESS(status)) {
|
|
int32_t i = c - data->setVariablesBase;
|
|
set = (i < setVariablesVector->size()) ?
|
|
(UnicodeSet*) setVariablesVector->elementAt(i) : 0;
|
|
}
|
|
}
|
|
|
|
//----------------------------------------------------------------------
|
|
// END ParseData
|
|
//----------------------------------------------------------------------
|
|
|
|
TransliterationRuleData*
|
|
TransliterationRuleParser::parse(const UnicodeString& rules,
|
|
RuleBasedTransliterator::Direction direction,
|
|
ParseError* parseError) {
|
|
TransliterationRuleParser parser(rules, direction, parseError);
|
|
parser.parseRules();
|
|
if (U_FAILURE(parser.status)) {
|
|
delete parser.data;
|
|
parser.data = 0;
|
|
}
|
|
return parser.data;
|
|
}
|
|
|
|
/**
|
|
* @param rules list of rules, separated by newline characters
|
|
* @exception IllegalArgumentException if there is a syntax error in the
|
|
* rules
|
|
*/
|
|
TransliterationRuleParser::TransliterationRuleParser(
|
|
const UnicodeString& theRules,
|
|
RuleBasedTransliterator::Direction theDirection,
|
|
ParseError* theParseError) :
|
|
rules(theRules), direction(theDirection), data(0), parseError(theParseError) {
|
|
parseData = new ParseData(0, &setVariablesVector);
|
|
}
|
|
|
|
/**
|
|
* Destructor.
|
|
*/
|
|
TransliterationRuleParser::~TransliterationRuleParser() {
|
|
delete parseData;
|
|
}
|
|
|
|
/**
|
|
* Parse the given string as a sequence of rules, separated by newline
|
|
* characters ('\n'), and cause this object to implement those rules. Any
|
|
* previous rules are discarded. Typically this method is called exactly
|
|
* once, during construction.
|
|
* @exception IllegalArgumentException if there is a syntax error in the
|
|
* rules
|
|
*/
|
|
void TransliterationRuleParser::parseRules(void) {
|
|
status = U_ZERO_ERROR;
|
|
|
|
delete data;
|
|
data = new TransliterationRuleData(status);
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
|
|
parseData->data = data;
|
|
setVariablesVector.removeAllElements();
|
|
if (parseError != 0) {
|
|
parseError->code = 0;
|
|
}
|
|
determineVariableRange();
|
|
|
|
int32_t pos = 0;
|
|
int32_t limit = rules.length();
|
|
while (pos < limit && U_SUCCESS(status)) {
|
|
UChar c = rules.charAt(pos++);
|
|
if (Unicode::isWhitespace(c)) {
|
|
// Ignore leading whitespace. Note that this is not
|
|
// Unicode spaces, but Java spaces -- a subset,
|
|
// representing whitespace likely to be seen in code.
|
|
continue;
|
|
}
|
|
// Skip lines starting with the comment character
|
|
if (c == RULE_COMMENT_CHAR) {
|
|
pos = rules.indexOf((UChar)0x000A /*\n*/, pos) + 1;
|
|
if (pos == 0) {
|
|
break; // No "\n" found; rest of rule is a commnet
|
|
}
|
|
continue; // Either fall out or restart with next line
|
|
}
|
|
// We've found the start of a rule. c is its first
|
|
// character, and pos points past c. Lexically parse the
|
|
// rule into component pieces.
|
|
pos = parseRule(--pos, limit);
|
|
}
|
|
|
|
// Convert the set vector to an array
|
|
data->setVariablesLength = setVariablesVector.size();
|
|
data->setVariables = new UnicodeSet*[data->setVariablesLength];
|
|
// orphanElement removes the given element and shifts all other
|
|
// elements down. For performance (and code clarity) we work from
|
|
// the end back to index 0.
|
|
for (int32_t i=data->setVariablesLength; i>0; ) {
|
|
--i;
|
|
data->setVariables[i] =
|
|
(UnicodeSet*) setVariablesVector.orphanElementAt(i);
|
|
}
|
|
|
|
// Index the rules
|
|
if (U_SUCCESS(status)) {
|
|
data->ruleSet.freeze(*data, status);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* MAIN PARSER. Parse the next rule in the given rule string, starting
|
|
* at pos. Return the index after the last character parsed. Do not
|
|
* parse characters at or after limit.
|
|
*
|
|
* Important: The character at pos must be a non-whitespace character
|
|
* that is not the comment character.
|
|
*
|
|
* This method handles quoting, escaping, and whitespace removal. It
|
|
* parses the end-of-rule character. It recognizes context and cursor
|
|
* indicators. Once it does a lexical breakdown of the rule at pos, it
|
|
* creates a rule object and adds it to our rule list.
|
|
*/
|
|
int32_t TransliterationRuleParser::parseRule(int32_t pos, int32_t limit) {
|
|
// Locate the left side, operator, and right side
|
|
int32_t start = pos;
|
|
UChar op = 0;
|
|
|
|
UnicodeString buf;
|
|
int32_t cursor = -1; // position of cursor in buf
|
|
int32_t ante = -1; // position of ante context marker ')' in buf
|
|
int32_t post = -1; // position of post context marker '(' in buf
|
|
int32_t postClose = -1; // position of post context close ')' in buf
|
|
|
|
// Assigned to buf and its adjuncts after the LHS has been
|
|
// parsed. Thereafter, buf etc. refer to the RHS.
|
|
UnicodeString left;
|
|
int32_t leftCursor = -1, leftAnte = -1, leftPost = -1, leftPostClose = -1;
|
|
|
|
UnicodeString scratch;
|
|
|
|
while (pos < limit) {
|
|
UChar c = rules.charAt(pos++);
|
|
if (Unicode::isWhitespace(c)) {
|
|
// Ignore whitespace. Note that this is not Unicode
|
|
// spaces, but Java spaces -- a subset, representing
|
|
// whitespace likely to be seen in code.
|
|
continue;
|
|
}
|
|
// Handle escapes
|
|
if (c == ESCAPE) {
|
|
if (pos == limit) {
|
|
return syntaxError(RuleBasedTransliterator::TRAILING_BACKSLASH, rules, start);
|
|
}
|
|
// Parse \uXXXX escapes
|
|
c = rules.charAt(pos++);
|
|
if (c == 0x0075/*u*/) {
|
|
if ((pos+4) > limit) {
|
|
return syntaxError(RuleBasedTransliterator::MALFORMED_UNICODE_ESCAPE, rules, start);
|
|
}
|
|
c = (UChar)0x0000;
|
|
for (int32_t plim=pos+4; pos<plim; ++pos) { // [sic]
|
|
int32_t digit = Unicode::digit(rules.charAt(pos), 16);
|
|
if (digit<0) {
|
|
return syntaxError(RuleBasedTransliterator::MALFORMED_UNICODE_ESCAPE, rules, start);
|
|
}
|
|
c = (UChar) ((c << 4) | digit);
|
|
}
|
|
}
|
|
|
|
buf.append(c);
|
|
continue;
|
|
}
|
|
// Handle quoted matter
|
|
if (c == QUOTE) {
|
|
int32_t iq = rules.indexOf(QUOTE, pos);
|
|
if (iq == pos) {
|
|
buf.append(c); // Parse [''] outside quotes as [']
|
|
++pos;
|
|
} else {
|
|
/* This loop picks up a segment of quoted text of the
|
|
* form 'aaaa' each time through. If this segment
|
|
* hasn't really ended ('aaaa''bbbb') then it keeps
|
|
* looping, each time adding on a new segment. When it
|
|
* reaches the final quote it breaks.
|
|
*/
|
|
for (;;) {
|
|
if (iq < 0) {
|
|
return syntaxError(RuleBasedTransliterator::UNTERMINATED_QUOTE, rules, start);
|
|
}
|
|
scratch.truncate(0);
|
|
rules.extractBetween(pos, iq, scratch);
|
|
buf.append(scratch);
|
|
pos = iq+1;
|
|
if (pos < limit && rules.charAt(pos) == QUOTE) {
|
|
// Parse [''] inside quotes as [']
|
|
iq = rules.indexOf(QUOTE, pos+1);
|
|
// Continue looping
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
if (OPERATORS.indexOf(c) >= 0) {
|
|
if (op != 0) {
|
|
return syntaxError(RuleBasedTransliterator::UNQUOTED_SPECIAL, rules, start);
|
|
}
|
|
// Found an operator char. Check for forward-reverse operator.
|
|
if (c == REVERSE_RULE_OP &&
|
|
(pos < limit && rules.charAt(pos) == FORWARD_RULE_OP)) {
|
|
++pos;
|
|
op = FWDREV_RULE_OP;
|
|
} else {
|
|
op = c;
|
|
}
|
|
left = buf; // lhs
|
|
leftCursor = cursor;
|
|
leftAnte = ante;
|
|
leftPost = post;
|
|
leftPostClose = postClose;
|
|
|
|
buf.truncate(0);
|
|
cursor = ante = post = postClose = -1;
|
|
continue;
|
|
}
|
|
if (c == END_OF_RULE) {
|
|
break;
|
|
}
|
|
switch (c) {
|
|
case VARIABLE_REF_OPEN:
|
|
{
|
|
int32_t j = rules.indexOf(VARIABLE_REF_CLOSE, pos);
|
|
if (pos == j || j < 0) { // empty or unterminated
|
|
return syntaxError(RuleBasedTransliterator::MALFORMED_VARIABLE_REFERENCE, rules, start);
|
|
}
|
|
scratch.truncate(0);
|
|
rules.extractBetween(pos, j, scratch);
|
|
pos = j+1;
|
|
UChar v = data->lookupVariable(scratch, status);
|
|
if (U_FAILURE(status)) {
|
|
return syntaxError(RuleBasedTransliterator::UNDEFINED_VARIABLE, rules, start);
|
|
}
|
|
buf.append(v);
|
|
}
|
|
break;
|
|
case CONTEXT_OPEN:
|
|
if (post >= 0) {
|
|
return syntaxError(RuleBasedTransliterator::MULTIPLE_POST_CONTEXTS, rules, start);
|
|
}
|
|
// Ignore CONTEXT_OPEN if buffer length is zero -- that means
|
|
// this is the optional opening delimiter for the ante context.
|
|
if (buf.length() > 0) {
|
|
post = buf.length();
|
|
}
|
|
break;
|
|
case CONTEXT_CLOSE:
|
|
if (postClose >= 0) {
|
|
return syntaxError(RuleBasedTransliterator::UNEXPECTED_CLOSE_CONTEXT, rules, start);
|
|
}
|
|
if (post >= 0) {
|
|
// This is probably the optional closing delimiter
|
|
// for the post context; save the pos and check later.
|
|
postClose = buf.length();
|
|
} else if (ante >= 0) {
|
|
return syntaxError(RuleBasedTransliterator::MULTIPLE_ANTE_CONTEXTS, rules, start);
|
|
} else {
|
|
ante = buf.length();
|
|
}
|
|
break;
|
|
case SET_OPEN: {
|
|
ParsePosition pp(pos-1); // Backup to opening '['
|
|
buf.append(registerSet(new UnicodeSet(rules, pp, *parseData, status)));
|
|
if (U_FAILURE(status)) {
|
|
return syntaxError(RuleBasedTransliterator::MALFORMED_SET, rules, start);
|
|
}
|
|
pos = pp.getIndex(); }
|
|
break;
|
|
case VARIABLE_REF_CLOSE:
|
|
case SET_CLOSE:
|
|
return syntaxError(RuleBasedTransliterator::UNQUOTED_SPECIAL, rules, start);
|
|
case CURSOR_POS:
|
|
if (cursor >= 0) {
|
|
return syntaxError(RuleBasedTransliterator::MULTIPLE_CURSORS, rules, start);
|
|
}
|
|
cursor = buf.length();
|
|
break;
|
|
default:
|
|
buf.append(c);
|
|
break;
|
|
}
|
|
}
|
|
if (op == 0) {
|
|
return syntaxError(RuleBasedTransliterator::MISSING_OPERATOR, rules, start);
|
|
}
|
|
|
|
// Check context close parameters
|
|
if ((leftPostClose >= 0 && leftPostClose != left.length()) ||
|
|
(postClose >= 0 && postClose != buf.length())) {
|
|
return syntaxError(RuleBasedTransliterator::TEXT_AFTER_CLOSE_CONTEXT, rules, start);
|
|
}
|
|
|
|
// Context is only allowed on the input side; that is, the left side
|
|
// for forward rules. Cursors are only allowed on the output side;
|
|
// that is, the right side for forward rules. Bidirectional rules
|
|
// ignore elements that do not apply.
|
|
|
|
switch (op) {
|
|
case VARIABLE_DEF_OP:
|
|
// LHS is the name. RHS is a single character, either a literal
|
|
// or a set (already parsed). If RHS is longer than one
|
|
// character, it is either a multi-character string, or multiple
|
|
// sets, or a mixture of chars and sets -- syntax error.
|
|
if (buf.length() != 1) {
|
|
return syntaxError(RuleBasedTransliterator::MALFORMED_RHS, rules, start);
|
|
}
|
|
if (data->isVariableDefined(left)) {
|
|
return syntaxError(RuleBasedTransliterator::DUPLICATE_VARIABLE_DEFINITION, rules, start);
|
|
}
|
|
data->defineVariable(left, buf.charAt(0), status);
|
|
break;
|
|
|
|
case FORWARD_RULE_OP:
|
|
if (direction == RuleBasedTransliterator::FORWARD) {
|
|
if (ante >= 0 || post >= 0 || leftCursor >= 0) {
|
|
return syntaxError(RuleBasedTransliterator::MALFORMED_RULE, rules, start);
|
|
}
|
|
data->ruleSet.addRule(new TransliterationRule(
|
|
left, leftAnte, leftPost,
|
|
buf, cursor, status), status);
|
|
} // otherwise ignore the rule; it's not the direction we want
|
|
break;
|
|
|
|
case REVERSE_RULE_OP:
|
|
if (direction == RuleBasedTransliterator::REVERSE) {
|
|
if (leftAnte >= 0 || leftPost >= 0 || cursor >= 0) {
|
|
return syntaxError(RuleBasedTransliterator::MALFORMED_RULE, rules, start);
|
|
}
|
|
data->ruleSet.addRule(new TransliterationRule(
|
|
buf, ante, post,
|
|
left, leftCursor, status), status);
|
|
} // otherwise ignore the rule; it's not the direction we want
|
|
break;
|
|
|
|
case FWDREV_RULE_OP:
|
|
if (direction == RuleBasedTransliterator::FORWARD) {
|
|
// The output side is the right; trim off any context
|
|
if (post >= 0) {
|
|
buf.remove(post);
|
|
}
|
|
if (ante >= 0) {
|
|
buf.removeBetween(0, ante);
|
|
}
|
|
data->ruleSet.addRule(new TransliterationRule(
|
|
left, leftAnte, leftPost,
|
|
buf, cursor, status), status);
|
|
} else {
|
|
// The output side is the left; trim off any context
|
|
if (leftPost >= 0) {
|
|
left.remove(leftPost);
|
|
}
|
|
if (leftAnte >= 0) {
|
|
left.removeBetween(0, leftAnte);
|
|
}
|
|
data->ruleSet.addRule(new TransliterationRule(
|
|
buf, ante, post,
|
|
left, leftCursor, status), status);
|
|
}
|
|
break;
|
|
}
|
|
|
|
return pos;
|
|
}
|
|
|
|
/**
|
|
* Called by main parser upon syntax error. Search the rule string
|
|
* for the probable end of the rule. Of course, if the error is that
|
|
* the end of rule marker is missing, then the rule end will not be found.
|
|
* In any case the rule start will be correctly reported.
|
|
* @param msg error description
|
|
* @param rule pattern string
|
|
* @param start position of first character of current rule
|
|
*/
|
|
int32_t TransliterationRuleParser::syntaxError(int32_t parseErrorCode,
|
|
const UnicodeString& rule,
|
|
int32_t start) {
|
|
if (parseError != 0) {
|
|
parseError->code = parseErrorCode;
|
|
parseError->line = 0; // We don't return a line #
|
|
parseError->offset = start; // Character offset from rule start
|
|
int32_t end = quotedIndexOf(rule, start, rule.length(), END_OF_RULE);
|
|
if (end < 0) {
|
|
end = rule.length();
|
|
}
|
|
rule.extractBetween(start, end, parseError->context); // Current rule
|
|
}
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
return start;
|
|
}
|
|
|
|
/**
|
|
* Allocate a private-use substitution character for the given set,
|
|
* register it in the setVariables hash, and return the substitution
|
|
* character.
|
|
*/
|
|
UChar TransliterationRuleParser::registerSet(UnicodeSet* adoptedSet) {
|
|
if (variableNext >= variableLimit) {
|
|
// throw new RuntimeException("Private use variables exhausted");
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
return 0;
|
|
}
|
|
setVariablesVector.addElement(adoptedSet);
|
|
return variableNext++;
|
|
}
|
|
|
|
/**
|
|
* Determines what part of the private use region of Unicode we can use for
|
|
* variable stand-ins. The correct way to do this is as follows: Parse each
|
|
* rule, and for forward and reverse rules, take the FROM expression, and
|
|
* make a hash of all characters used. The TO expression should be ignored.
|
|
* When done, everything not in the hash is available for use. In practice,
|
|
* this method may employ some other algorithm for improved speed.
|
|
*/
|
|
void TransliterationRuleParser::determineVariableRange(void) {
|
|
UnicodeRange privateUse(0xE000, 0x1900); // Private use area
|
|
|
|
UnicodeRange* r = privateUse.largestUnusedSubrange(rules);
|
|
|
|
data->setVariablesBase = variableNext = variableLimit = (UChar) 0;
|
|
|
|
if (r != 0) {
|
|
data->setVariablesBase = variableNext = r->start;
|
|
variableLimit = (UChar) (r->start + r->length);
|
|
delete r;
|
|
}
|
|
|
|
if (variableNext >= variableLimit) {
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns the index of a character, ignoring quoted text.
|
|
* For example, in the string "abc'hide'h", the 'h' in "hide" will not be
|
|
* found by a search for 'h'.
|
|
*/
|
|
int32_t TransliterationRuleParser::quotedIndexOf(const UnicodeString& text,
|
|
int32_t start, int32_t limit,
|
|
UChar charToFind) {
|
|
for (int32_t i=start; i<limit; ++i) {
|
|
UChar c = text.charAt(i);
|
|
if (c == ESCAPE) {
|
|
++i;
|
|
} else if (c == QUOTE) {
|
|
while (++i < limit
|
|
&& text.charAt(i) != QUOTE) {}
|
|
} else if (c == charToFind) {
|
|
return i;
|
|
}
|
|
}
|
|
return -1;
|
|
}
|