/* ********************************************************************** * Copyright (C) 1999-2006, International Business Machines * Corporation and others. All Rights Reserved. ********************************************************************** * Date Name Description * 11/17/99 aliu Creation. ********************************************************************** */ #include "unicode/utypes.h" #if !UCONFIG_NO_TRANSLITERATION #include "unicode/uobject.h" #include "unicode/parseerr.h" #include "unicode/parsepos.h" #include "unicode/putil.h" #include "unicode/uchar.h" #include "unicode/ustring.h" #include "unicode/uniset.h" #include "cstring.h" #include "funcrepl.h" #include "hash.h" #include "quant.h" #include "rbt.h" #include "rbt_data.h" #include "rbt_pars.h" #include "rbt_rule.h" #include "strmatch.h" #include "strrepl.h" #include "unicode/symtable.h" #include "tridpars.h" #include "uvector.h" #include "hash.h" #include "util.h" #include "cmemory.h" #include "uprops.h" #include "putilimp.h" // Operators #define VARIABLE_DEF_OP ((UChar)0x003D) /*=*/ #define FORWARD_RULE_OP ((UChar)0x003E) /*>*/ #define REVERSE_RULE_OP ((UChar)0x003C) /*<*/ #define FWDREV_RULE_OP ((UChar)0x007E) /*~*/ // internal rep of <> op // Other special characters #define QUOTE ((UChar)0x0027) /*'*/ #define ESCAPE ((UChar)0x005C) /*\*/ #define END_OF_RULE ((UChar)0x003B) /*;*/ #define RULE_COMMENT_CHAR ((UChar)0x0023) /*#*/ #define SEGMENT_OPEN ((UChar)0x0028) /*(*/ #define SEGMENT_CLOSE ((UChar)0x0029) /*)*/ #define CONTEXT_ANTE ((UChar)0x007B) /*{*/ #define CONTEXT_POST ((UChar)0x007D) /*}*/ #define CURSOR_POS ((UChar)0x007C) /*|*/ #define CURSOR_OFFSET ((UChar)0x0040) /*@*/ #define ANCHOR_START ((UChar)0x005E) /*^*/ #define KLEENE_STAR ((UChar)0x002A) /***/ #define ONE_OR_MORE ((UChar)0x002B) /*+*/ #define ZERO_OR_ONE ((UChar)0x003F) /*?*/ #define DOT ((UChar)46) /*.*/ static const UChar DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]"; 91, 94, 91, 58, 90, 112, 58, 93, 91, 58, 90, 108, 58, 93, 92, 114, 92, 110, 36, 93, 0 }; // A function is denoted &Source-Target/Variant(text) #define FUNCTION ((UChar)38) /*&*/ // Aliases for some of the syntax characters. These are provided so // transliteration rules can be expressed in XML without clashing with // XML syntax characters '<', '>', and '&'. #define ALT_REVERSE_RULE_OP ((UChar)0x2190) // Left Arrow #define ALT_FORWARD_RULE_OP ((UChar)0x2192) // Right Arrow #define ALT_FWDREV_RULE_OP ((UChar)0x2194) // Left Right Arrow #define ALT_FUNCTION ((UChar)0x2206) // Increment (~Greek Capital Delta) // Special characters disallowed at the top level static const UChar ILLEGAL_TOP[] = {41,0}; // ")" // Special characters disallowed within a segment static const UChar ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@" // Special characters disallowed within a function argument static const UChar ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@" // By definition, the ANCHOR_END special character is a // trailing SymbolTable.SYMBOL_REF character. // private static final char ANCHOR_END = '$'; static const UChar gOPERATORS[] = { // "=><" VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, 0 }; static const UChar HALF_ENDERS[] = { // "=><;" VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, END_OF_RULE, 0 }; // These are also used in Transliterator::toRules() static const int32_t ID_TOKEN_LEN = 2; static const UChar ID_TOKEN[] = { 0x3A, 0x3A }; // ':', ':' /* commented out until we do real ::BEGIN/::END functionality static const int32_t BEGIN_TOKEN_LEN = 5; static const UChar BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN' static const int32_t END_TOKEN_LEN = 3; static const UChar END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END' */ U_NAMESPACE_BEGIN //---------------------------------------------------------------------- // BEGIN ParseData //---------------------------------------------------------------------- /** * This class implements the SymbolTable interface. It is used * during parsing to give UnicodeSet access to variables that * have been defined so far. Note that it uses variablesVector, * _not_ data.setVariables. */ class ParseData : public UMemory, public SymbolTable { public: const TransliterationRuleData* data; // alias const UVector* variablesVector; // alias const Hashtable* variableNames; // alias ParseData(const TransliterationRuleData* data = 0, const UVector* variablesVector = 0, const Hashtable* variableNames = 0); virtual const UnicodeString* lookup(const UnicodeString& s) const; virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const; virtual UnicodeString parseReference(const UnicodeString& text, ParsePosition& pos, int32_t limit) const; /** * Return true if the given character is a matcher standin or a plain * character (non standin). */ UBool isMatcher(UChar32 ch); /** * Return true if the given character is a replacer standin or a plain * character (non standin). */ UBool isReplacer(UChar32 ch); private: ParseData(const ParseData &other); // forbid copying of this class ParseData &operator=(const ParseData &other); // forbid copying of this class }; ParseData::ParseData(const TransliterationRuleData* d, const UVector* sets, const Hashtable* vNames) : data(d), variablesVector(sets), variableNames(vNames) {} /** * Implement SymbolTable API. */ const UnicodeString* ParseData::lookup(const UnicodeString& name) const { return (const UnicodeString*) variableNames->get(name); } /** * Implement SymbolTable API. */ const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const { // Note that we cannot use data.lookupSet() because the // set array has not been constructed yet. const UnicodeFunctor* set = NULL; int32_t i = ch - data->variablesBase; if (i >= 0 && i < variablesVector->size()) { int32_t i = ch - data->variablesBase; set = (i < variablesVector->size()) ? (UnicodeFunctor*) variablesVector->elementAt(i) : 0; } return set; } /** * Implement SymbolTable API. Parse out a symbol reference * name. */ UnicodeString ParseData::parseReference(const UnicodeString& text, ParsePosition& pos, int32_t limit) const { int32_t start = pos.getIndex(); int32_t i = start; UnicodeString result; while (i < limit) { UChar c = text.charAt(i); if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) { break; } ++i; } if (i == start) { // No valid name chars return result; // Indicate failure with empty string } pos.setIndex(i); text.extractBetween(start, i, result); return result; } UBool ParseData::isMatcher(UChar32 ch) { // Note that we cannot use data.lookup() because the // set array has not been constructed yet. int32_t i = ch - data->variablesBase; if (i >= 0 && i < variablesVector->size()) { UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); return f != NULL && f->toMatcher() != NULL; } return TRUE; } /** * Return true if the given character is a replacer standin or a plain * character (non standin). */ UBool ParseData::isReplacer(UChar32 ch) { // Note that we cannot use data.lookup() because the // set array has not been constructed yet. int i = ch - data->variablesBase; if (i >= 0 && i < variablesVector->size()) { UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); return f != NULL && f->toReplacer() != NULL; } return TRUE; } //---------------------------------------------------------------------- // BEGIN RuleHalf //---------------------------------------------------------------------- /** * A class representing one side of a rule. This class knows how to * parse half of a rule. It is tightly coupled to the method * RuleBasedTransliterator.Parser.parseRule(). */ class RuleHalf : public UMemory { public: UnicodeString text; int32_t cursor; // position of cursor in text int32_t ante; // position of ante context marker '{' in text int32_t post; // position of post context marker '}' in text // Record the offset to the cursor either to the left or to the // right of the key. This is indicated by characters on the output // side that allow the cursor to be positioned arbitrarily within // the matching text. For example, abc{def} > | @@@ xyz; changes // def to xyz and moves the cursor to before abc. Offset characters // must be at the start or end, and they cannot move the cursor past // the ante- or postcontext text. Placeholders are only valid in // output text. The length of the ante and post context is // determined at runtime, because of supplementals and quantifiers. int32_t cursorOffset; // only nonzero on output side // Position of first CURSOR_OFFSET on _right_. This will be -1 // for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc. int32_t cursorOffsetPos; UBool anchorStart; UBool anchorEnd; /** * The segment number from 1..n of the next '(' we see * during parsing; 1-based. */ int32_t nextSegmentNumber; TransliteratorParser& parser; //-------------------------------------------------- // Methods RuleHalf(TransliteratorParser& parser); ~RuleHalf(); int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status); int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, UnicodeString& buf, const UnicodeString& illegal, UBool isSegment, UErrorCode& status); /** * Remove context. */ void removeContext(); /** * Return true if this half looks like valid output, that is, does not * contain quantifiers or other special input-only elements. */ UBool isValidOutput(TransliteratorParser& parser); /** * Return true if this half looks like valid input, that is, does not * contain functions or other special output-only elements. */ UBool isValidInput(TransliteratorParser& parser); int syntaxError(UErrorCode code, const UnicodeString& rule, int32_t start, UErrorCode& status) { return parser.syntaxError(code, rule, start, status); } private: // Disallowed methods; no impl. RuleHalf(const RuleHalf&); RuleHalf& operator=(const RuleHalf&); }; RuleHalf::RuleHalf(TransliteratorParser& p) : parser(p) { cursor = -1; ante = -1; post = -1; cursorOffset = 0; cursorOffsetPos = 0; anchorStart = anchorEnd = FALSE; nextSegmentNumber = 1; } RuleHalf::~RuleHalf() { } /** * Parse one side of a rule, stopping at either the limit, * the END_OF_RULE character, or an operator. * @return the index after the terminating character, or * if limit was reached, limit */ int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { int32_t start = pos; text.truncate(0); pos = parseSection(rule, pos, limit, text, ILLEGAL_TOP, FALSE, status); if (cursorOffset > 0 && cursor != cursorOffsetPos) { return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); } return pos; } /** * Parse a section of one side of a rule, stopping at either * the limit, the END_OF_RULE character, an operator, or a * segment close character. This method parses both a * top-level rule half and a segment within such a rule half. * It calls itself recursively to parse segments and nested * segments. * @param buf buffer into which to accumulate the rule pattern * characters, either literal characters from the rule or * standins for UnicodeMatcher objects including segments. * @param illegal the set of special characters that is illegal during * this parse. * @param isSegment if true, then we've already seen a '(' and * pos on entry points right after it. Accumulate everything * up to the closing ')', put it in a segment matcher object, * generate a standin for it, and add the standin to buf. As * a side effect, update the segments vector with a reference * to the segment matcher. This works recursively for nested * segments. If isSegment is false, just accumulate * characters into buf. * @return the index after the terminating character, or * if limit was reached, limit */ int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, UnicodeString& buf, const UnicodeString& illegal, UBool isSegment, UErrorCode& status) { int32_t start = pos; ParsePosition pp; UnicodeString scratch; UBool done = FALSE; int32_t quoteStart = -1; // Most recent 'single quoted string' int32_t quoteLimit = -1; int32_t varStart = -1; // Most recent $variableReference int32_t varLimit = -1; int32_t bufStart = buf.length(); while (pos < limit && !done) { // Since all syntax characters are in the BMP, fetching // 16-bit code units suffices here. UChar c = rule.charAt(pos++); if (uprv_isRuleWhiteSpace(c)) { // Ignore whitespace. Note that this is not Unicode // spaces, but Java spaces -- a subset, representing // whitespace likely to be seen in code. continue; } if (u_strchr(HALF_ENDERS, c) != NULL) { if (isSegment) { // Unclosed segment return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status); } break; } if (anchorEnd) { // Text after a presumed end anchor is a syntax err return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status); } if (UnicodeSet::resemblesPattern(rule, pos-1)) { pp.setIndex(pos-1); // Backup to opening '[' buf.append(parser.parseSet(rule, pp, status)); if (U_FAILURE(status)) { return syntaxError(U_MALFORMED_SET, rule, start, status); } pos = pp.getIndex(); continue; } // Handle escapes if (c == ESCAPE) { if (pos == limit) { return syntaxError(U_TRAILING_BACKSLASH, rule, start, status); } UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\' if (escaped == (UChar32) -1) { return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status); } if (!parser.checkVariableRange(escaped)) { return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status); } buf.append(escaped); continue; } // Handle quoted matter if (c == QUOTE) { int32_t iq = rule.indexOf(QUOTE, pos); if (iq == pos) { buf.append(c); // Parse [''] outside quotes as ['] ++pos; } else { /* This loop picks up a run of quoted text of the * form 'aaaa' each time through. If this run * hasn't really ended ('aaaa''bbbb') then it keeps * looping, each time adding on a new run. When it * reaches the final quote it breaks. */ quoteStart = buf.length(); for (;;) { if (iq < 0) { return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status); } scratch.truncate(0); rule.extractBetween(pos, iq, scratch); buf.append(scratch); pos = iq+1; if (pos < limit && rule.charAt(pos) == QUOTE) { // Parse [''] inside quotes as ['] iq = rule.indexOf(QUOTE, pos+1); // Continue looping } else { break; } } quoteLimit = buf.length(); for (iq=quoteStart; iq= 0) { syntaxError(U_ILLEGAL_CHARACTER, rule, start, status); } switch (c) { //------------------------------------------------------ // Elements allowed within and out of segments //------------------------------------------------------ case ANCHOR_START: if (buf.length() == 0 && !anchorStart) { anchorStart = TRUE; } else { return syntaxError(U_MISPLACED_ANCHOR_START, rule, start, status); } break; case SEGMENT_OPEN: { // bufSegStart is the offset in buf to the first // character of the segment we are parsing. int32_t bufSegStart = buf.length(); // Record segment number now, since nextSegmentNumber // will be incremented during the call to parseSection // if there are nested segments. int32_t segmentNumber = nextSegmentNumber++; // 1-based // Parse the segment pos = parseSection(rule, pos, limit, buf, ILLEGAL_SEG, TRUE, status); // After parsing a segment, the relevant characters are // in buf, starting at offset bufSegStart. Extract them // into a string matcher, and replace them with a // standin for that matcher. StringMatcher* m = new StringMatcher(buf, bufSegStart, buf.length(), segmentNumber, *parser.curData); // Record and associate object and segment number parser.setSegmentObject(segmentNumber, m, status); buf.truncate(bufSegStart); buf.append(parser.getSegmentStandin(segmentNumber, status)); } break; case FUNCTION: case ALT_FUNCTION: { int32_t iref = pos; TransliteratorIDParser::SingleID* single = TransliteratorIDParser::parseFilterID(rule, iref); // The next character MUST be a segment open if (single == NULL || !ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) { return syntaxError(U_INVALID_FUNCTION, rule, start, status); } Transliterator *t = single->createInstance(); delete single; if (t == NULL) { return syntaxError(U_INVALID_FUNCTION, rule, start, status); } // bufSegStart is the offset in buf to the first // character of the segment we are parsing. int32_t bufSegStart = buf.length(); // Parse the segment pos = parseSection(rule, iref, limit, buf, ILLEGAL_FUNC, TRUE, status); // After parsing a segment, the relevant characters are // in buf, starting at offset bufSegStart. UnicodeString output; buf.extractBetween(bufSegStart, buf.length(), output); FunctionReplacer *r = new FunctionReplacer(t, new StringReplacer(output, parser.curData)); // Replace the buffer contents with a stand-in buf.truncate(bufSegStart); buf.append(parser.generateStandInFor(r, status)); } break; case SymbolTable::SYMBOL_REF: // Handle variable references and segment references "$1" .. "$9" { // A variable reference must be followed immediately // by a Unicode identifier start and zero or more // Unicode identifier part characters, or by a digit // 1..9 if it is a segment reference. if (pos == limit) { // A variable ref character at the end acts as // an anchor to the context limit, as in perl. anchorEnd = TRUE; break; } // Parse "$1" "$2" .. "$9" .. (no upper limit) c = rule.charAt(pos); int32_t r = u_digit(c, 10); if (r >= 1 && r <= 9) { r = ICU_Utility::parseNumber(rule, pos, 10); if (r < 0) { return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status); } buf.append(parser.getSegmentStandin(r, status)); } else { pp.setIndex(pos); UnicodeString name = parser.parseData-> parseReference(rule, pp, limit); if (name.length() == 0) { // This means the '$' was not followed by a // valid name. Try to interpret it as an // end anchor then. If this also doesn't work // (if we see a following character) then signal // an error. anchorEnd = TRUE; break; } pos = pp.getIndex(); // If this is a variable definition statement, // then the LHS variable will be undefined. In // that case appendVariableDef() will append the // special placeholder char variableLimit-1. varStart = buf.length(); parser.appendVariableDef(name, buf, status); varLimit = buf.length(); } } break; case DOT: buf.append(parser.getDotStandIn(status)); break; case KLEENE_STAR: case ONE_OR_MORE: case ZERO_OR_ONE: // Quantifiers. We handle single characters, quoted strings, // variable references, and segments. // a+ matches aaa // 'foo'+ matches foofoofoo // $v+ matches xyxyxy if $v == xy // (seg)+ matches segsegseg { if (isSegment && buf.length() == bufStart) { // The */+ immediately follows '(' return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status); } int32_t qstart, qlimit; // The */+ follows an isolated character or quote // or variable reference if (buf.length() == quoteLimit) { // The */+ follows a 'quoted string' qstart = quoteStart; qlimit = quoteLimit; } else if (buf.length() == varLimit) { // The */+ follows a $variableReference qstart = varStart; qlimit = varLimit; } else { // The */+ follows a single character, possibly // a segment standin qstart = buf.length() - 1; qlimit = qstart + 1; } UnicodeFunctor *m = new StringMatcher(buf, qstart, qlimit, 0, *parser.curData); int32_t min = 0; int32_t max = Quantifier::MAX; switch (c) { case ONE_OR_MORE: min = 1; break; case ZERO_OR_ONE: min = 0; max = 1; break; // case KLEENE_STAR: // do nothing -- min, max already set } m = new Quantifier(m, min, max); buf.truncate(qstart); buf.append(parser.generateStandInFor(m, status)); } break; //------------------------------------------------------ // Elements allowed ONLY WITHIN segments //------------------------------------------------------ case SEGMENT_CLOSE: // assert(isSegment); // We're done parsing a segment. done = TRUE; break; //------------------------------------------------------ // Elements allowed ONLY OUTSIDE segments //------------------------------------------------------ case CONTEXT_ANTE: if (ante >= 0) { return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status); } ante = buf.length(); break; case CONTEXT_POST: if (post >= 0) { return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status); } post = buf.length(); break; case CURSOR_POS: if (cursor >= 0) { return syntaxError(U_MULTIPLE_CURSORS, rule, start, status); } cursor = buf.length(); break; case CURSOR_OFFSET: if (cursorOffset < 0) { if (buf.length() > 0) { return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); } --cursorOffset; } else if (cursorOffset > 0) { if (buf.length() != cursorOffsetPos || cursor >= 0) { return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); } ++cursorOffset; } else { if (cursor == 0 && buf.length() == 0) { cursorOffset = -1; } else if (cursor < 0) { cursorOffsetPos = buf.length(); cursorOffset = 1; } else { return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); } } break; //------------------------------------------------------ // Non-special characters //------------------------------------------------------ default: // Disallow unquoted characters other than [0-9A-Za-z] // in the printable ASCII range. These characters are // reserved for possible future use. if (c >= 0x0021 && c <= 0x007E && !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) || (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) || (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) { return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); } buf.append(c); break; } } return pos; } /** * Remove context. */ void RuleHalf::removeContext() { //text = text.substring(ante < 0 ? 0 : ante, // post < 0 ? text.length() : post); if (post >= 0) { text.remove(post); } if (ante >= 0) { text.removeBetween(0, ante); } ante = post = -1; anchorStart = anchorEnd = FALSE; } /** * Return true if this half looks like valid output, that is, does not * contain quantifiers or other special input-only elements. */ UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) { for (int32_t i=0; iisReplacer(c)) { return FALSE; } } return TRUE; } /** * Return true if this half looks like valid input, that is, does not * contain functions or other special output-only elements. */ UBool RuleHalf::isValidInput(TransliteratorParser& transParser) { for (int32_t i=0; iisMatcher(c)) { return FALSE; } } return TRUE; } //---------------------------------------------------------------------- // PUBLIC API //---------------------------------------------------------------------- /** * Constructor. */ TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) : dataVector(statusReturn), idBlockVector(statusReturn), variablesVector(statusReturn), segmentObjects(statusReturn) { idBlockVector.setDeleter(uhash_deleteUnicodeString); curData = NULL; compoundFilter = NULL; parseData = NULL; variableNames.setValueDeleter(uhash_deleteUnicodeString); } /** * Destructor. */ TransliteratorParser::~TransliteratorParser() { while (!dataVector.isEmpty()) delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); delete compoundFilter; delete parseData; while (!variablesVector.isEmpty()) delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); } void TransliteratorParser::parse(const UnicodeString& rules, UTransDirection transDirection, UParseError& pe, UErrorCode& ec) { if (U_SUCCESS(ec)) { parseRules(rules, transDirection, ec); pe = parseError; } } /** * Return the compound filter parsed by parse(). Caller owns result. */ UnicodeSet* TransliteratorParser::orphanCompoundFilter() { UnicodeSet* f = compoundFilter; compoundFilter = NULL; return f; } //---------------------------------------------------------------------- // Private implementation //---------------------------------------------------------------------- /** * Parse the given string as a sequence of rules, separated by newline * characters ('\n'), and cause this object to implement those rules. Any * previous rules are discarded. Typically this method is called exactly * once, during construction. * @exception IllegalArgumentException if there is a syntax error in the * rules */ void TransliteratorParser::parseRules(const UnicodeString& rule, UTransDirection theDirection, UErrorCode& status) { // Clear error struct uprv_memset(&parseError, 0, sizeof(parseError)); parseError.line = parseError.offset = -1; UBool parsingIDs = TRUE; int32_t ruleCount = 0; while (!dataVector.isEmpty()) { delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); } if (U_FAILURE(status)) { return; } idBlockVector.removeAllElements(); curData = NULL; direction = theDirection; ruleCount = 0; delete compoundFilter; compoundFilter = NULL; while (!variablesVector.isEmpty()) { delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); } variableNames.removeAll(); parseData = new ParseData(0, &variablesVector, &variableNames); if (parseData == NULL) { status = U_MEMORY_ALLOCATION_ERROR; return; } dotStandIn = (UChar) -1; UnicodeString str; // scratch UnicodeString idBlockResult; int32_t pos = 0; int32_t limit = rule.length(); // The compound filter offset is an index into idBlockResult. // If it is 0, then the compound filter occurred at the start, // and it is the offset to the _start_ of the compound filter // pattern. Otherwise it is the offset to the _limit_ of the // compound filter pattern within idBlockResult. compoundFilter = NULL; int32_t compoundFilterOffset = -1; while (pos < limit && U_SUCCESS(status)) { UChar c = rule.charAt(pos++); if (uprv_isRuleWhiteSpace(c)) { // Ignore leading whitespace. continue; } // Skip lines starting with the comment character if (c == RULE_COMMENT_CHAR) { pos = rule.indexOf((UChar)0x000A /*\n*/, pos) + 1; if (pos == 0) { break; // No "\n" found; rest of rule is a commnet } continue; // Either fall out or restart with next line } // skip empty rules if (c == END_OF_RULE) continue; // keep track of how many rules we've seen ++ruleCount; // We've found the start of a rule or ID. c is its first // character, and pos points past c. --pos; // Look for an ID token. Must have at least ID_TOKEN_LEN + 1 // chars left. if ((pos + ID_TOKEN_LEN + 1) <= limit && rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) { pos += ID_TOKEN_LEN; c = rule.charAt(pos); while (uprv_isRuleWhiteSpace(c) && pos < limit) { ++pos; c = rule.charAt(pos); } int32_t p = pos; if (!parsingIDs) { if (curData != NULL) { if (direction == UTRANS_FORWARD) dataVector.addElement(curData, status); else dataVector.insertElementAt(curData, 0, status); curData = NULL; } parsingIDs = TRUE; } TransliteratorIDParser::SingleID* id = TransliteratorIDParser::parseSingleID(rule, p, direction, status); if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) { // Successful ::ID parse. if (direction == UTRANS_FORWARD) { idBlockResult.append(id->canonID).append(END_OF_RULE); } else { idBlockResult.insert(0, END_OF_RULE); idBlockResult.insert(0, id->canonID); } } else { // Couldn't parse an ID. Try to parse a global filter int32_t withParens = -1; UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, NULL); if (f != NULL) { if (ICU_Utility::parseChar(rule, p, END_OF_RULE) && (direction == UTRANS_FORWARD) == (withParens == 0)) { if (compoundFilter != NULL) { // Multiple compound filters syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status); delete f; } else { compoundFilter = f; compoundFilterOffset = ruleCount; } } else { delete f; } } else { // Invalid ::id // Can be parsed as neither an ID nor a global filter syntaxError(U_INVALID_ID, rule, pos, status); } } delete id; pos = p; } else { if (parsingIDs) { if (direction == UTRANS_FORWARD) idBlockVector.addElement(new UnicodeString(idBlockResult), status); else idBlockVector.insertElementAt(new UnicodeString(idBlockResult), 0, status); idBlockResult.remove(); parsingIDs = FALSE; curData = new TransliterationRuleData(status); parseData->data = curData; // By default, rules use part of the private use area // E000..F8FF for variables and other stand-ins. Currently // the range F000..F8FF is typically sufficient. The 'use // variable range' pragma allows rule sets to modify this. setVariableRange(0xF000, 0xF8FF, status); } if (resemblesPragma(rule, pos, limit)) { int32_t ppp = parsePragma(rule, pos, limit, status); if (ppp < 0) { syntaxError(U_MALFORMED_PRAGMA, rule, pos, status); } pos = ppp; // Parse a rule } else { pos = parseRule(rule, pos, limit, status); } } } if (parsingIDs && idBlockResult.length() > 0) { if (direction == UTRANS_FORWARD) idBlockVector.addElement(new UnicodeString(idBlockResult), status); else idBlockVector.insertElementAt(new UnicodeString(idBlockResult), 0, status); } else if (!parsingIDs && curData != NULL) { if (direction == UTRANS_FORWARD) dataVector.addElement(curData, status); else dataVector.insertElementAt(curData, 0, status); } if (U_SUCCESS(status)) { // Convert the set vector to an array int32_t i, dataVectorSize = dataVector.size(); for (i = 0; i < dataVectorSize; i++) { TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); data->variablesLength = variablesVector.size(); if (data->variablesLength == 0) { data->variables = 0; } else { data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*)); data->variablesAreOwned = (i == 0); } for (int32_t j = 0; j < data->variablesLength; j++) { data->variables[j] = ((UnicodeSet*)variablesVector.elementAt(j)); } data->variableNames.removeAll(); int32_t pos = -1; const UHashElement* he = variableNames.nextElement(pos); while (he != NULL) { data->variableNames.put(*((UnicodeString*)(he->key.pointer)), ((UnicodeString*)(he->value.pointer))->clone(), status); he = variableNames.nextElement(pos); } } variablesVector.removeAllElements(); // keeps them from getting deleted when we succeed // Index the rules if (compoundFilter != NULL) { if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) || (direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) { status = U_MISPLACED_COMPOUND_FILTER; } } for (i = 0; i < dataVectorSize; i++) { TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); data->ruleSet.freeze(parseError, status); } if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) { idBlockVector.removeElementAt(0); } } } /** * Set the variable range to [start, end] (inclusive). */ void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) { if (start > end || start < 0 || end > 0xFFFF) { status = U_MALFORMED_PRAGMA; return; } curData->variablesBase = (UChar) start; if (dataVector.size() == 0) { variableNext = (UChar) start; variableLimit = (UChar) (end + 1); } } /** * Assert that the given character is NOT within the variable range. * If it is, return FALSE. This is neccesary to ensure that the * variable range does not overlap characters used in a rule. */ UBool TransliteratorParser::checkVariableRange(UChar32 ch) const { return !(ch >= curData->variablesBase && ch < variableLimit); } /** * Set the maximum backup to 'backup', in response to a pragma * statement. */ void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) { //TODO Finish } /** * Begin normalizing all rules using the given mode, in response * to a pragma statement. */ void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) { //TODO Finish } static const UChar PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use " static const UChar PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;" static const UChar PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;" static const UChar PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;" static const UChar PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;" /** * Return true if the given rule looks like a pragma. * @param pos offset to the first non-whitespace character * of the rule. * @param limit pointer past the last character of the rule. */ UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) { // Must start with /use\s/i return ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_USE, NULL) >= 0; } /** * Parse a pragma. This method assumes resemblesPragma() has * already returned true. * @param pos offset to the first non-whitespace character * of the rule. * @param limit pointer past the last character of the rule. * @return the position index after the final ';' of the pragma, * or -1 on failure. */ int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { int32_t array[2]; // resemblesPragma() has already returned true, so we // know that pos points to /use\s/i; we can skip 4 characters // immediately pos += 4; // Here are the pragmas we recognize: // use variable range 0xE000 0xEFFF; // use maximum backup 16; // use nfd rules; // use nfc rules; int p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_VARIABLE_RANGE, array); if (p >= 0) { setVariableRange(array[0], array[1], status); return p; } p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_MAXIMUM_BACKUP, array); if (p >= 0) { pragmaMaximumBackup(array[0]); return p; } p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_NFD_RULES, NULL); if (p >= 0) { pragmaNormalizeRules(UNORM_NFD); return p; } p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_NFC_RULES, NULL); if (p >= 0) { pragmaNormalizeRules(UNORM_NFC); return p; } // Syntax error: unable to parse pragma return -1; } /** * MAIN PARSER. Parse the next rule in the given rule string, starting * at pos. Return the index after the last character parsed. Do not * parse characters at or after limit. * * Important: The character at pos must be a non-whitespace character * that is not the comment character. * * This method handles quoting, escaping, and whitespace removal. It * parses the end-of-rule character. It recognizes context and cursor * indicators. Once it does a lexical breakdown of the rule at pos, it * creates a rule object and adds it to our rule list. */ int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { // Locate the left side, operator, and right side int32_t start = pos; UChar op = 0; int32_t i; // Set up segments data segmentStandins.truncate(0); segmentObjects.removeAllElements(); // Use pointers to automatics to make swapping possible. RuleHalf _left(*this), _right(*this); RuleHalf* left = &_left; RuleHalf* right = &_right; undefinedVariableName.remove(); pos = left->parse(rule, pos, limit, status); if (U_FAILURE(status)) { return start; } if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == NULL) { return syntaxError(U_MISSING_OPERATOR, rule, start, status); } ++pos; // Found an operator char. Check for forward-reverse operator. if (op == REVERSE_RULE_OP && (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) { ++pos; op = FWDREV_RULE_OP; } // Translate alternate op characters. switch (op) { case ALT_FORWARD_RULE_OP: op = FORWARD_RULE_OP; break; case ALT_REVERSE_RULE_OP: op = REVERSE_RULE_OP; break; case ALT_FWDREV_RULE_OP: op = FWDREV_RULE_OP; break; } pos = right->parse(rule, pos, limit, status); if (U_FAILURE(status)) { return start; } if (pos < limit) { if (rule.charAt(--pos) == END_OF_RULE) { ++pos; } else { // RuleHalf parser must have terminated at an operator return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); } } if (op == VARIABLE_DEF_OP) { // LHS is the name. RHS is a single character, either a literal // or a set (already parsed). If RHS is longer than one // character, it is either a multi-character string, or multiple // sets, or a mixture of chars and sets -- syntax error. // We expect to see a single undefined variable (the one being // defined). if (undefinedVariableName.length() == 0) { // "Missing '$' or duplicate definition" return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status); } if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) { // "Malformed LHS" return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); } if (left->anchorStart || left->anchorEnd || right->anchorStart || right->anchorEnd) { return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); } // We allow anything on the right, including an empty string. UnicodeString* value = new UnicodeString(right->text); variableNames.put(undefinedVariableName, value, status); ++variableLimit; return pos; } // If this is not a variable definition rule, we shouldn't have // any undefined variable names. if (undefinedVariableName.length() != 0) { return syntaxError(// "Undefined variable $" + undefinedVariableName, U_UNDEFINED_VARIABLE, rule, start, status); } // Verify segments if (segmentStandins.length() > segmentObjects.size()) { syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status); } for (i=0; iremoveContext(); left->cursor = -1; left->cursorOffset = 0; } // Normalize context if (left->ante < 0) { left->ante = 0; } if (left->post < 0) { left->post = left->text.length(); } // Context is only allowed on the input side. Cursors are only // allowed on the output side. Segment delimiters can only appear // on the left, and references on the right. Cursor offset // cannot appear without an explicit cursor. Cursor offset // cannot place the cursor outside the limits of the context. // Anchors are only allowed on the input side. if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 || (right->cursorOffset != 0 && right->cursor < 0) || // - The following two checks were used to ensure that the // - the cursor offset stayed within the ante- or postcontext. // - However, with the addition of quantifiers, we have to // - allow arbitrary cursor offsets and do runtime checking. //(right->cursorOffset > (left->text.length() - left->post)) || //(-right->cursorOffset > left->ante) || right->anchorStart || right->anchorEnd || !left->isValidInput(*this) || !right->isValidOutput(*this) || left->ante > left->post) { return syntaxError(U_MALFORMED_RULE, rule, start, status); } // Flatten segment objects vector to an array UnicodeFunctor** segmentsArray = NULL; if (segmentObjects.size() > 0) { segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *)); segmentObjects.toArray((void**) segmentsArray); } curData->ruleSet.addRule(new TransliterationRule( left->text, left->ante, left->post, right->text, right->cursor, right->cursorOffset, segmentsArray, segmentObjects.size(), left->anchorStart, left->anchorEnd, curData, status), status); return pos; } /** * Called by main parser upon syntax error. Search the rule string * for the probable end of the rule. Of course, if the error is that * the end of rule marker is missing, then the rule end will not be found. * In any case the rule start will be correctly reported. * @param msg error description * @param rule pattern string * @param start position of first character of current rule */ int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode, const UnicodeString& rule, int32_t pos, UErrorCode& status) { parseError.offset = pos; parseError.line = 0 ; /* we are not using line numbers */ // for pre-context const int32_t LEN = U_PARSE_CONTEXT_LEN - 1; int32_t start = uprv_max(pos - LEN, 0); int32_t stop = pos; rule.extract(start,stop-start,parseError.preContext); //null terminate the buffer parseError.preContext[stop-start] = 0; //for post-context start = pos; stop = uprv_min(pos + LEN, rule.length()); rule.extract(start,stop-start,parseError.postContext); //null terminate the buffer parseError.postContext[stop-start]= 0; status = (UErrorCode)parseErrorCode; return pos; } /** * Parse a UnicodeSet out, store it, and return the stand-in character * used to represent it. */ UChar TransliteratorParser::parseSet(const UnicodeString& rule, ParsePosition& pos, UErrorCode& status) { UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status); set->compact(); return generateStandInFor(set, status); } /** * Generate and return a stand-in for a new UnicodeFunctor. Store * the matcher (adopt it). */ UChar TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) { // assert(obj != null); // Look up previous stand-in, if any. This is a short list // (typical n is 0, 1, or 2); linear search is optimal. for (int32_t i=0; ivariablesBase + i); } } if (variableNext >= variableLimit) { delete adopted; status = U_VARIABLE_RANGE_EXHAUSTED; return 0; } variablesVector.addElement(adopted, status); return variableNext++; } /** * Return the standin for segment seg (1-based). */ UChar TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) { // Special character used to indicate an empty spot UChar empty = curData->variablesBase - 1; while (segmentStandins.length() < seg) { segmentStandins.append(empty); } UChar c = segmentStandins.charAt(seg-1); if (c == empty) { if (variableNext >= variableLimit) { status = U_VARIABLE_RANGE_EXHAUSTED; return 0; } c = variableNext++; // Set a placeholder in the master variables vector that will be // filled in later by setSegmentObject(). We know that we will get // called first because setSegmentObject() will call us. variablesVector.addElement((void*) NULL, status); segmentStandins.setCharAt(seg-1, c); } return c; } /** * Set the object for segment seg (1-based). */ void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) { // Since we call parseSection() recursively, nested // segments will result in segment i+1 getting parsed // and stored before segment i; be careful with the // vector handling here. if (segmentObjects.size() < seg) { segmentObjects.setSize(seg); } int32_t index = getSegmentStandin(seg, status) - curData->variablesBase; if (segmentObjects.elementAt(seg-1) != NULL || variablesVector.elementAt(index) != NULL) { // should never happen status = U_INTERNAL_TRANSLITERATOR_ERROR; return; } segmentObjects.setElementAt(adopted, seg-1); variablesVector.setElementAt(adopted, index); } /** * Return the stand-in for the dot set. It is allocated the first * time and reused thereafter. */ UChar TransliteratorParser::getDotStandIn(UErrorCode& status) { if (dotStandIn == (UChar) -1) { dotStandIn = generateStandInFor(new UnicodeSet(DOT_SET, status), status); } return dotStandIn; } /** * Append the value of the given variable name to the given * UnicodeString. */ void TransliteratorParser::appendVariableDef(const UnicodeString& name, UnicodeString& buf, UErrorCode& status) { const UnicodeString* s = (const UnicodeString*) variableNames.get(name); if (s == NULL) { // We allow one undefined variable so that variable definition // statements work. For the first undefined variable we return // the special placeholder variableLimit-1, and save the variable // name. if (undefinedVariableName.length() == 0) { undefinedVariableName = name; if (variableNext >= variableLimit) { // throw new RuntimeException("Private use variables exhausted"); status = U_ILLEGAL_ARGUMENT_ERROR; return; } buf.append((UChar) --variableLimit); } else { //throw new IllegalArgumentException("Undefined variable $" // + name); status = U_ILLEGAL_ARGUMENT_ERROR; return; } } else { buf.append(*s); } } /** * Glue method to get around access restrictions in C++. */ Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) { return Transliterator::createBasicInstance(id, canonID); } U_NAMESPACE_END U_CAPI int32_t utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status) { U_NAMESPACE_USE //const UChar *sourceStart = source; const UChar *targetStart = target; const UChar *sourceLimit = source+sourceLen; UChar *targetLimit = target+sourceLen; UChar32 c = 0; UBool quoted = FALSE; int32_t index; uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR); /* read the rules into the buffer */ while (source < sourceLimit) { index=0; U16_NEXT_UNSAFE(source, index, c); source+=index; if(c == QUOTE) { quoted = (UBool)!quoted; } else if (!quoted) { if (c == RULE_COMMENT_CHAR) { /* skip comments and all preceding spaces */ while (targetStart < target && *(target - 1) == 0x0020) { target--; } do { c = *(source++); } while (c != CR && c != LF); } else if (c == ESCAPE) { UChar32 c2 = *source; if (c2 == CR || c2 == LF) { /* A backslash at the end of a line. */ /* Since we're stripping lines, ignore the backslash. */ source++; continue; } if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */ int32_t escapeOffset = 0; UnicodeString escapedStr(source, 5); c2 = escapedStr.unescapeAt(escapeOffset); if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0) { *status = U_PARSE_ERROR; return 0; } if (!uprv_isRuleWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) { /* It was escaped for a reason. Write what it was suppose to be. */ source+=5; c = c2; } } else if (c2 == QUOTE) { /* \' seen. Make sure we don't do anything when we see it again. */ quoted = (UBool)!quoted; } } } if (c == CR || c == LF) { /* ignore spaces carriage returns, and all leading spaces on the next line. * and line feed unless in the form \uXXXX */ quoted = FALSE; while (source < sourceLimit) { c = *(source); if (c != CR && c != LF && c != 0x0020) { break; } source++; } continue; } /* Append UChar * after dissembling if c > 0xffff*/ index=0; U16_APPEND_UNSAFE(target, index, c); target+=index; } if (target < targetLimit) { *target = 0; } return (int32_t)(target-targetStart); } #endif /* #if !UCONFIG_NO_TRANSLITERATION */