1748 lines
62 KiB
C++
1748 lines
62 KiB
C++
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*
|
|
**********************************************************************
|
|
* Copyright (C) 1999-2016, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
**********************************************************************
|
|
* Date Name Description
|
|
* 11/17/99 aliu Creation.
|
|
**********************************************************************
|
|
*/
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_TRANSLITERATION
|
|
|
|
#include "unicode/uobject.h"
|
|
#include "unicode/parseerr.h"
|
|
#include "unicode/parsepos.h"
|
|
#include "unicode/putil.h"
|
|
#include "unicode/uchar.h"
|
|
#include "unicode/ustring.h"
|
|
#include "unicode/uniset.h"
|
|
#include "unicode/utf16.h"
|
|
#include "cstring.h"
|
|
#include "funcrepl.h"
|
|
#include "hash.h"
|
|
#include "quant.h"
|
|
#include "rbt.h"
|
|
#include "rbt_data.h"
|
|
#include "rbt_pars.h"
|
|
#include "rbt_rule.h"
|
|
#include "strmatch.h"
|
|
#include "strrepl.h"
|
|
#include "unicode/symtable.h"
|
|
#include "tridpars.h"
|
|
#include "uvector.h"
|
|
#include "hash.h"
|
|
#include "patternprops.h"
|
|
#include "util.h"
|
|
#include "cmemory.h"
|
|
#include "uprops.h"
|
|
#include "putilimp.h"
|
|
|
|
// Operators
|
|
#define VARIABLE_DEF_OP ((UChar)0x003D) /*=*/
|
|
#define FORWARD_RULE_OP ((UChar)0x003E) /*>*/
|
|
#define REVERSE_RULE_OP ((UChar)0x003C) /*<*/
|
|
#define FWDREV_RULE_OP ((UChar)0x007E) /*~*/ // internal rep of <> op
|
|
|
|
// Other special characters
|
|
#define QUOTE ((UChar)0x0027) /*'*/
|
|
#define ESCAPE ((UChar)0x005C) /*\*/
|
|
#define END_OF_RULE ((UChar)0x003B) /*;*/
|
|
#define RULE_COMMENT_CHAR ((UChar)0x0023) /*#*/
|
|
|
|
#define SEGMENT_OPEN ((UChar)0x0028) /*(*/
|
|
#define SEGMENT_CLOSE ((UChar)0x0029) /*)*/
|
|
#define CONTEXT_ANTE ((UChar)0x007B) /*{*/
|
|
#define CONTEXT_POST ((UChar)0x007D) /*}*/
|
|
#define CURSOR_POS ((UChar)0x007C) /*|*/
|
|
#define CURSOR_OFFSET ((UChar)0x0040) /*@*/
|
|
#define ANCHOR_START ((UChar)0x005E) /*^*/
|
|
#define KLEENE_STAR ((UChar)0x002A) /***/
|
|
#define ONE_OR_MORE ((UChar)0x002B) /*+*/
|
|
#define ZERO_OR_ONE ((UChar)0x003F) /*?*/
|
|
|
|
#define DOT ((UChar)46) /*.*/
|
|
|
|
static const UChar DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]";
|
|
91, 94, 91, 58, 90, 112, 58, 93, 91, 58, 90,
|
|
108, 58, 93, 92, 114, 92, 110, 36, 93, 0
|
|
};
|
|
|
|
// A function is denoted &Source-Target/Variant(text)
|
|
#define FUNCTION ((UChar)38) /*&*/
|
|
|
|
// Aliases for some of the syntax characters. These are provided so
|
|
// transliteration rules can be expressed in XML without clashing with
|
|
// XML syntax characters '<', '>', and '&'.
|
|
#define ALT_REVERSE_RULE_OP ((UChar)0x2190) // Left Arrow
|
|
#define ALT_FORWARD_RULE_OP ((UChar)0x2192) // Right Arrow
|
|
#define ALT_FWDREV_RULE_OP ((UChar)0x2194) // Left Right Arrow
|
|
#define ALT_FUNCTION ((UChar)0x2206) // Increment (~Greek Capital Delta)
|
|
|
|
// Special characters disallowed at the top level
|
|
static const UChar ILLEGAL_TOP[] = {41,0}; // ")"
|
|
|
|
// Special characters disallowed within a segment
|
|
static const UChar ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@"
|
|
|
|
// Special characters disallowed within a function argument
|
|
static const UChar ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@"
|
|
|
|
// By definition, the ANCHOR_END special character is a
|
|
// trailing SymbolTable.SYMBOL_REF character.
|
|
// private static final char ANCHOR_END = '$';
|
|
|
|
static const UChar gOPERATORS[] = { // "=><"
|
|
VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
|
|
ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
|
|
0
|
|
};
|
|
|
|
static const UChar HALF_ENDERS[] = { // "=><;"
|
|
VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
|
|
ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
|
|
END_OF_RULE,
|
|
0
|
|
};
|
|
|
|
// These are also used in Transliterator::toRules()
|
|
static const int32_t ID_TOKEN_LEN = 2;
|
|
static const UChar ID_TOKEN[] = { 0x3A, 0x3A }; // ':', ':'
|
|
|
|
/*
|
|
commented out until we do real ::BEGIN/::END functionality
|
|
static const int32_t BEGIN_TOKEN_LEN = 5;
|
|
static const UChar BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN'
|
|
|
|
static const int32_t END_TOKEN_LEN = 3;
|
|
static const UChar END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END'
|
|
*/
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
//----------------------------------------------------------------------
|
|
// BEGIN ParseData
|
|
//----------------------------------------------------------------------
|
|
|
|
/**
|
|
* This class implements the SymbolTable interface. It is used
|
|
* during parsing to give UnicodeSet access to variables that
|
|
* have been defined so far. Note that it uses variablesVector,
|
|
* _not_ data.setVariables.
|
|
*/
|
|
class ParseData : public UMemory, public SymbolTable {
|
|
public:
|
|
const TransliterationRuleData* data; // alias
|
|
|
|
const UVector* variablesVector; // alias
|
|
|
|
const Hashtable* variableNames; // alias
|
|
|
|
ParseData(const TransliterationRuleData* data = 0,
|
|
const UVector* variablesVector = 0,
|
|
const Hashtable* variableNames = 0);
|
|
|
|
virtual ~ParseData();
|
|
|
|
virtual const UnicodeString* lookup(const UnicodeString& s) const;
|
|
|
|
virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const;
|
|
|
|
virtual UnicodeString parseReference(const UnicodeString& text,
|
|
ParsePosition& pos, int32_t limit) const;
|
|
/**
|
|
* Return true if the given character is a matcher standin or a plain
|
|
* character (non standin).
|
|
*/
|
|
UBool isMatcher(UChar32 ch);
|
|
|
|
/**
|
|
* Return true if the given character is a replacer standin or a plain
|
|
* character (non standin).
|
|
*/
|
|
UBool isReplacer(UChar32 ch);
|
|
|
|
private:
|
|
ParseData(const ParseData &other); // forbid copying of this class
|
|
ParseData &operator=(const ParseData &other); // forbid copying of this class
|
|
};
|
|
|
|
ParseData::ParseData(const TransliterationRuleData* d,
|
|
const UVector* sets,
|
|
const Hashtable* vNames) :
|
|
data(d), variablesVector(sets), variableNames(vNames) {}
|
|
|
|
ParseData::~ParseData() {}
|
|
|
|
/**
|
|
* Implement SymbolTable API.
|
|
*/
|
|
const UnicodeString* ParseData::lookup(const UnicodeString& name) const {
|
|
return (const UnicodeString*) variableNames->get(name);
|
|
}
|
|
|
|
/**
|
|
* Implement SymbolTable API.
|
|
*/
|
|
const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const {
|
|
// Note that we cannot use data.lookupSet() because the
|
|
// set array has not been constructed yet.
|
|
const UnicodeFunctor* set = NULL;
|
|
int32_t i = ch - data->variablesBase;
|
|
if (i >= 0 && i < variablesVector->size()) {
|
|
int32_t i = ch - data->variablesBase;
|
|
set = (i < variablesVector->size()) ?
|
|
(UnicodeFunctor*) variablesVector->elementAt(i) : 0;
|
|
}
|
|
return set;
|
|
}
|
|
|
|
/**
|
|
* Implement SymbolTable API. Parse out a symbol reference
|
|
* name.
|
|
*/
|
|
UnicodeString ParseData::parseReference(const UnicodeString& text,
|
|
ParsePosition& pos, int32_t limit) const {
|
|
int32_t start = pos.getIndex();
|
|
int32_t i = start;
|
|
UnicodeString result;
|
|
while (i < limit) {
|
|
UChar c = text.charAt(i);
|
|
if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) {
|
|
break;
|
|
}
|
|
++i;
|
|
}
|
|
if (i == start) { // No valid name chars
|
|
return result; // Indicate failure with empty string
|
|
}
|
|
pos.setIndex(i);
|
|
text.extractBetween(start, i, result);
|
|
return result;
|
|
}
|
|
|
|
UBool ParseData::isMatcher(UChar32 ch) {
|
|
// Note that we cannot use data.lookup() because the
|
|
// set array has not been constructed yet.
|
|
int32_t i = ch - data->variablesBase;
|
|
if (i >= 0 && i < variablesVector->size()) {
|
|
UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
|
|
return f != NULL && f->toMatcher() != NULL;
|
|
}
|
|
return TRUE;
|
|
}
|
|
|
|
/**
|
|
* Return true if the given character is a replacer standin or a plain
|
|
* character (non standin).
|
|
*/
|
|
UBool ParseData::isReplacer(UChar32 ch) {
|
|
// Note that we cannot use data.lookup() because the
|
|
// set array has not been constructed yet.
|
|
int i = ch - data->variablesBase;
|
|
if (i >= 0 && i < variablesVector->size()) {
|
|
UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
|
|
return f != NULL && f->toReplacer() != NULL;
|
|
}
|
|
return TRUE;
|
|
}
|
|
|
|
//----------------------------------------------------------------------
|
|
// BEGIN RuleHalf
|
|
//----------------------------------------------------------------------
|
|
|
|
/**
|
|
* A class representing one side of a rule. This class knows how to
|
|
* parse half of a rule. It is tightly coupled to the method
|
|
* RuleBasedTransliterator.Parser.parseRule().
|
|
*/
|
|
class RuleHalf : public UMemory {
|
|
|
|
public:
|
|
|
|
UnicodeString text;
|
|
|
|
int32_t cursor; // position of cursor in text
|
|
int32_t ante; // position of ante context marker '{' in text
|
|
int32_t post; // position of post context marker '}' in text
|
|
|
|
// Record the offset to the cursor either to the left or to the
|
|
// right of the key. This is indicated by characters on the output
|
|
// side that allow the cursor to be positioned arbitrarily within
|
|
// the matching text. For example, abc{def} > | @@@ xyz; changes
|
|
// def to xyz and moves the cursor to before abc. Offset characters
|
|
// must be at the start or end, and they cannot move the cursor past
|
|
// the ante- or postcontext text. Placeholders are only valid in
|
|
// output text. The length of the ante and post context is
|
|
// determined at runtime, because of supplementals and quantifiers.
|
|
int32_t cursorOffset; // only nonzero on output side
|
|
|
|
// Position of first CURSOR_OFFSET on _right_. This will be -1
|
|
// for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc.
|
|
int32_t cursorOffsetPos;
|
|
|
|
UBool anchorStart;
|
|
UBool anchorEnd;
|
|
|
|
/**
|
|
* The segment number from 1..n of the next '(' we see
|
|
* during parsing; 1-based.
|
|
*/
|
|
int32_t nextSegmentNumber;
|
|
|
|
TransliteratorParser& parser;
|
|
|
|
//--------------------------------------------------
|
|
// Methods
|
|
|
|
RuleHalf(TransliteratorParser& parser);
|
|
~RuleHalf();
|
|
|
|
int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status);
|
|
|
|
int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
|
|
UnicodeString& buf,
|
|
const UnicodeString& illegal,
|
|
UBool isSegment,
|
|
UErrorCode& status);
|
|
|
|
/**
|
|
* Remove context.
|
|
*/
|
|
void removeContext();
|
|
|
|
/**
|
|
* Return true if this half looks like valid output, that is, does not
|
|
* contain quantifiers or other special input-only elements.
|
|
*/
|
|
UBool isValidOutput(TransliteratorParser& parser);
|
|
|
|
/**
|
|
* Return true if this half looks like valid input, that is, does not
|
|
* contain functions or other special output-only elements.
|
|
*/
|
|
UBool isValidInput(TransliteratorParser& parser);
|
|
|
|
int syntaxError(UErrorCode code,
|
|
const UnicodeString& rule,
|
|
int32_t start,
|
|
UErrorCode& status) {
|
|
return parser.syntaxError(code, rule, start, status);
|
|
}
|
|
|
|
private:
|
|
// Disallowed methods; no impl.
|
|
RuleHalf(const RuleHalf&);
|
|
RuleHalf& operator=(const RuleHalf&);
|
|
};
|
|
|
|
RuleHalf::RuleHalf(TransliteratorParser& p) :
|
|
parser(p)
|
|
{
|
|
cursor = -1;
|
|
ante = -1;
|
|
post = -1;
|
|
cursorOffset = 0;
|
|
cursorOffsetPos = 0;
|
|
anchorStart = anchorEnd = FALSE;
|
|
nextSegmentNumber = 1;
|
|
}
|
|
|
|
RuleHalf::~RuleHalf() {
|
|
}
|
|
|
|
/**
|
|
* Parse one side of a rule, stopping at either the limit,
|
|
* the END_OF_RULE character, or an operator.
|
|
* @return the index after the terminating character, or
|
|
* if limit was reached, limit
|
|
*/
|
|
int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
|
|
int32_t start = pos;
|
|
text.truncate(0);
|
|
pos = parseSection(rule, pos, limit, text, UnicodeString(TRUE, ILLEGAL_TOP, -1), FALSE, status);
|
|
|
|
if (cursorOffset > 0 && cursor != cursorOffsetPos) {
|
|
return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
|
|
}
|
|
|
|
return pos;
|
|
}
|
|
|
|
/**
|
|
* Parse a section of one side of a rule, stopping at either
|
|
* the limit, the END_OF_RULE character, an operator, or a
|
|
* segment close character. This method parses both a
|
|
* top-level rule half and a segment within such a rule half.
|
|
* It calls itself recursively to parse segments and nested
|
|
* segments.
|
|
* @param buf buffer into which to accumulate the rule pattern
|
|
* characters, either literal characters from the rule or
|
|
* standins for UnicodeMatcher objects including segments.
|
|
* @param illegal the set of special characters that is illegal during
|
|
* this parse.
|
|
* @param isSegment if true, then we've already seen a '(' and
|
|
* pos on entry points right after it. Accumulate everything
|
|
* up to the closing ')', put it in a segment matcher object,
|
|
* generate a standin for it, and add the standin to buf. As
|
|
* a side effect, update the segments vector with a reference
|
|
* to the segment matcher. This works recursively for nested
|
|
* segments. If isSegment is false, just accumulate
|
|
* characters into buf.
|
|
* @return the index after the terminating character, or
|
|
* if limit was reached, limit
|
|
*/
|
|
int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
|
|
UnicodeString& buf,
|
|
const UnicodeString& illegal,
|
|
UBool isSegment, UErrorCode& status) {
|
|
int32_t start = pos;
|
|
ParsePosition pp;
|
|
UnicodeString scratch;
|
|
UBool done = FALSE;
|
|
int32_t quoteStart = -1; // Most recent 'single quoted string'
|
|
int32_t quoteLimit = -1;
|
|
int32_t varStart = -1; // Most recent $variableReference
|
|
int32_t varLimit = -1;
|
|
int32_t bufStart = buf.length();
|
|
|
|
while (pos < limit && !done) {
|
|
// Since all syntax characters are in the BMP, fetching
|
|
// 16-bit code units suffices here.
|
|
UChar c = rule.charAt(pos++);
|
|
if (PatternProps::isWhiteSpace(c)) {
|
|
// Ignore whitespace. Note that this is not Unicode
|
|
// spaces, but Java spaces -- a subset, representing
|
|
// whitespace likely to be seen in code.
|
|
continue;
|
|
}
|
|
if (u_strchr(HALF_ENDERS, c) != NULL) {
|
|
if (isSegment) {
|
|
// Unclosed segment
|
|
return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status);
|
|
}
|
|
break;
|
|
}
|
|
if (anchorEnd) {
|
|
// Text after a presumed end anchor is a syntax err
|
|
return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status);
|
|
}
|
|
if (UnicodeSet::resemblesPattern(rule, pos-1)) {
|
|
pp.setIndex(pos-1); // Backup to opening '['
|
|
buf.append(parser.parseSet(rule, pp, status));
|
|
if (U_FAILURE(status)) {
|
|
return syntaxError(U_MALFORMED_SET, rule, start, status);
|
|
}
|
|
pos = pp.getIndex();
|
|
continue;
|
|
}
|
|
// Handle escapes
|
|
if (c == ESCAPE) {
|
|
if (pos == limit) {
|
|
return syntaxError(U_TRAILING_BACKSLASH, rule, start, status);
|
|
}
|
|
UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\'
|
|
if (escaped == (UChar32) -1) {
|
|
return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status);
|
|
}
|
|
if (!parser.checkVariableRange(escaped)) {
|
|
return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
|
|
}
|
|
buf.append(escaped);
|
|
continue;
|
|
}
|
|
// Handle quoted matter
|
|
if (c == QUOTE) {
|
|
int32_t iq = rule.indexOf(QUOTE, pos);
|
|
if (iq == pos) {
|
|
buf.append(c); // Parse [''] outside quotes as [']
|
|
++pos;
|
|
} else {
|
|
/* This loop picks up a run of quoted text of the
|
|
* form 'aaaa' each time through. If this run
|
|
* hasn't really ended ('aaaa''bbbb') then it keeps
|
|
* looping, each time adding on a new run. When it
|
|
* reaches the final quote it breaks.
|
|
*/
|
|
quoteStart = buf.length();
|
|
for (;;) {
|
|
if (iq < 0) {
|
|
return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status);
|
|
}
|
|
scratch.truncate(0);
|
|
rule.extractBetween(pos, iq, scratch);
|
|
buf.append(scratch);
|
|
pos = iq+1;
|
|
if (pos < limit && rule.charAt(pos) == QUOTE) {
|
|
// Parse [''] inside quotes as [']
|
|
iq = rule.indexOf(QUOTE, pos+1);
|
|
// Continue looping
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
quoteLimit = buf.length();
|
|
|
|
for (iq=quoteStart; iq<quoteLimit; ++iq) {
|
|
if (!parser.checkVariableRange(buf.charAt(iq))) {
|
|
return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
|
|
}
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (!parser.checkVariableRange(c)) {
|
|
return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
|
|
}
|
|
|
|
if (illegal.indexOf(c) >= 0) {
|
|
syntaxError(U_ILLEGAL_CHARACTER, rule, start, status);
|
|
}
|
|
|
|
switch (c) {
|
|
|
|
//------------------------------------------------------
|
|
// Elements allowed within and out of segments
|
|
//------------------------------------------------------
|
|
case ANCHOR_START:
|
|
if (buf.length() == 0 && !anchorStart) {
|
|
anchorStart = TRUE;
|
|
} else {
|
|
return syntaxError(U_MISPLACED_ANCHOR_START,
|
|
rule, start, status);
|
|
}
|
|
break;
|
|
case SEGMENT_OPEN:
|
|
{
|
|
// bufSegStart is the offset in buf to the first
|
|
// character of the segment we are parsing.
|
|
int32_t bufSegStart = buf.length();
|
|
|
|
// Record segment number now, since nextSegmentNumber
|
|
// will be incremented during the call to parseSection
|
|
// if there are nested segments.
|
|
int32_t segmentNumber = nextSegmentNumber++; // 1-based
|
|
|
|
// Parse the segment
|
|
pos = parseSection(rule, pos, limit, buf, UnicodeString(TRUE, ILLEGAL_SEG, -1), TRUE, status);
|
|
|
|
// After parsing a segment, the relevant characters are
|
|
// in buf, starting at offset bufSegStart. Extract them
|
|
// into a string matcher, and replace them with a
|
|
// standin for that matcher.
|
|
StringMatcher* m =
|
|
new StringMatcher(buf, bufSegStart, buf.length(),
|
|
segmentNumber, *parser.curData);
|
|
if (m == NULL) {
|
|
return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
|
|
}
|
|
|
|
// Record and associate object and segment number
|
|
parser.setSegmentObject(segmentNumber, m, status);
|
|
buf.truncate(bufSegStart);
|
|
buf.append(parser.getSegmentStandin(segmentNumber, status));
|
|
}
|
|
break;
|
|
case FUNCTION:
|
|
case ALT_FUNCTION:
|
|
{
|
|
int32_t iref = pos;
|
|
TransliteratorIDParser::SingleID* single =
|
|
TransliteratorIDParser::parseFilterID(rule, iref);
|
|
// The next character MUST be a segment open
|
|
if (single == NULL ||
|
|
!ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) {
|
|
return syntaxError(U_INVALID_FUNCTION, rule, start, status);
|
|
}
|
|
|
|
Transliterator *t = single->createInstance();
|
|
delete single;
|
|
if (t == NULL) {
|
|
return syntaxError(U_INVALID_FUNCTION, rule, start, status);
|
|
}
|
|
|
|
// bufSegStart is the offset in buf to the first
|
|
// character of the segment we are parsing.
|
|
int32_t bufSegStart = buf.length();
|
|
|
|
// Parse the segment
|
|
pos = parseSection(rule, iref, limit, buf, UnicodeString(TRUE, ILLEGAL_FUNC, -1), TRUE, status);
|
|
|
|
// After parsing a segment, the relevant characters are
|
|
// in buf, starting at offset bufSegStart.
|
|
UnicodeString output;
|
|
buf.extractBetween(bufSegStart, buf.length(), output);
|
|
FunctionReplacer *r =
|
|
new FunctionReplacer(t, new StringReplacer(output, parser.curData));
|
|
if (r == NULL) {
|
|
return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
|
|
}
|
|
|
|
// Replace the buffer contents with a stand-in
|
|
buf.truncate(bufSegStart);
|
|
buf.append(parser.generateStandInFor(r, status));
|
|
}
|
|
break;
|
|
case SymbolTable::SYMBOL_REF:
|
|
// Handle variable references and segment references "$1" .. "$9"
|
|
{
|
|
// A variable reference must be followed immediately
|
|
// by a Unicode identifier start and zero or more
|
|
// Unicode identifier part characters, or by a digit
|
|
// 1..9 if it is a segment reference.
|
|
if (pos == limit) {
|
|
// A variable ref character at the end acts as
|
|
// an anchor to the context limit, as in perl.
|
|
anchorEnd = TRUE;
|
|
break;
|
|
}
|
|
// Parse "$1" "$2" .. "$9" .. (no upper limit)
|
|
c = rule.charAt(pos);
|
|
int32_t r = u_digit(c, 10);
|
|
if (r >= 1 && r <= 9) {
|
|
r = ICU_Utility::parseNumber(rule, pos, 10);
|
|
if (r < 0) {
|
|
return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE,
|
|
rule, start, status);
|
|
}
|
|
buf.append(parser.getSegmentStandin(r, status));
|
|
} else {
|
|
pp.setIndex(pos);
|
|
UnicodeString name = parser.parseData->
|
|
parseReference(rule, pp, limit);
|
|
if (name.length() == 0) {
|
|
// This means the '$' was not followed by a
|
|
// valid name. Try to interpret it as an
|
|
// end anchor then. If this also doesn't work
|
|
// (if we see a following character) then signal
|
|
// an error.
|
|
anchorEnd = TRUE;
|
|
break;
|
|
}
|
|
pos = pp.getIndex();
|
|
// If this is a variable definition statement,
|
|
// then the LHS variable will be undefined. In
|
|
// that case appendVariableDef() will append the
|
|
// special placeholder char variableLimit-1.
|
|
varStart = buf.length();
|
|
parser.appendVariableDef(name, buf, status);
|
|
varLimit = buf.length();
|
|
}
|
|
}
|
|
break;
|
|
case DOT:
|
|
buf.append(parser.getDotStandIn(status));
|
|
break;
|
|
case KLEENE_STAR:
|
|
case ONE_OR_MORE:
|
|
case ZERO_OR_ONE:
|
|
// Quantifiers. We handle single characters, quoted strings,
|
|
// variable references, and segments.
|
|
// a+ matches aaa
|
|
// 'foo'+ matches foofoofoo
|
|
// $v+ matches xyxyxy if $v == xy
|
|
// (seg)+ matches segsegseg
|
|
{
|
|
if (isSegment && buf.length() == bufStart) {
|
|
// The */+ immediately follows '('
|
|
return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status);
|
|
}
|
|
|
|
int32_t qstart, qlimit;
|
|
// The */+ follows an isolated character or quote
|
|
// or variable reference
|
|
if (buf.length() == quoteLimit) {
|
|
// The */+ follows a 'quoted string'
|
|
qstart = quoteStart;
|
|
qlimit = quoteLimit;
|
|
} else if (buf.length() == varLimit) {
|
|
// The */+ follows a $variableReference
|
|
qstart = varStart;
|
|
qlimit = varLimit;
|
|
} else {
|
|
// The */+ follows a single character, possibly
|
|
// a segment standin
|
|
qstart = buf.length() - 1;
|
|
qlimit = qstart + 1;
|
|
}
|
|
|
|
UnicodeFunctor *m =
|
|
new StringMatcher(buf, qstart, qlimit, 0, *parser.curData);
|
|
if (m == NULL) {
|
|
return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
|
|
}
|
|
int32_t min = 0;
|
|
int32_t max = Quantifier::MAX;
|
|
switch (c) {
|
|
case ONE_OR_MORE:
|
|
min = 1;
|
|
break;
|
|
case ZERO_OR_ONE:
|
|
min = 0;
|
|
max = 1;
|
|
break;
|
|
// case KLEENE_STAR:
|
|
// do nothing -- min, max already set
|
|
}
|
|
m = new Quantifier(m, min, max);
|
|
if (m == NULL) {
|
|
return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
|
|
}
|
|
buf.truncate(qstart);
|
|
buf.append(parser.generateStandInFor(m, status));
|
|
}
|
|
break;
|
|
|
|
//------------------------------------------------------
|
|
// Elements allowed ONLY WITHIN segments
|
|
//------------------------------------------------------
|
|
case SEGMENT_CLOSE:
|
|
// assert(isSegment);
|
|
// We're done parsing a segment.
|
|
done = TRUE;
|
|
break;
|
|
|
|
//------------------------------------------------------
|
|
// Elements allowed ONLY OUTSIDE segments
|
|
//------------------------------------------------------
|
|
case CONTEXT_ANTE:
|
|
if (ante >= 0) {
|
|
return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status);
|
|
}
|
|
ante = buf.length();
|
|
break;
|
|
case CONTEXT_POST:
|
|
if (post >= 0) {
|
|
return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status);
|
|
}
|
|
post = buf.length();
|
|
break;
|
|
case CURSOR_POS:
|
|
if (cursor >= 0) {
|
|
return syntaxError(U_MULTIPLE_CURSORS, rule, start, status);
|
|
}
|
|
cursor = buf.length();
|
|
break;
|
|
case CURSOR_OFFSET:
|
|
if (cursorOffset < 0) {
|
|
if (buf.length() > 0) {
|
|
return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
|
|
}
|
|
--cursorOffset;
|
|
} else if (cursorOffset > 0) {
|
|
if (buf.length() != cursorOffsetPos || cursor >= 0) {
|
|
return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
|
|
}
|
|
++cursorOffset;
|
|
} else {
|
|
if (cursor == 0 && buf.length() == 0) {
|
|
cursorOffset = -1;
|
|
} else if (cursor < 0) {
|
|
cursorOffsetPos = buf.length();
|
|
cursorOffset = 1;
|
|
} else {
|
|
return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
|
|
}
|
|
}
|
|
break;
|
|
|
|
|
|
//------------------------------------------------------
|
|
// Non-special characters
|
|
//------------------------------------------------------
|
|
default:
|
|
// Disallow unquoted characters other than [0-9A-Za-z]
|
|
// in the printable ASCII range. These characters are
|
|
// reserved for possible future use.
|
|
if (c >= 0x0021 && c <= 0x007E &&
|
|
!((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) ||
|
|
(c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) ||
|
|
(c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) {
|
|
return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
|
|
}
|
|
buf.append(c);
|
|
break;
|
|
}
|
|
}
|
|
|
|
return pos;
|
|
}
|
|
|
|
/**
|
|
* Remove context.
|
|
*/
|
|
void RuleHalf::removeContext() {
|
|
//text = text.substring(ante < 0 ? 0 : ante,
|
|
// post < 0 ? text.length() : post);
|
|
if (post >= 0) {
|
|
text.remove(post);
|
|
}
|
|
if (ante >= 0) {
|
|
text.removeBetween(0, ante);
|
|
}
|
|
ante = post = -1;
|
|
anchorStart = anchorEnd = FALSE;
|
|
}
|
|
|
|
/**
|
|
* Return true if this half looks like valid output, that is, does not
|
|
* contain quantifiers or other special input-only elements.
|
|
*/
|
|
UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) {
|
|
for (int32_t i=0; i<text.length(); ) {
|
|
UChar32 c = text.char32At(i);
|
|
i += U16_LENGTH(c);
|
|
if (!transParser.parseData->isReplacer(c)) {
|
|
return FALSE;
|
|
}
|
|
}
|
|
return TRUE;
|
|
}
|
|
|
|
/**
|
|
* Return true if this half looks like valid input, that is, does not
|
|
* contain functions or other special output-only elements.
|
|
*/
|
|
UBool RuleHalf::isValidInput(TransliteratorParser& transParser) {
|
|
for (int32_t i=0; i<text.length(); ) {
|
|
UChar32 c = text.char32At(i);
|
|
i += U16_LENGTH(c);
|
|
if (!transParser.parseData->isMatcher(c)) {
|
|
return FALSE;
|
|
}
|
|
}
|
|
return TRUE;
|
|
}
|
|
|
|
//----------------------------------------------------------------------
|
|
// PUBLIC API
|
|
//----------------------------------------------------------------------
|
|
|
|
/**
|
|
* Constructor.
|
|
*/
|
|
TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) :
|
|
dataVector(statusReturn),
|
|
idBlockVector(statusReturn),
|
|
variablesVector(statusReturn),
|
|
segmentObjects(statusReturn)
|
|
{
|
|
idBlockVector.setDeleter(uprv_deleteUObject);
|
|
curData = NULL;
|
|
compoundFilter = NULL;
|
|
parseData = NULL;
|
|
variableNames.setValueDeleter(uprv_deleteUObject);
|
|
}
|
|
|
|
/**
|
|
* Destructor.
|
|
*/
|
|
TransliteratorParser::~TransliteratorParser() {
|
|
while (!dataVector.isEmpty())
|
|
delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
|
|
delete compoundFilter;
|
|
delete parseData;
|
|
while (!variablesVector.isEmpty())
|
|
delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
|
|
}
|
|
|
|
void
|
|
TransliteratorParser::parse(const UnicodeString& rules,
|
|
UTransDirection transDirection,
|
|
UParseError& pe,
|
|
UErrorCode& ec) {
|
|
if (U_SUCCESS(ec)) {
|
|
parseRules(rules, transDirection, ec);
|
|
pe = parseError;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Return the compound filter parsed by parse(). Caller owns result.
|
|
*/
|
|
UnicodeSet* TransliteratorParser::orphanCompoundFilter() {
|
|
UnicodeSet* f = compoundFilter;
|
|
compoundFilter = NULL;
|
|
return f;
|
|
}
|
|
|
|
//----------------------------------------------------------------------
|
|
// Private implementation
|
|
//----------------------------------------------------------------------
|
|
|
|
/**
|
|
* Parse the given string as a sequence of rules, separated by newline
|
|
* characters ('\n'), and cause this object to implement those rules. Any
|
|
* previous rules are discarded. Typically this method is called exactly
|
|
* once, during construction.
|
|
* @exception IllegalArgumentException if there is a syntax error in the
|
|
* rules
|
|
*/
|
|
void TransliteratorParser::parseRules(const UnicodeString& rule,
|
|
UTransDirection theDirection,
|
|
UErrorCode& status)
|
|
{
|
|
// Clear error struct
|
|
uprv_memset(&parseError, 0, sizeof(parseError));
|
|
parseError.line = parseError.offset = -1;
|
|
|
|
UBool parsingIDs = TRUE;
|
|
int32_t ruleCount = 0;
|
|
|
|
while (!dataVector.isEmpty()) {
|
|
delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
|
|
}
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
|
|
idBlockVector.removeAllElements();
|
|
curData = NULL;
|
|
direction = theDirection;
|
|
ruleCount = 0;
|
|
|
|
delete compoundFilter;
|
|
compoundFilter = NULL;
|
|
|
|
while (!variablesVector.isEmpty()) {
|
|
delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
|
|
}
|
|
variableNames.removeAll();
|
|
parseData = new ParseData(0, &variablesVector, &variableNames);
|
|
if (parseData == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
|
|
dotStandIn = (UChar) -1;
|
|
|
|
UnicodeString *tempstr = NULL; // used for memory allocation error checking
|
|
UnicodeString str; // scratch
|
|
UnicodeString idBlockResult;
|
|
int32_t pos = 0;
|
|
int32_t limit = rule.length();
|
|
|
|
// The compound filter offset is an index into idBlockResult.
|
|
// If it is 0, then the compound filter occurred at the start,
|
|
// and it is the offset to the _start_ of the compound filter
|
|
// pattern. Otherwise it is the offset to the _limit_ of the
|
|
// compound filter pattern within idBlockResult.
|
|
compoundFilter = NULL;
|
|
int32_t compoundFilterOffset = -1;
|
|
|
|
while (pos < limit && U_SUCCESS(status)) {
|
|
UChar c = rule.charAt(pos++);
|
|
if (PatternProps::isWhiteSpace(c)) {
|
|
// Ignore leading whitespace.
|
|
continue;
|
|
}
|
|
// Skip lines starting with the comment character
|
|
if (c == RULE_COMMENT_CHAR) {
|
|
pos = rule.indexOf((UChar)0x000A /*\n*/, pos) + 1;
|
|
if (pos == 0) {
|
|
break; // No "\n" found; rest of rule is a commnet
|
|
}
|
|
continue; // Either fall out or restart with next line
|
|
}
|
|
|
|
// skip empty rules
|
|
if (c == END_OF_RULE)
|
|
continue;
|
|
|
|
// keep track of how many rules we've seen
|
|
++ruleCount;
|
|
|
|
// We've found the start of a rule or ID. c is its first
|
|
// character, and pos points past c.
|
|
--pos;
|
|
// Look for an ID token. Must have at least ID_TOKEN_LEN + 1
|
|
// chars left.
|
|
if ((pos + ID_TOKEN_LEN + 1) <= limit &&
|
|
rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) {
|
|
pos += ID_TOKEN_LEN;
|
|
c = rule.charAt(pos);
|
|
while (PatternProps::isWhiteSpace(c) && pos < limit) {
|
|
++pos;
|
|
c = rule.charAt(pos);
|
|
}
|
|
|
|
int32_t p = pos;
|
|
|
|
if (!parsingIDs) {
|
|
if (curData != NULL) {
|
|
if (direction == UTRANS_FORWARD)
|
|
dataVector.addElement(curData, status);
|
|
else
|
|
dataVector.insertElementAt(curData, 0, status);
|
|
curData = NULL;
|
|
}
|
|
parsingIDs = TRUE;
|
|
}
|
|
|
|
TransliteratorIDParser::SingleID* id =
|
|
TransliteratorIDParser::parseSingleID(rule, p, direction, status);
|
|
if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) {
|
|
// Successful ::ID parse.
|
|
|
|
if (direction == UTRANS_FORWARD) {
|
|
idBlockResult.append(id->canonID).append(END_OF_RULE);
|
|
} else {
|
|
idBlockResult.insert(0, END_OF_RULE);
|
|
idBlockResult.insert(0, id->canonID);
|
|
}
|
|
|
|
} else {
|
|
// Couldn't parse an ID. Try to parse a global filter
|
|
int32_t withParens = -1;
|
|
UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, NULL);
|
|
if (f != NULL) {
|
|
if (ICU_Utility::parseChar(rule, p, END_OF_RULE)
|
|
&& (direction == UTRANS_FORWARD) == (withParens == 0))
|
|
{
|
|
if (compoundFilter != NULL) {
|
|
// Multiple compound filters
|
|
syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status);
|
|
delete f;
|
|
} else {
|
|
compoundFilter = f;
|
|
compoundFilterOffset = ruleCount;
|
|
}
|
|
} else {
|
|
delete f;
|
|
}
|
|
} else {
|
|
// Invalid ::id
|
|
// Can be parsed as neither an ID nor a global filter
|
|
syntaxError(U_INVALID_ID, rule, pos, status);
|
|
}
|
|
}
|
|
delete id;
|
|
pos = p;
|
|
} else {
|
|
if (parsingIDs) {
|
|
tempstr = new UnicodeString(idBlockResult);
|
|
// NULL pointer check
|
|
if (tempstr == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
if (direction == UTRANS_FORWARD)
|
|
idBlockVector.addElement(tempstr, status);
|
|
else
|
|
idBlockVector.insertElementAt(tempstr, 0, status);
|
|
idBlockResult.remove();
|
|
parsingIDs = FALSE;
|
|
curData = new TransliterationRuleData(status);
|
|
// NULL pointer check
|
|
if (curData == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
parseData->data = curData;
|
|
|
|
// By default, rules use part of the private use area
|
|
// E000..F8FF for variables and other stand-ins. Currently
|
|
// the range F000..F8FF is typically sufficient. The 'use
|
|
// variable range' pragma allows rule sets to modify this.
|
|
setVariableRange(0xF000, 0xF8FF, status);
|
|
}
|
|
|
|
if (resemblesPragma(rule, pos, limit)) {
|
|
int32_t ppp = parsePragma(rule, pos, limit, status);
|
|
if (ppp < 0) {
|
|
syntaxError(U_MALFORMED_PRAGMA, rule, pos, status);
|
|
}
|
|
pos = ppp;
|
|
// Parse a rule
|
|
} else {
|
|
pos = parseRule(rule, pos, limit, status);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (parsingIDs && idBlockResult.length() > 0) {
|
|
tempstr = new UnicodeString(idBlockResult);
|
|
// NULL pointer check
|
|
if (tempstr == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
if (direction == UTRANS_FORWARD)
|
|
idBlockVector.addElement(tempstr, status);
|
|
else
|
|
idBlockVector.insertElementAt(tempstr, 0, status);
|
|
}
|
|
else if (!parsingIDs && curData != NULL) {
|
|
if (direction == UTRANS_FORWARD)
|
|
dataVector.addElement(curData, status);
|
|
else
|
|
dataVector.insertElementAt(curData, 0, status);
|
|
}
|
|
|
|
if (U_SUCCESS(status)) {
|
|
// Convert the set vector to an array
|
|
int32_t i, dataVectorSize = dataVector.size();
|
|
for (i = 0; i < dataVectorSize; i++) {
|
|
TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
|
|
data->variablesLength = variablesVector.size();
|
|
if (data->variablesLength == 0) {
|
|
data->variables = 0;
|
|
} else {
|
|
data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*));
|
|
// NULL pointer check
|
|
if (data->variables == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
data->variablesAreOwned = (i == 0);
|
|
}
|
|
|
|
for (int32_t j = 0; j < data->variablesLength; j++) {
|
|
data->variables[j] =
|
|
static_cast<UnicodeFunctor *>(variablesVector.elementAt(j));
|
|
}
|
|
|
|
data->variableNames.removeAll();
|
|
int32_t pos = UHASH_FIRST;
|
|
const UHashElement* he = variableNames.nextElement(pos);
|
|
while (he != NULL) {
|
|
UnicodeString* tempus = (UnicodeString*)(((UnicodeString*)(he->value.pointer))->clone());
|
|
if (tempus == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
data->variableNames.put(*((UnicodeString*)(he->key.pointer)),
|
|
tempus, status);
|
|
he = variableNames.nextElement(pos);
|
|
}
|
|
}
|
|
variablesVector.removeAllElements(); // keeps them from getting deleted when we succeed
|
|
|
|
// Index the rules
|
|
if (compoundFilter != NULL) {
|
|
if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) ||
|
|
(direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) {
|
|
status = U_MISPLACED_COMPOUND_FILTER;
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < dataVectorSize; i++) {
|
|
TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
|
|
data->ruleSet.freeze(parseError, status);
|
|
}
|
|
if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) {
|
|
idBlockVector.removeElementAt(0);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Set the variable range to [start, end] (inclusive).
|
|
*/
|
|
void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) {
|
|
if (start > end || start < 0 || end > 0xFFFF) {
|
|
status = U_MALFORMED_PRAGMA;
|
|
return;
|
|
}
|
|
|
|
curData->variablesBase = (UChar) start;
|
|
if (dataVector.size() == 0) {
|
|
variableNext = (UChar) start;
|
|
variableLimit = (UChar) (end + 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Assert that the given character is NOT within the variable range.
|
|
* If it is, return FALSE. This is neccesary to ensure that the
|
|
* variable range does not overlap characters used in a rule.
|
|
*/
|
|
UBool TransliteratorParser::checkVariableRange(UChar32 ch) const {
|
|
return !(ch >= curData->variablesBase && ch < variableLimit);
|
|
}
|
|
|
|
/**
|
|
* Set the maximum backup to 'backup', in response to a pragma
|
|
* statement.
|
|
*/
|
|
void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) {
|
|
//TODO Finish
|
|
}
|
|
|
|
/**
|
|
* Begin normalizing all rules using the given mode, in response
|
|
* to a pragma statement.
|
|
*/
|
|
void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) {
|
|
//TODO Finish
|
|
}
|
|
|
|
static const UChar PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use "
|
|
|
|
static const UChar PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;"
|
|
|
|
static const UChar PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;"
|
|
|
|
static const UChar PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;"
|
|
|
|
static const UChar PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;"
|
|
|
|
/**
|
|
* Return true if the given rule looks like a pragma.
|
|
* @param pos offset to the first non-whitespace character
|
|
* of the rule.
|
|
* @param limit pointer past the last character of the rule.
|
|
*/
|
|
UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) {
|
|
// Must start with /use\s/i
|
|
return ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_USE, 4), NULL) >= 0;
|
|
}
|
|
|
|
/**
|
|
* Parse a pragma. This method assumes resemblesPragma() has
|
|
* already returned true.
|
|
* @param pos offset to the first non-whitespace character
|
|
* of the rule.
|
|
* @param limit pointer past the last character of the rule.
|
|
* @return the position index after the final ';' of the pragma,
|
|
* or -1 on failure.
|
|
*/
|
|
int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
|
|
int32_t array[2];
|
|
|
|
// resemblesPragma() has already returned true, so we
|
|
// know that pos points to /use\s/i; we can skip 4 characters
|
|
// immediately
|
|
pos += 4;
|
|
|
|
// Here are the pragmas we recognize:
|
|
// use variable range 0xE000 0xEFFF;
|
|
// use maximum backup 16;
|
|
// use nfd rules;
|
|
// use nfc rules;
|
|
int p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_VARIABLE_RANGE, -1), array);
|
|
if (p >= 0) {
|
|
setVariableRange(array[0], array[1], status);
|
|
return p;
|
|
}
|
|
|
|
p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_MAXIMUM_BACKUP, -1), array);
|
|
if (p >= 0) {
|
|
pragmaMaximumBackup(array[0]);
|
|
return p;
|
|
}
|
|
|
|
p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFD_RULES, -1), NULL);
|
|
if (p >= 0) {
|
|
pragmaNormalizeRules(UNORM_NFD);
|
|
return p;
|
|
}
|
|
|
|
p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFC_RULES, -1), NULL);
|
|
if (p >= 0) {
|
|
pragmaNormalizeRules(UNORM_NFC);
|
|
return p;
|
|
}
|
|
|
|
// Syntax error: unable to parse pragma
|
|
return -1;
|
|
}
|
|
|
|
/**
|
|
* MAIN PARSER. Parse the next rule in the given rule string, starting
|
|
* at pos. Return the index after the last character parsed. Do not
|
|
* parse characters at or after limit.
|
|
*
|
|
* Important: The character at pos must be a non-whitespace character
|
|
* that is not the comment character.
|
|
*
|
|
* This method handles quoting, escaping, and whitespace removal. It
|
|
* parses the end-of-rule character. It recognizes context and cursor
|
|
* indicators. Once it does a lexical breakdown of the rule at pos, it
|
|
* creates a rule object and adds it to our rule list.
|
|
*/
|
|
int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
|
|
// Locate the left side, operator, and right side
|
|
int32_t start = pos;
|
|
UChar op = 0;
|
|
int32_t i;
|
|
|
|
// Set up segments data
|
|
segmentStandins.truncate(0);
|
|
segmentObjects.removeAllElements();
|
|
|
|
// Use pointers to automatics to make swapping possible.
|
|
RuleHalf _left(*this), _right(*this);
|
|
RuleHalf* left = &_left;
|
|
RuleHalf* right = &_right;
|
|
|
|
undefinedVariableName.remove();
|
|
pos = left->parse(rule, pos, limit, status);
|
|
if (U_FAILURE(status)) {
|
|
return start;
|
|
}
|
|
|
|
if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == NULL) {
|
|
return syntaxError(U_MISSING_OPERATOR, rule, start, status);
|
|
}
|
|
++pos;
|
|
|
|
// Found an operator char. Check for forward-reverse operator.
|
|
if (op == REVERSE_RULE_OP &&
|
|
(pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) {
|
|
++pos;
|
|
op = FWDREV_RULE_OP;
|
|
}
|
|
|
|
// Translate alternate op characters.
|
|
switch (op) {
|
|
case ALT_FORWARD_RULE_OP:
|
|
op = FORWARD_RULE_OP;
|
|
break;
|
|
case ALT_REVERSE_RULE_OP:
|
|
op = REVERSE_RULE_OP;
|
|
break;
|
|
case ALT_FWDREV_RULE_OP:
|
|
op = FWDREV_RULE_OP;
|
|
break;
|
|
}
|
|
|
|
pos = right->parse(rule, pos, limit, status);
|
|
if (U_FAILURE(status)) {
|
|
return start;
|
|
}
|
|
|
|
if (pos < limit) {
|
|
if (rule.charAt(--pos) == END_OF_RULE) {
|
|
++pos;
|
|
} else {
|
|
// RuleHalf parser must have terminated at an operator
|
|
return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
|
|
}
|
|
}
|
|
|
|
if (op == VARIABLE_DEF_OP) {
|
|
// LHS is the name. RHS is a single character, either a literal
|
|
// or a set (already parsed). If RHS is longer than one
|
|
// character, it is either a multi-character string, or multiple
|
|
// sets, or a mixture of chars and sets -- syntax error.
|
|
|
|
// We expect to see a single undefined variable (the one being
|
|
// defined).
|
|
if (undefinedVariableName.length() == 0) {
|
|
// "Missing '$' or duplicate definition"
|
|
return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status);
|
|
}
|
|
if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) {
|
|
// "Malformed LHS"
|
|
return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
|
|
}
|
|
if (left->anchorStart || left->anchorEnd ||
|
|
right->anchorStart || right->anchorEnd) {
|
|
return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
|
|
}
|
|
// We allow anything on the right, including an empty string.
|
|
UnicodeString* value = new UnicodeString(right->text);
|
|
// NULL pointer check
|
|
if (value == NULL) {
|
|
return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
|
|
}
|
|
variableNames.put(undefinedVariableName, value, status);
|
|
++variableLimit;
|
|
return pos;
|
|
}
|
|
|
|
// If this is not a variable definition rule, we shouldn't have
|
|
// any undefined variable names.
|
|
if (undefinedVariableName.length() != 0) {
|
|
return syntaxError(// "Undefined variable $" + undefinedVariableName,
|
|
U_UNDEFINED_VARIABLE,
|
|
rule, start, status);
|
|
}
|
|
|
|
// Verify segments
|
|
if (segmentStandins.length() > segmentObjects.size()) {
|
|
syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status);
|
|
}
|
|
for (i=0; i<segmentStandins.length(); ++i) {
|
|
if (segmentStandins.charAt(i) == 0) {
|
|
syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
|
|
}
|
|
}
|
|
for (i=0; i<segmentObjects.size(); ++i) {
|
|
if (segmentObjects.elementAt(i) == NULL) {
|
|
syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
|
|
}
|
|
}
|
|
|
|
// If the direction we want doesn't match the rule
|
|
// direction, do nothing.
|
|
if (op != FWDREV_RULE_OP &&
|
|
((direction == UTRANS_FORWARD) != (op == FORWARD_RULE_OP))) {
|
|
return pos;
|
|
}
|
|
|
|
// Transform the rule into a forward rule by swapping the
|
|
// sides if necessary.
|
|
if (direction == UTRANS_REVERSE) {
|
|
left = &_right;
|
|
right = &_left;
|
|
}
|
|
|
|
// Remove non-applicable elements in forward-reverse
|
|
// rules. Bidirectional rules ignore elements that do not
|
|
// apply.
|
|
if (op == FWDREV_RULE_OP) {
|
|
right->removeContext();
|
|
left->cursor = -1;
|
|
left->cursorOffset = 0;
|
|
}
|
|
|
|
// Normalize context
|
|
if (left->ante < 0) {
|
|
left->ante = 0;
|
|
}
|
|
if (left->post < 0) {
|
|
left->post = left->text.length();
|
|
}
|
|
|
|
// Context is only allowed on the input side. Cursors are only
|
|
// allowed on the output side. Segment delimiters can only appear
|
|
// on the left, and references on the right. Cursor offset
|
|
// cannot appear without an explicit cursor. Cursor offset
|
|
// cannot place the cursor outside the limits of the context.
|
|
// Anchors are only allowed on the input side.
|
|
if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 ||
|
|
(right->cursorOffset != 0 && right->cursor < 0) ||
|
|
// - The following two checks were used to ensure that the
|
|
// - the cursor offset stayed within the ante- or postcontext.
|
|
// - However, with the addition of quantifiers, we have to
|
|
// - allow arbitrary cursor offsets and do runtime checking.
|
|
//(right->cursorOffset > (left->text.length() - left->post)) ||
|
|
//(-right->cursorOffset > left->ante) ||
|
|
right->anchorStart || right->anchorEnd ||
|
|
!left->isValidInput(*this) || !right->isValidOutput(*this) ||
|
|
left->ante > left->post) {
|
|
|
|
return syntaxError(U_MALFORMED_RULE, rule, start, status);
|
|
}
|
|
|
|
// Flatten segment objects vector to an array
|
|
UnicodeFunctor** segmentsArray = NULL;
|
|
if (segmentObjects.size() > 0) {
|
|
segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *));
|
|
// Null pointer check
|
|
if (segmentsArray == NULL) {
|
|
return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
|
|
}
|
|
segmentObjects.toArray((void**) segmentsArray);
|
|
}
|
|
TransliterationRule* temptr = new TransliterationRule(
|
|
left->text, left->ante, left->post,
|
|
right->text, right->cursor, right->cursorOffset,
|
|
segmentsArray,
|
|
segmentObjects.size(),
|
|
left->anchorStart, left->anchorEnd,
|
|
curData,
|
|
status);
|
|
//Null pointer check
|
|
if (temptr == NULL) {
|
|
uprv_free(segmentsArray);
|
|
return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
|
|
}
|
|
|
|
curData->ruleSet.addRule(temptr, status);
|
|
|
|
return pos;
|
|
}
|
|
|
|
/**
|
|
* Called by main parser upon syntax error. Search the rule string
|
|
* for the probable end of the rule. Of course, if the error is that
|
|
* the end of rule marker is missing, then the rule end will not be found.
|
|
* In any case the rule start will be correctly reported.
|
|
* @param msg error description
|
|
* @param rule pattern string
|
|
* @param start position of first character of current rule
|
|
*/
|
|
int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode,
|
|
const UnicodeString& rule,
|
|
int32_t pos,
|
|
UErrorCode& status)
|
|
{
|
|
parseError.offset = pos;
|
|
parseError.line = 0 ; /* we are not using line numbers */
|
|
|
|
// for pre-context
|
|
const int32_t LEN = U_PARSE_CONTEXT_LEN - 1;
|
|
int32_t start = uprv_max(pos - LEN, 0);
|
|
int32_t stop = pos;
|
|
|
|
rule.extract(start,stop-start,parseError.preContext);
|
|
//null terminate the buffer
|
|
parseError.preContext[stop-start] = 0;
|
|
|
|
//for post-context
|
|
start = pos;
|
|
stop = uprv_min(pos + LEN, rule.length());
|
|
|
|
rule.extract(start,stop-start,parseError.postContext);
|
|
//null terminate the buffer
|
|
parseError.postContext[stop-start]= 0;
|
|
|
|
status = (UErrorCode)parseErrorCode;
|
|
return pos;
|
|
|
|
}
|
|
|
|
/**
|
|
* Parse a UnicodeSet out, store it, and return the stand-in character
|
|
* used to represent it.
|
|
*/
|
|
UChar TransliteratorParser::parseSet(const UnicodeString& rule,
|
|
ParsePosition& pos,
|
|
UErrorCode& status) {
|
|
UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status);
|
|
// Null pointer check
|
|
if (set == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return (UChar)0x0000; // Return empty character with error.
|
|
}
|
|
set->compact();
|
|
return generateStandInFor(set, status);
|
|
}
|
|
|
|
/**
|
|
* Generate and return a stand-in for a new UnicodeFunctor. Store
|
|
* the matcher (adopt it).
|
|
*/
|
|
UChar TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) {
|
|
// assert(obj != null);
|
|
|
|
// Look up previous stand-in, if any. This is a short list
|
|
// (typical n is 0, 1, or 2); linear search is optimal.
|
|
for (int32_t i=0; i<variablesVector.size(); ++i) {
|
|
if (variablesVector.elementAt(i) == adopted) { // [sic] pointer comparison
|
|
return (UChar) (curData->variablesBase + i);
|
|
}
|
|
}
|
|
|
|
if (variableNext >= variableLimit) {
|
|
delete adopted;
|
|
status = U_VARIABLE_RANGE_EXHAUSTED;
|
|
return 0;
|
|
}
|
|
variablesVector.addElement(adopted, status);
|
|
return variableNext++;
|
|
}
|
|
|
|
/**
|
|
* Return the standin for segment seg (1-based).
|
|
*/
|
|
UChar TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) {
|
|
// Special character used to indicate an empty spot
|
|
UChar empty = curData->variablesBase - 1;
|
|
while (segmentStandins.length() < seg) {
|
|
segmentStandins.append(empty);
|
|
}
|
|
UChar c = segmentStandins.charAt(seg-1);
|
|
if (c == empty) {
|
|
if (variableNext >= variableLimit) {
|
|
status = U_VARIABLE_RANGE_EXHAUSTED;
|
|
return 0;
|
|
}
|
|
c = variableNext++;
|
|
// Set a placeholder in the master variables vector that will be
|
|
// filled in later by setSegmentObject(). We know that we will get
|
|
// called first because setSegmentObject() will call us.
|
|
variablesVector.addElement((void*) NULL, status);
|
|
segmentStandins.setCharAt(seg-1, c);
|
|
}
|
|
return c;
|
|
}
|
|
|
|
/**
|
|
* Set the object for segment seg (1-based).
|
|
*/
|
|
void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) {
|
|
// Since we call parseSection() recursively, nested
|
|
// segments will result in segment i+1 getting parsed
|
|
// and stored before segment i; be careful with the
|
|
// vector handling here.
|
|
if (segmentObjects.size() < seg) {
|
|
segmentObjects.setSize(seg, status);
|
|
}
|
|
int32_t index = getSegmentStandin(seg, status) - curData->variablesBase;
|
|
if (segmentObjects.elementAt(seg-1) != NULL ||
|
|
variablesVector.elementAt(index) != NULL) {
|
|
// should never happen
|
|
status = U_INTERNAL_TRANSLITERATOR_ERROR;
|
|
return;
|
|
}
|
|
segmentObjects.setElementAt(adopted, seg-1);
|
|
variablesVector.setElementAt(adopted, index);
|
|
}
|
|
|
|
/**
|
|
* Return the stand-in for the dot set. It is allocated the first
|
|
* time and reused thereafter.
|
|
*/
|
|
UChar TransliteratorParser::getDotStandIn(UErrorCode& status) {
|
|
if (dotStandIn == (UChar) -1) {
|
|
UnicodeSet* tempus = new UnicodeSet(UnicodeString(TRUE, DOT_SET, -1), status);
|
|
// Null pointer check.
|
|
if (tempus == NULL) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return (UChar)0x0000;
|
|
}
|
|
dotStandIn = generateStandInFor(tempus, status);
|
|
}
|
|
return dotStandIn;
|
|
}
|
|
|
|
/**
|
|
* Append the value of the given variable name to the given
|
|
* UnicodeString.
|
|
*/
|
|
void TransliteratorParser::appendVariableDef(const UnicodeString& name,
|
|
UnicodeString& buf,
|
|
UErrorCode& status) {
|
|
const UnicodeString* s = (const UnicodeString*) variableNames.get(name);
|
|
if (s == NULL) {
|
|
// We allow one undefined variable so that variable definition
|
|
// statements work. For the first undefined variable we return
|
|
// the special placeholder variableLimit-1, and save the variable
|
|
// name.
|
|
if (undefinedVariableName.length() == 0) {
|
|
undefinedVariableName = name;
|
|
if (variableNext >= variableLimit) {
|
|
// throw new RuntimeException("Private use variables exhausted");
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
return;
|
|
}
|
|
buf.append((UChar) --variableLimit);
|
|
} else {
|
|
//throw new IllegalArgumentException("Undefined variable $"
|
|
// + name);
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
return;
|
|
}
|
|
} else {
|
|
buf.append(*s);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Glue method to get around access restrictions in C++.
|
|
*/
|
|
/*Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) {
|
|
return Transliterator::createBasicInstance(id, canonID);
|
|
}*/
|
|
|
|
U_NAMESPACE_END
|
|
|
|
U_CAPI int32_t
|
|
utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status) {
|
|
U_NAMESPACE_USE
|
|
|
|
//const UChar *sourceStart = source;
|
|
const UChar *targetStart = target;
|
|
const UChar *sourceLimit = source+sourceLen;
|
|
UChar *targetLimit = target+sourceLen;
|
|
UChar32 c = 0;
|
|
UBool quoted = FALSE;
|
|
int32_t index;
|
|
|
|
uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR);
|
|
|
|
/* read the rules into the buffer */
|
|
while (source < sourceLimit)
|
|
{
|
|
index=0;
|
|
U16_NEXT_UNSAFE(source, index, c);
|
|
source+=index;
|
|
if(c == QUOTE) {
|
|
quoted = (UBool)!quoted;
|
|
}
|
|
else if (!quoted) {
|
|
if (c == RULE_COMMENT_CHAR) {
|
|
/* skip comments and all preceding spaces */
|
|
while (targetStart < target && *(target - 1) == 0x0020) {
|
|
target--;
|
|
}
|
|
do {
|
|
if (source == sourceLimit) {
|
|
c = U_SENTINEL;
|
|
break;
|
|
}
|
|
c = *(source++);
|
|
}
|
|
while (c != CR && c != LF);
|
|
if (c < 0) {
|
|
break;
|
|
}
|
|
}
|
|
else if (c == ESCAPE && source < sourceLimit) {
|
|
UChar32 c2 = *source;
|
|
if (c2 == CR || c2 == LF) {
|
|
/* A backslash at the end of a line. */
|
|
/* Since we're stripping lines, ignore the backslash. */
|
|
source++;
|
|
continue;
|
|
}
|
|
if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */
|
|
int32_t escapeOffset = 0;
|
|
UnicodeString escapedStr(source, 5);
|
|
c2 = escapedStr.unescapeAt(escapeOffset);
|
|
|
|
if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0)
|
|
{
|
|
*status = U_PARSE_ERROR;
|
|
return 0;
|
|
}
|
|
if (!PatternProps::isWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) {
|
|
/* It was escaped for a reason. Write what it was suppose to be. */
|
|
source+=5;
|
|
c = c2;
|
|
}
|
|
}
|
|
else if (c2 == QUOTE) {
|
|
/* \' seen. Make sure we don't do anything when we see it again. */
|
|
quoted = (UBool)!quoted;
|
|
}
|
|
}
|
|
}
|
|
if (c == CR || c == LF)
|
|
{
|
|
/* ignore spaces carriage returns, and all leading spaces on the next line.
|
|
* and line feed unless in the form \uXXXX
|
|
*/
|
|
quoted = FALSE;
|
|
while (source < sourceLimit) {
|
|
c = *(source);
|
|
if (c != CR && c != LF && c != 0x0020) {
|
|
break;
|
|
}
|
|
source++;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
/* Append UChar * after dissembling if c > 0xffff*/
|
|
index=0;
|
|
U16_APPEND_UNSAFE(target, index, c);
|
|
target+=index;
|
|
}
|
|
if (target < targetLimit) {
|
|
*target = 0;
|
|
}
|
|
return (int32_t)(target-targetStart);
|
|
}
|
|
|
|
#endif /* #if !UCONFIG_NO_TRANSLITERATION */
|