1347 lines
46 KiB
C++
1347 lines
46 KiB
C++
/*
|
|
***************************************************************************
|
|
* Copyright (C) 1999-2004 International Business Machines Corporation *
|
|
* and others. All rights reserved. *
|
|
***************************************************************************
|
|
*/
|
|
//
|
|
// file: rbbi.c Contains the implementation of the rule based break iterator
|
|
// runtime engine and the API implementation for
|
|
// class RuleBasedBreakIterator
|
|
//
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_BREAK_ITERATION
|
|
|
|
#include "unicode/rbbi.h"
|
|
#include "unicode/schriter.h"
|
|
#include "unicode/udata.h"
|
|
#include "unicode/uclean.h"
|
|
#include "rbbidata.h"
|
|
#include "rbbirb.h"
|
|
#include "cmemory.h"
|
|
#include "cstring.h"
|
|
|
|
#include "uassert.h"
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
|
|
static const int16_t START_STATE = 1; // The state number of the starting state
|
|
static const int16_t STOP_STATE = 0; // The state-transition value indicating "stop"
|
|
|
|
|
|
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
|
|
|
|
|
|
//=======================================================================
|
|
// constructors
|
|
//=======================================================================
|
|
|
|
/**
|
|
* Constructs a RuleBasedBreakIterator that uses the already-created
|
|
* tables object that is passed in as a parameter.
|
|
*/
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status)
|
|
{
|
|
init();
|
|
fData = new RBBIDataWrapper(data, status); // status checked in constructor
|
|
if (U_FAILURE(status)) {return;}
|
|
if(fData == 0) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
}
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Constructor from a UDataMemory handle to precompiled break rules
|
|
// stored in an ICU data file.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status)
|
|
{
|
|
init();
|
|
fData = new RBBIDataWrapper(udm, status); // status checked in constructor
|
|
if (U_FAILURE(status)) {return;}
|
|
if(fData == 0) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Constructor from a set of rules supplied as a string.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules,
|
|
UParseError &parseError,
|
|
UErrorCode &status)
|
|
{
|
|
u_init(&status); // Just in case ICU is not yet initialized
|
|
init();
|
|
if (U_FAILURE(status)) {return;}
|
|
RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)
|
|
RBBIRuleBuilder::createRuleBasedBreakIterator(rules, parseError, status);
|
|
// Note: This is a bit awkward. The RBBI ruleBuilder has a factory method that
|
|
// creates and returns a complete RBBI. From here, in a constructor, we
|
|
// can't just return the object created by the builder factory, hence
|
|
// the assignment of the factory created object to "this".
|
|
if (U_SUCCESS(status)) {
|
|
*this = *bi;
|
|
delete bi;
|
|
}
|
|
}
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Default Constructor. Create an empty shell that can be set up later.
|
|
// Used when creating a RuleBasedBreakIterator from a set
|
|
// of rules.
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator() {
|
|
init();
|
|
}
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// Copy constructor. Will produce a break iterator with the same behavior,
|
|
// and which iterates over the same text, as the one passed in.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other)
|
|
: BreakIterator(other)
|
|
{
|
|
this->init();
|
|
*this = other;
|
|
}
|
|
|
|
|
|
/**
|
|
* Destructor
|
|
*/
|
|
RuleBasedBreakIterator::~RuleBasedBreakIterator() {
|
|
delete fText;
|
|
fText = NULL;
|
|
if (fData != NULL) {
|
|
fData->removeReference();
|
|
fData = NULL;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Assignment operator. Sets this iterator to have the same behavior,
|
|
* and iterate over the same text, as the one passed in.
|
|
*/
|
|
RuleBasedBreakIterator&
|
|
RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) {
|
|
if (this == &that) {
|
|
return *this;
|
|
}
|
|
delete fText;
|
|
fText = NULL;
|
|
if (that.fText != NULL) {
|
|
fText = that.fText->clone();
|
|
}
|
|
|
|
if (fData != NULL) {
|
|
fData->removeReference();
|
|
fData = NULL;
|
|
}
|
|
if (that.fData != NULL) {
|
|
fData = that.fData->addReference();
|
|
}
|
|
fTrace = that.fTrace;
|
|
|
|
return *this;
|
|
}
|
|
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// init() Shared initialization routine. Used by all the constructors.
|
|
// Initializes all fields, leaving the object in a consistent state.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
UBool RuleBasedBreakIterator::fTrace = FALSE;
|
|
void RuleBasedBreakIterator::init() {
|
|
|
|
fText = NULL;
|
|
fData = NULL;
|
|
fLastRuleStatusIndex = 0;
|
|
fLastStatusIndexValid = TRUE;
|
|
fDictionaryCharCount = 0;
|
|
|
|
#ifdef RBBI_DEBUG
|
|
static UBool debugInitDone = FALSE;
|
|
if (debugInitDone == FALSE) {
|
|
char *debugEnv = getenv("U_RBBIDEBUG");
|
|
if (debugEnv && uprv_strstr(debugEnv, "trace")) {
|
|
fTrace = TRUE;
|
|
}
|
|
debugInitDone = TRUE;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
//
|
|
// clone - Returns a newly-constructed RuleBasedBreakIterator with the same
|
|
// behavior, and iterating over the same text, as this one.
|
|
// Virtual function: does the right thing with subclasses.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
BreakIterator*
|
|
RuleBasedBreakIterator::clone(void) const {
|
|
return new RuleBasedBreakIterator(*this);
|
|
}
|
|
|
|
/**
|
|
* Equality operator. Returns TRUE if both BreakIterators are of the
|
|
* same class, have the same behavior, and iterate over the same text.
|
|
*/
|
|
UBool
|
|
RuleBasedBreakIterator::operator==(const BreakIterator& that) const {
|
|
UBool r = FALSE;
|
|
if (that.getDynamicClassID() != getDynamicClassID()) {
|
|
return r;
|
|
}
|
|
|
|
const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that;
|
|
if (fText == that2.fText ||
|
|
(fText != NULL && that2.fText != NULL && *that2.fText == *fText)) {
|
|
if (that2.fData == fData ||
|
|
(fData != NULL && that2.fData != NULL && *that2.fData == *fData)) {
|
|
r = TRUE;
|
|
}
|
|
}
|
|
return r;
|
|
}
|
|
|
|
/**
|
|
* Compute a hash code for this BreakIterator
|
|
* @return A hash code
|
|
*/
|
|
int32_t
|
|
RuleBasedBreakIterator::hashCode(void) const {
|
|
int32_t hash = 0;
|
|
if (fData != NULL) {
|
|
hash = fData->hashCode();
|
|
}
|
|
return hash;
|
|
}
|
|
|
|
/**
|
|
* Returns the description used to create this iterator
|
|
*/
|
|
const UnicodeString&
|
|
RuleBasedBreakIterator::getRules() const {
|
|
if (fData != NULL) {
|
|
return fData->getRuleSourceString();
|
|
} else {
|
|
static const UnicodeString *s;
|
|
if (s == NULL) {
|
|
// TODO: something more elegant here.
|
|
// perhaps API should return the string by value.
|
|
// Note: thread unsafe init & leak are semi-ok, better than
|
|
// what was before. Sould be cleaned up, though.
|
|
s = new UnicodeString;
|
|
}
|
|
return *s;
|
|
}
|
|
}
|
|
|
|
//=======================================================================
|
|
// BreakIterator overrides
|
|
//=======================================================================
|
|
|
|
/**
|
|
* Return a CharacterIterator over the text being analyzed. This version
|
|
* of this method returns the actual CharacterIterator we're using internally.
|
|
* Changing the state of this iterator can have undefined consequences. If
|
|
* you need to change it, clone it first.
|
|
* @return An iterator over the text being analyzed.
|
|
*/
|
|
const CharacterIterator&
|
|
RuleBasedBreakIterator::getText() const {
|
|
RuleBasedBreakIterator* nonConstThis = (RuleBasedBreakIterator*)this;
|
|
|
|
// The iterator is initialized pointing to no text at all, so if this
|
|
// function is called while we're in that state, we have to fudge an
|
|
// an iterator to return.
|
|
if (nonConstThis->fText == NULL) {
|
|
nonConstThis->fText = new StringCharacterIterator(UnicodeString());
|
|
}
|
|
return *nonConstThis->fText;
|
|
}
|
|
|
|
/**
|
|
* Set the iterator to analyze a new piece of text. This function resets
|
|
* the current iteration position to the beginning of the text.
|
|
* @param newText An iterator over the text to analyze.
|
|
*/
|
|
void
|
|
RuleBasedBreakIterator::adoptText(CharacterIterator* newText) {
|
|
reset();
|
|
delete fText;
|
|
fText = newText;
|
|
this->first();
|
|
}
|
|
|
|
/**
|
|
* Set the iterator to analyze a new piece of text. This function resets
|
|
* the current iteration position to the beginning of the text.
|
|
* @param newText An iterator over the text to analyze.
|
|
*/
|
|
void
|
|
RuleBasedBreakIterator::setText(const UnicodeString& newText) {
|
|
reset();
|
|
if (fText != NULL && fText->getDynamicClassID()
|
|
== StringCharacterIterator::getStaticClassID()) {
|
|
((StringCharacterIterator*)fText)->setText(newText);
|
|
}
|
|
else {
|
|
delete fText;
|
|
fText = new StringCharacterIterator(newText);
|
|
}
|
|
this->first();
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
* Sets the current iteration position to the beginning of the text.
|
|
* (i.e., the CharacterIterator's starting offset).
|
|
* @return The offset of the beginning of the text.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::first(void) {
|
|
reset();
|
|
fLastRuleStatusIndex = 0;
|
|
fLastStatusIndexValid = TRUE;
|
|
if (fText == NULL)
|
|
return BreakIterator::DONE;
|
|
|
|
fText->first();
|
|
return fText->getIndex();
|
|
}
|
|
|
|
/**
|
|
* Sets the current iteration position to the end of the text.
|
|
* (i.e., the CharacterIterator's ending offset).
|
|
* @return The text's past-the-end offset.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::last(void) {
|
|
reset();
|
|
if (fText == NULL) {
|
|
fLastRuleStatusIndex = 0;
|
|
fLastStatusIndexValid = TRUE;
|
|
return BreakIterator::DONE;
|
|
}
|
|
|
|
// I'm not sure why, but t.last() returns the offset of the last character,
|
|
// rather than the past-the-end offset
|
|
//
|
|
// (It's so a loop like for(p=it.last(); p!=DONE; p=it.previous()) ...
|
|
// will work correctly.)
|
|
|
|
|
|
fLastStatusIndexValid = FALSE;
|
|
int32_t pos = fText->endIndex();
|
|
fText->setIndex(pos);
|
|
|
|
return pos;
|
|
}
|
|
|
|
/**
|
|
* Advances the iterator either forward or backward the specified number of steps.
|
|
* Negative values move backward, and positive values move forward. This is
|
|
* equivalent to repeatedly calling next() or previous().
|
|
* @param n The number of steps to move. The sign indicates the direction
|
|
* (negative is backwards, and positive is forwards).
|
|
* @return The character offset of the boundary position n boundaries away from
|
|
* the current one.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::next(int32_t n) {
|
|
int32_t result = current();
|
|
while (n > 0) {
|
|
result = handleNext();
|
|
--n;
|
|
}
|
|
while (n < 0) {
|
|
result = previous();
|
|
++n;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Advances the iterator to the next boundary position.
|
|
* @return The position of the first boundary after this one.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::next(void) {
|
|
return handleNext();
|
|
}
|
|
|
|
/**
|
|
* Advances the iterator backwards, to the last boundary preceding this one.
|
|
* @return The position of the last boundary position preceding this one.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::previous(void) {
|
|
// if we're already sitting at the beginning of the text, return DONE
|
|
if (fText == NULL || current() == fText->startIndex()) {
|
|
fLastRuleStatusIndex = 0;
|
|
fLastStatusIndexValid = TRUE;
|
|
return BreakIterator::DONE;
|
|
}
|
|
|
|
if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) {
|
|
return handlePrevious(fData->fReverseTable);
|
|
}
|
|
|
|
// old rule syntax
|
|
// set things up. handlePrevious() will back us up to some valid
|
|
// break position before the current position (we back our internal
|
|
// iterator up one step to prevent handlePrevious() from returning
|
|
// the current position), but not necessarily the last one before
|
|
// where we started
|
|
|
|
int32_t start = current();
|
|
|
|
fText->previous32();
|
|
int32_t lastResult = handlePrevious();
|
|
int32_t result = lastResult;
|
|
int32_t lastTag = 0;
|
|
UBool breakTagValid = FALSE;
|
|
|
|
// iterate forward from the known break position until we pass our
|
|
// starting point. The last break position before the starting
|
|
// point is our return value
|
|
|
|
for (;;) {
|
|
result = handleNext();
|
|
if (result == BreakIterator::DONE || result >= start) {
|
|
break;
|
|
}
|
|
lastResult = result;
|
|
lastTag = fLastRuleStatusIndex;
|
|
breakTagValid = TRUE;
|
|
}
|
|
|
|
// fLastBreakTag wants to have the value for section of text preceding
|
|
// the result position that we are to return (in lastResult.) If
|
|
// the backwards rules overshot and the above loop had to do two or more
|
|
// handleNext()s to move up to the desired return position, we will have a valid
|
|
// tag value. But, if handlePrevious() took us to exactly the correct result positon,
|
|
// we wont have a tag value for that position, which is only set by handleNext().
|
|
|
|
// set the current iteration position to be the last break position
|
|
// before where we started, and then return that value
|
|
fText->setIndex(lastResult);
|
|
fLastRuleStatusIndex = lastTag; // for use by getRuleStatus()
|
|
fLastStatusIndexValid = breakTagValid;
|
|
return lastResult;
|
|
}
|
|
|
|
/**
|
|
* Sets the iterator to refer to the first boundary position following
|
|
* the specified position.
|
|
* @offset The position from which to begin searching for a break position.
|
|
* @return The position of the first break after the current position.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::following(int32_t offset) {
|
|
// if the offset passed in is already past the end of the text,
|
|
// just return DONE; if it's before the beginning, return the
|
|
// text's starting offset
|
|
fLastRuleStatusIndex = 0;
|
|
fLastStatusIndexValid = TRUE;
|
|
if (fText == NULL || offset >= fText->endIndex()) {
|
|
last();
|
|
return next();
|
|
}
|
|
else if (offset < fText->startIndex()) {
|
|
return first();
|
|
}
|
|
|
|
// otherwise, set our internal iteration position (temporarily)
|
|
// to the position passed in. If this is the _beginning_ position,
|
|
// then we can just use next() to get our return value
|
|
|
|
int32_t result = 0;
|
|
|
|
if (fData->fSafeRevTable != NULL) {
|
|
// new rule syntax
|
|
/// todo synwee
|
|
fText->setIndex(offset);
|
|
// move forward one codepoint to prepare for moving back to a
|
|
// safe point.
|
|
// this handles offset being between a supplementary character
|
|
fText->next32();
|
|
// handlePrevious will move most of the time to < 1 boundary away
|
|
handlePrevious(fData->fSafeRevTable);
|
|
int32_t result = next();
|
|
while (result <= offset) {
|
|
result = next();
|
|
}
|
|
return result;
|
|
}
|
|
if (fData->fSafeFwdTable != NULL) {
|
|
// backup plan if forward safe table is not available
|
|
fText->setIndex(offset);
|
|
fText->previous32();
|
|
// handle next will give result >= offset
|
|
handleNext(fData->fSafeFwdTable);
|
|
// previous will give result 0 or 1 boundary away from offset,
|
|
// most of the time
|
|
// we have to
|
|
int32_t oldresult = previous();
|
|
while (oldresult > offset) {
|
|
int32_t result = previous();
|
|
if (result <= offset) {
|
|
return oldresult;
|
|
}
|
|
oldresult = result;
|
|
}
|
|
int32_t result = next();
|
|
if (result <= offset) {
|
|
return next();
|
|
}
|
|
return result;
|
|
}
|
|
// otherwise, we have to sync up first. Use handlePrevious() to back
|
|
// us up to a known break position before the specified position (if
|
|
// we can determine that the specified position is a break position,
|
|
// we don't back up at all). This may or may not be the last break
|
|
// position at or before our starting position. Advance forward
|
|
// from here until we've passed the starting position. The position
|
|
// we stop on will be the first break position after the specified one.
|
|
// old rule syntax
|
|
|
|
fText->setIndex(offset);
|
|
if (offset == fText->startIndex()) {
|
|
return handleNext();
|
|
}
|
|
result = previous();
|
|
|
|
while (result != BreakIterator::DONE && result <= offset) {
|
|
result = next();
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Sets the iterator to refer to the last boundary position before the
|
|
* specified position.
|
|
* @offset The position to begin searching for a break from.
|
|
* @return The position of the last boundary before the starting position.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
|
|
// if the offset passed in is already past the end of the text,
|
|
// just return DONE; if it's before the beginning, return the
|
|
|
|
// text's starting offset
|
|
if (fText == NULL || offset > fText->endIndex()) {
|
|
// return BreakIterator::DONE;
|
|
return last();
|
|
}
|
|
else if (offset < fText->startIndex()) {
|
|
return first();
|
|
}
|
|
|
|
// if we start by updating the current iteration position to the
|
|
// position specified by the caller, we can just use previous()
|
|
// to carry out this operation
|
|
|
|
if (fData->fSafeFwdTable != NULL) {
|
|
/// todo synwee
|
|
// new rule syntax
|
|
fText->setIndex(offset);
|
|
// move backwards one codepoint to prepare for moving forwards to a
|
|
// safe point.
|
|
// this handles offset being between a supplementary character
|
|
// TODO: would it be better to just check for being in the middle of a surrogate pair,
|
|
// rather than adjusting the position unconditionally?
|
|
// (Change would interact with safe rules.)
|
|
fText->previous32();
|
|
handleNext(fData->fSafeFwdTable);
|
|
int32_t result = fText->getIndex();
|
|
while (result >= offset) {
|
|
result = previous();
|
|
}
|
|
return result;
|
|
}
|
|
if (fData->fSafeRevTable != NULL) {
|
|
// backup plan if forward safe table is not available
|
|
fText->setIndex(offset);
|
|
fText->next32();
|
|
// handle previous will give result <= offset
|
|
handlePrevious(fData->fSafeRevTable);
|
|
|
|
// next will give result 0 or 1 boundary away from offset,
|
|
// most of the time
|
|
// we have to
|
|
int32_t oldresult = next();
|
|
while (oldresult < offset) {
|
|
int32_t result = next();
|
|
if (result >= offset) {
|
|
return oldresult;
|
|
}
|
|
oldresult = result;
|
|
}
|
|
int32_t result = previous();
|
|
if (result >= offset) {
|
|
return previous();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// old rule syntax
|
|
fText->setIndex(offset);
|
|
return previous();
|
|
}
|
|
|
|
/**
|
|
* Returns true if the specfied position is a boundary position. As a side
|
|
* effect, leaves the iterator pointing to the first boundary position at
|
|
* or after "offset".
|
|
* @param offset the offset to check.
|
|
* @return True if "offset" is a boundary position.
|
|
*/
|
|
UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
|
|
// the beginning index of the iterator is always a boundary position by definition
|
|
if (fText == NULL || offset == fText->startIndex()) {
|
|
first(); // For side effects on current position, tag values.
|
|
return TRUE;
|
|
}
|
|
|
|
if (offset == fText->endIndex()) {
|
|
last(); // For side effects on current position, tag values.
|
|
return TRUE;
|
|
}
|
|
|
|
// out-of-range indexes are never boundary positions
|
|
if (offset < fText->startIndex()) {
|
|
first(); // For side effects on current position, tag values.
|
|
return FALSE;
|
|
}
|
|
|
|
if (offset > fText->endIndex()) {
|
|
last(); // For side effects on current position, tag values.
|
|
return FALSE;
|
|
}
|
|
|
|
// otherwise, we can use following() on the position before the specified
|
|
// one and return true if the position we get back is the one the user
|
|
// specified
|
|
return following(offset - 1) == offset;
|
|
}
|
|
|
|
/**
|
|
* Returns the current iteration position.
|
|
* @return The current iteration position.
|
|
*/
|
|
int32_t RuleBasedBreakIterator::current(void) const {
|
|
return (fText != NULL) ? fText->getIndex() : BreakIterator::DONE;
|
|
}
|
|
|
|
//=======================================================================
|
|
// implementation
|
|
//=======================================================================
|
|
|
|
|
|
//-----------------------------------------------------------------------------------
|
|
//
|
|
// handleNext()
|
|
// This method is the actual implementation of the next() method. All iteration
|
|
// vectors through here. This method initializes the state machine to state 1
|
|
// and advances through the text character by character until we reach the end
|
|
// of the text or the state machine transitions to state 0. We update our return
|
|
// value every time the state machine passes through an accepting state.
|
|
//
|
|
//-----------------------------------------------------------------------------------
|
|
int32_t RuleBasedBreakIterator::handleNext() {
|
|
return handleNext(fData->fForwardTable);
|
|
}
|
|
|
|
int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
|
|
if (fTrace) {
|
|
RBBIDebugPuts("Handle Next pos char state category");
|
|
}
|
|
|
|
// No matter what, handleNext alway correctly sets the break tag value.
|
|
fLastStatusIndexValid = TRUE;
|
|
|
|
// if we're already at the end of the text, return DONE.
|
|
if (fText == NULL || fData == NULL || fText->hasNext() == FALSE) {
|
|
fLastRuleStatusIndex = 0;
|
|
return BreakIterator::DONE;
|
|
}
|
|
|
|
int32_t initialPosition = fText->getIndex();
|
|
int32_t result = initialPosition;
|
|
int32_t lookaheadResult = 0;
|
|
|
|
// Initialize the state machine. Begin in state 1
|
|
int32_t state = START_STATE;
|
|
int16_t category;
|
|
UChar32 c = fText->current32();
|
|
RBBIStateTableRow *row;
|
|
int32_t lookaheadStatus = 0;
|
|
int32_t lookaheadTagIdx = 0;
|
|
|
|
fLastRuleStatusIndex = 0;
|
|
|
|
row = (RBBIStateTableRow *) // Point to starting row of state table.
|
|
(statetable->fTableData + (statetable->fRowLen * state));
|
|
|
|
// Character Category fetch for starting character.
|
|
// See comments on character category code within loop, below.
|
|
UTRIE_GET16(&fData->fTrie, c, category);
|
|
if ((category & 0x4000) != 0) {
|
|
fDictionaryCharCount++;
|
|
category &= ~0x4000;
|
|
}
|
|
|
|
// loop until we reach the end of the text or transition to state 0
|
|
for (;;) {
|
|
if (c == CharacterIterator::DONE && fText->hasNext()==FALSE) {
|
|
// Reached end of input string.
|
|
// Note: CharacterIterator::DONE is 0xffff, which is also a legal
|
|
// character value. Check for DONE first, because it's quicker,
|
|
// but also need to check fText->hasNext() to be certain.
|
|
|
|
if (lookaheadResult > result) {
|
|
// We ran off the end of the string with a pending look-ahead match.
|
|
// Treat this as if the look-ahead condition had been met, and return
|
|
// the match at the / position from the look-ahead rule.
|
|
result = lookaheadResult;
|
|
fLastRuleStatusIndex = lookaheadTagIdx;
|
|
lookaheadStatus = 0;
|
|
} else if (result == initialPosition) {
|
|
// Ran off end, no match found.
|
|
// move forward one
|
|
fText->setIndex(initialPosition);
|
|
fText->next32();
|
|
fText->getIndex();
|
|
}
|
|
break;
|
|
}
|
|
// look up the current character's character category, which tells us
|
|
// which column in the state table to look at.
|
|
// Note: the 16 in UTRIE_GET16 refers to the size of the data being returned,
|
|
// not the size of the character going in, which is a UChar32.
|
|
//
|
|
UTRIE_GET16(&fData->fTrie, c, category);
|
|
|
|
// Check the dictionary bit in the character's category.
|
|
// Counter is only used by dictionary based iterators (subclasses).
|
|
// Chars that need to be handled by a dictionary have a flag bit set
|
|
// in their category values.
|
|
//
|
|
if ((category & 0x4000) != 0) {
|
|
fDictionaryCharCount++;
|
|
// And off the dictionary flag bit.
|
|
category &= ~0x4000;
|
|
}
|
|
|
|
#ifdef RBBI_DEBUG
|
|
if (fTrace) {
|
|
RBBIDebugPrintf(" %4d ", fText->getIndex());
|
|
if (0x20<=c && c<0x7f) {
|
|
RBBIDebugPrintf("\"%c\" ", c);
|
|
} else {
|
|
RBBIDebugPrintf("%5x ", c);
|
|
}
|
|
RBBIDebugPrintf("%3d %3d\n", state, category);
|
|
}
|
|
#endif
|
|
|
|
// look up a state transition in the state table
|
|
state = row->fNextState[category];
|
|
row = (RBBIStateTableRow *)
|
|
(statetable->fTableData + (statetable->fRowLen * state));
|
|
|
|
// Get the next character. Doing it here positions the iterator
|
|
// to the correct position for recording matches in the code that
|
|
// follows.
|
|
c = fText->next32();
|
|
|
|
if (row->fAccepting == -1) {
|
|
// Match found, common case, could have lookahead so we move on to check it
|
|
result = fText->getIndex();
|
|
/// added
|
|
fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values.
|
|
}
|
|
|
|
if (row->fLookAhead != 0) {
|
|
if (lookaheadStatus != 0
|
|
&& row->fAccepting == lookaheadStatus) {
|
|
// Lookahead match is completed. Set the result accordingly, but only
|
|
// if no other rule has matched further in the mean time.
|
|
result = lookaheadResult;
|
|
fLastRuleStatusIndex = lookaheadTagIdx;
|
|
lookaheadStatus = 0;
|
|
/// i think we have to back up to read the lookahead character again
|
|
/// fText->setIndex(lookaheadResult);
|
|
/// TODO: this is a simple hack since reverse rules only have simple
|
|
/// lookahead rules that we can definitely break out from.
|
|
/// we need to make the lookahead rules not chain eventually.
|
|
/// return result;
|
|
/// this is going to be the longest match again
|
|
goto continueOn;
|
|
}
|
|
|
|
int32_t r = fText->getIndex();
|
|
lookaheadResult = r;
|
|
lookaheadStatus = row->fLookAhead;
|
|
lookaheadTagIdx = row->fTagIdx;
|
|
goto continueOn;
|
|
}
|
|
|
|
|
|
if (row->fAccepting == 0) {
|
|
// No match, nothing of interest happening, common case.
|
|
goto continueOn;
|
|
}
|
|
|
|
lookaheadStatus = 0; // clear out any pending look-ahead matches.
|
|
|
|
continueOn:
|
|
if (state == STOP_STATE) {
|
|
// This is the normal exit from the lookup state machine.
|
|
// We have advanced through the string until it is certain that no
|
|
// longer match is possible, no matter what characters follow.
|
|
break;
|
|
}
|
|
}
|
|
|
|
// The state machine is done. Check whether it found a match...
|
|
|
|
// If the iterator failed to advance in the match engine, force it ahead by one.
|
|
// (This really indicates a defect in the break rules. They should always match
|
|
// at least one character.)
|
|
if (result == initialPosition) {
|
|
result = fText->setIndex(initialPosition);
|
|
fText ->next32();
|
|
result = fText->getIndex();
|
|
}
|
|
|
|
// Leave the iterator at our result position.
|
|
fText->setIndex(result);
|
|
if (fTrace) {
|
|
RBBIDebugPrintf("result = %d\n\n", result);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
//----------------------------------------------------------------
|
|
//
|
|
// handlePrevious(void) This is the variant used with old style rules
|
|
// (Overshoot to a safe point, then move forward)
|
|
//
|
|
//----------------------------------------------------------------
|
|
int32_t RuleBasedBreakIterator::handlePrevious(void) {
|
|
if (fText == NULL || fData == NULL) {
|
|
return 0;
|
|
}
|
|
if (fData->fReverseTable == NULL) {
|
|
return fText->setToStart();
|
|
}
|
|
|
|
int32_t state = START_STATE;
|
|
int32_t category;
|
|
int32_t lastCategory = 0;
|
|
int32_t result = fText->getIndex();
|
|
int32_t lookaheadStatus = 0;
|
|
int32_t lookaheadResult = 0;
|
|
int32_t lookaheadTagIdx = 0;
|
|
UChar32 c = fText->current32();
|
|
RBBIStateTableRow *row;
|
|
|
|
row = (RBBIStateTableRow *)
|
|
(this->fData->fReverseTable->fTableData + (state * fData->fReverseTable->fRowLen));
|
|
UTRIE_GET16(&fData->fTrie, c, category);
|
|
if ((category & 0x4000) != 0) {
|
|
fDictionaryCharCount++;
|
|
category &= ~0x4000;
|
|
}
|
|
|
|
if (fTrace) {
|
|
RBBIDebugPuts("Handle Prev pos char state category");
|
|
}
|
|
|
|
// loop until we reach the beginning of the text or transition to state 0
|
|
for (;;) {
|
|
if (c == CharacterIterator::DONE && fText->hasPrevious()==FALSE) {
|
|
break;
|
|
}
|
|
|
|
// save the last character's category and look up the current
|
|
// character's category
|
|
lastCategory = category;
|
|
UTRIE_GET16(&fData->fTrie, c, category);
|
|
|
|
// Check the dictionary bit in the character's category.
|
|
// Counter is only used by dictionary based iterators.
|
|
//
|
|
if ((category & 0x4000) != 0) {
|
|
fDictionaryCharCount++;
|
|
category &= ~0x4000;
|
|
}
|
|
|
|
#ifdef RBBI_DEBUG
|
|
if (fTrace) {
|
|
RBBIDebugPrintf(" %4d ", fText->getIndex());
|
|
if (0x20<=c && c<0x7f) {
|
|
RBBIDebugPrintf("\"%c\" ", c);
|
|
} else {
|
|
RBBIDebugPrintf("%5x ", c);
|
|
}
|
|
RBBIDebugPrintf("%3d %3d\n", state, category);
|
|
}
|
|
#endif
|
|
|
|
// look up a state transition in the backwards state table
|
|
state = row->fNextState[category];
|
|
row = (RBBIStateTableRow *)
|
|
(this->fData->fReverseTable->fTableData + (state * fData->fReverseTable->fRowLen));
|
|
|
|
if (row->fAccepting == 0 && row->fLookAhead == 0) {
|
|
// No match, nothing of interest happening, common case.
|
|
goto continueOn;
|
|
}
|
|
|
|
if (row->fAccepting == -1) {
|
|
// Match found, common case, no lookahead involved.
|
|
result = fText->getIndex();
|
|
lookaheadStatus = 0; // clear out any pending look-ahead matches.
|
|
goto continueOn;
|
|
}
|
|
|
|
if (row->fAccepting == 0 && row->fLookAhead != 0) {
|
|
// Lookahead match point. Remember it, but only if no other rule
|
|
// has unconditionally matched to this point.
|
|
// TODO: handle case where there's a pending match from a different rule
|
|
// where lookaheadStatus != 0 && lookaheadStatus != row->fLookAhead.
|
|
int32_t r = fText->getIndex();
|
|
if (r > result) {
|
|
lookaheadResult = r;
|
|
lookaheadStatus = row->fLookAhead;
|
|
lookaheadTagIdx = row->fTagIdx;
|
|
}
|
|
goto continueOn;
|
|
}
|
|
|
|
if (row->fAccepting != 0 && row->fLookAhead != 0) {
|
|
// Lookahead match is completed. Set the result accordingly, but only
|
|
// if no other rule has matched further in the mean time.
|
|
if (lookaheadResult > result) {
|
|
U_ASSERT(row->fAccepting == lookaheadStatus); // TODO: handle this case
|
|
// of overlapping lookahead matches.
|
|
result = lookaheadResult;
|
|
fLastRuleStatusIndex = lookaheadTagIdx;
|
|
lookaheadStatus = 0;
|
|
}
|
|
goto continueOn;
|
|
}
|
|
|
|
continueOn:
|
|
if (state == STOP_STATE) {
|
|
break;
|
|
}
|
|
|
|
// then advance one character backwards
|
|
c = fText->previous32();
|
|
}
|
|
|
|
// Note: the result postion isn't what is returned to the user by previous(),
|
|
// but where the implementation of previous() turns around and
|
|
// starts iterating forward again.
|
|
if (c == CharacterIterator::DONE && fText->hasPrevious()==FALSE) {
|
|
result = fText->startIndex();
|
|
}
|
|
fText->setIndex(result);
|
|
|
|
return result;
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------------
|
|
//
|
|
// handlePrevious()
|
|
//
|
|
// This method backs the iterator back up to a "safe position" in the text.
|
|
// This is a position that we know, without any context, may be any position
|
|
// not more than 2 breaks away. Occasionally, the position may be less than
|
|
// one break away.
|
|
// The various calling methods then iterate forward from this safe position to
|
|
// the appropriate position to return.
|
|
//
|
|
// The logic of this function is very similar to handleNext(), above.
|
|
//
|
|
//-----------------------------------------------------------------------------------
|
|
int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) {
|
|
if (fText == NULL || statetable == NULL) {
|
|
return 0;
|
|
}
|
|
// break tag is no longer valid after icu switched to exact backwards
|
|
// positioning.
|
|
fLastStatusIndexValid = FALSE;
|
|
if (statetable == NULL) {
|
|
return fText->setToStart();
|
|
}
|
|
|
|
int32_t state = START_STATE;
|
|
int32_t category;
|
|
int32_t lastCategory = 0;
|
|
UBool hasPassedStartText = !fText->hasPrevious();
|
|
UChar32 c = fText->previous32();
|
|
// previous character
|
|
int32_t result = fText->getIndex();
|
|
int32_t lookaheadStatus = 0;
|
|
int32_t lookaheadResult = 0;
|
|
int32_t lookaheadTagIdx = 0;
|
|
UBool lookAheadHardBreak = (statetable->fFlags & RBBI_LOOKAHEAD_HARD_BREAK) != 0;
|
|
|
|
RBBIStateTableRow *row;
|
|
|
|
row = (RBBIStateTableRow *)
|
|
(statetable->fTableData + (state * statetable->fRowLen));
|
|
UTRIE_GET16(&fData->fTrie, c, category);
|
|
if ((category & 0x4000) != 0) {
|
|
fDictionaryCharCount++;
|
|
category &= ~0x4000;
|
|
}
|
|
|
|
if (fTrace) {
|
|
RBBIDebugPuts("Handle Prev pos char state category");
|
|
}
|
|
|
|
// loop until we reach the beginning of the text or transition to state 0
|
|
for (;;) {
|
|
// if (c == CharacterIterator::DONE && fText->hasPrevious()==FALSE) {
|
|
if (hasPassedStartText) {
|
|
// if we have already considered the start of the text
|
|
if (row->fLookAhead != 0 && lookaheadResult == 0) {
|
|
result = 0;
|
|
}
|
|
break;
|
|
}
|
|
|
|
// save the last character's category and look up the current
|
|
// character's category
|
|
lastCategory = category;
|
|
UTRIE_GET16(&fData->fTrie, c, category);
|
|
|
|
// Check the dictionary bit in the character's category.
|
|
// Counter is only used by dictionary based iterators.
|
|
//
|
|
if ((category & 0x4000) != 0) {
|
|
fDictionaryCharCount++;
|
|
category &= ~0x4000;
|
|
}
|
|
|
|
#ifdef RBBI_DEBUG
|
|
if (fTrace) {
|
|
RBBIDebugPrintf(" %4d ", fText->getIndex());
|
|
if (0x20<=c && c<0x7f) {
|
|
RBBIDebugPrintf("\"%c\" ", c);
|
|
} else {
|
|
RBBIDebugPrintf("%5x ", c);
|
|
}
|
|
RBBIDebugPrintf("%3d %3d\n", state, category);
|
|
}
|
|
#endif
|
|
|
|
// look up a state transition in the backwards state table
|
|
state = row->fNextState[category];
|
|
row = (RBBIStateTableRow *)
|
|
(statetable->fTableData + (state * statetable->fRowLen));
|
|
|
|
if (row->fAccepting == -1) {
|
|
// Match found, common case, could have lookahead so we move on to check it
|
|
result = fText->getIndex();
|
|
/// added
|
|
fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) value.
|
|
}
|
|
|
|
if (row->fLookAhead != 0) {
|
|
if (lookaheadStatus != 0
|
|
&& row->fAccepting == lookaheadStatus) {
|
|
// Lookahead match is completed. Set the result accordingly, but only
|
|
// if no other rule has matched further in the mean time.
|
|
result = lookaheadResult;
|
|
fLastRuleStatusIndex = lookaheadTagIdx;
|
|
lookaheadStatus = 0;
|
|
/// i think we have to back up to read the lookahead character again
|
|
/// fText->setIndex(lookaheadResult);
|
|
/// TODO: this is a simple hack since reverse rules only have simple
|
|
/// lookahead rules that we can definitely break out from.
|
|
/// we need to make the lookahead rules not chain eventually.
|
|
/// return result;
|
|
/// this is going to be the longest match again
|
|
|
|
/// syn wee todo hard coded for line breaks stuff
|
|
/// needs to provide a tag in rules to ensure a stop.
|
|
|
|
if (lookAheadHardBreak) {
|
|
fText->setIndex(result);
|
|
return result;
|
|
}
|
|
category = lastCategory;
|
|
fText->setIndex(result);
|
|
|
|
goto continueOn;
|
|
}
|
|
|
|
int32_t r = fText->getIndex();
|
|
lookaheadResult = r;
|
|
lookaheadStatus = row->fLookAhead;
|
|
fLastRuleStatusIndex = row->fTagIdx;
|
|
goto continueOn;
|
|
}
|
|
|
|
// not lookahead
|
|
if (row->fAccepting == 0) {
|
|
// No match, nothing of interest happening, common case.
|
|
goto continueOn;
|
|
}
|
|
|
|
lookaheadStatus = 0; // clear out any pending look-ahead matches.
|
|
|
|
continueOn:
|
|
if (state == STOP_STATE) {
|
|
break;
|
|
}
|
|
|
|
// then advance one character backwards
|
|
hasPassedStartText = !fText->hasPrevious();
|
|
c = fText->previous32();
|
|
}
|
|
|
|
// Note: the result postion isn't what is returned to the user by previous(),
|
|
// but where the implementation of previous() turns around and
|
|
// starts iterating forward again.
|
|
fText->setIndex(result);
|
|
|
|
return result;
|
|
}
|
|
|
|
|
|
void
|
|
RuleBasedBreakIterator::reset()
|
|
{
|
|
// Base-class version of this function is a no-op.
|
|
// Subclasses may override with their own reset behavior.
|
|
}
|
|
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// getRuleStatus() Return the break rule tag associated with the current
|
|
// iterator position. If the iterator arrived at its current
|
|
// position by iterating forwards, the value will have been
|
|
// cached by the handleNext() function.
|
|
//
|
|
// If no cached status value is available, the status is
|
|
// found by doing a previous() followed by a next(), which
|
|
// leaves the iterator where it started, and computes the
|
|
// status while doing the next().
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
void RuleBasedBreakIterator::makeRuleStatusValid() {
|
|
if (fLastStatusIndexValid == FALSE) {
|
|
// No cached status is available.
|
|
if (fText == NULL || current() == fText->startIndex()) {
|
|
// At start of text, or there is no text. Status is always zero.
|
|
fLastRuleStatusIndex = 0;
|
|
fLastStatusIndexValid = TRUE;
|
|
} else {
|
|
// Not at start of text. Find status the tedious way.
|
|
int32_t pa = current();
|
|
previous();
|
|
int32_t pb = next();
|
|
if (pa != pb) {
|
|
// note: the if (pa != pb) test is here only to eliminate warnings for
|
|
// unused local variables on gcc. Logically, it isn't needed.
|
|
U_ASSERT(pa == pb);
|
|
}
|
|
}
|
|
}
|
|
U_ASSERT(fLastStatusIndexValid == TRUE);
|
|
U_ASSERT(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fData->fStatusMaxIdx);
|
|
}
|
|
|
|
|
|
int32_t RuleBasedBreakIterator::getRuleStatus() const {
|
|
RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
|
|
nonConstThis->makeRuleStatusValid();
|
|
|
|
// fLastRuleStatusIndex indexes to the start of the appropriate status record
|
|
// (the number of status values.)
|
|
// This function returns the last (largest) of the array of status values.
|
|
int32_t idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex];
|
|
int32_t tagVal = fData->fRuleStatusTable[idx];
|
|
|
|
return tagVal;
|
|
}
|
|
|
|
|
|
|
|
|
|
int32_t RuleBasedBreakIterator::getRuleStatusVec(
|
|
int32_t *fillInVec, int32_t capacity, UErrorCode &status)
|
|
{
|
|
if (U_FAILURE(status)) {
|
|
return 0;
|
|
}
|
|
|
|
RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
|
|
nonConstThis->makeRuleStatusValid();
|
|
int32_t numVals = fData->fRuleStatusTable[fLastRuleStatusIndex];
|
|
int32_t numValsToCopy = numVals;
|
|
if (numVals > capacity) {
|
|
status = U_BUFFER_OVERFLOW_ERROR;
|
|
numValsToCopy = capacity;
|
|
}
|
|
int i;
|
|
for (i=0; i<numValsToCopy; i++) {
|
|
fillInVec[i] = fData->fRuleStatusTable[fLastRuleStatusIndex + i + 1];
|
|
}
|
|
return numVals;
|
|
}
|
|
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// getBinaryRules Access to the compiled form of the rules,
|
|
// for use by build system tools that save the data
|
|
// for standard iterator types.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
const uint8_t *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) {
|
|
const uint8_t *retPtr = NULL;
|
|
length = 0;
|
|
|
|
if (fData != NULL) {
|
|
retPtr = (const uint8_t *)fData->fHeader;
|
|
length = fData->fHeader->fLength;
|
|
}
|
|
return retPtr;
|
|
}
|
|
|
|
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// BufferClone TODO: In my (Andy) opinion, this function should be deprecated.
|
|
// Saving one heap allocation isn't worth the trouble.
|
|
// Cloning shouldn't be done in tight loops, and
|
|
// making the clone copy involves other heap operations anyway.
|
|
// And the application code for correctly dealing with buffer
|
|
// size problems and the eventual object destruction is ugly.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
BreakIterator * RuleBasedBreakIterator::createBufferClone(void *stackBuffer,
|
|
int32_t &bufferSize,
|
|
UErrorCode &status)
|
|
{
|
|
if (U_FAILURE(status)){
|
|
return NULL;
|
|
}
|
|
|
|
//
|
|
// If user buffer size is zero this is a preflight operation to
|
|
// obtain the needed buffer size, allowing for worst case misalignment.
|
|
//
|
|
if (bufferSize == 0) {
|
|
bufferSize = sizeof(RuleBasedBreakIterator) + U_ALIGNMENT_OFFSET_UP(0);
|
|
return NULL;
|
|
}
|
|
|
|
|
|
//
|
|
// Check the alignment and size of the user supplied buffer.
|
|
// Allocate heap memory if the user supplied memory is insufficient.
|
|
//
|
|
char *buf = (char *)stackBuffer;
|
|
uint32_t s = bufferSize;
|
|
|
|
if (stackBuffer == NULL) {
|
|
s = 0; // Ignore size, force allocation if user didn't give us a buffer.
|
|
}
|
|
if (U_ALIGNMENT_OFFSET(stackBuffer) != 0) {
|
|
uint32_t offsetUp = (uint32_t)U_ALIGNMENT_OFFSET_UP(buf);
|
|
s -= offsetUp;
|
|
buf += offsetUp;
|
|
}
|
|
if (s < sizeof(RuleBasedBreakIterator)) {
|
|
buf = (char *) new RuleBasedBreakIterator;
|
|
if (buf == 0) {
|
|
status = U_MEMORY_ALLOCATION_ERROR;
|
|
return NULL;
|
|
}
|
|
status = U_SAFECLONE_ALLOCATED_WARNING;
|
|
}
|
|
|
|
//
|
|
// Clone the object.
|
|
// TODO: using an overloaded operator new to directly initialize the
|
|
// copy in the user's buffer would be better, but it doesn't seem
|
|
// to get along with namespaces. Investigate why.
|
|
//
|
|
// The memcpy is only safe with an empty (default constructed)
|
|
// break iterator. Use on others can screw up reference counts
|
|
// to data. memcpy-ing objects is not really a good idea...
|
|
//
|
|
RuleBasedBreakIterator localIter; // Empty break iterator, source for memcpy
|
|
RuleBasedBreakIterator *clone = (RuleBasedBreakIterator *)buf;
|
|
uprv_memcpy(clone, &localIter, sizeof(RuleBasedBreakIterator)); // clone = empty, but initialized, iterator.
|
|
*clone = *this; // clone = the real one we want.
|
|
if (status != U_SAFECLONE_ALLOCATED_WARNING) {
|
|
clone->fBufferClone = TRUE;
|
|
}
|
|
|
|
return clone;
|
|
}
|
|
|
|
|
|
|
|
//-------------------------------------------------------------------------------
|
|
//
|
|
// isDictionaryChar Return true if the category lookup for this char
|
|
// indicates that it is in the set of dictionary lookup
|
|
// chars.
|
|
//
|
|
// This function is intended for use by dictionary based
|
|
// break iterators.
|
|
//
|
|
//-------------------------------------------------------------------------------
|
|
UBool RuleBasedBreakIterator::isDictionaryChar(UChar32 c) {
|
|
if (fData == NULL) {
|
|
return FALSE;
|
|
}
|
|
uint16_t category;
|
|
UTRIE_GET16(&fData->fTrie, c, category);
|
|
return (category & 0x4000) != 0;
|
|
}
|
|
|
|
|
|
|
|
U_NAMESPACE_END
|
|
|
|
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
|