/* ********************************************************************** * Copyright (C) 2001-2008 IBM and others. All rights reserved. ********************************************************************** * Date Name Description * 07/02/2001 synwee Creation. ********************************************************************** */ #include "unicode/utypes.h" #if !UCONFIG_NO_COLLATION #include "unicode/usearch.h" #include "unicode/ustring.h" #include "unicode/uchar.h" #include "unormimp.h" #include "ucol_imp.h" #include "usrchimp.h" #include "cmemory.h" #include "ucln_in.h" #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) // internal definition --------------------------------------------------- #define LAST_BYTE_MASK_ 0xFF #define SECOND_LAST_BYTE_SHIFT_ 8 #define SUPPLEMENTARY_MIN_VALUE_ 0x10000 static const uint16_t *FCD_ = NULL; // internal methods ------------------------------------------------- /** * Fast collation element iterator setOffset. * This function does not check for bounds. * @param coleiter collation element iterator * @param offset to set */ static inline void setColEIterOffset(UCollationElements *elems, int32_t offset) { collIterate *ci = &(elems->iteratordata_); ci->pos = ci->string + offset; ci->CEpos = ci->toReturn = ci->extendCEs ? ci->extendCEs : ci->CEs; if (ci->flags & UCOL_ITER_INNORMBUF) { ci->flags = ci->origFlags; } ci->fcdPosition = NULL; } /** * Getting the mask for collation strength * @param strength collation strength * @return collation element mask */ static inline uint32_t getMask(UCollationStrength strength) { switch (strength) { case UCOL_PRIMARY: return UCOL_PRIMARYORDERMASK; case UCOL_SECONDARY: return UCOL_SECONDARYORDERMASK | UCOL_PRIMARYORDERMASK; default: return UCOL_TERTIARYORDERMASK | UCOL_SECONDARYORDERMASK | UCOL_PRIMARYORDERMASK; } } /** * This is to squeeze the 21bit ces into a 256 table * @param ce collation element * @return collapsed version of the collation element */ static inline int hash(uint32_t ce) { // the old value UCOL_PRIMARYORDER(ce) % MAX_TABLE_SIZE_ does not work // well with the new collation where most of the latin 1 characters // are of the value xx000xxx. their hashes will most of the time be 0 // to be discussed on the hash algo. return UCOL_PRIMARYORDER(ce) % MAX_TABLE_SIZE_; } U_CDECL_BEGIN static UBool U_CALLCONV usearch_cleanup(void) { FCD_ = NULL; return TRUE; } U_CDECL_END /** * Initializing the fcd tables. * Internal method, status assumed to be a success. * @param status output error if any, caller to check status before calling * method, status assumed to be success when passed in. */ static inline void initializeFCD(UErrorCode *status) { if (FCD_ == NULL) { FCD_ = unorm_getFCDTrie(status); ucln_i18n_registerCleanup(UCLN_I18N_USEARCH, usearch_cleanup); } } /** * Gets the fcd value for a character at the argument index. * This method takes into accounts of the supplementary characters. * @param str UTF16 string where character for fcd retrieval resides * @param offset position of the character whose fcd is to be retrieved, to be * overwritten with the next character position, taking * surrogate characters into consideration. * @param strlength length of the argument string * @return fcd value */ static uint16_t getFCD(const UChar *str, int32_t *offset, int32_t strlength) { int32_t temp = *offset; uint16_t result; UChar ch = str[temp]; result = unorm_getFCD16(FCD_, ch); temp ++; if (result && temp != strlength && UTF_IS_FIRST_SURROGATE(ch)) { ch = str[temp]; if (UTF_IS_SECOND_SURROGATE(ch)) { result = unorm_getFCD16FromSurrogatePair(FCD_, result, ch); temp ++; } else { result = 0; } } *offset = temp; return result; } /** * Getting the modified collation elements taking into account the collation * attributes * @param strsrch string search data * @param sourcece * @return the modified collation element */ static inline int32_t getCE(const UStringSearch *strsrch, uint32_t sourcece) { // note for tertiary we can't use the collator->tertiaryMask, that // is a preprocessed mask that takes into account case options. since // we are only concerned with exact matches, we don't need that. sourcece &= strsrch->ceMask; if (strsrch->toShift) { // alternate handling here, since only the 16 most significant digits // is only used, we can safely do a compare without masking // if the ce is a variable, we mask and get only the primary values // no shifting to quartenary is required since all primary values // less than variabletop will need to be masked off anyway. if (strsrch->variableTop > sourcece) { if (strsrch->strength == UCOL_QUATERNARY) { sourcece &= UCOL_PRIMARYORDERMASK; } else { sourcece = UCOL_IGNORABLE; } } } return sourcece; } /** * Allocate a memory and returns NULL if it failed. * Internal method, status assumed to be a success. * @param size to allocate * @param status output error if any, caller to check status before calling * method, status assumed to be success when passed in. * @return newly allocated array, NULL otherwise */ static inline void * allocateMemory(uint32_t size, UErrorCode *status) { uint32_t *result = (uint32_t *)uprv_malloc(size); if (result == NULL) { *status = U_MEMORY_ALLOCATION_ERROR; } return result; } /** * Adds a uint32_t value to a destination array. * Creates a new array if we run out of space. The caller will have to * manually deallocate the newly allocated array. * Internal method, status assumed to be success, caller has to check status * before calling this method. destination not to be NULL and has at least * size destinationlength. * @param destination target array * @param offset destination offset to add value * @param destinationlength target array size, return value for the new size * @param value to be added * @param increments incremental size expected * @param status output error if any, caller to check status before calling * method, status assumed to be success when passed in. * @return new destination array, destination if there was no new allocation */ static inline int32_t * addTouint32_tArray(int32_t *destination, uint32_t offset, uint32_t *destinationlength, uint32_t value, uint32_t increments, UErrorCode *status) { uint32_t newlength = *destinationlength; if (offset + 1 == newlength) { newlength += increments; int32_t *temp = (int32_t *)allocateMemory( sizeof(int32_t) * newlength, status); if (U_FAILURE(*status)) { return NULL; } uprv_memcpy(temp, destination, sizeof(int32_t) * offset); *destinationlength = newlength; destination = temp; } destination[offset] = value; return destination; } /** * Initializing the ce table for a pattern. * Stores non-ignorable collation keys. * Table size will be estimated by the size of the pattern text. Table * expansion will be perform as we go along. Adding 1 to ensure that the table * size definitely increases. * Internal method, status assumed to be a success. * @param strsrch string search data * @param status output error if any, caller to check status before calling * method, status assumed to be success when passed in. * @return total number of expansions */ static inline uint16_t initializePatternCETable(UStringSearch *strsrch, UErrorCode *status) { UPattern *pattern = &(strsrch->pattern); uint32_t cetablesize = INITIAL_ARRAY_SIZE_; int32_t *cetable = pattern->CEBuffer; uint32_t patternlength = pattern->textLength; UCollationElements *coleiter = strsrch->utilIter; if (coleiter == NULL) { coleiter = ucol_openElements(strsrch->collator, pattern->text, patternlength, status); // status will be checked in ucol_next(..) later and if it is an // error UCOL_NULLORDER the result of ucol_next(..) and 0 will be // returned. strsrch->utilIter = coleiter; } else { uprv_init_collIterate(strsrch->collator, pattern->text, pattern->textLength, &coleiter->iteratordata_); } if (pattern->CE != cetable && pattern->CE) { uprv_free(pattern->CE); } uint16_t offset = 0; uint16_t result = 0; int32_t ce; while ((ce = ucol_next(coleiter, status)) != UCOL_NULLORDER && U_SUCCESS(*status)) { uint32_t newce = getCE(strsrch, ce); if (newce) { int32_t *temp = addTouint32_tArray(cetable, offset, &cetablesize, newce, patternlength - ucol_getOffset(coleiter) + 1, status); if (U_FAILURE(*status)) { return 0; } offset ++; if (cetable != temp && cetable != pattern->CEBuffer) { uprv_free(cetable); } cetable = temp; } result += (uint16_t)(ucol_getMaxExpansion(coleiter, ce) - 1); } cetable[offset] = 0; pattern->CE = cetable; pattern->CELength = offset; return result; } /** * Initializes the pattern struct. * Internal method, status assumed to be success. * @param strsrch UStringSearch data storage * @param status output error if any, caller to check status before calling * method, status assumed to be success when passed in. * @return expansionsize the total expansion size of the pattern */ static inline int16_t initializePattern(UStringSearch *strsrch, UErrorCode *status) { UPattern *pattern = &(strsrch->pattern); const UChar *patterntext = pattern->text; int32_t length = pattern->textLength; int32_t index = 0; // Since the strength is primary, accents are ignored in the pattern. if (strsrch->strength == UCOL_PRIMARY) { pattern->hasPrefixAccents = 0; pattern->hasSuffixAccents = 0; } else { pattern->hasPrefixAccents = getFCD(patterntext, &index, length) >> SECOND_LAST_BYTE_SHIFT_; index = length; UTF_BACK_1(patterntext, 0, index); pattern->hasSuffixAccents = getFCD(patterntext, &index, length) & LAST_BYTE_MASK_; } // since intializePattern is an internal method status is a success. return initializePatternCETable(strsrch, status); } /** * Initializing shift tables, with the default values. * If a corresponding default value is 0, the shift table is not set. * @param shift table for forwards shift * @param backshift table for backwards shift * @param cetable table containing pattern ce * @param cesize size of the pattern ces * @param expansionsize total size of the expansions * @param defaultforward the default forward value * @param defaultbackward the default backward value */ static inline void setShiftTable(int16_t shift[], int16_t backshift[], int32_t *cetable, int32_t cesize, int16_t expansionsize, int16_t defaultforward, int16_t defaultbackward) { // estimate the value to shift. to do that we estimate the smallest // number of characters to give the relevant ces, ie approximately // the number of ces minus their expansion, since expansions can come // from a character. int32_t count; for (count = 0; count < MAX_TABLE_SIZE_; count ++) { shift[count] = defaultforward; } cesize --; // down to the last index for (count = 0; count < cesize; count ++) { // number of ces from right of array to the count int temp = defaultforward - count - 1; shift[hash(cetable[count])] = temp > 1 ? temp : 1; } shift[hash(cetable[cesize])] = 1; // for ignorables we just shift by one. see test examples. shift[hash(0)] = 1; for (count = 0; count < MAX_TABLE_SIZE_; count ++) { backshift[count] = defaultbackward; } for (count = cesize; count > 0; count --) { // the original value count does not seem to work backshift[hash(cetable[count])] = count > expansionsize ? (int16_t)(count - expansionsize) : 1; } backshift[hash(cetable[0])] = 1; backshift[hash(0)] = 1; } /** * Building of the pattern collation element list and the boyer moore strsrch * table. * The canonical match will only be performed after the default match fails. * For both cases we need to remember the size of the composed and decomposed * versions of the string. Since the Boyer-Moore shift calculations shifts by * a number of characters in the text and tries to match the pattern from that * offset, the shift value can not be too large in case we miss some * characters. To choose a right shift size, we estimate the NFC form of the * and use its size as a shift guide. The NFC form should be the small * possible representation of the pattern. Anyways, we'll err on the smaller * shift size. Hence the calculation for minlength. * Canonical match will be performed slightly differently. We'll split the * pattern into 3 parts, the prefix accents (PA), the middle string bounded by * the first and last base character (MS), the ending accents (EA). Matches * will be done on MS first, and only when we match MS then some processing * will be required for the prefix and end accents in order to determine if * they match PA and EA. Hence the default shift values * for the canonical match will take the size of either end's accent into * consideration. Forwards search will take the end accents into consideration * for the default shift values and the backwards search will take the prefix * accents into consideration. * If pattern has no non-ignorable ce, we return a illegal argument error. * Internal method, status assumed to be success. * @param strsrch UStringSearch data storage * @param status for output errors if it occurs, status is assumed to be a * success when it is passed in. */ static inline void initialize(UStringSearch *strsrch, UErrorCode *status) { int16_t expandlength = initializePattern(strsrch, status); if (U_SUCCESS(*status) && strsrch->pattern.CELength > 0) { UPattern *pattern = &strsrch->pattern; int32_t cesize = pattern->CELength; int16_t minlength = cesize > expandlength ? (int16_t)cesize - expandlength : 1; pattern->defaultShiftSize = minlength; setShiftTable(pattern->shift, pattern->backShift, pattern->CE, cesize, expandlength, minlength, minlength); return; } strsrch->pattern.defaultShiftSize = 0; } /** * Check to make sure that the match length is at the end of the character by * using the breakiterator. * @param strsrch string search data * @param start target text start offset * @param end target text end offset */ static void checkBreakBoundary(const UStringSearch *strsrch, int32_t * /*start*/, int32_t *end) { #if !UCONFIG_NO_BREAK_ITERATION UBreakIterator *breakiterator = strsrch->search->internalBreakIter; if (breakiterator) { int32_t matchend = *end; //int32_t matchstart = *start; if (!ubrk_isBoundary(breakiterator, matchend)) { *end = ubrk_following(breakiterator, matchend); } /* Check the start of the matched text to make sure it doesn't have any accents * before it. This code may not be necessary and so it is commented out */ /*if (!ubrk_isBoundary(breakiterator, matchstart) && !ubrk_isBoundary(breakiterator, matchstart-1)) { *start = ubrk_preceding(breakiterator, matchstart); }*/ } #endif } /** * Determine whether the target text in UStringSearch bounded by the offset * start and end is one or more whole units of text as * determined by the breakiterator in UStringSearch. * @param strsrch string search data * @param start target text start offset * @param end target text end offset */ static UBool isBreakUnit(const UStringSearch *strsrch, int32_t start, int32_t end) { #if !UCONFIG_NO_BREAK_ITERATION UBreakIterator *breakiterator = strsrch->search->breakIter; //TODO: Add here. if (breakiterator) { int32_t startindex = ubrk_first(breakiterator); int32_t endindex = ubrk_last(breakiterator); // out-of-range indexes are never boundary positions if (start < startindex || start > endindex || end < startindex || end > endindex) { return FALSE; } // otherwise, we can use following() on the position before the // specified one and return true of the position we get back is the // one the user specified UBool result = (start == startindex || ubrk_following(breakiterator, start - 1) == start) && (end == endindex || ubrk_following(breakiterator, end - 1) == end); if (result) { // iterates the individual ces UCollationElements *coleiter = strsrch->utilIter; const UChar *text = strsrch->search->text + start; UErrorCode status = U_ZERO_ERROR; ucol_setText(coleiter, text, end - start, &status); for (int32_t count = 0; count < strsrch->pattern.CELength; count ++) { int32_t ce = getCE(strsrch, ucol_next(coleiter, &status)); if (ce == UCOL_IGNORABLE) { count --; continue; } if (U_FAILURE(status) || ce != strsrch->pattern.CE[count]) { return FALSE; } } int32_t nextce = ucol_next(coleiter, &status); while (ucol_getOffset(coleiter) == (end - start) && getCE(strsrch, nextce) == UCOL_IGNORABLE) { nextce = ucol_next(coleiter, &status); } if (ucol_getOffset(coleiter) == (end - start) && nextce != UCOL_NULLORDER) { // extra collation elements at the end of the match return FALSE; } } return result; } #endif return TRUE; } /** * Getting the next base character offset if current offset is an accent, * or the current offset if the current character contains a base character. * accents the following base character will be returned * @param text string * @param textoffset current offset * @param textlength length of text string * @return the next base character or the current offset * if the current character is contains a base character. */ static inline int32_t getNextBaseOffset(const UChar *text, int32_t textoffset, int32_t textlength) { if (textoffset < textlength) { int32_t temp = textoffset; if (getFCD(text, &temp, textlength) >> SECOND_LAST_BYTE_SHIFT_) { while (temp < textlength) { int32_t result = temp; if ((getFCD(text, &temp, textlength) >> SECOND_LAST_BYTE_SHIFT_) == 0) { return result; } } return textlength; } } return textoffset; } /** * Gets the next base character offset depending on the string search pattern * data * @param strsrch string search data * @param textoffset current offset, one offset away from the last character * to search for. * @return start index of the next base character or the current offset * if the current character is contains a base character. */ static inline int32_t getNextUStringSearchBaseOffset(UStringSearch *strsrch, int32_t textoffset) { int32_t textlength = strsrch->search->textLength; if (strsrch->pattern.hasSuffixAccents && textoffset < textlength) { int32_t temp = textoffset; const UChar *text = strsrch->search->text; UTF_BACK_1(text, 0, temp); if (getFCD(text, &temp, textlength) & LAST_BYTE_MASK_) { return getNextBaseOffset(text, textoffset, textlength); } } return textoffset; } /** * Shifting the collation element iterator position forward to prepare for * a following match. If the last character is a unsafe character, we'll only * shift by 1 to capture contractions, normalization etc. * Internal method, status assumed to be success. * @param text strsrch string search data * @param textoffset start text position to do search * @param ce the text ce which failed the match. * @param patternceindex index of the ce within the pattern ce buffer which * failed the match * @return final offset */ static inline int32_t shiftForward(UStringSearch *strsrch, int32_t textoffset, int32_t ce, int32_t patternceindex) { UPattern *pattern = &(strsrch->pattern); if (ce != UCOL_NULLORDER) { int32_t shift = pattern->shift[hash(ce)]; // this is to adjust for characters in the middle of the // substring for matching that failed. int32_t adjust = pattern->CELength - patternceindex; if (adjust > 1 && shift >= adjust) { shift -= adjust - 1; } textoffset += shift; } else { textoffset += pattern->defaultShiftSize; } textoffset = getNextUStringSearchBaseOffset(strsrch, textoffset); // check for unsafe characters // * if it is the start or middle of a contraction: to be done after // a initial match is found // * thai or lao base consonant character: similar to contraction // * high surrogate character: similar to contraction // * next character is a accent: shift to the next base character return textoffset; } /** * sets match not found * @param strsrch string search data */ static inline void setMatchNotFound(UStringSearch *strsrch) { // this method resets the match result regardless of the error status. strsrch->search->matchedIndex = USEARCH_DONE; strsrch->search->matchedLength = 0; if (strsrch->search->isForwardSearching) { setColEIterOffset(strsrch->textIter, strsrch->search->textLength); } else { setColEIterOffset(strsrch->textIter, 0); } } /** * Gets the offset to the next safe point in text. * ie. not the middle of a contraction, swappable characters or supplementary * characters. * @param collator collation sata * @param text string to work with * @param textoffset offset in string * @param textlength length of text string * @return offset to the next safe character */ static inline int32_t getNextSafeOffset(const UCollator *collator, const UChar *text, int32_t textoffset, int32_t textlength) { int32_t result = textoffset; // first contraction character while (result != textlength && ucol_unsafeCP(text[result], collator)) { result ++; } return result; } /** * This checks for accents in the potential match started with a . * composite character. * This is really painful... we have to check that composite character do not * have any extra accents. We have to normalize the potential match and find * the immediate decomposed character before the match. * The first composite character would have been taken care of by the fcd * checks in checkForwardExactMatch. * This is the slow path after the fcd of the first character and * the last character has been checked by checkForwardExactMatch and we * determine that the potential match has extra non-ignorable preceding * ces. * E.g. looking for \u0301 acute in \u01FA A ring above and acute, * checkExtraMatchAccent should fail since there is a middle ring in \u01FA * Note here that accents checking are slow and cautioned in the API docs. * Internal method, status assumed to be a success, caller should check status * before calling this method * @param strsrch string search data * @param start index of the potential unfriendly composite character * @param end index of the potential unfriendly composite character * @param status output error status if any. * @return TRUE if there is non-ignorable accents before at the beginning * of the match, FALSE otherwise. */ static UBool checkExtraMatchAccents(const UStringSearch *strsrch, int32_t start, int32_t end, UErrorCode *status) { UBool result = FALSE; if (strsrch->pattern.hasPrefixAccents) { int32_t length = end - start; int32_t offset = 0; const UChar *text = strsrch->search->text + start; UTF_FWD_1(text, offset, length); // we are only concerned with the first composite character if (unorm_quickCheck(text, offset, UNORM_NFD, status) == UNORM_NO) { int32_t safeoffset = getNextSafeOffset(strsrch->collator, text, 0, length); if (safeoffset != length) { safeoffset ++; } UChar *norm = NULL; UChar buffer[INITIAL_ARRAY_SIZE_]; int32_t size = unorm_normalize(text, safeoffset, UNORM_NFD, 0, buffer, INITIAL_ARRAY_SIZE_, status); if (U_FAILURE(*status)) { return FALSE; } if (size >= INITIAL_ARRAY_SIZE_) { norm = (UChar *)allocateMemory((size + 1) * sizeof(UChar), status); // if allocation failed, status will be set to // U_MEMORY_ALLOCATION_ERROR and unorm_normalize internally // checks for it. size = unorm_normalize(text, safeoffset, UNORM_NFD, 0, norm, size, status); if (U_FAILURE(*status) && norm != NULL) { uprv_free(norm); return FALSE; } } else { norm = buffer; } UCollationElements *coleiter = strsrch->utilIter; ucol_setText(coleiter, norm, size, status); uint32_t firstce = strsrch->pattern.CE[0]; UBool ignorable = TRUE; uint32_t ce = UCOL_IGNORABLE; while (U_SUCCESS(*status) && ce != firstce && ce != (uint32_t)UCOL_NULLORDER) { offset = ucol_getOffset(coleiter); if (ce != firstce && ce != UCOL_IGNORABLE) { ignorable = FALSE; } ce = ucol_next(coleiter, status); } UChar32 codepoint; UTF_PREV_CHAR(norm, 0, offset, codepoint); result = !ignorable && (u_getCombiningClass(codepoint) != 0); if (norm != buffer) { uprv_free(norm); } } } return result; } /** * Used by exact matches, checks if there are accents before the match. * This is really painful... we have to check that composite characters at * the start of the matches have to not have any extra accents. * We check the FCD of the character first, if it starts with an accent and * the first pattern ce does not match the first ce of the character, we bail. * Otherwise we try normalizing the first composite * character and find the immediate decomposed character before the match to * see if it is an non-ignorable accent. * Now normalizing the first composite character is enough because we ensure * that when the match is passed in here with extra beginning ces, the * first or last ce that match has to occur within the first character. * E.g. looking for \u0301 acute in \u01FA A ring above and acute, * checkExtraMatchAccent should fail since there is a middle ring in \u01FA * Note here that accents checking are slow and cautioned in the API docs. * @param strsrch string search data * @param start offset * @param end offset * @return TRUE if there are accents on either side of the match, * FALSE otherwise */ static UBool hasAccentsBeforeMatch(const UStringSearch *strsrch, int32_t start, int32_t end) { if (strsrch->pattern.hasPrefixAccents) { UCollationElements *coleiter = strsrch->textIter; UErrorCode status = U_ZERO_ERROR; // we have been iterating forwards previously uint32_t ignorable = TRUE; int32_t firstce = strsrch->pattern.CE[0]; setColEIterOffset(coleiter, start); int32_t ce = getCE(strsrch, ucol_next(coleiter, &status)); if (U_FAILURE(status)) { return TRUE; } while (ce != firstce) { if (ce != UCOL_IGNORABLE) { ignorable = FALSE; } ce = getCE(strsrch, ucol_next(coleiter, &status)); if (U_FAILURE(status)) { return TRUE; } } if (!ignorable && inNormBuf(coleiter)) { // within normalization buffer, discontiguous handled here return TRUE; } // within text int32_t temp = start; // original code // accent = (getFCD(strsrch->search->text, &temp, // strsrch->search->textLength) // >> SECOND_LAST_BYTE_SHIFT_); // however this code does not work well with VC7 .net in release mode. // maybe the inlines for getFCD combined with shifting has bugs in // VC7. anyways this is a work around. UBool accent = getFCD(strsrch->search->text, &temp, strsrch->search->textLength) > 0xFF; if (!accent) { return checkExtraMatchAccents(strsrch, start, end, &status); } if (!ignorable) { return TRUE; } if (start > 0) { temp = start; UTF_BACK_1(strsrch->search->text, 0, temp); if (getFCD(strsrch->search->text, &temp, strsrch->search->textLength) & LAST_BYTE_MASK_) { setColEIterOffset(coleiter, start); ce = ucol_previous(coleiter, &status); if (U_FAILURE(status) || (ce != UCOL_NULLORDER && ce != UCOL_IGNORABLE)) { return TRUE; } } } } return FALSE; } /** * Used by exact matches, checks if there are accents bounding the match. * Note this is the initial boundary check. If the potential match * starts or ends with composite characters, the accents in those * characters will be determined later. * Not doing backwards iteration here, since discontiguos contraction for * backwards collation element iterator, use up too many characters. * E.g. looking for \u030A ring in \u01FA A ring above and acute, * should fail since there is a acute at the end of \u01FA * Note here that accents checking are slow and cautioned in the API docs. * @param strsrch string search data * @param start offset of match * @param end end offset of the match * @return TRUE if there are accents on either side of the match, * FALSE otherwise */ static UBool hasAccentsAfterMatch(const UStringSearch *strsrch, int32_t start, int32_t end) { if (strsrch->pattern.hasSuffixAccents) { const UChar *text = strsrch->search->text; int32_t temp = end; int32_t textlength = strsrch->search->textLength; UTF_BACK_1(text, 0, temp); if (getFCD(text, &temp, textlength) & LAST_BYTE_MASK_) { int32_t firstce = strsrch->pattern.CE[0]; UCollationElements *coleiter = strsrch->textIter; UErrorCode status = U_ZERO_ERROR; setColEIterOffset(coleiter, start); while (getCE(strsrch, ucol_next(coleiter, &status)) != firstce) { if (U_FAILURE(status)) { return TRUE; } } int32_t count = 1; while (count < strsrch->pattern.CELength) { if (getCE(strsrch, ucol_next(coleiter, &status)) == UCOL_IGNORABLE) { // Thai can give an ignorable here. count --; } if (U_FAILURE(status)) { return TRUE; } count ++; } int32_t ce = ucol_next(coleiter, &status); if (U_FAILURE(status)) { return TRUE; } if (ce != UCOL_NULLORDER && ce != UCOL_IGNORABLE) { ce = getCE(strsrch, ce); } if (ce != UCOL_NULLORDER && ce != UCOL_IGNORABLE) { if (ucol_getOffset(coleiter) <= end) { return TRUE; } if (getFCD(text, &end, textlength) >> SECOND_LAST_BYTE_SHIFT_) { return TRUE; } } } } return FALSE; } /** * Checks if the offset runs out of the text string * @param offset * @param textlength of the text string * @return TRUE if offset is out of bounds, FALSE otherwise */ static inline UBool isOutOfBounds(int32_t textlength, int32_t offset) { return offset < 0 || offset > textlength; } /** * Checks for identical match * @param strsrch string search data * @param start offset of possible match * @param end offset of possible match * @return TRUE if identical match is found */ static inline UBool checkIdentical(const UStringSearch *strsrch, int32_t start, int32_t end) { UChar t2[32], p2[32]; int32_t length = end - start; if (strsrch->strength != UCOL_IDENTICAL) { return TRUE; } UErrorCode status = U_ZERO_ERROR, status2 = U_ZERO_ERROR; int32_t decomplength = unorm_decompose(t2, LENGTHOF(t2), strsrch->search->text + start, length, FALSE, 0, &status); // use separate status2 in case of buffer overflow if (decomplength != unorm_decompose(p2, LENGTHOF(p2), strsrch->pattern.text, strsrch->pattern.textLength, FALSE, 0, &status2)) { return FALSE; // lengths are different } // compare contents UChar *text, *pattern; if(U_SUCCESS(status)) { text = t2; pattern = p2; } else if(status==U_BUFFER_OVERFLOW_ERROR) { status = U_ZERO_ERROR; // allocate one buffer for both decompositions text = (UChar *)uprv_malloc(decomplength * 2 * U_SIZEOF_UCHAR); // Check for allocation failure. if (text == NULL) { return FALSE; } pattern = text + decomplength; unorm_decompose(text, decomplength, strsrch->search->text + start, length, FALSE, 0, &status); unorm_decompose(pattern, decomplength, strsrch->pattern.text, strsrch->pattern.textLength, FALSE, 0, &status); } else { // NFD failed, make sure that u_memcmp() does not overrun t2 & p2 // and that we don't uprv_free() an undefined text pointer text = pattern = t2; decomplength = 0; } UBool result = (UBool)(u_memcmp(pattern, text, decomplength) == 0); if(text != t2) { uprv_free(text); } // return FALSE if NFD failed return U_SUCCESS(status) && result; } /** * Checks to see if the match is repeated * @param strsrch string search data * @param start new match start index * @param end new match end index * @return TRUE if the the match is repeated, FALSE otherwise */ static inline UBool checkRepeatedMatch(UStringSearch *strsrch, int32_t start, int32_t end) { int32_t lastmatchindex = strsrch->search->matchedIndex; UBool result; if (lastmatchindex == USEARCH_DONE) { return FALSE; } if (strsrch->search->isForwardSearching) { result = start <= lastmatchindex; } else { result = start >= lastmatchindex; } if (!result && !strsrch->search->isOverlap) { if (strsrch->search->isForwardSearching) { result = start < lastmatchindex + strsrch->search->matchedLength; } else { result = end > lastmatchindex; } } return result; } /** * Gets the collation element iterator's current offset. * @param coleiter collation element iterator * @param forwards flag TRUE if we are moving in th forwards direction * @return current offset */ static inline int32_t getColElemIterOffset(const UCollationElements *coleiter, UBool forwards) { int32_t result = ucol_getOffset(coleiter); // intricacies of the the backwards collation element iterator if (!forwards && inNormBuf(coleiter) && !isFCDPointerNull(coleiter)) { result ++; } return result; } /** * Checks match for contraction. * If the match ends with a partial contraction we fail. * If the match starts too far off (because of backwards iteration) we try to * chip off the extra characters depending on whether a breakiterator has * been used. * Internal method, error assumed to be success, caller has to check status * before calling this method. * @param strsrch string search data * @param start offset of potential match, to be modified if necessary * @param end offset of potential match, to be modified if necessary * @param status output error status if any * @return TRUE if match passes the contraction test, FALSE otherwise */ static UBool checkNextExactContractionMatch(UStringSearch *strsrch, int32_t *start, int32_t *end, UErrorCode *status) { UCollationElements *coleiter = strsrch->textIter; int32_t textlength = strsrch->search->textLength; int32_t temp = *start; const UCollator *collator = strsrch->collator; const UChar *text = strsrch->search->text; // This part checks if either ends of the match contains potential // contraction. If so we'll have to iterate through them // The start contraction needs to be checked since ucol_previous dumps // all characters till the first safe character into the buffer. // *start + 1 is used to test for the unsafe characters instead of *start // because ucol_prev takes all unsafe characters till the first safe // character ie *start. so by testing *start + 1, we can estimate if // excess prefix characters has been included in the potential search // results. if ((*end < textlength && ucol_unsafeCP(text[*end], collator)) || (*start + 1 < textlength && ucol_unsafeCP(text[*start + 1], collator))) { int32_t expansion = getExpansionPrefix(coleiter); UBool expandflag = expansion > 0; setColEIterOffset(coleiter, *start); while (expansion > 0) { // getting rid of the redundant ce, caused by setOffset. // since backward contraction/expansion may have extra ces if we // are in the normalization buffer, hasAccentsBeforeMatch would // have taken care of it. // E.g. the character \u01FA will have an expansion of 3, but if // we are only looking for acute and ring \u030A and \u0301, we'll // have to skip the first ce in the expansion buffer. ucol_next(coleiter, status); if (U_FAILURE(*status)) { return FALSE; } if (ucol_getOffset(coleiter) != temp) { *start = temp; temp = ucol_getOffset(coleiter); } expansion --; } int32_t *patternce = strsrch->pattern.CE; int32_t patterncelength = strsrch->pattern.CELength; int32_t count = 0; while (count < patterncelength) { int32_t ce = getCE(strsrch, ucol_next(coleiter, status)); if (ce == UCOL_IGNORABLE) { continue; } if (expandflag && count == 0 && ucol_getOffset(coleiter) != temp) { *start = temp; temp = ucol_getOffset(coleiter); } if (U_FAILURE(*status) || ce != patternce[count]) { (*end) ++; *end = getNextUStringSearchBaseOffset(strsrch, *end); return FALSE; } count ++; } } return TRUE; } /** * Checks and sets the match information if found. * Checks *