2016-06-15 18:58:17 +00:00
|
|
|
// Copyright (C) 2016 and later: Unicode, Inc. and others.
|
|
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
2001-12-06 01:23:47 +00:00
|
|
|
/*
|
|
|
|
******************************************************************************
|
|
|
|
*
|
2016-05-31 21:45:07 +00:00
|
|
|
* Copyright (C) 2001-2012, International Business Machines
|
|
|
|
* Corporation and others. All Rights Reserved.
|
2001-12-06 01:23:47 +00:00
|
|
|
*
|
|
|
|
******************************************************************************
|
2011-05-26 20:05:39 +00:00
|
|
|
* file name: utrie.cpp
|
2001-12-06 01:23:47 +00:00
|
|
|
* encoding: US-ASCII
|
|
|
|
* tab size: 8 (not used)
|
|
|
|
* indentation:4
|
|
|
|
*
|
|
|
|
* created on: 2001oct20
|
|
|
|
* created by: Markus W. Scherer
|
|
|
|
*
|
|
|
|
* This is a common implementation of a "folded" trie.
|
|
|
|
* It is a kind of compressed, serializable table of 16- or 32-bit values associated with
|
|
|
|
* Unicode code points (0..0x10ffff).
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef UTRIE_DEBUG
|
|
|
|
# include <stdio.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#include "cmemory.h"
|
|
|
|
#include "utrie.h"
|
|
|
|
|
2004-12-31 13:43:57 +00:00
|
|
|
/* miscellaneous ------------------------------------------------------------ */
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
#undef ABS
|
|
|
|
#define ABS(x) ((x)>=0 ? (x) : -(x))
|
|
|
|
|
2011-05-26 20:05:39 +00:00
|
|
|
static inline UBool
|
2004-12-31 13:43:57 +00:00
|
|
|
equal_uint32(const uint32_t *s, const uint32_t *t, int32_t length) {
|
|
|
|
while(length>0 && *s==*t) {
|
|
|
|
++s;
|
|
|
|
++t;
|
|
|
|
--length;
|
|
|
|
}
|
|
|
|
return (UBool)(length==0);
|
|
|
|
}
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/* Building a trie ----------------------------------------------------------*/
|
|
|
|
|
|
|
|
U_CAPI UNewTrie * U_EXPORT2
|
2001-12-16 21:15:09 +00:00
|
|
|
utrie_open(UNewTrie *fillIn,
|
|
|
|
uint32_t *aliasData, int32_t maxDataLength,
|
2003-07-24 23:23:19 +00:00
|
|
|
uint32_t initialValue, uint32_t leadUnitValue,
|
|
|
|
UBool latin1Linear) {
|
2001-12-06 01:23:47 +00:00
|
|
|
UNewTrie *trie;
|
2001-12-16 21:15:09 +00:00
|
|
|
int32_t i, j;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
if( maxDataLength<UTRIE_DATA_BLOCK_LENGTH ||
|
|
|
|
(latin1Linear && maxDataLength<1024)
|
|
|
|
) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(fillIn!=NULL) {
|
|
|
|
trie=fillIn;
|
|
|
|
} else {
|
|
|
|
trie=(UNewTrie *)uprv_malloc(sizeof(UNewTrie));
|
|
|
|
if(trie==NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
uprv_memset(trie, 0, sizeof(UNewTrie));
|
|
|
|
trie->isAllocated= (UBool)(fillIn==NULL);
|
|
|
|
|
|
|
|
if(aliasData!=NULL) {
|
|
|
|
trie->data=aliasData;
|
|
|
|
trie->isDataAllocated=FALSE;
|
|
|
|
} else {
|
|
|
|
trie->data=(uint32_t *)uprv_malloc(maxDataLength*4);
|
|
|
|
if(trie->data==NULL) {
|
|
|
|
uprv_free(trie);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
trie->isDataAllocated=TRUE;
|
|
|
|
}
|
|
|
|
|
2001-12-16 21:15:09 +00:00
|
|
|
/* preallocate and reset the first data block (block index 0) */
|
|
|
|
j=UTRIE_DATA_BLOCK_LENGTH;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
if(latin1Linear) {
|
2001-12-16 21:15:09 +00:00
|
|
|
/* preallocate and reset the first block (number 0) and Latin-1 (U+0000..U+00ff) after that */
|
|
|
|
/* made sure above that maxDataLength>=1024 */
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/* set indexes to point to consecutive data blocks */
|
|
|
|
i=0;
|
|
|
|
do {
|
|
|
|
/* do this at least for trie->index[0] even if that block is only partly used for Latin-1 */
|
|
|
|
trie->index[i++]=j;
|
|
|
|
j+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
} while(i<(256>>UTRIE_SHIFT));
|
2001-12-16 21:15:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* reset the initially allocated blocks to the initial value */
|
|
|
|
trie->dataLength=j;
|
|
|
|
while(j>0) {
|
|
|
|
trie->data[--j]=initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
|
2003-07-24 23:23:19 +00:00
|
|
|
trie->leadUnitValue=leadUnitValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
trie->indexLength=UTRIE_MAX_INDEX_LENGTH;
|
|
|
|
trie->dataCapacity=maxDataLength;
|
|
|
|
trie->isLatin1Linear=latin1Linear;
|
|
|
|
trie->isCompacted=FALSE;
|
|
|
|
return trie;
|
|
|
|
}
|
|
|
|
|
|
|
|
U_CAPI UNewTrie * U_EXPORT2
|
|
|
|
utrie_clone(UNewTrie *fillIn, const UNewTrie *other, uint32_t *aliasData, int32_t aliasDataCapacity) {
|
|
|
|
UNewTrie *trie;
|
|
|
|
UBool isDataAllocated;
|
|
|
|
|
|
|
|
/* do not clone if other is not valid or already compacted */
|
|
|
|
if(other==NULL || other->data==NULL || other->isCompacted) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* clone data */
|
|
|
|
if(aliasData!=NULL && aliasDataCapacity>=other->dataCapacity) {
|
|
|
|
isDataAllocated=FALSE;
|
|
|
|
} else {
|
|
|
|
aliasDataCapacity=other->dataCapacity;
|
|
|
|
aliasData=(uint32_t *)uprv_malloc(other->dataCapacity*4);
|
|
|
|
if(aliasData==NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
isDataAllocated=TRUE;
|
|
|
|
}
|
|
|
|
|
2003-07-24 23:23:19 +00:00
|
|
|
trie=utrie_open(fillIn, aliasData, aliasDataCapacity,
|
|
|
|
other->data[0], other->leadUnitValue,
|
|
|
|
other->isLatin1Linear);
|
2001-12-06 01:23:47 +00:00
|
|
|
if(trie==NULL) {
|
|
|
|
uprv_free(aliasData);
|
|
|
|
} else {
|
|
|
|
uprv_memcpy(trie->index, other->index, sizeof(trie->index));
|
2001-12-12 11:16:36 +00:00
|
|
|
uprv_memcpy(trie->data, other->data, other->dataLength*4);
|
|
|
|
trie->dataLength=other->dataLength;
|
2001-12-06 01:23:47 +00:00
|
|
|
trie->isDataAllocated=isDataAllocated;
|
|
|
|
}
|
|
|
|
|
|
|
|
return trie;
|
|
|
|
}
|
|
|
|
|
|
|
|
U_CAPI void U_EXPORT2
|
|
|
|
utrie_close(UNewTrie *trie) {
|
|
|
|
if(trie!=NULL) {
|
|
|
|
if(trie->isDataAllocated) {
|
|
|
|
uprv_free(trie->data);
|
|
|
|
trie->data=NULL;
|
|
|
|
}
|
|
|
|
if(trie->isAllocated) {
|
|
|
|
uprv_free(trie);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
U_CAPI uint32_t * U_EXPORT2
|
|
|
|
utrie_getData(UNewTrie *trie, int32_t *pLength) {
|
|
|
|
if(trie==NULL || pLength==NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
*pLength=trie->dataLength;
|
|
|
|
return trie->data;
|
|
|
|
}
|
|
|
|
|
2003-08-05 00:20:30 +00:00
|
|
|
static int32_t
|
|
|
|
utrie_allocDataBlock(UNewTrie *trie) {
|
|
|
|
int32_t newBlock, newTop;
|
|
|
|
|
|
|
|
newBlock=trie->dataLength;
|
|
|
|
newTop=newBlock+UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
if(newTop>trie->dataCapacity) {
|
|
|
|
/* out of memory in the data array */
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
trie->dataLength=newTop;
|
|
|
|
return newBlock;
|
|
|
|
}
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/**
|
|
|
|
* No error checking for illegal arguments.
|
|
|
|
*
|
|
|
|
* @return -1 if no new data block available (out of memory in data array)
|
|
|
|
* @internal
|
|
|
|
*/
|
|
|
|
static int32_t
|
|
|
|
utrie_getDataBlock(UNewTrie *trie, UChar32 c) {
|
2003-08-05 00:20:30 +00:00
|
|
|
int32_t indexValue, newBlock;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
c>>=UTRIE_SHIFT;
|
|
|
|
indexValue=trie->index[c];
|
|
|
|
if(indexValue>0) {
|
|
|
|
return indexValue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* allocate a new data block */
|
2003-08-05 00:20:30 +00:00
|
|
|
newBlock=utrie_allocDataBlock(trie);
|
|
|
|
if(newBlock<0) {
|
2001-12-06 01:23:47 +00:00
|
|
|
/* out of memory in the data array */
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
trie->index[c]=newBlock;
|
|
|
|
|
2001-12-16 21:15:09 +00:00
|
|
|
/* copy-on-write for a block from a setRange() */
|
|
|
|
uprv_memcpy(trie->data+newBlock, trie->data-indexValue, 4*UTRIE_DATA_BLOCK_LENGTH);
|
2001-12-06 01:23:47 +00:00
|
|
|
return newBlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @return TRUE if the value was successfully set
|
|
|
|
*/
|
|
|
|
U_CAPI UBool U_EXPORT2
|
|
|
|
utrie_set32(UNewTrie *trie, UChar32 c, uint32_t value) {
|
|
|
|
int32_t block;
|
|
|
|
|
|
|
|
/* valid, uncompacted trie and valid c? */
|
|
|
|
if(trie==NULL || trie->isCompacted || (uint32_t)c>0x10ffff) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
|
|
|
|
block=utrie_getDataBlock(trie, c);
|
|
|
|
if(block<0) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
|
|
|
|
trie->data[block+(c&UTRIE_MASK)]=value;
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
|
|
|
|
U_CAPI uint32_t U_EXPORT2
|
|
|
|
utrie_get32(UNewTrie *trie, UChar32 c, UBool *pInBlockZero) {
|
|
|
|
int32_t block;
|
|
|
|
|
|
|
|
/* valid, uncompacted trie and valid c? */
|
|
|
|
if(trie==NULL || trie->isCompacted || (uint32_t)c>0x10ffff) {
|
|
|
|
if(pInBlockZero!=NULL) {
|
|
|
|
*pInBlockZero=TRUE;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
block=trie->index[c>>UTRIE_SHIFT];
|
|
|
|
if(pInBlockZero!=NULL) {
|
|
|
|
*pInBlockZero= (UBool)(block==0);
|
|
|
|
}
|
|
|
|
|
|
|
|
return trie->data[ABS(block)+(c&UTRIE_MASK)];
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @internal
|
|
|
|
*/
|
|
|
|
static void
|
2001-12-16 21:15:09 +00:00
|
|
|
utrie_fillBlock(uint32_t *block, UChar32 start, UChar32 limit,
|
|
|
|
uint32_t value, uint32_t initialValue, UBool overwrite) {
|
2001-12-06 01:23:47 +00:00
|
|
|
uint32_t *pLimit;
|
|
|
|
|
|
|
|
pLimit=block+limit;
|
|
|
|
block+=start;
|
|
|
|
if(overwrite) {
|
|
|
|
while(block<pLimit) {
|
|
|
|
*block++=value;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
while(block<pLimit) {
|
2001-12-16 21:15:09 +00:00
|
|
|
if(*block==initialValue) {
|
2001-12-06 01:23:47 +00:00
|
|
|
*block=value;
|
|
|
|
}
|
|
|
|
++block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
U_CAPI UBool U_EXPORT2
|
|
|
|
utrie_setRange32(UNewTrie *trie, UChar32 start, UChar32 limit, uint32_t value, UBool overwrite) {
|
|
|
|
/*
|
|
|
|
* repeat value in [start..limit[
|
|
|
|
* mark index values for repeat-data blocks by setting bit 31 of the index values
|
|
|
|
* fill around existing values if any, if(overwrite)
|
|
|
|
*/
|
2001-12-16 21:15:09 +00:00
|
|
|
uint32_t initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
int32_t block, rest, repeatBlock;
|
|
|
|
|
|
|
|
/* valid, uncompacted trie and valid indexes? */
|
|
|
|
if( trie==NULL || trie->isCompacted ||
|
|
|
|
(uint32_t)start>0x10ffff || (uint32_t)limit>0x110000 || start>limit
|
|
|
|
) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
if(start==limit) {
|
|
|
|
return TRUE; /* nothing to do */
|
|
|
|
}
|
|
|
|
|
2001-12-16 21:15:09 +00:00
|
|
|
initialValue=trie->data[0];
|
2001-12-06 01:23:47 +00:00
|
|
|
if(start&UTRIE_MASK) {
|
|
|
|
UChar32 nextStart;
|
|
|
|
|
|
|
|
/* set partial block at [start..following block boundary[ */
|
|
|
|
block=utrie_getDataBlock(trie, start);
|
|
|
|
if(block<0) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
|
|
|
|
nextStart=(start+UTRIE_DATA_BLOCK_LENGTH)&~UTRIE_MASK;
|
|
|
|
if(nextStart<=limit) {
|
2001-12-16 21:15:09 +00:00
|
|
|
utrie_fillBlock(trie->data+block, start&UTRIE_MASK, UTRIE_DATA_BLOCK_LENGTH,
|
|
|
|
value, initialValue, overwrite);
|
2001-12-06 01:23:47 +00:00
|
|
|
start=nextStart;
|
|
|
|
} else {
|
2001-12-16 21:15:09 +00:00
|
|
|
utrie_fillBlock(trie->data+block, start&UTRIE_MASK, limit&UTRIE_MASK,
|
|
|
|
value, initialValue, overwrite);
|
2001-12-06 01:23:47 +00:00
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* number of positions in the last, partial block */
|
|
|
|
rest=limit&UTRIE_MASK;
|
|
|
|
|
|
|
|
/* round down limit to a block boundary */
|
|
|
|
limit&=~UTRIE_MASK;
|
|
|
|
|
|
|
|
/* iterate over all-value blocks */
|
2001-12-16 21:15:09 +00:00
|
|
|
if(value==initialValue) {
|
2001-12-06 01:23:47 +00:00
|
|
|
repeatBlock=0;
|
|
|
|
} else {
|
|
|
|
repeatBlock=-1;
|
|
|
|
}
|
|
|
|
while(start<limit) {
|
|
|
|
/* get index value */
|
|
|
|
block=trie->index[start>>UTRIE_SHIFT];
|
|
|
|
if(block>0) {
|
|
|
|
/* already allocated, fill in value */
|
2001-12-16 21:15:09 +00:00
|
|
|
utrie_fillBlock(trie->data+block, 0, UTRIE_DATA_BLOCK_LENGTH, value, initialValue, overwrite);
|
2001-12-06 01:23:47 +00:00
|
|
|
} else if(trie->data[-block]!=value && (block==0 || overwrite)) {
|
|
|
|
/* set the repeatBlock instead of the current block 0 or range block */
|
|
|
|
if(repeatBlock>=0) {
|
|
|
|
trie->index[start>>UTRIE_SHIFT]=-repeatBlock;
|
|
|
|
} else {
|
|
|
|
/* create and set and fill the repeatBlock */
|
|
|
|
repeatBlock=utrie_getDataBlock(trie, start);
|
|
|
|
if(repeatBlock<0) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* set the negative block number to indicate that it is a repeat block */
|
|
|
|
trie->index[start>>UTRIE_SHIFT]=-repeatBlock;
|
2001-12-16 21:15:09 +00:00
|
|
|
utrie_fillBlock(trie->data+repeatBlock, 0, UTRIE_DATA_BLOCK_LENGTH, value, initialValue, TRUE);
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
start+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(rest>0) {
|
|
|
|
/* set partial block at [last block boundary..limit[ */
|
|
|
|
block=utrie_getDataBlock(trie, start);
|
|
|
|
if(block<0) {
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
|
2001-12-16 21:15:09 +00:00
|
|
|
utrie_fillBlock(trie->data+block, 0, rest, value, initialValue, overwrite);
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int32_t
|
2009-04-15 01:57:02 +00:00
|
|
|
_findSameIndexBlock(const int32_t *idx, int32_t indexLength,
|
2001-12-06 01:23:47 +00:00
|
|
|
int32_t otherBlock) {
|
|
|
|
int32_t block, i;
|
|
|
|
|
|
|
|
for(block=UTRIE_BMP_INDEX_LENGTH; block<indexLength; block+=UTRIE_SURROGATE_BLOCK_COUNT) {
|
|
|
|
for(i=0; i<UTRIE_SURROGATE_BLOCK_COUNT; ++i) {
|
2009-04-15 01:57:02 +00:00
|
|
|
if(idx[block+i]!=idx[otherBlock+i]) {
|
2001-12-06 01:23:47 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(i==UTRIE_SURROGATE_BLOCK_COUNT) {
|
|
|
|
return block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return indexLength;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fold the normalization data for supplementary code points into
|
|
|
|
* a compact area on top of the BMP-part of the trie index,
|
|
|
|
* with the lead surrogates indexing this compact area.
|
|
|
|
*
|
|
|
|
* Duplicate the index values for lead surrogates:
|
|
|
|
* From inside the BMP area, where some may be overridden with folded values,
|
|
|
|
* to just after the BMP area, where they can be retrieved for
|
|
|
|
* code point lookups.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
utrie_fold(UNewTrie *trie, UNewTrieGetFoldedValue *getFoldedValue, UErrorCode *pErrorCode) {
|
|
|
|
int32_t leadIndexes[UTRIE_SURROGATE_BLOCK_COUNT];
|
2009-04-15 01:57:02 +00:00
|
|
|
int32_t *idx;
|
2001-12-06 01:23:47 +00:00
|
|
|
uint32_t value;
|
|
|
|
UChar32 c;
|
|
|
|
int32_t indexLength, block;
|
2008-10-22 19:50:07 +00:00
|
|
|
#ifdef UTRIE_DEBUG
|
|
|
|
int countLeadCUWithData=0;
|
|
|
|
#endif
|
2001-12-06 01:23:47 +00:00
|
|
|
|
2009-04-15 01:57:02 +00:00
|
|
|
idx=trie->index;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/* copy the lead surrogate indexes into a temporary array */
|
2009-04-15 01:57:02 +00:00
|
|
|
uprv_memcpy(leadIndexes, idx+(0xd800>>UTRIE_SHIFT), 4*UTRIE_SURROGATE_BLOCK_COUNT);
|
2001-12-06 01:23:47 +00:00
|
|
|
|
2003-07-24 23:23:19 +00:00
|
|
|
/*
|
|
|
|
* set all values for lead surrogate code *units* to leadUnitValue
|
2003-08-05 00:20:30 +00:00
|
|
|
* so that, by default, runtime lookups will find no data for associated
|
2003-07-24 23:23:19 +00:00
|
|
|
* supplementary code points, unless there is data for such code points
|
|
|
|
* which will result in a non-zero folding value below that is set for
|
|
|
|
* the respective lead units
|
|
|
|
*
|
2003-08-05 00:20:30 +00:00
|
|
|
* the above saved the indexes for surrogate code *points*
|
|
|
|
* fill the indexes with simplified code from utrie_setRange32()
|
2003-07-24 23:23:19 +00:00
|
|
|
*/
|
2003-08-05 00:20:30 +00:00
|
|
|
if(trie->leadUnitValue==trie->data[0]) {
|
|
|
|
block=0; /* leadUnitValue==initialValue, use all-initial-value block */
|
|
|
|
} else {
|
|
|
|
/* create and fill the repeatBlock */
|
|
|
|
block=utrie_allocDataBlock(trie);
|
|
|
|
if(block<0) {
|
|
|
|
/* data table overflow */
|
|
|
|
*pErrorCode=U_MEMORY_ALLOCATION_ERROR;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
utrie_fillBlock(trie->data+block, 0, UTRIE_DATA_BLOCK_LENGTH, trie->leadUnitValue, trie->data[0], TRUE);
|
|
|
|
block=-block; /* negative block number to indicate that it is a repeat block */
|
|
|
|
}
|
|
|
|
for(c=(0xd800>>UTRIE_SHIFT); c<(0xdc00>>UTRIE_SHIFT); ++c) {
|
|
|
|
trie->index[c]=block;
|
2003-07-24 23:23:19 +00:00
|
|
|
}
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/*
|
|
|
|
* Fold significant index values into the area just after the BMP indexes.
|
|
|
|
* In case the first lead surrogate has significant data,
|
|
|
|
* its index block must be used first (in which case the folding is a no-op).
|
|
|
|
* Later all folded index blocks are moved up one to insert the copied
|
|
|
|
* lead surrogate indexes.
|
|
|
|
*/
|
|
|
|
indexLength=UTRIE_BMP_INDEX_LENGTH;
|
|
|
|
|
|
|
|
/* search for any index (stage 1) entries for supplementary code points */
|
|
|
|
for(c=0x10000; c<0x110000;) {
|
2009-04-15 01:57:02 +00:00
|
|
|
if(idx[c>>UTRIE_SHIFT]!=0) {
|
2001-12-06 01:23:47 +00:00
|
|
|
/* there is data, treat the full block for a lead surrogate */
|
|
|
|
c&=~0x3ff;
|
|
|
|
|
|
|
|
#ifdef UTRIE_DEBUG
|
2008-10-22 19:50:07 +00:00
|
|
|
++countLeadCUWithData;
|
|
|
|
/* printf("supplementary data for lead surrogate U+%04lx\n", (long)(0xd7c0+(c>>10))); */
|
2001-12-06 01:23:47 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/* is there an identical index block? */
|
2009-04-15 01:57:02 +00:00
|
|
|
block=_findSameIndexBlock(idx, indexLength, c>>UTRIE_SHIFT);
|
2001-12-06 01:23:47 +00:00
|
|
|
|
2003-07-24 23:23:19 +00:00
|
|
|
/*
|
|
|
|
* get a folded value for [c..c+0x400[ and,
|
|
|
|
* if different from the value for the lead surrogate code point,
|
|
|
|
* set it for the lead surrogate code unit
|
|
|
|
*/
|
2001-12-06 01:23:47 +00:00
|
|
|
value=getFoldedValue(trie, c, block+UTRIE_SURROGATE_BLOCK_COUNT);
|
2003-07-24 23:23:19 +00:00
|
|
|
if(value!=utrie_get32(trie, U16_LEAD(c), NULL)) {
|
|
|
|
if(!utrie_set32(trie, U16_LEAD(c), value)) {
|
2001-12-06 01:23:47 +00:00
|
|
|
/* data table overflow */
|
|
|
|
*pErrorCode=U_MEMORY_ALLOCATION_ERROR;
|
|
|
|
return;
|
|
|
|
}
|
2003-08-04 22:25:18 +00:00
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/* if we did not find an identical index block... */
|
|
|
|
if(block==indexLength) {
|
|
|
|
/* move the actual index (stage 1) entries from the supplementary position to the new one */
|
2009-04-15 01:57:02 +00:00
|
|
|
uprv_memmove(idx+indexLength,
|
|
|
|
idx+(c>>UTRIE_SHIFT),
|
2001-12-06 01:23:47 +00:00
|
|
|
4*UTRIE_SURROGATE_BLOCK_COUNT);
|
|
|
|
indexLength+=UTRIE_SURROGATE_BLOCK_COUNT;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
c+=0x400;
|
|
|
|
} else {
|
|
|
|
c+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
}
|
|
|
|
}
|
2008-10-22 19:50:07 +00:00
|
|
|
#ifdef UTRIE_DEBUG
|
|
|
|
if(countLeadCUWithData>0) {
|
|
|
|
printf("supplementary data for %d lead surrogates\n", countLeadCUWithData);
|
|
|
|
}
|
|
|
|
#endif
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* index array overflow?
|
|
|
|
* This is to guarantee that a folding offset is of the form
|
|
|
|
* UTRIE_BMP_INDEX_LENGTH+n*UTRIE_SURROGATE_BLOCK_COUNT with n=0..1023.
|
|
|
|
* If the index is too large, then n>=1024 and more than 10 bits are necessary.
|
|
|
|
*
|
|
|
|
* In fact, it can only ever become n==1024 with completely unfoldable data and
|
|
|
|
* the additional block of duplicated values for lead surrogates.
|
|
|
|
*/
|
|
|
|
if(indexLength>=UTRIE_MAX_INDEX_LENGTH) {
|
|
|
|
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* make space for the lead surrogate index block and
|
|
|
|
* insert it between the BMP indexes and the folded ones
|
|
|
|
*/
|
2009-04-15 01:57:02 +00:00
|
|
|
uprv_memmove(idx+UTRIE_BMP_INDEX_LENGTH+UTRIE_SURROGATE_BLOCK_COUNT,
|
|
|
|
idx+UTRIE_BMP_INDEX_LENGTH,
|
2001-12-06 01:23:47 +00:00
|
|
|
4*(indexLength-UTRIE_BMP_INDEX_LENGTH));
|
2009-04-15 01:57:02 +00:00
|
|
|
uprv_memcpy(idx+UTRIE_BMP_INDEX_LENGTH,
|
2001-12-06 01:23:47 +00:00
|
|
|
leadIndexes,
|
|
|
|
4*UTRIE_SURROGATE_BLOCK_COUNT);
|
|
|
|
indexLength+=UTRIE_SURROGATE_BLOCK_COUNT;
|
|
|
|
|
|
|
|
#ifdef UTRIE_DEBUG
|
|
|
|
printf("trie index count: BMP %ld all Unicode %ld folded %ld\n",
|
|
|
|
UTRIE_BMP_INDEX_LENGTH, (long)UTRIE_MAX_INDEX_LENGTH, indexLength);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
trie->indexLength=indexLength;
|
2002-02-26 02:06:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set a value in the trie index map to indicate which data block
|
|
|
|
* is referenced and which one is not.
|
|
|
|
* utrie_compact() will remove data blocks that are not used at all.
|
|
|
|
* Set
|
|
|
|
* - 0 if it is used
|
|
|
|
* - -1 if it is not used
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
_findUnusedBlocks(UNewTrie *trie) {
|
|
|
|
int32_t i;
|
|
|
|
|
|
|
|
/* fill the entire map with "not used" */
|
|
|
|
uprv_memset(trie->map, 0xff, (UTRIE_MAX_BUILD_TIME_DATA_LENGTH>>UTRIE_SHIFT)*4);
|
|
|
|
|
|
|
|
/* mark each block that _is_ used with 0 */
|
|
|
|
for(i=0; i<trie->indexLength; ++i) {
|
|
|
|
trie->map[ABS(trie->index[i])>>UTRIE_SHIFT]=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* never move the all-initial-value block 0 */
|
|
|
|
trie->map[0]=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int32_t
|
|
|
|
_findSameDataBlock(const uint32_t *data, int32_t dataLength,
|
|
|
|
int32_t otherBlock, int32_t step) {
|
2004-12-31 13:43:57 +00:00
|
|
|
int32_t block;
|
2002-02-26 02:06:12 +00:00
|
|
|
|
|
|
|
/* ensure that we do not even partially get past dataLength */
|
|
|
|
dataLength-=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
|
2002-03-04 01:17:45 +00:00
|
|
|
for(block=0; block<=dataLength; block+=step) {
|
2004-12-31 13:43:57 +00:00
|
|
|
if(equal_uint32(data+block, data+otherBlock, UTRIE_DATA_BLOCK_LENGTH)) {
|
2002-02-26 02:06:12 +00:00
|
|
|
return block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return -1;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Compact a folded build-time trie.
|
|
|
|
*
|
|
|
|
* The compaction
|
2002-02-26 02:06:12 +00:00
|
|
|
* - removes blocks that are identical with earlier ones
|
|
|
|
* - overlaps adjacent blocks as much as possible (if overlap==TRUE)
|
|
|
|
* - moves blocks in steps of the data granularity
|
2004-12-31 13:43:57 +00:00
|
|
|
* - moves and overlaps blocks that overlap with multiple values in the overlap region
|
2001-12-06 01:23:47 +00:00
|
|
|
*
|
|
|
|
* It does not
|
|
|
|
* - try to move and overlap blocks that are not already adjacent
|
|
|
|
*/
|
|
|
|
static void
|
2002-02-26 02:06:12 +00:00
|
|
|
utrie_compact(UNewTrie *trie, UBool overlap, UErrorCode *pErrorCode) {
|
2004-12-31 13:43:57 +00:00
|
|
|
int32_t i, start, newStart, overlapStart;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* valid, uncompacted trie? */
|
|
|
|
if(trie==NULL) {
|
|
|
|
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if(trie->isCompacted) {
|
|
|
|
return; /* nothing left to do */
|
|
|
|
}
|
|
|
|
|
|
|
|
/* compaction */
|
|
|
|
|
2002-02-26 02:06:12 +00:00
|
|
|
/* initialize the index map with "block is used/unused" flags */
|
|
|
|
_findUnusedBlocks(trie);
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/* if Latin-1 is preallocated and linear, then do not compact Latin-1 data */
|
|
|
|
if(trie->isLatin1Linear && UTRIE_SHIFT<=8) {
|
|
|
|
overlapStart=UTRIE_DATA_BLOCK_LENGTH+256;
|
|
|
|
} else {
|
|
|
|
overlapStart=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
}
|
|
|
|
|
|
|
|
newStart=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
for(start=newStart; start<trie->dataLength;) {
|
|
|
|
/*
|
|
|
|
* start: index of first entry of current block
|
|
|
|
* newStart: index where the current block is to be moved
|
2004-12-31 13:43:57 +00:00
|
|
|
* (right after current end of already-compacted data)
|
2001-12-06 01:23:47 +00:00
|
|
|
*/
|
|
|
|
|
2002-02-26 02:06:12 +00:00
|
|
|
/* skip blocks that are not used */
|
|
|
|
if(trie->map[start>>UTRIE_SHIFT]<0) {
|
|
|
|
/* advance start to the next block */
|
|
|
|
start+=UTRIE_DATA_BLOCK_LENGTH;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
2004-12-31 13:43:57 +00:00
|
|
|
/* leave newStart with the previous block! */
|
2002-02-26 02:06:12 +00:00
|
|
|
continue;
|
|
|
|
}
|
2001-12-06 01:23:47 +00:00
|
|
|
|
2002-02-26 02:06:12 +00:00
|
|
|
/* search for an identical block */
|
|
|
|
if( start>=overlapStart &&
|
|
|
|
(i=_findSameDataBlock(trie->data, newStart, start,
|
|
|
|
overlap ? UTRIE_DATA_GRANULARITY : UTRIE_DATA_BLOCK_LENGTH))
|
|
|
|
>=0
|
|
|
|
) {
|
|
|
|
/* found an identical block, set the other block's index value for the current block */
|
|
|
|
trie->map[start>>UTRIE_SHIFT]=i;
|
|
|
|
|
|
|
|
/* advance start to the next block */
|
|
|
|
start+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
|
2004-12-31 13:43:57 +00:00
|
|
|
/* leave newStart with the previous block! */
|
2002-02-26 02:06:12 +00:00
|
|
|
continue;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* see if the beginning of this block can be overlapped with the end of the previous block */
|
2004-12-31 13:43:57 +00:00
|
|
|
if(overlap && start>=overlapStart) {
|
|
|
|
/* look for maximum overlap (modulo granularity) with the previous, adjacent block */
|
|
|
|
for(i=UTRIE_DATA_BLOCK_LENGTH-UTRIE_DATA_GRANULARITY;
|
|
|
|
i>0 && !equal_uint32(trie->data+(newStart-i), trie->data+start, i);
|
|
|
|
i-=UTRIE_DATA_GRANULARITY) {}
|
2001-12-06 01:23:47 +00:00
|
|
|
} else {
|
|
|
|
i=0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(i>0) {
|
|
|
|
/* some overlap */
|
|
|
|
trie->map[start>>UTRIE_SHIFT]=newStart-i;
|
|
|
|
|
|
|
|
/* move the non-overlapping indexes to their new positions */
|
|
|
|
start+=i;
|
|
|
|
for(i=UTRIE_DATA_BLOCK_LENGTH-i; i>0; --i) {
|
|
|
|
trie->data[newStart++]=trie->data[start++];
|
|
|
|
}
|
|
|
|
} else if(newStart<start) {
|
|
|
|
/* no overlap, just move the indexes to their new positions */
|
|
|
|
trie->map[start>>UTRIE_SHIFT]=newStart;
|
|
|
|
for(i=UTRIE_DATA_BLOCK_LENGTH; i>0; --i) {
|
|
|
|
trie->data[newStart++]=trie->data[start++];
|
|
|
|
}
|
|
|
|
} else /* no overlap && newStart==start */ {
|
|
|
|
trie->map[start>>UTRIE_SHIFT]=start;
|
|
|
|
newStart+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
start=newStart;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* now adjust the index (stage 1) table */
|
|
|
|
for(i=0; i<trie->indexLength; ++i) {
|
|
|
|
trie->index[i]=trie->map[ABS(trie->index[i])>>UTRIE_SHIFT];
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef UTRIE_DEBUG
|
|
|
|
/* we saved some space */
|
|
|
|
printf("compacting trie: count of 32-bit words %lu->%lu\n",
|
|
|
|
(long)trie->dataLength, (long)newStart);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
trie->dataLength=newStart;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* serialization ------------------------------------------------------------ */
|
|
|
|
|
2004-09-06 15:38:09 +00:00
|
|
|
/*
|
|
|
|
* Default function for the folding value:
|
|
|
|
* Just store the offset (16 bits) if there is any non-initial-value entry.
|
|
|
|
*
|
|
|
|
* The offset parameter is never 0.
|
|
|
|
* Returning the offset itself is safe for UTRIE_SHIFT>=5 because
|
|
|
|
* for UTRIE_SHIFT==5 the maximum index length is UTRIE_MAX_INDEX_LENGTH==0x8800
|
|
|
|
* which fits into 16-bit trie values;
|
|
|
|
* for higher UTRIE_SHIFT, UTRIE_MAX_INDEX_LENGTH decreases.
|
|
|
|
*
|
|
|
|
* Theoretically, it would be safer for all possible UTRIE_SHIFT including
|
|
|
|
* those of 4 and lower to return offset>>UTRIE_SURROGATE_BLOCK_BITS
|
|
|
|
* which would always result in a value of 0x40..0x43f
|
|
|
|
* (start/end 1k blocks of supplementary Unicode code points).
|
|
|
|
* However, this would be uglier, and would not work for some existing
|
|
|
|
* binary data file formats.
|
|
|
|
*
|
|
|
|
* Also, we do not plan to change UTRIE_SHIFT because it would change binary
|
|
|
|
* data file formats, and we would probably not make it smaller because of
|
|
|
|
* the then even larger BMP index length even for empty tries.
|
|
|
|
*/
|
|
|
|
static uint32_t U_CALLCONV
|
|
|
|
defaultGetFoldedValue(UNewTrie *trie, UChar32 start, int32_t offset) {
|
|
|
|
uint32_t value, initialValue;
|
|
|
|
UChar32 limit;
|
|
|
|
UBool inBlockZero;
|
|
|
|
|
|
|
|
initialValue=trie->data[0];
|
|
|
|
limit=start+0x400;
|
|
|
|
while(start<limit) {
|
|
|
|
value=utrie_get32(trie, start, &inBlockZero);
|
|
|
|
if(inBlockZero) {
|
|
|
|
start+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
} else if(value!=initialValue) {
|
|
|
|
return (uint32_t)offset;
|
|
|
|
} else {
|
|
|
|
++start;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
U_CAPI int32_t U_EXPORT2
|
2003-04-23 21:15:12 +00:00
|
|
|
utrie_serialize(UNewTrie *trie, void *dt, int32_t capacity,
|
2001-12-06 01:23:47 +00:00
|
|
|
UNewTrieGetFoldedValue *getFoldedValue,
|
|
|
|
UBool reduceTo16Bits,
|
|
|
|
UErrorCode *pErrorCode) {
|
|
|
|
UTrieHeader *header;
|
|
|
|
uint32_t *p;
|
|
|
|
uint16_t *dest16;
|
|
|
|
int32_t i, length;
|
2003-04-23 21:15:12 +00:00
|
|
|
uint8_t* data = NULL;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/* argument check */
|
|
|
|
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2004-09-06 15:38:09 +00:00
|
|
|
if(trie==NULL || capacity<0 || (capacity>0 && dt==NULL)) {
|
2001-12-06 01:23:47 +00:00
|
|
|
*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
|
|
|
|
return 0;
|
|
|
|
}
|
2004-09-06 15:38:09 +00:00
|
|
|
if(getFoldedValue==NULL) {
|
|
|
|
getFoldedValue=defaultGetFoldedValue;
|
|
|
|
}
|
|
|
|
|
2003-04-23 21:15:12 +00:00
|
|
|
data = (uint8_t*)dt;
|
2001-12-06 01:23:47 +00:00
|
|
|
/* fold and compact if necessary, also checks that indexLength is within limits */
|
|
|
|
if(!trie->isCompacted) {
|
2002-02-26 02:06:12 +00:00
|
|
|
/* compact once without overlap to improve folding */
|
|
|
|
utrie_compact(trie, FALSE, pErrorCode);
|
|
|
|
|
|
|
|
/* fold the supplementary part of the index array */
|
2001-12-06 01:23:47 +00:00
|
|
|
utrie_fold(trie, getFoldedValue, pErrorCode);
|
2002-02-26 02:06:12 +00:00
|
|
|
|
|
|
|
/* compact again with overlap for minimum data array length */
|
|
|
|
utrie_compact(trie, TRUE, pErrorCode);
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
trie->isCompacted=TRUE;
|
|
|
|
if(U_FAILURE(*pErrorCode)) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* is dataLength within limits? */
|
|
|
|
if( (reduceTo16Bits ? (trie->dataLength+trie->indexLength) : trie->dataLength) >= UTRIE_MAX_DATA_LENGTH) {
|
|
|
|
*pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
length=sizeof(UTrieHeader)+2*trie->indexLength;
|
|
|
|
if(reduceTo16Bits) {
|
|
|
|
length+=2*trie->dataLength;
|
|
|
|
} else {
|
|
|
|
length+=4*trie->dataLength;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(length>capacity) {
|
|
|
|
return length; /* preflighting */
|
|
|
|
}
|
|
|
|
|
2008-10-22 19:50:07 +00:00
|
|
|
#ifdef UTRIE_DEBUG
|
|
|
|
printf("**UTrieLengths(serialize)** index:%6ld data:%6ld serialized:%6ld\n",
|
|
|
|
(long)trie->indexLength, (long)trie->dataLength, (long)length);
|
|
|
|
#endif
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/* set the header fields */
|
|
|
|
header=(UTrieHeader *)data;
|
|
|
|
data+=sizeof(UTrieHeader);
|
|
|
|
|
|
|
|
header->signature=0x54726965; /* "Trie" */
|
|
|
|
header->options=UTRIE_SHIFT | (UTRIE_INDEX_SHIFT<<UTRIE_OPTIONS_INDEX_SHIFT);
|
|
|
|
|
|
|
|
if(!reduceTo16Bits) {
|
|
|
|
header->options|=UTRIE_OPTIONS_DATA_IS_32_BIT;
|
|
|
|
}
|
|
|
|
if(trie->isLatin1Linear) {
|
|
|
|
header->options|=UTRIE_OPTIONS_LATIN1_IS_LINEAR;
|
|
|
|
}
|
|
|
|
|
|
|
|
header->indexLength=trie->indexLength;
|
|
|
|
header->dataLength=trie->dataLength;
|
|
|
|
|
|
|
|
/* write the index (stage 1) array and the 16/32-bit data (stage 2) array */
|
|
|
|
if(reduceTo16Bits) {
|
|
|
|
/* write 16-bit index values shifted right by UTRIE_INDEX_SHIFT, after adding indexLength */
|
|
|
|
p=(uint32_t *)trie->index;
|
|
|
|
dest16=(uint16_t *)data;
|
|
|
|
for(i=trie->indexLength; i>0; --i) {
|
|
|
|
*dest16++=(uint16_t)((*p++ + trie->indexLength)>>UTRIE_INDEX_SHIFT);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* write 16-bit data values */
|
|
|
|
p=trie->data;
|
|
|
|
for(i=trie->dataLength; i>0; --i) {
|
|
|
|
*dest16++=(uint16_t)*p++;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* write 16-bit index values shifted right by UTRIE_INDEX_SHIFT */
|
|
|
|
p=(uint32_t *)trie->index;
|
|
|
|
dest16=(uint16_t *)data;
|
|
|
|
for(i=trie->indexLength; i>0; --i) {
|
|
|
|
*dest16++=(uint16_t)(*p++ >> UTRIE_INDEX_SHIFT);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* write 32-bit data values */
|
|
|
|
uprv_memcpy(dest16, trie->data, 4*trie->dataLength);
|
|
|
|
}
|
|
|
|
|
|
|
|
return length;
|
|
|
|
}
|
|
|
|
|
2004-09-06 15:38:09 +00:00
|
|
|
/* inverse to defaultGetFoldedValue() */
|
2005-04-28 21:53:38 +00:00
|
|
|
U_CAPI int32_t U_EXPORT2
|
|
|
|
utrie_defaultGetFoldingOffset(uint32_t data) {
|
2004-09-06 15:38:09 +00:00
|
|
|
return (int32_t)data;
|
|
|
|
}
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
U_CAPI int32_t U_EXPORT2
|
2003-04-23 21:15:12 +00:00
|
|
|
utrie_unserialize(UTrie *trie, const void *data, int32_t length, UErrorCode *pErrorCode) {
|
2005-02-23 00:54:19 +00:00
|
|
|
const UTrieHeader *header;
|
|
|
|
const uint16_t *p16;
|
2001-12-06 01:23:47 +00:00
|
|
|
uint32_t options;
|
|
|
|
|
|
|
|
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* enough data for a trie header? */
|
2011-05-26 20:05:39 +00:00
|
|
|
if(length<(int32_t)sizeof(UTrieHeader)) {
|
2001-12-06 01:23:47 +00:00
|
|
|
*pErrorCode=U_INVALID_FORMAT_ERROR;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* check the signature */
|
2005-02-23 00:54:19 +00:00
|
|
|
header=(const UTrieHeader *)data;
|
2001-12-06 01:23:47 +00:00
|
|
|
if(header->signature!=0x54726965) {
|
|
|
|
*pErrorCode=U_INVALID_FORMAT_ERROR;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* get the options and check the shift values */
|
|
|
|
options=header->options;
|
|
|
|
if( (options&UTRIE_OPTIONS_SHIFT_MASK)!=UTRIE_SHIFT ||
|
|
|
|
((options>>UTRIE_OPTIONS_INDEX_SHIFT)&UTRIE_OPTIONS_SHIFT_MASK)!=UTRIE_INDEX_SHIFT
|
|
|
|
) {
|
|
|
|
*pErrorCode=U_INVALID_FORMAT_ERROR;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
trie->isLatin1Linear= (UBool)((options&UTRIE_OPTIONS_LATIN1_IS_LINEAR)!=0);
|
|
|
|
|
|
|
|
/* get the length values */
|
|
|
|
trie->indexLength=header->indexLength;
|
|
|
|
trie->dataLength=header->dataLength;
|
|
|
|
|
2004-09-06 15:38:09 +00:00
|
|
|
length-=(int32_t)sizeof(UTrieHeader);
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/* enough data for the index? */
|
|
|
|
if(length<2*trie->indexLength) {
|
|
|
|
*pErrorCode=U_INVALID_FORMAT_ERROR;
|
|
|
|
return -1;
|
|
|
|
}
|
2005-02-23 00:54:19 +00:00
|
|
|
p16=(const uint16_t *)(header+1);
|
2001-12-06 01:23:47 +00:00
|
|
|
trie->index=p16;
|
|
|
|
p16+=trie->indexLength;
|
|
|
|
length-=2*trie->indexLength;
|
|
|
|
|
|
|
|
/* get the data */
|
|
|
|
if(options&UTRIE_OPTIONS_DATA_IS_32_BIT) {
|
|
|
|
if(length<4*trie->dataLength) {
|
|
|
|
*pErrorCode=U_INVALID_FORMAT_ERROR;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
trie->data32=(const uint32_t *)p16;
|
2001-12-16 21:15:09 +00:00
|
|
|
trie->initialValue=trie->data32[0];
|
2004-09-06 15:38:09 +00:00
|
|
|
length=(int32_t)sizeof(UTrieHeader)+2*trie->indexLength+4*trie->dataLength;
|
2001-12-06 01:23:47 +00:00
|
|
|
} else {
|
|
|
|
if(length<2*trie->dataLength) {
|
|
|
|
*pErrorCode=U_INVALID_FORMAT_ERROR;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* the "data16" data is used via the index pointer */
|
|
|
|
trie->data32=NULL;
|
2001-12-16 21:15:09 +00:00
|
|
|
trie->initialValue=trie->index[trie->indexLength];
|
2004-09-06 15:38:09 +00:00
|
|
|
length=(int32_t)sizeof(UTrieHeader)+2*trie->indexLength+2*trie->dataLength;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
2004-09-06 15:38:09 +00:00
|
|
|
|
2005-04-28 21:53:38 +00:00
|
|
|
trie->getFoldingOffset=utrie_defaultGetFoldingOffset;
|
2004-09-06 15:38:09 +00:00
|
|
|
|
|
|
|
return length;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
|
2005-02-23 00:54:19 +00:00
|
|
|
U_CAPI int32_t U_EXPORT2
|
|
|
|
utrie_unserializeDummy(UTrie *trie,
|
|
|
|
void *data, int32_t length,
|
|
|
|
uint32_t initialValue, uint32_t leadUnitValue,
|
|
|
|
UBool make16BitTrie,
|
|
|
|
UErrorCode *pErrorCode) {
|
|
|
|
uint16_t *p16;
|
|
|
|
int32_t actualLength, latin1Length, i, limit;
|
|
|
|
uint16_t block;
|
|
|
|
|
|
|
|
if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* calculate the actual size of the dummy trie data */
|
|
|
|
|
|
|
|
/* max(Latin-1, block 0) */
|
2012-05-03 05:50:26 +00:00
|
|
|
latin1Length= 256; /*UTRIE_SHIFT<=8 ? 256 : UTRIE_DATA_BLOCK_LENGTH;*/
|
2005-02-23 00:54:19 +00:00
|
|
|
|
|
|
|
trie->indexLength=UTRIE_BMP_INDEX_LENGTH+UTRIE_SURROGATE_BLOCK_COUNT;
|
|
|
|
trie->dataLength=latin1Length;
|
|
|
|
if(leadUnitValue!=initialValue) {
|
|
|
|
trie->dataLength+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
}
|
|
|
|
|
|
|
|
actualLength=trie->indexLength*2;
|
|
|
|
if(make16BitTrie) {
|
|
|
|
actualLength+=trie->dataLength*2;
|
|
|
|
} else {
|
|
|
|
actualLength+=trie->dataLength*4;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* enough space for the dummy trie? */
|
|
|
|
if(length<actualLength) {
|
|
|
|
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
|
|
|
|
return actualLength;
|
|
|
|
}
|
|
|
|
|
|
|
|
trie->isLatin1Linear=TRUE;
|
|
|
|
trie->initialValue=initialValue;
|
|
|
|
|
|
|
|
/* fill the index and data arrays */
|
|
|
|
p16=(uint16_t *)data;
|
|
|
|
trie->index=p16;
|
|
|
|
|
|
|
|
if(make16BitTrie) {
|
|
|
|
/* indexes to block 0 */
|
|
|
|
block=(uint16_t)(trie->indexLength>>UTRIE_INDEX_SHIFT);
|
|
|
|
limit=trie->indexLength;
|
|
|
|
for(i=0; i<limit; ++i) {
|
|
|
|
p16[i]=block;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(leadUnitValue!=initialValue) {
|
|
|
|
/* indexes for lead surrogate code units to the block after Latin-1 */
|
|
|
|
block+=(uint16_t)(latin1Length>>UTRIE_INDEX_SHIFT);
|
|
|
|
i=0xd800>>UTRIE_SHIFT;
|
|
|
|
limit=0xdc00>>UTRIE_SHIFT;
|
|
|
|
for(; i<limit; ++i) {
|
|
|
|
p16[i]=block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
trie->data32=NULL;
|
|
|
|
|
|
|
|
/* Latin-1 data */
|
|
|
|
p16+=trie->indexLength;
|
|
|
|
for(i=0; i<latin1Length; ++i) {
|
|
|
|
p16[i]=(uint16_t)initialValue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* data for lead surrogate code units */
|
|
|
|
if(leadUnitValue!=initialValue) {
|
|
|
|
limit=latin1Length+UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
for(/* i=latin1Length */; i<limit; ++i) {
|
|
|
|
p16[i]=(uint16_t)leadUnitValue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
uint32_t *p32;
|
|
|
|
|
|
|
|
/* indexes to block 0 */
|
|
|
|
uprv_memset(p16, 0, trie->indexLength*2);
|
|
|
|
|
|
|
|
if(leadUnitValue!=initialValue) {
|
|
|
|
/* indexes for lead surrogate code units to the block after Latin-1 */
|
|
|
|
block=(uint16_t)(latin1Length>>UTRIE_INDEX_SHIFT);
|
|
|
|
i=0xd800>>UTRIE_SHIFT;
|
|
|
|
limit=0xdc00>>UTRIE_SHIFT;
|
|
|
|
for(; i<limit; ++i) {
|
|
|
|
p16[i]=block;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
trie->data32=p32=(uint32_t *)(p16+trie->indexLength);
|
|
|
|
|
|
|
|
/* Latin-1 data */
|
|
|
|
for(i=0; i<latin1Length; ++i) {
|
|
|
|
p32[i]=initialValue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* data for lead surrogate code units */
|
|
|
|
if(leadUnitValue!=initialValue) {
|
|
|
|
limit=latin1Length+UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
for(/* i=latin1Length */; i<limit; ++i) {
|
|
|
|
p32[i]=leadUnitValue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-04-28 21:53:38 +00:00
|
|
|
trie->getFoldingOffset=utrie_defaultGetFoldingOffset;
|
2005-02-23 00:54:19 +00:00
|
|
|
|
|
|
|
return actualLength;
|
|
|
|
}
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/* enumeration -------------------------------------------------------------- */
|
|
|
|
|
|
|
|
/* default UTrieEnumValue() returns the input value itself */
|
|
|
|
static uint32_t U_CALLCONV
|
2011-05-26 20:05:39 +00:00
|
|
|
enumSameValue(const void * /*context*/, uint32_t value) {
|
2001-12-06 01:23:47 +00:00
|
|
|
return value;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Enumerate all ranges of code points with the same relevant values.
|
|
|
|
* The values are transformed from the raw trie entries by the enumValue function.
|
|
|
|
*/
|
|
|
|
U_CAPI void U_EXPORT2
|
2004-09-06 15:38:09 +00:00
|
|
|
utrie_enum(const UTrie *trie,
|
2001-12-06 01:23:47 +00:00
|
|
|
UTrieEnumValue *enumValue, UTrieEnumRange *enumRange, const void *context) {
|
|
|
|
const uint32_t *data32;
|
2009-04-15 01:57:02 +00:00
|
|
|
const uint16_t *idx;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
2001-12-16 21:15:09 +00:00
|
|
|
uint32_t value, prevValue, initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
UChar32 c, prev;
|
2008-09-01 11:45:56 +00:00
|
|
|
int32_t l, i, j, block, prevBlock, nullBlock, offset;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/* check arguments */
|
|
|
|
if(trie==NULL || trie->index==NULL || enumRange==NULL) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if(enumValue==NULL) {
|
|
|
|
enumValue=enumSameValue;
|
|
|
|
}
|
|
|
|
|
2009-04-15 01:57:02 +00:00
|
|
|
idx=trie->index;
|
2001-12-06 01:23:47 +00:00
|
|
|
data32=trie->data32;
|
|
|
|
|
2001-12-16 21:15:09 +00:00
|
|
|
/* get the enumeration value that corresponds to an initial-value trie data entry */
|
|
|
|
initialValue=enumValue(context, trie->initialValue);
|
2001-12-06 01:23:47 +00:00
|
|
|
|
2008-09-01 11:45:56 +00:00
|
|
|
if(data32==NULL) {
|
|
|
|
nullBlock=trie->indexLength;
|
|
|
|
} else {
|
|
|
|
nullBlock=0;
|
|
|
|
}
|
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/* set variables for previous range */
|
2008-09-01 11:45:56 +00:00
|
|
|
prevBlock=nullBlock;
|
2001-12-06 01:23:47 +00:00
|
|
|
prev=0;
|
2001-12-16 21:15:09 +00:00
|
|
|
prevValue=initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
|
|
|
|
/* enumerate BMP - the main loop enumerates data blocks */
|
|
|
|
for(i=0, c=0; c<=0xffff; ++i) {
|
|
|
|
if(c==0xd800) {
|
|
|
|
/* skip lead surrogate code _units_, go to lead surr. code _points_ */
|
|
|
|
i=UTRIE_BMP_INDEX_LENGTH;
|
|
|
|
} else if(c==0xdc00) {
|
|
|
|
/* go back to regular BMP code points */
|
|
|
|
i=c>>UTRIE_SHIFT;
|
|
|
|
}
|
|
|
|
|
2009-04-15 01:57:02 +00:00
|
|
|
block=idx[i]<<UTRIE_INDEX_SHIFT;
|
2001-12-06 01:23:47 +00:00
|
|
|
if(block==prevBlock) {
|
|
|
|
/* the block is the same as the previous one, and filled with value */
|
|
|
|
c+=UTRIE_DATA_BLOCK_LENGTH;
|
2008-09-01 11:45:56 +00:00
|
|
|
} else if(block==nullBlock) {
|
2001-12-16 21:15:09 +00:00
|
|
|
/* this is the all-initial-value block */
|
|
|
|
if(prevValue!=initialValue) {
|
2001-12-06 01:23:47 +00:00
|
|
|
if(prev<c) {
|
2002-01-12 00:10:01 +00:00
|
|
|
if(!enumRange(context, prev, c, prevValue)) {
|
|
|
|
return;
|
|
|
|
}
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
2008-09-01 11:45:56 +00:00
|
|
|
prevBlock=nullBlock;
|
2001-12-06 01:23:47 +00:00
|
|
|
prev=c;
|
2001-12-16 21:15:09 +00:00
|
|
|
prevValue=initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
c+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
} else {
|
|
|
|
prevBlock=block;
|
|
|
|
for(j=0; j<UTRIE_DATA_BLOCK_LENGTH; ++j) {
|
2009-04-15 01:57:02 +00:00
|
|
|
value=enumValue(context, data32!=NULL ? data32[block+j] : idx[block+j]);
|
2001-12-06 01:23:47 +00:00
|
|
|
if(value!=prevValue) {
|
|
|
|
if(prev<c) {
|
2002-01-12 00:10:01 +00:00
|
|
|
if(!enumRange(context, prev, c, prevValue)) {
|
|
|
|
return;
|
|
|
|
}
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
if(j>0) {
|
2008-09-01 11:45:56 +00:00
|
|
|
/* the block is not filled with all the same value */
|
2001-12-06 01:23:47 +00:00
|
|
|
prevBlock=-1;
|
|
|
|
}
|
|
|
|
prev=c;
|
|
|
|
prevValue=value;
|
|
|
|
}
|
|
|
|
++c;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* enumerate supplementary code points */
|
|
|
|
for(l=0xd800; l<0xdc00;) {
|
|
|
|
/* lead surrogate access */
|
2009-04-15 01:57:02 +00:00
|
|
|
offset=idx[l>>UTRIE_SHIFT]<<UTRIE_INDEX_SHIFT;
|
2008-09-01 11:45:56 +00:00
|
|
|
if(offset==nullBlock) {
|
2003-08-04 22:25:18 +00:00
|
|
|
/* no entries for a whole block of lead surrogates */
|
|
|
|
if(prevValue!=initialValue) {
|
|
|
|
if(prev<c) {
|
|
|
|
if(!enumRange(context, prev, c, prevValue)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2008-09-01 11:45:56 +00:00
|
|
|
prevBlock=nullBlock;
|
2003-08-04 22:25:18 +00:00
|
|
|
prev=c;
|
|
|
|
prevValue=initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
2003-08-04 22:25:18 +00:00
|
|
|
|
|
|
|
l+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
c+=UTRIE_DATA_BLOCK_LENGTH<<10;
|
|
|
|
continue;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
|
2009-04-15 01:57:02 +00:00
|
|
|
value= data32!=NULL ? data32[offset+(l&UTRIE_MASK)] : idx[offset+(l&UTRIE_MASK)];
|
2003-08-04 22:25:18 +00:00
|
|
|
|
2001-12-06 01:23:47 +00:00
|
|
|
/* enumerate trail surrogates for this lead surrogate */
|
|
|
|
offset=trie->getFoldingOffset(value);
|
|
|
|
if(offset<=0) {
|
|
|
|
/* no data for this lead surrogate */
|
2001-12-16 21:15:09 +00:00
|
|
|
if(prevValue!=initialValue) {
|
2001-12-06 01:23:47 +00:00
|
|
|
if(prev<c) {
|
2002-01-12 00:10:01 +00:00
|
|
|
if(!enumRange(context, prev, c, prevValue)) {
|
|
|
|
return;
|
|
|
|
}
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
2008-09-01 11:45:56 +00:00
|
|
|
prevBlock=nullBlock;
|
2001-12-06 01:23:47 +00:00
|
|
|
prev=c;
|
2001-12-16 21:15:09 +00:00
|
|
|
prevValue=initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* nothing else to do for the supplementary code points for this lead surrogate */
|
|
|
|
c+=0x400;
|
|
|
|
} else {
|
|
|
|
/* enumerate code points for this lead surrogate */
|
|
|
|
i=offset;
|
|
|
|
offset+=UTRIE_SURROGATE_BLOCK_COUNT;
|
|
|
|
do {
|
|
|
|
/* copy of most of the body of the BMP loop */
|
2009-04-15 01:57:02 +00:00
|
|
|
block=idx[i]<<UTRIE_INDEX_SHIFT;
|
2001-12-06 01:23:47 +00:00
|
|
|
if(block==prevBlock) {
|
|
|
|
/* the block is the same as the previous one, and filled with value */
|
|
|
|
c+=UTRIE_DATA_BLOCK_LENGTH;
|
2008-09-01 11:45:56 +00:00
|
|
|
} else if(block==nullBlock) {
|
2001-12-16 21:15:09 +00:00
|
|
|
/* this is the all-initial-value block */
|
|
|
|
if(prevValue!=initialValue) {
|
2001-12-06 01:23:47 +00:00
|
|
|
if(prev<c) {
|
2002-01-12 00:10:01 +00:00
|
|
|
if(!enumRange(context, prev, c, prevValue)) {
|
|
|
|
return;
|
|
|
|
}
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
2008-09-01 11:45:56 +00:00
|
|
|
prevBlock=nullBlock;
|
2001-12-06 01:23:47 +00:00
|
|
|
prev=c;
|
2001-12-16 21:15:09 +00:00
|
|
|
prevValue=initialValue;
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
c+=UTRIE_DATA_BLOCK_LENGTH;
|
|
|
|
} else {
|
|
|
|
prevBlock=block;
|
|
|
|
for(j=0; j<UTRIE_DATA_BLOCK_LENGTH; ++j) {
|
2009-04-15 01:57:02 +00:00
|
|
|
value=enumValue(context, data32!=NULL ? data32[block+j] : idx[block+j]);
|
2001-12-06 01:23:47 +00:00
|
|
|
if(value!=prevValue) {
|
|
|
|
if(prev<c) {
|
2002-01-12 00:10:01 +00:00
|
|
|
if(!enumRange(context, prev, c, prevValue)) {
|
|
|
|
return;
|
|
|
|
}
|
2001-12-06 01:23:47 +00:00
|
|
|
}
|
|
|
|
if(j>0) {
|
2008-09-01 11:45:56 +00:00
|
|
|
/* the block is not filled with all the same value */
|
2001-12-06 01:23:47 +00:00
|
|
|
prevBlock=-1;
|
|
|
|
}
|
|
|
|
prev=c;
|
|
|
|
prevValue=value;
|
|
|
|
}
|
|
|
|
++c;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} while(++i<offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
++l;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* deliver last range */
|
|
|
|
enumRange(context, prev, c, prevValue);
|
|
|
|
}
|