254 lines
10 KiB
C++
254 lines
10 KiB
C++
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*
|
|
*******************************************************************************
|
|
* Copyright (C) 2013-2015, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
*******************************************************************************
|
|
* collationdatareader.h
|
|
*
|
|
* created on: 2013feb07
|
|
* created by: Markus W. Scherer
|
|
*/
|
|
|
|
#ifndef __COLLATIONDATAREADER_H__
|
|
#define __COLLATIONDATAREADER_H__
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_COLLATION
|
|
|
|
#include "unicode/udata.h"
|
|
|
|
struct UDataMemory;
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
struct CollationTailoring;
|
|
|
|
/**
|
|
* Collation binary data reader.
|
|
*/
|
|
struct U_I18N_API CollationDataReader /* all static */ {
|
|
// The following constants are also copied into source/common/ucol_swp.cpp.
|
|
// Keep them in sync!
|
|
enum {
|
|
/**
|
|
* Number of int32_t indexes.
|
|
*
|
|
* Can be 2 if there are only options.
|
|
* Can be 7 or 8 if there are only options and a script reordering.
|
|
* The loader treats any index>=indexes[IX_INDEXES_LENGTH] as 0.
|
|
*/
|
|
IX_INDEXES_LENGTH, // 0
|
|
/**
|
|
* Bits 31..24: numericPrimary, for numeric collation
|
|
* 23..16: fast Latin format version (0 = no fast Latin table)
|
|
* 15.. 0: options bit set
|
|
*/
|
|
IX_OPTIONS,
|
|
IX_RESERVED2,
|
|
IX_RESERVED3,
|
|
|
|
/** Array offset to Jamo CE32s in ce32s[], or <0 if none. */
|
|
IX_JAMO_CE32S_START, // 4
|
|
|
|
// Byte offsets from the start of the data, after the generic header.
|
|
// The indexes[] are at byte offset 0, other data follows.
|
|
// Each data item is aligned properly.
|
|
// The data items should be in descending order of unit size,
|
|
// to minimize the need for padding.
|
|
// Each item's byte length is given by the difference between its offset and
|
|
// the next index/offset value.
|
|
/** Byte offset to int32_t reorderCodes[]. */
|
|
IX_REORDER_CODES_OFFSET,
|
|
/**
|
|
* Byte offset to uint8_t reorderTable[].
|
|
* Empty table if <256 bytes (padding only).
|
|
* Otherwise 256 bytes or more (with padding).
|
|
*/
|
|
IX_REORDER_TABLE_OFFSET,
|
|
/** Byte offset to the collation trie. Its length is a multiple of 8 bytes. */
|
|
IX_TRIE_OFFSET,
|
|
|
|
IX_RESERVED8_OFFSET, // 8
|
|
/** Byte offset to int64_t ces[]. */
|
|
IX_CES_OFFSET,
|
|
IX_RESERVED10_OFFSET,
|
|
/** Byte offset to uint32_t ce32s[]. */
|
|
IX_CE32S_OFFSET,
|
|
|
|
/** Byte offset to uint32_t rootElements[]. */
|
|
IX_ROOT_ELEMENTS_OFFSET, // 12
|
|
/** Byte offset to UChar *contexts[]. */
|
|
IX_CONTEXTS_OFFSET,
|
|
/** Byte offset to uint16_t [] with serialized unsafeBackwardSet. */
|
|
IX_UNSAFE_BWD_OFFSET,
|
|
/** Byte offset to uint16_t fastLatinTable[]. */
|
|
IX_FAST_LATIN_TABLE_OFFSET,
|
|
|
|
/** Byte offset to uint16_t scripts[]. */
|
|
IX_SCRIPTS_OFFSET, // 16
|
|
/**
|
|
* Byte offset to UBool compressibleBytes[].
|
|
* Empty table if <256 bytes (padding only).
|
|
* Otherwise 256 bytes or more (with padding).
|
|
*/
|
|
IX_COMPRESSIBLE_BYTES_OFFSET,
|
|
IX_RESERVED18_OFFSET,
|
|
IX_TOTAL_SIZE
|
|
};
|
|
|
|
static void read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
|
|
CollationTailoring &tailoring, UErrorCode &errorCode);
|
|
|
|
static UBool U_CALLCONV
|
|
isAcceptable(void *context, const char *type, const char *name, const UDataInfo *pInfo);
|
|
|
|
private:
|
|
CollationDataReader(); // no constructor
|
|
};
|
|
|
|
/*
|
|
* Format of collation data (ucadata.icu, binary data in coll/ *.res files).
|
|
* Format version 5.
|
|
*
|
|
* The root collation data is stored in the ucadata.icu file.
|
|
* Tailorings are stored inside .res resource bundle files, with a complete file header.
|
|
*
|
|
* Collation data begins with a standard ICU data file header
|
|
* (DataHeader, see ucmndata.h and unicode/udata.h).
|
|
* The UDataInfo.dataVersion field contains the UCA and other version numbers,
|
|
* see the comments for CollationTailoring.version.
|
|
*
|
|
* After the header, the file contains the following parts.
|
|
* Constants are defined as enum values of the CollationDataReader class.
|
|
* See also the Collation class.
|
|
*
|
|
* int32_t indexes[indexesLength];
|
|
* The indexes array has variable length.
|
|
* Some tailorings only need the length and the options,
|
|
* others only add reorderCodes and the reorderTable,
|
|
* some need to store mappings.
|
|
* Only as many indexes are stored as needed to read all of the data.
|
|
*
|
|
* Index 0: indexesLength
|
|
* Index 1: numericPrimary, CollationFastLatin::VERSION, and options: see IX_OPTIONS
|
|
* Index 2..3: Unused/reserved/0.
|
|
* Index 4: Index into the ce32s array where the CE32s of the conjoining Jamo
|
|
* are stored in a short, contiguous part of the ce32s array.
|
|
*
|
|
* Indexes 5..19 are byte offsets in ascending order.
|
|
* Each byte offset marks the start of the next part in the data file,
|
|
* and the end of the previous one.
|
|
* When two consecutive byte offsets are the same (or too short),
|
|
* then the corresponding part is empty.
|
|
* Byte offsets are offsets from after the header,
|
|
* that is, from the beginning of the indexes[].
|
|
* Each part starts at an offset with proper alignment for its data.
|
|
* If necessary, the previous part may include padding bytes to achieve this alignment.
|
|
* The last byte offset that is stored in the indexes indicates the total size of the data
|
|
* (starting with the indexes).
|
|
*
|
|
* int32_t reorderCodes[]; -- empty in root
|
|
* The list of script and reordering codes.
|
|
*
|
|
* Beginning with format version 5, this array may optionally
|
|
* have trailing entries with a full list of reorder ranges
|
|
* as described for CollationSettings::reorderRanges.
|
|
*
|
|
* Script or reorder codes are first and do not exceed 16-bit values.
|
|
* Range limits are stored in the upper 16 bits, and are never 0.
|
|
* Split this array into reorder codes and ranges at the first entry
|
|
* with non-zero upper 16 bits.
|
|
*
|
|
* If the ranges are missing but needed for split-reordered primary lead bytes,
|
|
* then they are regenerated at load time.
|
|
*
|
|
* uint8_t reorderTable[256]; -- empty in root; can be longer to include padding bytes
|
|
* Primary-weight lead byte permutation table.
|
|
* Normally present when the reorderCodes are, but can be built at load time.
|
|
*
|
|
* Beginning with format version 5, a 0 entry at a non-zero index
|
|
* (which is otherwise an illegal value)
|
|
* means that the primary lead byte is "split"
|
|
* (there are different offsets for primaries that share that lead byte)
|
|
* and the reordering offset must be determined via the reorder ranges
|
|
* that are either stored as part of the reorderCodes array
|
|
* or regenerated at load time.
|
|
*
|
|
* UTrie2 trie; -- see utrie2_impl.h and utrie2.h
|
|
* The trie holds the main collation data. Each code point is mapped to a 32-bit value.
|
|
* It encodes a simple collation element (CE) in compact form, unless bits 7..6 are both set,
|
|
* in which case it is a special CE32 and contains a 4-bit tag and further data.
|
|
* See the Collation class for details.
|
|
*
|
|
* The trie has a value for each lead surrogate code unit with some bits encoding
|
|
* collective properties of the 1024 supplementary characters whose UTF-16 form starts with
|
|
* the lead surrogate. See Collation::LEAD_SURROGATE_TAG..
|
|
*
|
|
* int64_t ces[];
|
|
* 64-bit CEs and expansions that cannot be stored in a more compact form.
|
|
*
|
|
* uint32_t ce32s[];
|
|
* CE32s for expansions in compact form, and for characters whose trie values
|
|
* contain special data.
|
|
*
|
|
* uint32_t rootElements[]; -- empty in all tailorings
|
|
* Compact storage for all of the CEs that occur in the root collation.
|
|
* See the CollationRootElements class.
|
|
*
|
|
* UChar *contexts[];
|
|
* Serialized UCharsTrie structures with prefix (pre-context) and contraction mappings.
|
|
*
|
|
* uint16_t unsafeBackwardSet[]; -- see UnicodeSet::serialize()
|
|
* Serialized form of characters that are unsafe when iterating backwards,
|
|
* and at the end of an identical string prefix.
|
|
* Back up to a safe character.
|
|
* Lead surrogates are "unsafe" when any of their corresponding supplementary
|
|
* code points are unsafe.
|
|
* Does not include [:^lccc=0:][:^tccc=0:].
|
|
* For each tailoring, the root unsafeBackwardSet is subtracted.
|
|
* (As a result, in many tailorings no set needs to be stored.)
|
|
*
|
|
* uint16_t fastLatinTable[];
|
|
* Optional optimization for Latin text.
|
|
* See the CollationFastLatin class.
|
|
*
|
|
* uint16_t scripts[]; -- empty in all tailorings
|
|
* Format version 5:
|
|
* uint16_t numScripts;
|
|
* uint16_t scriptsIndex[numScripts+16];
|
|
* uint16_t scriptStarts[];
|
|
* See CollationData::numScripts etc.
|
|
*
|
|
* Format version 4:
|
|
* Table of the reordering groups with their first and last lead bytes,
|
|
* and their script and reordering codes.
|
|
* See CollationData::scripts.
|
|
*
|
|
* UBool compressibleBytes[]; -- empty in all tailorings
|
|
* Flag for getSortKey(), indicating primary weight lead bytes that are compressible.
|
|
*
|
|
* -----------------
|
|
* Changes for formatVersion 5 (ICU 55)
|
|
*
|
|
* Reordering moves single scripts, not groups of scripts.
|
|
* Reorder ranges are optionally appended to the reorderCodes,
|
|
* and a 0 entry in the reorderTable indicates a split lead byte.
|
|
* The scripts data has a new format.
|
|
*
|
|
* The rootElements may contain secondary and tertiary weights below common=05.
|
|
* (Used for small Hiragana letters.)
|
|
* Where is occurs, there is also an explicit unit with common secondary & tertiary weights.
|
|
* There are no other data structure changes, but builder code needs to be able to handle such data.
|
|
*
|
|
* The collation element for the merge separator code point U+FFFE
|
|
* does not necessarily have special, unique secondary/tertiary weights any more.
|
|
*/
|
|
|
|
U_NAMESPACE_END
|
|
|
|
#endif // !UCONFIG_NO_COLLATION
|
|
#endif // __COLLATIONDATAREADER_H__
|