mirror of
https://github.com/google/brotli.git
synced 2024-11-29 14:41:05 +00:00
35e69fc7cf
* New feature: "Large Window Brotli" By setting special encoder/decoder flag it is now possible to extend LZ-window up to 30 bits; though produced stream will not be RFC7932 compliant. Added new dictionary generator - "DSH". It combines speed of "Sieve" and quality of "DM". Plus utilities to prepare train corpora (remove unique strings). Improved compression ratio: now two sub-blocks could be stitched: the last copy command could be extended to span the next sub-block. Fixed compression ineffectiveness caused by floating numbers rounding and wrong cost heuristic. Other C changes: - combined / moved `context.h` to `common` - moved transforms to `common` - unified some aspects of code formatting - added an abstraction for encoder (static) dictionary - moved default allocator/deallocator functions to `common` brotli CLI: - window size is auto-adjusted if not specified explicitly Java: - added "eager" decoding both to JNI wrapper and pure decoder - huge speed-up of `DictionaryData` initialization * Add dictionaryless compressed dictionary * Fix `sources.lst` * Fix `sources.lst` and add a note that `libtool` is also required. * Update setup.py * Fix `EagerStreamTest` * Fix BUILD file * Add missing `libdivsufsort` dependency * Fix "unused parameter" warning.
86 lines
2.2 KiB
C
86 lines
2.2 KiB
C
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
|
|
Distributed under MIT license.
|
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
*/
|
|
|
|
/* Heuristics for deciding about the UTF8-ness of strings. */
|
|
|
|
#include "./utf8_util.h"
|
|
|
|
#include <brotli/types.h>
|
|
|
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
static size_t BrotliParseAsUTF8(
|
|
int* symbol, const uint8_t* input, size_t size) {
|
|
/* ASCII */
|
|
if ((input[0] & 0x80) == 0) {
|
|
*symbol = input[0];
|
|
if (*symbol > 0) {
|
|
return 1;
|
|
}
|
|
}
|
|
/* 2-byte UTF8 */
|
|
if (size > 1u &&
|
|
(input[0] & 0xE0) == 0xC0 &&
|
|
(input[1] & 0xC0) == 0x80) {
|
|
*symbol = (((input[0] & 0x1F) << 6) |
|
|
(input[1] & 0x3F));
|
|
if (*symbol > 0x7F) {
|
|
return 2;
|
|
}
|
|
}
|
|
/* 3-byte UFT8 */
|
|
if (size > 2u &&
|
|
(input[0] & 0xF0) == 0xE0 &&
|
|
(input[1] & 0xC0) == 0x80 &&
|
|
(input[2] & 0xC0) == 0x80) {
|
|
*symbol = (((input[0] & 0x0F) << 12) |
|
|
((input[1] & 0x3F) << 6) |
|
|
(input[2] & 0x3F));
|
|
if (*symbol > 0x7FF) {
|
|
return 3;
|
|
}
|
|
}
|
|
/* 4-byte UFT8 */
|
|
if (size > 3u &&
|
|
(input[0] & 0xF8) == 0xF0 &&
|
|
(input[1] & 0xC0) == 0x80 &&
|
|
(input[2] & 0xC0) == 0x80 &&
|
|
(input[3] & 0xC0) == 0x80) {
|
|
*symbol = (((input[0] & 0x07) << 18) |
|
|
((input[1] & 0x3F) << 12) |
|
|
((input[2] & 0x3F) << 6) |
|
|
(input[3] & 0x3F));
|
|
if (*symbol > 0xFFFF && *symbol <= 0x10FFFF) {
|
|
return 4;
|
|
}
|
|
}
|
|
/* Not UTF8, emit a special symbol above the UTF8-code space */
|
|
*symbol = 0x110000 | input[0];
|
|
return 1;
|
|
}
|
|
|
|
/* Returns 1 if at least min_fraction of the data is UTF8-encoded.*/
|
|
BROTLI_BOOL BrotliIsMostlyUTF8(
|
|
const uint8_t* data, const size_t pos, const size_t mask,
|
|
const size_t length, const double min_fraction) {
|
|
size_t size_utf8 = 0;
|
|
size_t i = 0;
|
|
while (i < length) {
|
|
int symbol;
|
|
size_t bytes_read =
|
|
BrotliParseAsUTF8(&symbol, &data[(pos + i) & mask], length - i);
|
|
i += bytes_read;
|
|
if (symbol < 0x110000) size_utf8 += bytes_read;
|
|
}
|
|
return TO_BROTLI_BOOL(size_utf8 > min_fraction * (double)length);
|
|
}
|
|
|
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
} /* extern "C" */
|
|
#endif
|