Merge branch 'dev' of github.com:senhuang42/zstd into new_path_for_cdict
This commit is contained in:
Sen Huang
commit
0c00455ea6
@ -54,7 +54,10 @@ matrix:
|
||||
- name: Trusty (clang-3.8 + MSan + Test Zstd)
|
||||
script:
|
||||
- make clang38install
|
||||
- CC=clang-3.8 make clean msan-test-zstd
|
||||
# External libraries must be turned off when using MSAN tests,
|
||||
# because they are not msan-instrumented,
|
||||
# so any data coming from these libraries is always considered "uninitialized"
|
||||
- CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
||||
|
||||
- name: Trusty (Minimal Decompressor Macros)
|
||||
script:
|
||||
|
||||
17
appveyor.yml
17
appveyor.yml
@ -1,3 +1,7 @@
|
||||
# Following tests are run _only_ on master branch
|
||||
# To reproduce these tests, it's possible to push into a branch `appveyorTest`
|
||||
# or a branch `visual*`, they will intentionnally trigger `master` tests
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
branches:
|
||||
@ -176,6 +180,11 @@
|
||||
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
|
||||
)
|
||||
|
||||
|
||||
# The following tests are for regular pushes
|
||||
# into `dev` or some feature branch
|
||||
# There run less tests, for shorter feedback loop
|
||||
|
||||
-
|
||||
version: 1.0.{build}
|
||||
environment:
|
||||
@ -249,3 +258,11 @@
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
|
||||
)
|
||||
|
||||
|
||||
test_script:
|
||||
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%HOST%]==[mingw] (
|
||||
set "CC=%COMPILER%" &&
|
||||
make check
|
||||
)
|
||||
|
||||
2
doc/educational_decoder/.gitignore
vendored
Normal file
2
doc/educational_decoder/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
# Build artifacts
|
||||
harness
|
||||
@ -7,8 +7,15 @@
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
ZSTD ?= zstd # requires zstd installation on local system
|
||||
ZSTD ?= zstd # note: requires zstd installation on local system
|
||||
|
||||
UNAME?= $(shell uname)
|
||||
ifeq ($(UNAME), SunOS)
|
||||
DIFF ?= gdiff
|
||||
else
|
||||
DIFF ?= diff
|
||||
endif
|
||||
|
||||
HARNESS_FILES=*.c
|
||||
|
||||
MULTITHREAD_LDFLAGS = -pthread
|
||||
@ -29,26 +36,27 @@ harness: $(HARNESS_FILES)
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
clean:
|
||||
@$(RM) -f harness
|
||||
@$(RM) -rf harness.dSYM
|
||||
@$(RM) harness
|
||||
@$(RM) -rf harness.dSYM # MacOS specific
|
||||
|
||||
test: harness
|
||||
#
|
||||
# Testing single-file decompression with educational decoder
|
||||
#
|
||||
@$(ZSTD) README.md -o tmp.zst
|
||||
@$(ZSTD) -f README.md -o tmp.zst
|
||||
@./harness tmp.zst tmp
|
||||
@$(DIFF) -s tmp README.md
|
||||
@$(RM) -f tmp*
|
||||
@$(RM) tmp*
|
||||
#
|
||||
# Testing dictionary decompression with education decoder
|
||||
#
|
||||
# note : files are presented multiple for training, to reach minimum threshold
|
||||
@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
|
||||
harness.c zstd_decompress.c zstd_decompress.h README.md \
|
||||
harness.c zstd_decompress.c zstd_decompress.h README.md
|
||||
@$(ZSTD) -D dictionary README.md -o tmp.zst
|
||||
harness.c zstd_decompress.c zstd_decompress.h README.md \
|
||||
-o dictionary
|
||||
@$(ZSTD) -f README.md -D dictionary -o tmp.zst
|
||||
@./harness tmp.zst tmp dictionary
|
||||
@$(DIFF) -s tmp README.md
|
||||
@$(RM) -f tmp* dictionary
|
||||
@$(RM) tmp* dictionary
|
||||
@$(MAKE) clean
|
||||
|
||||
@ -21,88 +21,90 @@ typedef unsigned char u8;
|
||||
// Protect against allocating too much memory for output
|
||||
#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
|
||||
|
||||
u8 *input;
|
||||
u8 *output;
|
||||
u8 *dict;
|
||||
|
||||
size_t read_file(const char *path, u8 **ptr) {
|
||||
FILE *f = fopen(path, "rb");
|
||||
static size_t read_file(const char *path, u8 **ptr)
|
||||
{
|
||||
FILE* const f = fopen(path, "rb");
|
||||
if (!f) {
|
||||
fprintf(stderr, "failed to open file %s\n", path);
|
||||
fprintf(stderr, "failed to open file %s \n", path);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
fseek(f, 0L, SEEK_END);
|
||||
size_t size = (size_t)ftell(f);
|
||||
size_t const size = (size_t)ftell(f);
|
||||
rewind(f);
|
||||
|
||||
*ptr = malloc(size);
|
||||
if (!ptr) {
|
||||
fprintf(stderr, "failed to allocate memory to hold %s\n", path);
|
||||
fprintf(stderr, "failed to allocate memory to hold %s \n", path);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
size_t pos = 0;
|
||||
while (!feof(f)) {
|
||||
size_t read = fread(&(*ptr)[pos], 1, size, f);
|
||||
if (ferror(f)) {
|
||||
fprintf(stderr, "error while reading file %s\n", path);
|
||||
exit(1);
|
||||
}
|
||||
pos += read;
|
||||
size_t const read = fread(*ptr, 1, size, f);
|
||||
if (read != size) { /* must read everything in one pass */
|
||||
fprintf(stderr, "error while reading file %s \n", path);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
|
||||
return pos;
|
||||
return read;
|
||||
}
|
||||
|
||||
void write_file(const char *path, const u8 *ptr, size_t size) {
|
||||
FILE *f = fopen(path, "wb");
|
||||
static void write_file(const char *path, const u8 *ptr, size_t size)
|
||||
{
|
||||
FILE* const f = fopen(path, "wb");
|
||||
if (!f) {
|
||||
fprintf(stderr, "failed to open file %s \n", path);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
size_t written = 0;
|
||||
while (written < size) {
|
||||
written += fwrite(&ptr[written], 1, size, f);
|
||||
written += fwrite(ptr+written, 1, size, f);
|
||||
if (ferror(f)) {
|
||||
fprintf(stderr, "error while writing file %s\n", path);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
} }
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (argc < 3) {
|
||||
fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary]\n",
|
||||
fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary] \n",
|
||||
argv[0]);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t input_size = read_file(argv[1], &input);
|
||||
u8* input;
|
||||
size_t const input_size = read_file(argv[1], &input);
|
||||
|
||||
u8* dict = NULL;
|
||||
size_t dict_size = 0;
|
||||
if (argc >= 4) {
|
||||
dict_size = read_file(argv[3], &dict);
|
||||
}
|
||||
|
||||
size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size);
|
||||
if (decompressed_size == (size_t)-1) {
|
||||
decompressed_size = MAX_COMPRESSION_RATIO * input_size;
|
||||
size_t out_capacity = ZSTD_get_decompressed_size(input, input_size);
|
||||
if (out_capacity == (size_t)-1) {
|
||||
out_capacity = MAX_COMPRESSION_RATIO * input_size;
|
||||
fprintf(stderr, "WARNING: Compressed data does not contain "
|
||||
"decompressed size, going to assume the compression "
|
||||
"ratio is at most %d (decompressed size of at most "
|
||||
"%zu)\n",
|
||||
MAX_COMPRESSION_RATIO, decompressed_size);
|
||||
"%u) \n",
|
||||
MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
|
||||
}
|
||||
if (decompressed_size > MAX_OUTPUT_SIZE) {
|
||||
if (out_capacity > MAX_OUTPUT_SIZE) {
|
||||
fprintf(stderr,
|
||||
"Required output size too large for this implementation\n");
|
||||
"Required output size too large for this implementation \n");
|
||||
return 1;
|
||||
}
|
||||
output = malloc(decompressed_size);
|
||||
|
||||
u8* const output = malloc(out_capacity);
|
||||
if (!output) {
|
||||
fprintf(stderr, "failed to allocate memory\n");
|
||||
fprintf(stderr, "failed to allocate memory \n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -110,16 +112,17 @@ int main(int argc, char **argv) {
|
||||
if (dict) {
|
||||
parse_dictionary(parsed_dict, dict, dict_size);
|
||||
}
|
||||
size_t decompressed =
|
||||
ZSTD_decompress_with_dict(output, decompressed_size,
|
||||
input, input_size, parsed_dict);
|
||||
size_t const decompressed_size =
|
||||
ZSTD_decompress_with_dict(output, out_capacity,
|
||||
input, input_size,
|
||||
parsed_dict);
|
||||
|
||||
free_dictionary(parsed_dict);
|
||||
|
||||
write_file(argv[2], output, decompressed);
|
||||
write_file(argv[2], output, decompressed_size);
|
||||
|
||||
free(input);
|
||||
free(output);
|
||||
free(dict);
|
||||
input = output = dict = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -856,8 +856,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx,
|
||||
// Impossible
|
||||
IMPOSSIBLE();
|
||||
}
|
||||
if (regenerated_size > MAX_LITERALS_SIZE ||
|
||||
compressed_size >= regenerated_size) {
|
||||
if (regenerated_size > MAX_LITERALS_SIZE) {
|
||||
CORRUPTION();
|
||||
}
|
||||
|
||||
@ -1530,7 +1529,7 @@ void free_dictionary(dictionary_t *const dict) {
|
||||
/******* END DICTIONARY PARSING ***********************************************/
|
||||
|
||||
/******* IO STREAM OPERATIONS *************************************************/
|
||||
#define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream")
|
||||
|
||||
/// Reads `num` bits from a bitstream, and updates the internal offset
|
||||
static inline u64 IO_read_bits(istream_t *const in, const int num_bits) {
|
||||
if (num_bits > 64 || num_bits <= 0) {
|
||||
@ -1609,7 +1608,7 @@ static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len) {
|
||||
INP_SIZE();
|
||||
}
|
||||
if (in->bit_offset != 0) {
|
||||
UNALIGNED();
|
||||
ERROR("Attempting to operate on a non-byte aligned stream");
|
||||
}
|
||||
const u8 *const ptr = in->ptr;
|
||||
in->ptr += len;
|
||||
@ -1635,7 +1634,7 @@ static inline void IO_advance_input(istream_t *const in, size_t len) {
|
||||
INP_SIZE();
|
||||
}
|
||||
if (in->bit_offset != 0) {
|
||||
UNALIGNED();
|
||||
ERROR("Attempting to operate on a non-byte aligned stream");
|
||||
}
|
||||
|
||||
in->ptr += len;
|
||||
|
||||
@ -16,7 +16,7 @@ Distribution of this document is unlimited.
|
||||
|
||||
### Version
|
||||
|
||||
0.3.3 (16/08/19)
|
||||
0.3.4 (16/08/19)
|
||||
|
||||
|
||||
Introduction
|
||||
@ -1107,18 +1107,18 @@ It follows the following build rule :
|
||||
|
||||
The table has a size of `Table_Size = 1 << Accuracy_Log`.
|
||||
Each cell describes the symbol decoded,
|
||||
and instructions to get the next state.
|
||||
and instructions to get the next state (`Number_of_Bits` and `Baseline`).
|
||||
|
||||
Symbols are scanned in their natural order for "less than 1" probabilities.
|
||||
Symbols with this probability are being attributed a single cell,
|
||||
starting from the end of the table and retreating.
|
||||
These symbols define a full state reset, reading `Accuracy_Log` bits.
|
||||
|
||||
All remaining symbols are allocated in their natural order.
|
||||
Starting from symbol `0` and table position `0`,
|
||||
Then, all remaining symbols, sorted in natural order, are allocated cells.
|
||||
Starting from symbol `0` (if it exists), and table position `0`,
|
||||
each symbol gets allocated as many cells as its probability.
|
||||
Cell allocation is spreaded, not linear :
|
||||
each successor position follow this rule :
|
||||
each successor position follows this rule :
|
||||
|
||||
```
|
||||
position += (tableSize>>1) + (tableSize>>3) + 3;
|
||||
@ -1130,40 +1130,41 @@ A position is skipped if already occupied by a "less than 1" probability symbol.
|
||||
each position in the table, switching to the next symbol when enough
|
||||
states have been allocated to the current one.
|
||||
|
||||
The result is a list of state values.
|
||||
Each state will decode the current symbol.
|
||||
The process guarantees that the table is entirely filled.
|
||||
Each cell corresponds to a state value, which contains the symbol being decoded.
|
||||
|
||||
To get the `Number_of_Bits` and `Baseline` required for next state,
|
||||
it's first necessary to sort all states in their natural order.
|
||||
The lower states will need 1 more bit than higher ones.
|
||||
To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
|
||||
it's first necessary to sort all occurrences of each symbol in state order.
|
||||
Lower states will need 1 more bit than higher ones.
|
||||
The process is repeated for each symbol.
|
||||
|
||||
__Example__ :
|
||||
Presuming a symbol has a probability of 5.
|
||||
It receives 5 state values. States are sorted in natural order.
|
||||
Presuming a symbol has a probability of 5,
|
||||
it receives 5 cells, corresponding to 5 state values.
|
||||
These state values are then sorted in natural order.
|
||||
|
||||
Next power of 2 is 8.
|
||||
Space of probabilities is divided into 8 equal parts.
|
||||
Presuming the `Accuracy_Log` is 7, it defines 128 states.
|
||||
Next power of 2 after 5 is 8.
|
||||
Space of probabilities must be divided into 8 equal parts.
|
||||
Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
|
||||
Divided by 8, each share is 16 large.
|
||||
|
||||
In order to reach 8, 8-5=3 lowest states will count "double",
|
||||
doubling the number of shares (32 in width),
|
||||
requiring one more bit in the process.
|
||||
In order to reach 8 shares, 8-5=3 lowest states will count "double",
|
||||
doubling their shares (32 in width), hence requiring one more bit.
|
||||
|
||||
Baseline is assigned starting from the higher states using fewer bits,
|
||||
and proceeding naturally, then resuming at the first state,
|
||||
each takes its allocated width from Baseline.
|
||||
increasing at each state, then resuming at the first state,
|
||||
each state takes its allocated width from Baseline.
|
||||
|
||||
| state order | 0 | 1 | 2 | 3 | 4 |
|
||||
| ---------------- | ----- | ----- | ------ | ---- | ----- |
|
||||
| width | 32 | 32 | 32 | 16 | 16 |
|
||||
| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 |
|
||||
| range number | 2 | 4 | 6 | 0 | 1 |
|
||||
| `Baseline` | 32 | 64 | 96 | 0 | 16 |
|
||||
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
|
||||
| state value | 1 | 39 | 77 | 84 | 122 |
|
||||
| state order | 0 | 1 | 2 | 3 | 4 |
|
||||
| ---------------- | ----- | ----- | ------ | ---- | ------ |
|
||||
| width | 32 | 32 | 32 | 16 | 16 |
|
||||
| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 |
|
||||
| range number | 2 | 4 | 6 | 0 | 1 |
|
||||
| `Baseline` | 32 | 64 | 96 | 0 | 16 |
|
||||
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
|
||||
|
||||
The next state is determined from current state
|
||||
During decoding, the next state value is determined from current state value,
|
||||
by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
|
||||
|
||||
See [Appendix A] for the results of this process applied to the default distributions.
|
||||
@ -1657,6 +1658,7 @@ or at least provide a meaningful error code explaining for which reason it canno
|
||||
|
||||
Version changes
|
||||
---------------
|
||||
- 0.3.4 : clarifications for FSE decoding table
|
||||
- 0.3.3 : clarifications for field Block_Size
|
||||
- 0.3.2 : remove additional block size restriction on compressed blocks
|
||||
- 0.3.1 : minor clarification regarding offset history update rules
|
||||
|
||||
@ -796,7 +796,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
||||
Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
|
||||
It's a CPU consuming operation, with non-negligible impact on latency.
|
||||
If there is a need to use the same prefix multiple times, consider loadDictionary instead.
|
||||
Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
|
||||
Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
|
||||
Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation.
|
||||
</p></pre><BR>
|
||||
|
||||
@ -840,7 +840,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
||||
Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
|
||||
Prefix buffer must remain unmodified up to the end of frame,
|
||||
reached when ZSTD_decompressStream() returns 0.
|
||||
Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
||||
Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
|
||||
Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
|
||||
Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
|
||||
A full dictionary is more costly, as it requires building tables.
|
||||
@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
|
||||
<BR></pre>
|
||||
|
||||
<pre><b>typedef struct {
|
||||
unsigned int matchPos; </b>/* Match pos in dst */<b>
|
||||
</b>/* If seqDef.offset > 3, then this is seqDef.offset - 3<b>
|
||||
* If seqDef.offset < 3, then this is the corresponding repeat offset
|
||||
* But if seqDef.offset < 3 and litLength == 0, this is the
|
||||
* repeat offset before the corresponding repeat offset
|
||||
* And if seqDef.offset == 3 and litLength == 0, this is the
|
||||
* most recent repeat offset - 1
|
||||
*/
|
||||
unsigned int offset;
|
||||
unsigned int litLength; </b>/* Literal length */<b>
|
||||
unsigned int matchLength; </b>/* Match length */<b>
|
||||
</b>/* 0 when seq not rep and seqDef.offset otherwise<b>
|
||||
* when litLength == 0 this will be <= 4, otherwise <= 3 like normal
|
||||
*/
|
||||
unsigned int rep;
|
||||
} ZSTD_Sequence;
|
||||
</b></pre><BR>
|
||||
<pre><b>typedef struct {
|
||||
unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b>
|
||||
unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b>
|
||||
@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
or an error code (if srcSize is too small)
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
|
||||
size_t outSeqsSize, const void* src, size_t srcSize);
|
||||
</b><p> Extract sequences from the sequence store
|
||||
zc can be used to insert custom compression params.
|
||||
This function invokes ZSTD_compress2
|
||||
@return : number of sequences extracted
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
|
||||
|
||||
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
||||
@ -1322,7 +1349,7 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigne
|
||||
*
|
||||
* Creates of an internal CDict (incompatible with static CCtx), except if
|
||||
* dict == NULL or dictSize < 8, in which case no dict is used.
|
||||
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
|
||||
* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
|
||||
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
|
||||
*/
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
||||
@ -1337,7 +1364,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
|
||||
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
||||
*
|
||||
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
||||
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
|
||||
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
|
||||
*/
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
|
||||
@ -74,10 +74,50 @@ void __msan_poison(const volatile void *a, size_t size);
|
||||
intptr_t __msan_test_shadow(const volatile void *x, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined (MEMORY_SANITIZER)
|
||||
# define MEM_SKIP_MSAN __attribute__((no_sanitize("memory")))
|
||||
#else
|
||||
# define MEM_SKIP_MSAN
|
||||
/* detects whether we are being compiled under asan */
|
||||
#if defined (__has_feature)
|
||||
# if __has_feature(address_sanitizer)
|
||||
# define ADDRESS_SANITIZER 1
|
||||
# endif
|
||||
#elif defined(__SANITIZE_ADDRESS__)
|
||||
# define ADDRESS_SANITIZER 1
|
||||
#endif
|
||||
|
||||
#if defined (ADDRESS_SANITIZER)
|
||||
/* Not all platforms that support asan provide sanitizers/asan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
* include the header file... */
|
||||
|
||||
/**
|
||||
* Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
|
||||
*
|
||||
* This memory must be previously allocated by your program. Instrumented
|
||||
* code is forbidden from accessing addresses in this region until it is
|
||||
* unpoisoned. This function is not guaranteed to poison the entire region -
|
||||
* it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
|
||||
* alignment restrictions.
|
||||
*
|
||||
* \note This function is not thread-safe because no two threads can poison or
|
||||
* unpoison memory in the same memory region simultaneously.
|
||||
*
|
||||
* \param addr Start of memory region.
|
||||
* \param size Size of memory region. */
|
||||
void __asan_poison_memory_region(void const volatile *addr, size_t size);
|
||||
|
||||
/**
|
||||
* Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
|
||||
*
|
||||
* This memory must be previously allocated by your program. Accessing
|
||||
* addresses in this region is allowed until this region is poisoned again.
|
||||
* This function could unpoison a super-region of <c>[addr, addr+size)</c> due
|
||||
* to ASan alignment restrictions.
|
||||
*
|
||||
* \note This function is not thread-safe because no two threads can
|
||||
* poison or unpoison memory in the same memory region simultaneously.
|
||||
*
|
||||
* \param addr Start of memory region.
|
||||
* \param size Size of memory region. */
|
||||
void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
@ -127,9 +127,13 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
||||
ctx->queueTail = 0;
|
||||
ctx->numThreadsBusy = 0;
|
||||
ctx->queueEmpty = 1;
|
||||
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
|
||||
{
|
||||
int error = 0;
|
||||
error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
|
||||
error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
|
||||
error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
|
||||
if (error) { POOL_free(ctx); return NULL; }
|
||||
}
|
||||
ctx->shutdown = 0;
|
||||
/* Allocate space for the thread handles */
|
||||
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
||||
|
||||
@ -14,6 +14,8 @@
|
||||
* This file will hold wrapper for systems, which do not support pthreads
|
||||
*/
|
||||
|
||||
#include "threading.h"
|
||||
|
||||
/* create fake symbol to avoid empty translation unit warning */
|
||||
int g_ZSTD_threading_useless_symbol;
|
||||
|
||||
@ -28,7 +30,6 @@ int g_ZSTD_threading_useless_symbol;
|
||||
/* === Dependencies === */
|
||||
#include <process.h>
|
||||
#include <errno.h>
|
||||
#include "threading.h"
|
||||
|
||||
|
||||
/* === Implementation === */
|
||||
@ -73,3 +74,47 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
||||
}
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
||||
|
||||
#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
|
||||
{
|
||||
*mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t));
|
||||
if (!*mutex)
|
||||
return 1;
|
||||
return pthread_mutex_init(*mutex, attr);
|
||||
}
|
||||
|
||||
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
|
||||
{
|
||||
if (!*mutex)
|
||||
return 0;
|
||||
{
|
||||
int const ret = pthread_mutex_destroy(*mutex);
|
||||
free(*mutex);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
|
||||
{
|
||||
*cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t));
|
||||
if (!*cond)
|
||||
return 1;
|
||||
return pthread_cond_init(*cond, attr);
|
||||
}
|
||||
|
||||
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
|
||||
{
|
||||
if (!*cond)
|
||||
return 0;
|
||||
{
|
||||
int const ret = pthread_cond_destroy(*cond);
|
||||
free(*cond);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -13,6 +13,8 @@
|
||||
#ifndef THREADING_H_938743
|
||||
#define THREADING_H_938743
|
||||
|
||||
#include "debug.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
@ -75,10 +77,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
|
||||
*/
|
||||
|
||||
|
||||
#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
|
||||
#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
|
||||
/* === POSIX Systems === */
|
||||
# include <pthread.h>
|
||||
|
||||
#if DEBUGLEVEL < 1
|
||||
|
||||
#define ZSTD_pthread_mutex_t pthread_mutex_t
|
||||
#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
|
||||
#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
|
||||
@ -96,6 +100,33 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
|
||||
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
|
||||
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
|
||||
|
||||
#else /* DEBUGLEVEL >= 1 */
|
||||
|
||||
/* Debug implementation of threading.
|
||||
* In this implementation we use pointers for mutexes and condition variables.
|
||||
* This way, if we forget to init/destroy them the program will crash or ASAN
|
||||
* will report leaks.
|
||||
*/
|
||||
|
||||
#define ZSTD_pthread_mutex_t pthread_mutex_t*
|
||||
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr);
|
||||
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex);
|
||||
#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a))
|
||||
|
||||
#define ZSTD_pthread_cond_t pthread_cond_t*
|
||||
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr);
|
||||
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
|
||||
#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b))
|
||||
#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a))
|
||||
|
||||
#define ZSTD_pthread_t pthread_t
|
||||
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
|
||||
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
|
||||
|
||||
#endif
|
||||
|
||||
#else /* ZSTD_MULTITHREAD not defined */
|
||||
/* No multithreading support */
|
||||
|
||||
|
||||
@ -202,7 +202,7 @@ static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); }
|
||||
|
||||
typedef enum {
|
||||
ZSTD_no_overlap,
|
||||
ZSTD_overlap_src_before_dst,
|
||||
ZSTD_overlap_src_before_dst
|
||||
/* ZSTD_overlap_dst_before_src, */
|
||||
} ZSTD_overlap_e;
|
||||
|
||||
@ -247,20 +247,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTD_wildcopy8() :
|
||||
* The same as ZSTD_wildcopy(), but it can only overwrite 8 bytes, and works for
|
||||
* overlapping buffers that are at least 8 bytes apart.
|
||||
*/
|
||||
MEM_STATIC void ZSTD_wildcopy8(void* dst, const void* src, ptrdiff_t length)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = (BYTE*)op + length;
|
||||
do {
|
||||
COPY8(op, ip);
|
||||
} while (op < oend);
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private declarations
|
||||
|
||||
@ -131,15 +131,11 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
|
||||
{
|
||||
assert(cctx != NULL);
|
||||
assert(cctx->staticSize == 0);
|
||||
/* Only free workspace if cctx not in workspace, otherwise the workspace
|
||||
* will be freed when the cctx itself is freed. */
|
||||
if ((void*)cctx->workspace.workspace != (void*)cctx) {
|
||||
ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
|
||||
}
|
||||
ZSTD_clearAllDicts(cctx);
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
|
||||
#endif
|
||||
ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
|
||||
}
|
||||
|
||||
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
||||
@ -147,8 +143,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
||||
if (cctx==NULL) return 0; /* support free on NULL */
|
||||
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
|
||||
"not compatible with static CCtx");
|
||||
ZSTD_freeCCtxContent(cctx);
|
||||
ZSTD_free(cctx, cctx->customMem);
|
||||
{
|
||||
int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
|
||||
ZSTD_freeCCtxContent(cctx);
|
||||
if (!cctxInWorkspace) {
|
||||
ZSTD_free(cctx, cctx->customMem);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -1078,10 +1079,19 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
|
||||
size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
|
||||
size_t const hSize = ((size_t)1) << cParams->hashLog;
|
||||
U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
|
||||
size_t const h3Size = ((size_t)1) << hashLog3;
|
||||
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32)
|
||||
+ (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t));
|
||||
size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
|
||||
/* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't
|
||||
* surrounded by redzones in ASAN. */
|
||||
size_t const tableSpace = chainSize * sizeof(U32)
|
||||
+ hSize * sizeof(U32)
|
||||
+ h3Size * sizeof(U32);
|
||||
size_t const optPotentialSpace =
|
||||
ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32))
|
||||
+ ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32))
|
||||
+ ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32))
|
||||
+ ZSTD_cwksp_alloc_size((1<<Litbits) * sizeof(U32))
|
||||
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
|
||||
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
|
||||
size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
|
||||
? optPotentialSpace
|
||||
: 0;
|
||||
@ -1098,20 +1108,23 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
|
||||
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
|
||||
U32 const divider = (cParams.minMatch==3) ? 3 : 4;
|
||||
size_t const maxNbSeq = blockSize / divider;
|
||||
size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq;
|
||||
size_t const entropySpace = HUF_WORKSPACE_SIZE;
|
||||
size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t);
|
||||
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
|
||||
+ ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
|
||||
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
|
||||
size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE);
|
||||
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
|
||||
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1);
|
||||
|
||||
size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams);
|
||||
size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq);
|
||||
size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq));
|
||||
|
||||
size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace +
|
||||
matchStateSize + ldmSpace + ldmSeqSpace;
|
||||
size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx));
|
||||
|
||||
DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx));
|
||||
DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace);
|
||||
DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
|
||||
return sizeof(ZSTD_CCtx) + neededSpace;
|
||||
return cctxSpace + neededSpace;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1147,7 +1160,8 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
|
||||
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
|
||||
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
|
||||
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
|
||||
size_t const streamingSize = inBuffSize + outBuffSize;
|
||||
size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize)
|
||||
+ ZSTD_cwksp_alloc_size(outBuffSize);
|
||||
|
||||
return CCtxSize + streamingSize;
|
||||
}
|
||||
@ -1394,7 +1408,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
||||
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
|
||||
U32 const divider = (params.cParams.minMatch==3) ? 3 : 4;
|
||||
size_t const maxNbSeq = blockSize / divider;
|
||||
size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq;
|
||||
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
|
||||
+ ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
|
||||
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
|
||||
size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0;
|
||||
size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0;
|
||||
size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1);
|
||||
@ -1409,12 +1425,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
||||
ZSTD_cwksp_bump_oversized_duration(ws, 0);
|
||||
|
||||
/* Check if workspace is large enough, alloc a new one if needed */
|
||||
{ size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0;
|
||||
size_t const entropySpace = HUF_WORKSPACE_SIZE;
|
||||
size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t);
|
||||
size_t const bufferSpace = buffInSize + buffOutSize;
|
||||
{ size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
|
||||
size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE);
|
||||
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
|
||||
size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize);
|
||||
size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams);
|
||||
size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq);
|
||||
size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq));
|
||||
|
||||
size_t const neededSpace =
|
||||
cctxSpace +
|
||||
@ -1650,12 +1666,13 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
|
||||
/* copy tables */
|
||||
{ size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog);
|
||||
size_t const hSize = (size_t)1 << cdict_cParams->hashLog;
|
||||
size_t const tableSpace = (chainSize + hSize) * sizeof(U32);
|
||||
assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */
|
||||
assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize);
|
||||
assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */
|
||||
assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize);
|
||||
memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace); /* presumes all tables follow each other */
|
||||
|
||||
memcpy(cctx->blockState.matchState.hashTable,
|
||||
cdict->matchState.hashTable,
|
||||
hSize * sizeof(U32));
|
||||
memcpy(cctx->blockState.matchState.chainTable,
|
||||
cdict->matchState.chainTable,
|
||||
chainSize * sizeof(U32));
|
||||
}
|
||||
|
||||
/* Zero the hashTable3, since the cdict never fills it */
|
||||
@ -1742,10 +1759,16 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
|
||||
size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
|
||||
int const h3log = srcCCtx->blockState.matchState.hashLog3;
|
||||
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
|
||||
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */
|
||||
assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize);
|
||||
memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */
|
||||
|
||||
memcpy(dstCCtx->blockState.matchState.hashTable,
|
||||
srcCCtx->blockState.matchState.hashTable,
|
||||
hSize * sizeof(U32));
|
||||
memcpy(dstCCtx->blockState.matchState.chainTable,
|
||||
srcCCtx->blockState.matchState.chainTable,
|
||||
chainSize * sizeof(U32));
|
||||
memcpy(dstCCtx->blockState.matchState.hashTable3,
|
||||
srcCCtx->blockState.matchState.hashTable3,
|
||||
h3Size * sizeof(U32));
|
||||
}
|
||||
|
||||
ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace);
|
||||
@ -3140,8 +3163,11 @@ size_t ZSTD_estimateCDictSize_advanced(
|
||||
ZSTD_dictLoadMethod_e dictLoadMethod)
|
||||
{
|
||||
DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict));
|
||||
return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0)
|
||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void *)));
|
||||
return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
|
||||
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
|
||||
+ ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0)
|
||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
|
||||
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
|
||||
@ -3221,11 +3247,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
|
||||
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
||||
|
||||
{ size_t const workspaceSize =
|
||||
sizeof(ZSTD_CDict) +
|
||||
HUF_WORKSPACE_SIZE +
|
||||
ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
|
||||
ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
|
||||
ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) +
|
||||
(dictLoadMethod == ZSTD_dlm_byRef ? 0
|
||||
: ZSTD_cwksp_align(dictSize, sizeof(void*)));
|
||||
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
|
||||
void* const workspace = ZSTD_malloc(workspaceSize, customMem);
|
||||
ZSTD_cwksp ws;
|
||||
ZSTD_CDict* cdict;
|
||||
@ -3278,12 +3304,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
|
||||
{
|
||||
if (cdict==NULL) return 0; /* support free on NULL */
|
||||
{ ZSTD_customMem const cMem = cdict->customMem;
|
||||
/* Only free workspace if cdict not in workspace, otherwise the
|
||||
* workspace will be freed when the cdict itself is freed. */
|
||||
if ((void*)cdict->workspace.workspace != (void*)cdict) {
|
||||
ZSTD_cwksp_free(&cdict->workspace, cMem);
|
||||
int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict);
|
||||
ZSTD_cwksp_free(&cdict->workspace, cMem);
|
||||
if (!cdictInWorkspace) {
|
||||
ZSTD_free(cdict, cMem);
|
||||
}
|
||||
ZSTD_free(cdict, cMem);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@ -3309,8 +3334,11 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
|
||||
ZSTD_compressionParameters cParams)
|
||||
{
|
||||
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0);
|
||||
size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void*)))
|
||||
+ HUF_WORKSPACE_SIZE + matchStateSize;
|
||||
size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
|
||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
|
||||
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
|
||||
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
|
||||
+ matchStateSize;
|
||||
ZSTD_CDict* cdict;
|
||||
|
||||
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
|
||||
@ -3543,7 +3571,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
|
||||
/* ZSTD_initCStream_advanced() :
|
||||
* pledgedSrcSize must be exact.
|
||||
* if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
|
||||
* dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pss)
|
||||
|
||||
@ -34,6 +34,17 @@ extern "C" {
|
||||
* In which case, resize it down to free some memory */
|
||||
#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
|
||||
|
||||
/* Since the workspace is effectively its own little malloc implementation /
|
||||
* arena, when we run under ASAN, we should similarly insert redzones between
|
||||
* each internal element of the workspace, so ASAN will catch overruns that
|
||||
* reach outside an object but that stay inside the workspace.
|
||||
*
|
||||
* This defines the size of that redzone.
|
||||
*/
|
||||
#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE
|
||||
#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
|
||||
#endif
|
||||
|
||||
/*-*************************************
|
||||
* Structures
|
||||
***************************************/
|
||||
@ -166,6 +177,24 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
|
||||
return (size + mask) & ~mask;
|
||||
}
|
||||
|
||||
/**
|
||||
* Use this to determine how much space in the workspace we will consume to
|
||||
* allocate this object. (Normally it should be exactly the size of the object,
|
||||
* but under special conditions, like ASAN, where we pad each object, it might
|
||||
* be larger.)
|
||||
*
|
||||
* Since tables aren't currently redzoned, you don't need to call through this
|
||||
* to figure out how much space you need for the matchState tables. Everything
|
||||
* else is though.
|
||||
*/
|
||||
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
#else
|
||||
return size;
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
|
||||
ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) {
|
||||
assert(phase >= ws->phase);
|
||||
@ -192,6 +221,13 @@ MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether this object/buffer/etc was allocated in this workspace.
|
||||
*/
|
||||
MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) {
|
||||
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal function. Do not use directly.
|
||||
*/
|
||||
@ -201,8 +237,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
|
||||
void* bottom = ws->tableEnd;
|
||||
ZSTD_cwksp_internal_advance_phase(ws, phase);
|
||||
alloc = (BYTE *)ws->allocStart - bytes;
|
||||
DEBUGLOG(5, "cwksp: reserving %zd bytes, %zd bytes remaining",
|
||||
bytes, ZSTD_cwksp_available_space(ws) - bytes);
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* over-reserve space */
|
||||
alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
#endif
|
||||
|
||||
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
|
||||
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
assert(alloc >= bottom);
|
||||
if (alloc < bottom) {
|
||||
@ -214,6 +256,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
|
||||
ws->tableValidEnd = alloc;
|
||||
}
|
||||
ws->allocStart = alloc;
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
|
||||
* either size. */
|
||||
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
#endif
|
||||
|
||||
return alloc;
|
||||
}
|
||||
|
||||
@ -242,8 +292,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
|
||||
void* alloc = ws->tableEnd;
|
||||
void* end = (BYTE *)alloc + bytes;
|
||||
void* top = ws->allocStart;
|
||||
DEBUGLOG(5, "cwksp: reserving table %zd bytes, %zd bytes remaining",
|
||||
bytes, ZSTD_cwksp_available_space(ws) - bytes);
|
||||
|
||||
DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
|
||||
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
|
||||
assert((bytes & (sizeof(U32)-1)) == 0);
|
||||
ZSTD_cwksp_internal_advance_phase(ws, phase);
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
@ -254,6 +305,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
|
||||
return NULL;
|
||||
}
|
||||
ws->tableEnd = end;
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
#endif
|
||||
|
||||
return alloc;
|
||||
}
|
||||
|
||||
@ -262,12 +318,18 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
|
||||
*/
|
||||
MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
|
||||
size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
|
||||
void* start = ws->objectEnd;
|
||||
void* end = (BYTE*)start + roundedBytes;
|
||||
void* alloc = ws->objectEnd;
|
||||
void* end = (BYTE*)alloc + roundedBytes;
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* over-reserve space */
|
||||
end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
#endif
|
||||
|
||||
DEBUGLOG(5,
|
||||
"cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining",
|
||||
bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
|
||||
assert(((size_t)start & (sizeof(void*)-1)) == 0);
|
||||
"cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining",
|
||||
alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
|
||||
assert(((size_t)alloc & (sizeof(void*)-1)) == 0);
|
||||
assert((bytes & (sizeof(void*)-1)) == 0);
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
/* we must be in the first phase, no advance is possible */
|
||||
@ -279,7 +341,15 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
|
||||
ws->objectEnd = end;
|
||||
ws->tableEnd = end;
|
||||
ws->tableValidEnd = end;
|
||||
return start;
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
|
||||
* either size. */
|
||||
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
|
||||
__asan_unpoison_memory_region(alloc, bytes);
|
||||
#endif
|
||||
|
||||
return alloc;
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) {
|
||||
@ -331,6 +401,14 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
|
||||
*/
|
||||
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
|
||||
DEBUGLOG(4, "cwksp: clearing tables!");
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
{
|
||||
size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
|
||||
__asan_poison_memory_region(ws->objectEnd, size);
|
||||
}
|
||||
#endif
|
||||
|
||||
ws->tableEnd = ws->objectEnd;
|
||||
ZSTD_cwksp_assert_internal_consistency(ws);
|
||||
}
|
||||
@ -353,6 +431,13 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
{
|
||||
size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
|
||||
__asan_poison_memory_region(ws->objectEnd, size);
|
||||
}
|
||||
#endif
|
||||
|
||||
ws->tableEnd = ws->objectEnd;
|
||||
ws->allocStart = ws->workspaceEnd;
|
||||
ws->allocFailed = 0;
|
||||
@ -389,9 +474,10 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
|
||||
void *ptr = ws->workspace;
|
||||
DEBUGLOG(4, "cwksp: freeing workspace");
|
||||
ZSTD_free(ws->workspace, customMem);
|
||||
memset(ws, 0, sizeof(ZSTD_cwksp));
|
||||
ZSTD_free(ptr, customMem);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -404,7 +490,7 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
|
||||
return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace;
|
||||
return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
|
||||
}
|
||||
|
||||
MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
|
||||
|
||||
@ -49,9 +49,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params)
|
||||
{
|
||||
size_t const ldmHSize = ((size_t)1) << params.hashLog;
|
||||
size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog);
|
||||
size_t const ldmBucketSize =
|
||||
((size_t)1) << (params.hashLog - ldmBucketSizeLog);
|
||||
size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t);
|
||||
size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
|
||||
size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize)
|
||||
+ ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t));
|
||||
return params.enableLdm ? totalSize : 0;
|
||||
}
|
||||
|
||||
|
||||
@ -927,12 +927,18 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
|
||||
unsigned jobID;
|
||||
DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
|
||||
for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
|
||||
/* Copy the mutex/cond out */
|
||||
ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex;
|
||||
ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond;
|
||||
|
||||
DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start);
|
||||
ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
|
||||
mtctx->jobs[jobID].dstBuff = g_nullBuffer;
|
||||
mtctx->jobs[jobID].cSize = 0;
|
||||
|
||||
/* Clear the job description, but keep the mutex/cond */
|
||||
memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID]));
|
||||
mtctx->jobs[jobID].job_mutex = mutex;
|
||||
mtctx->jobs[jobID].job_cond = cond;
|
||||
}
|
||||
memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
|
||||
mtctx->inBuff.buffer = g_nullBuffer;
|
||||
mtctx->inBuff.filled = 0;
|
||||
mtctx->allJobsCompleted = 1;
|
||||
|
||||
16
lib/zstd.h
16
lib/zstd.h
@ -398,7 +398,7 @@ typedef enum {
|
||||
ZSTD_c_experimentalParam4=1001,
|
||||
ZSTD_c_experimentalParam5=1002,
|
||||
ZSTD_c_experimentalParam6=1003,
|
||||
ZSTD_c_experimentalParam7=1004,
|
||||
ZSTD_c_experimentalParam7=1004
|
||||
} ZSTD_cParameter;
|
||||
|
||||
typedef struct {
|
||||
@ -928,7 +928,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
|
||||
* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
|
||||
* It's a CPU consuming operation, with non-negligible impact on latency.
|
||||
* If there is a need to use the same prefix multiple times, consider loadDictionary instead.
|
||||
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
|
||||
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
|
||||
* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
|
||||
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
|
||||
const void* prefix, size_t prefixSize);
|
||||
@ -972,7 +972,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
* Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
|
||||
* Prefix buffer must remain unmodified up to the end of frame,
|
||||
* reached when ZSTD_decompressStream() returns 0.
|
||||
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
||||
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
|
||||
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
|
||||
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
|
||||
* A full dictionary is more costly, as it requires building tables.
|
||||
@ -1124,7 +1124,7 @@ typedef enum {
|
||||
|
||||
typedef enum {
|
||||
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
|
||||
ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
|
||||
ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
|
||||
} ZSTD_dictLoadMethod_e;
|
||||
|
||||
typedef enum {
|
||||
@ -1138,7 +1138,7 @@ typedef enum {
|
||||
* This question could be kept for later, when there are actually multiple formats to support,
|
||||
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
|
||||
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
|
||||
ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
|
||||
ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
|
||||
* Useful to save 4 bytes per generated frame.
|
||||
* Decoder cannot recognise automatically this format, requiring this instruction. */
|
||||
} ZSTD_format_e;
|
||||
@ -1188,7 +1188,7 @@ typedef enum {
|
||||
* levels will be compressed. */
|
||||
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
|
||||
* emitted if Huffman compression is not profitable. */
|
||||
ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
|
||||
ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
|
||||
} ZSTD_literalCompressionMode_e;
|
||||
|
||||
|
||||
@ -1677,7 +1677,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
|
||||
*
|
||||
* Creates of an internal CDict (incompatible with static CCtx), except if
|
||||
* dict == NULL or dictSize < 8, in which case no dict is used.
|
||||
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
|
||||
* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
|
||||
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
|
||||
@ -1692,7 +1692,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
|
||||
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
||||
*
|
||||
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
||||
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
|
||||
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
|
||||
@ -173,10 +173,13 @@ Benchmark arguments :
|
||||
```
|
||||
|
||||
#### Restricted usage of Environment Variables
|
||||
Using environment variables to set compression/decompression parameters has security implications. Therefore,
|
||||
we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
|
||||
Using environment variables to set parameters has security implications.
|
||||
Therefore, this avenue is intentionally restricted.
|
||||
Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
|
||||
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
|
||||
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
|
||||
Note that command line options will override corresponding environment variable settings.
|
||||
`ZSTD_CLEVEL` just replaces the default compression level (`3`).
|
||||
It can be overridden by corresponding command line arguments.
|
||||
|
||||
#### Long distance matching mode
|
||||
The long distance matching mode, enabled with `--long`, is designed to improve
|
||||
|
||||
@ -88,7 +88,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
|
||||
|
||||
#define EXM_THROW_INT(errorNum, ...) { \
|
||||
#define RETURN_ERROR_INT(errorNum, ...) { \
|
||||
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", errorNum); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
@ -401,9 +401,9 @@ BMK_benchMemAdvancedNoAlloc(
|
||||
BMK_initCCtxArgs cctxprep;
|
||||
BMK_initDCtxArgs dctxprep;
|
||||
|
||||
cbp.benchFn = local_defaultCompress;
|
||||
cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */
|
||||
cbp.benchPayload = cctx;
|
||||
cbp.initFn = local_initCCtx;
|
||||
cbp.initFn = local_initCCtx; /* BMK_initCCtx */
|
||||
cbp.initPayload = &cctxprep;
|
||||
cbp.errorFn = ZSTD_isError;
|
||||
cbp.blockCount = nbBlocks;
|
||||
@ -534,8 +534,8 @@ BMK_benchMemAdvancedNoAlloc(
|
||||
if (u==srcSize-1) { /* should never happen */
|
||||
DISPLAY("no difference detected\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
} /* for (u=0; u<srcSize; u++) */
|
||||
} /* if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) */
|
||||
} /* CRC Checking */
|
||||
|
||||
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
|
||||
@ -754,8 +754,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t pos = 0, totalSize = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
FILE* f;
|
||||
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); /* last file may be shortened */
|
||||
if (UTIL_isDirectory(fileNamesTable[n])) {
|
||||
DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
|
||||
fileSizes[n] = 0;
|
||||
@ -766,20 +765,20 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW_INT(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize;
|
||||
}
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
totalSize += (size_t)fileSize;
|
||||
fclose(f);
|
||||
}
|
||||
{ FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize;
|
||||
}
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
totalSize += (size_t)fileSize;
|
||||
fclose(f);
|
||||
} }
|
||||
|
||||
if (totalSize == 0) EXM_THROW_INT(12, "no data to bench");
|
||||
if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -205,7 +205,6 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||
|
||||
|
||||
|
||||
|
||||
#endif /* BENCH_ZSTD_H_3242387 */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
@ -284,9 +284,10 @@ void FIO_addAbortHandler()
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Parameters: Typedefs
|
||||
* Parameters: FIO_prefs_t
|
||||
***************************************/
|
||||
|
||||
/* typedef'd to FIO_prefs_t within fileio.h */
|
||||
struct FIO_prefs_s {
|
||||
|
||||
/* Algorithm preferences */
|
||||
@ -308,6 +309,7 @@ struct FIO_prefs_s {
|
||||
size_t streamSrcSize;
|
||||
size_t targetCBlockSize;
|
||||
int srcSizeHint;
|
||||
int testMode;
|
||||
ZSTD_literalCompressionMode_e literalCompressionMode;
|
||||
|
||||
/* IO preferences */
|
||||
@ -355,6 +357,7 @@ FIO_prefs_t* FIO_createPreferences(void)
|
||||
ret->streamSrcSize = 0;
|
||||
ret->targetCBlockSize = 0;
|
||||
ret->srcSizeHint = 0;
|
||||
ret->testMode = 0;
|
||||
ret->literalCompressionMode = ZSTD_lcm_auto;
|
||||
return ret;
|
||||
}
|
||||
@ -435,6 +438,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
|
||||
prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
|
||||
}
|
||||
|
||||
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
|
||||
prefs->testMode = (testMode!=0);
|
||||
}
|
||||
|
||||
void FIO_setLiteralCompressionMode(
|
||||
FIO_prefs_t* const prefs,
|
||||
ZSTD_literalCompressionMode_e mode) {
|
||||
@ -529,8 +536,12 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
|
||||
/** FIO_openDstFile() :
|
||||
* condition : `dstFileName` must be non-NULL.
|
||||
* @result : FILE* to `dstFileName`, or NULL if it fails */
|
||||
static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
|
||||
static FILE*
|
||||
FIO_openDstFile(FIO_prefs_t* const prefs,
|
||||
const char* srcFileName, const char* dstFileName)
|
||||
{
|
||||
if (prefs->testMode) return NULL; /* do not open file in test mode */
|
||||
|
||||
assert(dstFileName != NULL);
|
||||
if (!strcmp (dstFileName, stdoutmark)) {
|
||||
DISPLAYLEVEL(4,"Using stdout for output \n");
|
||||
@ -555,10 +566,14 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
|
||||
if (UTIL_isRegularFile(dstFileName)) {
|
||||
/* Check if destination file already exists */
|
||||
FILE* const fCheck = fopen( dstFileName, "rb" );
|
||||
#if !defined(_WIN32)
|
||||
/* this test does not work on Windows :
|
||||
* `NUL` and `nul` are detected as regular files */
|
||||
if (!strcmp(dstFileName, nulmark)) {
|
||||
EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
|
||||
dstFileName);
|
||||
}
|
||||
#endif
|
||||
if (fCheck != NULL) { /* dst file exists, authorization prompt */
|
||||
fclose(fCheck);
|
||||
if (!prefs->overwrite) {
|
||||
@ -648,7 +663,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
|
||||
DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
for (u = 0; u < nbFiles; ++u) {
|
||||
filename = strrchr(filenameTable[u], c[0]);
|
||||
if (filename == NULL) {
|
||||
@ -671,56 +686,50 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const char*
|
||||
extractFilename(const char* path, char separator)
|
||||
{
|
||||
const char* search = strrchr(path, separator);
|
||||
if (search == NULL) return path;
|
||||
return search+1;
|
||||
}
|
||||
|
||||
/* FIO_createFilename_fromOutDir() :
|
||||
* Takes a source file name and specified output directory, and
|
||||
* allocates memory for and returns a pointer to final path.
|
||||
* This function never returns an error (it may abort() in case of pb)
|
||||
*/
|
||||
static char*
|
||||
FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
|
||||
FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen)
|
||||
{
|
||||
const char* c, *filenameBegin;
|
||||
char* filename, *result;
|
||||
size_t finalPathLen;
|
||||
const char* filenameStart;
|
||||
char separator;
|
||||
char* result;
|
||||
|
||||
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
|
||||
c = "\\";
|
||||
#else
|
||||
c = "/";
|
||||
#endif
|
||||
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
|
||||
separator = '\\';
|
||||
#else
|
||||
separator = '/';
|
||||
#endif
|
||||
|
||||
finalPathLen = strlen(outDirName);
|
||||
filenameBegin = strrchr(srcFilename, c[0]);
|
||||
if (filenameBegin == NULL) {
|
||||
filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char));
|
||||
if (!filename) {
|
||||
EXM_THROW(30, "zstd: %s", strerror(errno));
|
||||
}
|
||||
strcpy(filename, srcFilename);
|
||||
} else {
|
||||
filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
|
||||
if (!filename) {
|
||||
EXM_THROW(30, "zstd: %s", strerror(errno));
|
||||
}
|
||||
strcpy(filename, filenameBegin+1);
|
||||
}
|
||||
filenameStart = extractFilename(path, separator);
|
||||
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
|
||||
filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */
|
||||
#endif
|
||||
|
||||
finalPathLen += strlen(filename);
|
||||
result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
|
||||
result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1);
|
||||
if (!result) {
|
||||
free(filename);
|
||||
EXM_THROW(30, "zstd: %s", strerror(errno));
|
||||
EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno));
|
||||
}
|
||||
|
||||
strcpy(result, outDirName);
|
||||
if (outDirName[strlen(outDirName)-1] == c[0]) {
|
||||
strcat(result, filename);
|
||||
} else {
|
||||
strcat(result, c);
|
||||
strcat(result, filename);
|
||||
memcpy(result, outDirName, strlen(outDirName));
|
||||
if (outDirName[strlen(outDirName)-1] == separator) {
|
||||
memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart));
|
||||
} else {
|
||||
memcpy(result + strlen(outDirName), &separator, 1);
|
||||
memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart));
|
||||
}
|
||||
|
||||
free(filename);
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -825,13 +834,12 @@ static void FIO_freeCResources(cRess_t ress)
|
||||
|
||||
#ifdef ZSTD_GZCOMPRESS
|
||||
static unsigned long long
|
||||
FIO_compressGzFrame(cRess_t* ress,
|
||||
FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */
|
||||
const char* srcFileName, U64 const srcFileSize,
|
||||
int compressionLevel, U64* readsize)
|
||||
{
|
||||
unsigned long long inFileSize = 0, outFileSize = 0;
|
||||
z_stream strm;
|
||||
int ret;
|
||||
|
||||
if (compressionLevel > Z_BEST_COMPRESSION)
|
||||
compressionLevel = Z_BEST_COMPRESSION;
|
||||
@ -840,11 +848,12 @@ FIO_compressGzFrame(cRess_t* ress,
|
||||
strm.zfree = Z_NULL;
|
||||
strm.opaque = Z_NULL;
|
||||
|
||||
ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
|
||||
{ int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
|
||||
15 /* maxWindowLogSize */ + 16 /* gzip only */,
|
||||
8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
|
||||
if (ret != Z_OK)
|
||||
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
|
||||
if (ret != Z_OK) {
|
||||
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
|
||||
} }
|
||||
|
||||
strm.next_in = 0;
|
||||
strm.avail_in = 0;
|
||||
@ -852,6 +861,7 @@ FIO_compressGzFrame(cRess_t* ress,
|
||||
strm.avail_out = (uInt)ress->dstBufferSize;
|
||||
|
||||
while (1) {
|
||||
int ret;
|
||||
if (strm.avail_in == 0) {
|
||||
size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
|
||||
if (inSize == 0) break;
|
||||
@ -862,32 +872,31 @@ FIO_compressGzFrame(cRess_t* ress,
|
||||
ret = deflate(&strm, Z_NO_FLUSH);
|
||||
if (ret != Z_OK)
|
||||
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
|
||||
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
|
||||
EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno));
|
||||
outFileSize += decompBytes;
|
||||
{ size_t const cSize = ress->dstBufferSize - strm.avail_out;
|
||||
if (cSize) {
|
||||
if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
|
||||
EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno));
|
||||
outFileSize += cSize;
|
||||
strm.next_out = (Bytef*)ress->dstBuffer;
|
||||
strm.avail_out = (uInt)ress->dstBufferSize;
|
||||
}
|
||||
}
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
} }
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
|
||||
(unsigned)(inFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100)
|
||||
else
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
|
||||
} else {
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
|
||||
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100);
|
||||
}
|
||||
} }
|
||||
|
||||
while (1) {
|
||||
ret = deflate(&strm, Z_FINISH);
|
||||
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
|
||||
EXM_THROW(75, "Write error : %s", strerror(errno));
|
||||
outFileSize += decompBytes;
|
||||
int const ret = deflate(&strm, Z_FINISH);
|
||||
{ size_t const cSize = ress->dstBufferSize - strm.avail_out;
|
||||
if (cSize) {
|
||||
if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
|
||||
EXM_THROW(75, "Write error : %s ", strerror(errno));
|
||||
outFileSize += cSize;
|
||||
strm.next_out = (Bytef*)ress->dstBuffer;
|
||||
strm.avail_out = (uInt)ress->dstBufferSize;
|
||||
} }
|
||||
@ -896,11 +905,11 @@ FIO_compressGzFrame(cRess_t* ress,
|
||||
EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
|
||||
}
|
||||
|
||||
ret = deflateEnd(&strm);
|
||||
if (ret != Z_OK)
|
||||
EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
|
||||
{ int const ret = deflateEnd(&strm);
|
||||
if (ret != Z_OK) {
|
||||
EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
|
||||
} }
|
||||
*readsize = inFileSize;
|
||||
|
||||
return outFileSize;
|
||||
}
|
||||
#endif
|
||||
@ -923,14 +932,14 @@ FIO_compressLzmaFrame(cRess_t* ress,
|
||||
if (plain_lzma) {
|
||||
lzma_options_lzma opt_lzma;
|
||||
if (lzma_lzma_preset(&opt_lzma, compressionLevel))
|
||||
EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
|
||||
EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName);
|
||||
ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
|
||||
if (ret != LZMA_OK)
|
||||
EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
|
||||
EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
|
||||
} else {
|
||||
ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
|
||||
if (ret != LZMA_OK)
|
||||
EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
|
||||
EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
|
||||
}
|
||||
|
||||
strm.next_in = 0;
|
||||
@ -950,11 +959,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
|
||||
ret = lzma_code(&strm, action);
|
||||
|
||||
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
|
||||
EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
|
||||
EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
|
||||
{ size_t const compBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (compBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
|
||||
EXM_THROW(73, "Write error : %s", strerror(errno));
|
||||
EXM_THROW(85, "Write error : %s", strerror(errno));
|
||||
outFileSize += compBytes;
|
||||
strm.next_out = (BYTE*)ress->dstBuffer;
|
||||
strm.avail_out = ress->dstBufferSize;
|
||||
@ -1493,7 +1502,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
|
||||
sfnSize = strlen(outDirFilename);
|
||||
assert(outDirFilename != NULL);
|
||||
}
|
||||
|
||||
|
||||
if (dfnbCapacity <= sfnSize+suffixSize+1) {
|
||||
/* resize buffer for dstName */
|
||||
free(dstFileNameBuffer);
|
||||
@ -1522,9 +1531,10 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
|
||||
* or into one file each (outFileName == NULL, but suffix != NULL),
|
||||
* or into a destination folder (specified with -O)
|
||||
*/
|
||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
|
||||
const char* outDirName, unsigned nbFiles,
|
||||
const char* outFileName, const char* suffix,
|
||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
const char** inFileNamesTable, unsigned nbFiles,
|
||||
const char* outDirName,
|
||||
const char* outFileName, const char* suffix,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters comprParams)
|
||||
{
|
||||
@ -1617,7 +1627,11 @@ static void FIO_freeDResources(dRess_t ress)
|
||||
|
||||
/** FIO_fwriteSparse() :
|
||||
* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
|
||||
static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
|
||||
static unsigned
|
||||
FIO_fwriteSparse(const FIO_prefs_t* const prefs,
|
||||
FILE* file,
|
||||
const void* buffer, size_t bufferSize,
|
||||
unsigned storedSkips)
|
||||
{
|
||||
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
|
||||
size_t bufferSizeT = bufferSize / sizeof(size_t);
|
||||
@ -1625,6 +1639,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
|
||||
const size_t* ptrT = bufferT;
|
||||
static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
|
||||
|
||||
if (prefs->testMode) return 0; /* do not output anything in test mode */
|
||||
|
||||
if (!prefs->sparseFileSupport) { /* normal write */
|
||||
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
|
||||
if (sizeCheck != bufferSize)
|
||||
@ -1637,7 +1653,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
|
||||
if (storedSkips > 1 GB) {
|
||||
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
|
||||
if (seekResult != 0)
|
||||
EXM_THROW(71, "1 GB skip error (sparse file support)");
|
||||
EXM_THROW(91, "1 GB skip error (sparse file support)");
|
||||
storedSkips -= 1 GB;
|
||||
}
|
||||
|
||||
@ -1653,13 +1669,13 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
|
||||
|
||||
if (nb0T != seg0SizeT) { /* not all 0s */
|
||||
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
|
||||
if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse");
|
||||
storedSkips = 0;
|
||||
seg0SizeT -= nb0T;
|
||||
ptrT += nb0T;
|
||||
{ size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
|
||||
if (sizeCheck != seg0SizeT)
|
||||
EXM_THROW(73, "Write error : cannot write decoded block : %s",
|
||||
EXM_THROW(93, "Write error : cannot write decoded block : %s",
|
||||
strerror(errno));
|
||||
} }
|
||||
ptrT += seg0SizeT;
|
||||
@ -1677,11 +1693,11 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
|
||||
if (restPtr != restEnd) {
|
||||
int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult)
|
||||
EXM_THROW(74, "Sparse skip error ; try --no-sparse");
|
||||
EXM_THROW(94, "Sparse skip error ; try --no-sparse");
|
||||
storedSkips = 0;
|
||||
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
|
||||
if (sizeCheck != (size_t)(restEnd - restPtr))
|
||||
EXM_THROW(75, "Write error : cannot write decoded end of block : %s",
|
||||
EXM_THROW(95, "Write error : cannot write decoded end of block : %s",
|
||||
strerror(errno));
|
||||
} } } }
|
||||
|
||||
@ -1689,8 +1705,9 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
|
||||
}
|
||||
|
||||
static void
|
||||
FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
|
||||
FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
|
||||
{
|
||||
if (prefs->testMode) assert(storedSkips == 0);
|
||||
if (storedSkips>0) {
|
||||
assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
|
||||
(void)prefs; /* assert can be disabled, in which case prefs becomes unused */
|
||||
@ -1707,7 +1724,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
|
||||
|
||||
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
|
||||
@return : 0 (no error) */
|
||||
static int FIO_passThrough(FIO_prefs_t* const prefs,
|
||||
static int FIO_passThrough(const FIO_prefs_t* const prefs,
|
||||
FILE* foutput, FILE* finput,
|
||||
void* buffer, size_t bufferSize,
|
||||
size_t alreadyLoaded)
|
||||
@ -1747,7 +1764,10 @@ static unsigned FIO_highbit64(unsigned long long v)
|
||||
|
||||
/* FIO_zstdErrorHelp() :
|
||||
* detailed error message when requested window size is too large */
|
||||
static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName)
|
||||
static void
|
||||
FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
|
||||
const dRess_t* ress,
|
||||
size_t err, const char* srcFileName)
|
||||
{
|
||||
ZSTD_frameHeader header;
|
||||
|
||||
@ -1779,12 +1799,10 @@ static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t er
|
||||
* @return : size of decoded zstd frame, or an error code
|
||||
*/
|
||||
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
|
||||
static unsigned long long FIO_decompressZstdFrame(
|
||||
FIO_prefs_t* const prefs,
|
||||
dRess_t* ress,
|
||||
FILE* finput,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded)
|
||||
static unsigned long long
|
||||
FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
|
||||
dRess_t* ress, FILE* finput,
|
||||
const char* srcFileName, U64 alreadyDecoded)
|
||||
{
|
||||
U64 frameSize = 0;
|
||||
U32 storedSkips = 0;
|
||||
@ -1848,13 +1866,16 @@ static unsigned long long FIO_decompressZstdFrame(
|
||||
|
||||
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
|
||||
FILE* srcFile, const char* srcFileName)
|
||||
static unsigned long long
|
||||
FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
|
||||
dRess_t* ress, FILE* srcFile,
|
||||
const char* srcFileName)
|
||||
{
|
||||
unsigned long long outFileSize = 0;
|
||||
z_stream strm;
|
||||
int flush = Z_NO_FLUSH;
|
||||
int decodingError = 0;
|
||||
unsigned storedSkips = 0;
|
||||
|
||||
strm.zalloc = Z_NULL;
|
||||
strm.zfree = Z_NULL;
|
||||
@ -1889,10 +1910,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
|
||||
}
|
||||
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
|
||||
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
|
||||
decodingError = 1; break;
|
||||
}
|
||||
storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = (Bytef*)ress->dstBuffer;
|
||||
strm.avail_out = (uInt)ress->dstBufferSize;
|
||||
@ -1909,19 +1927,24 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
|
||||
DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
|
||||
decodingError = 1;
|
||||
}
|
||||
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
|
||||
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
|
||||
static unsigned long long
|
||||
FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
|
||||
dRess_t* ress, FILE* srcFile,
|
||||
const char* srcFileName, int plain_lzma)
|
||||
{
|
||||
unsigned long long outFileSize = 0;
|
||||
lzma_stream strm = LZMA_STREAM_INIT;
|
||||
lzma_action action = LZMA_RUN;
|
||||
lzma_ret initRet;
|
||||
int decodingError = 0;
|
||||
unsigned storedSkips = 0;
|
||||
|
||||
strm.next_in = 0;
|
||||
strm.avail_in = 0;
|
||||
@ -1964,10 +1987,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
|
||||
}
|
||||
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
|
||||
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
|
||||
decodingError = 1; break;
|
||||
}
|
||||
storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = (BYTE*)ress->dstBuffer;
|
||||
strm.avail_out = ress->dstBufferSize;
|
||||
@ -1979,19 +1999,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
|
||||
memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
|
||||
ress->srcBufferLoaded = strm.avail_in;
|
||||
lzma_end(&strm);
|
||||
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
|
||||
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
|
||||
FILE* srcFile, const char* srcFileName)
|
||||
static unsigned long long
|
||||
FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
|
||||
dRess_t* ress, FILE* srcFile,
|
||||
const char* srcFileName)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
LZ4F_errorCode_t nextToLoad;
|
||||
LZ4F_decompressionContext_t dCtx;
|
||||
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
|
||||
int decodingError = 0;
|
||||
unsigned storedSkips = 0;
|
||||
|
||||
if (LZ4F_isError(errorCode)) {
|
||||
DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
|
||||
@ -2035,10 +2059,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
|
||||
|
||||
/* Write Block */
|
||||
if (decodedBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) {
|
||||
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
|
||||
decodingError = 1; nextToLoad = 0; break;
|
||||
}
|
||||
storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips);
|
||||
filesize += decodedBytes;
|
||||
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
|
||||
}
|
||||
@ -2059,6 +2080,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
|
||||
|
||||
LZ4F_freeDecompressionContext(dCtx);
|
||||
ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */
|
||||
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
|
||||
|
||||
return decodingError ? FIO_ERROR_FRAME_DECODING : filesize;
|
||||
}
|
||||
@ -2072,8 +2094,9 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
|
||||
* @return : 0 : OK
|
||||
* 1 : error
|
||||
*/
|
||||
static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile,
|
||||
const char* dstFileName, const char* srcFileName)
|
||||
static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
|
||||
dRess_t ress, FILE* srcFile,
|
||||
const char* dstFileName, const char* srcFileName)
|
||||
{
|
||||
unsigned readSomething = 0;
|
||||
unsigned long long filesize = 0;
|
||||
@ -2105,7 +2128,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
|
||||
filesize += frameSize;
|
||||
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
|
||||
unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName);
|
||||
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
|
||||
filesize += frameSize;
|
||||
#else
|
||||
@ -2115,7 +2138,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
|
||||
} else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
|
||||
|| (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
|
||||
unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD);
|
||||
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
|
||||
filesize += frameSize;
|
||||
#else
|
||||
@ -2124,7 +2147,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
|
||||
#endif
|
||||
} else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName);
|
||||
unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName);
|
||||
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
|
||||
filesize += frameSize;
|
||||
#else
|
||||
@ -2164,11 +2187,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
|
||||
int transfer_permissions = 0;
|
||||
int releaseDstFile = 0;
|
||||
|
||||
if (ress.dstFile == NULL) {
|
||||
if ((ress.dstFile == NULL) && (prefs->testMode==0)) {
|
||||
releaseDstFile = 1;
|
||||
|
||||
ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
|
||||
if (ress.dstFile==0) return 1;
|
||||
if (ress.dstFile==NULL) return 1;
|
||||
|
||||
/* Must only be added after FIO_openDstFile() succeeds.
|
||||
* Otherwise we may delete the destination file if it already exists,
|
||||
@ -2181,7 +2204,6 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
|
||||
transfer_permissions = 1;
|
||||
}
|
||||
|
||||
|
||||
result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
|
||||
|
||||
if (releaseDstFile) {
|
||||
@ -2278,7 +2300,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
|
||||
char* outDirFilename = NULL;
|
||||
size_t sfnSize = strlen(srcFileName);
|
||||
size_t suffixSize;
|
||||
|
||||
|
||||
const char* const suffixPtr = strrchr(srcFileName, '.');
|
||||
if (suffixPtr == NULL) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
|
||||
@ -2328,7 +2350,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
|
||||
dfnbCapacity = sfnSize + 20;
|
||||
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
|
||||
if (dstFileNameBuffer==NULL)
|
||||
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
|
||||
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
|
||||
}
|
||||
|
||||
/* return dst name == src name truncated from suffix */
|
||||
@ -2357,11 +2379,13 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
|
||||
if (outFileName) {
|
||||
unsigned u;
|
||||
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
|
||||
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
|
||||
if (!prefs->testMode) {
|
||||
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
|
||||
if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
|
||||
}
|
||||
for (u=0; u<nbFiles; u++)
|
||||
error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]);
|
||||
if (fclose(ress.dstFile))
|
||||
if ((!prefs->testMode) && (fclose(ress.dstFile)))
|
||||
EXM_THROW(72, "Write error : %s : cannot properly close output file",
|
||||
strerror(errno));
|
||||
} else {
|
||||
|
||||
@ -26,7 +26,7 @@ extern "C" {
|
||||
#define stdinmark "/*stdin*\\"
|
||||
#define stdoutmark "/*stdout*\\"
|
||||
#ifdef _WIN32
|
||||
# define nulmark "nul"
|
||||
# define nulmark "NUL"
|
||||
#else
|
||||
# define nulmark "/dev/null"
|
||||
#endif
|
||||
@ -74,6 +74,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
|
||||
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
|
||||
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
|
||||
void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
|
||||
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
|
||||
void FIO_setLiteralCompressionMode(
|
||||
FIO_prefs_t* const prefs,
|
||||
ZSTD_literalCompressionMode_e mode);
|
||||
@ -85,14 +86,14 @@ void FIO_setNotificationLevel(int level);
|
||||
* Single File functions
|
||||
***************************************/
|
||||
/** FIO_compressFilename() :
|
||||
@return : 0 == ok; 1 == pb with src file. */
|
||||
* @return : 0 == ok; 1 == pb with src file. */
|
||||
int FIO_compressFilename (FIO_prefs_t* const prefs,
|
||||
const char* outfilename, const char* infilename,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters comprParams);
|
||||
|
||||
/** FIO_decompressFilename() :
|
||||
@return : 0 == ok; 1 == pb with src file. */
|
||||
* @return : 0 == ok; 1 == pb with src file. */
|
||||
int FIO_decompressFilename (FIO_prefs_t* const prefs,
|
||||
const char* outfilename, const char* infilename, const char* dictFileName);
|
||||
|
||||
@ -103,15 +104,16 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
* Multiple File functions
|
||||
***************************************/
|
||||
/** FIO_compressMultipleFilenames() :
|
||||
@return : nb of missing files */
|
||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
|
||||
const char* outDirName, unsigned nbFiles,
|
||||
const char* outFileName, const char* suffix,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
* @return : nb of missing files */
|
||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
const char** inFileNamesTable, unsigned nbFiles,
|
||||
const char* outDirName,
|
||||
const char* outFileName, const char* suffix,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters comprParams);
|
||||
|
||||
/** FIO_decompressMultipleFilenames() :
|
||||
@return : nb of missing or skipped files */
|
||||
* @return : nb of missing or skipped files */
|
||||
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
const char** srcNamesTable, unsigned nbFiles,
|
||||
const char* outDirName,
|
||||
@ -119,10 +121,12 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
const char* dictFileName);
|
||||
|
||||
/* FIO_checkFilenameCollisions() :
|
||||
* Checks for and warns if thereå are any files that would have the same output path
|
||||
* Checks for and warns if there are any files that would have the same output path
|
||||
*/
|
||||
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
|
||||
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Advanced stuff (should actually be hosted elsewhere)
|
||||
***************************************/
|
||||
|
||||
@ -72,7 +72,8 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
|
||||
#else
|
||||
{
|
||||
/* (atime, mtime) */
|
||||
struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim };
|
||||
struct timespec timebuf[2] = { {0, UTIME_NOW} };
|
||||
timebuf[1] = statbuf->st_mtim;
|
||||
res += utimensat(AT_FDCWD, filename, timebuf, 0);
|
||||
}
|
||||
#endif
|
||||
@ -105,20 +106,23 @@ int UTIL_compareStr(const void *p1, const void *p2) {
|
||||
return strcmp(* (char * const *) p1, * (char * const *) p2);
|
||||
}
|
||||
|
||||
int UTIL_isSameFile(const char* file1, const char* file2)
|
||||
int UTIL_isSameFile(const char* fName1, const char* fName2)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
assert(fName1 != NULL); assert(fName2 != NULL);
|
||||
#if defined(_MSC_VER) || defined(_WIN32)
|
||||
/* note : Visual does not support file identification by inode.
|
||||
* inode does not work on Windows, even with a posix layer, like msys2.
|
||||
* The following work-around is limited to detecting exact name repetition only,
|
||||
* aka `filename` is considered different from `subdir/../filename` */
|
||||
return !strcmp(file1, file2);
|
||||
return !strcmp(fName1, fName2);
|
||||
#else
|
||||
stat_t file1Stat;
|
||||
stat_t file2Stat;
|
||||
return UTIL_getFileStat(file1, &file1Stat)
|
||||
&& UTIL_getFileStat(file2, &file2Stat)
|
||||
&& (file1Stat.st_dev == file2Stat.st_dev)
|
||||
&& (file1Stat.st_ino == file2Stat.st_ino);
|
||||
{ stat_t file1Stat;
|
||||
stat_t file2Stat;
|
||||
return UTIL_getFileStat(fName1, &file1Stat)
|
||||
&& UTIL_getFileStat(fName2, &file2Stat)
|
||||
&& (file1Stat.st_dev == file2Stat.st_dev)
|
||||
&& (file1Stat.st_ino == file2Stat.st_ino);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -239,19 +243,20 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
|
||||
DIR *dir;
|
||||
struct dirent *entry;
|
||||
char* path;
|
||||
int dirLength, fnameLength, pathLength, nbFiles = 0;
|
||||
size_t dirLength, fnameLength, pathLength;
|
||||
int nbFiles = 0;
|
||||
|
||||
if (!(dir = opendir(dirName))) {
|
||||
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
|
||||
return 0;
|
||||
}
|
||||
|
||||
dirLength = (int)strlen(dirName);
|
||||
dirLength = strlen(dirName);
|
||||
errno = 0;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp (entry->d_name, "..") == 0 ||
|
||||
strcmp (entry->d_name, ".") == 0) continue;
|
||||
fnameLength = (int)strlen(entry->d_name);
|
||||
fnameLength = strlen(entry->d_name);
|
||||
path = (char*) malloc(dirLength + fnameLength + 2);
|
||||
if (!path) { closedir(dir); return 0; }
|
||||
memcpy(path, dirName, dirLength);
|
||||
@ -273,7 +278,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
|
||||
} else {
|
||||
if (*bufStart + *pos + pathLength >= *bufEnd) {
|
||||
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
|
||||
assert(newListSize >= 0);
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
|
||||
*bufEnd = *bufStart + newListSize;
|
||||
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
|
||||
}
|
||||
@ -322,7 +328,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
|
||||
unsigned i, nbFiles;
|
||||
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
|
||||
char* bufend = buf + LIST_SIZE_INCREASE;
|
||||
const char** fileTable;
|
||||
|
||||
if (!buf) return NULL;
|
||||
|
||||
@ -331,36 +336,37 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
|
||||
size_t const len = strlen(inputNames[i]);
|
||||
if (buf + pos + len >= bufend) {
|
||||
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
|
||||
buf = (char*)UTIL_realloc(buf, newListSize);
|
||||
assert(newListSize >= 0);
|
||||
buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
|
||||
bufend = buf + newListSize;
|
||||
if (!buf) return NULL;
|
||||
}
|
||||
if (buf + pos + len < bufend) {
|
||||
memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */
|
||||
memcpy(buf+pos, inputNames[i], len+1); /* including final \0 */
|
||||
pos += len + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
} else {
|
||||
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
|
||||
nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
|
||||
if (buf == NULL) return NULL;
|
||||
} }
|
||||
|
||||
if (nbFiles == 0) { free(buf); return NULL; }
|
||||
|
||||
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
|
||||
if (!fileTable) { free(buf); return NULL; }
|
||||
{ const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
|
||||
if (!fileTable) { free(buf); return NULL; }
|
||||
|
||||
for (i=0, pos=0; i<nbFiles; i++) {
|
||||
fileTable[i] = buf + pos;
|
||||
pos += strlen(fileTable[i]) + 1;
|
||||
for (i = 0, pos = 0; i < nbFiles; i++) {
|
||||
fileTable[i] = buf + pos;
|
||||
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
|
||||
pos += strlen(fileTable[i]) + 1;
|
||||
}
|
||||
|
||||
*allocatedBuffer = buf;
|
||||
*allocatedNamesNb = nbFiles;
|
||||
|
||||
return fileTable;
|
||||
}
|
||||
|
||||
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
|
||||
|
||||
*allocatedBuffer = buf;
|
||||
*allocatedNamesNb = nbFiles;
|
||||
|
||||
return fileTable;
|
||||
}
|
||||
|
||||
|
||||
@ -393,8 +399,13 @@ int UTIL_countPhysicalCores(void)
|
||||
DWORD returnLength = 0;
|
||||
size_t byteOffset = 0;
|
||||
|
||||
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
|
||||
"GetLogicalProcessorInformation");
|
||||
#if defined(_MSC_VER)
|
||||
/* Visual Studio does not like the following cast */
|
||||
# pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */
|
||||
# pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */
|
||||
#endif
|
||||
glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
|
||||
"GetLogicalProcessorInformation");
|
||||
|
||||
if (glpi == NULL) {
|
||||
goto failed;
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTD" "1" "September 2019" "zstd 1.4.4" "User Commands"
|
||||
.TH "ZSTD" "1" "October 2019" "zstd 1.4.4" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
@ -206,6 +206,9 @@ add integrity check computed from uncompressed data (default: enabled)
|
||||
\fB\-\-\fR
|
||||
All arguments after \fB\-\-\fR are treated as files
|
||||
.
|
||||
.SS "Restricted usage of Environment Variables"
|
||||
Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR is supported currently, for setting compression level\. \fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. It can be overridden by corresponding command line arguments\.
|
||||
.
|
||||
.SH "DICTIONARY BUILDER"
|
||||
\fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\.
|
||||
.
|
||||
|
||||
@ -214,6 +214,16 @@ the last one takes effect.
|
||||
* `--`:
|
||||
All arguments after `--` are treated as files
|
||||
|
||||
### Restricted usage of Environment Variables
|
||||
|
||||
Using environment variables to set parameters has security implications.
|
||||
Therefore, this avenue is intentionally restricted.
|
||||
Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
|
||||
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
|
||||
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
|
||||
`ZSTD_CLEVEL` just replaces the default compression level (`3`).
|
||||
It can be overridden by corresponding command line arguments.
|
||||
|
||||
|
||||
DICTIONARY BUILDER
|
||||
------------------
|
||||
|
||||
@ -287,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
*/
|
||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
static int longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
@ -430,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
|
||||
static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
|
||||
{
|
||||
for ( ; ;) {
|
||||
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
DISPLAYLEVEL(4, "invalid compression parameter \n");
|
||||
return 0;
|
||||
}
|
||||
@ -526,7 +526,7 @@ static int init_cLevel(void) {
|
||||
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
|
||||
return ZSTDCLI_CLEVEL_DEFAULT;
|
||||
} else if (*ptr == 0) {
|
||||
return sign * absLevel;
|
||||
return sign * (int)absLevel;
|
||||
}
|
||||
}
|
||||
|
||||
@ -584,7 +584,7 @@ int main(int argCount, const char* argv[])
|
||||
int cLevelLast = -1000000000;
|
||||
unsigned recursive = 0;
|
||||
unsigned memLimit = 0;
|
||||
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
|
||||
const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*)); /* argCount >= 1 */
|
||||
unsigned filenameIdx = 0;
|
||||
const char* programName = argv[0];
|
||||
const char* outFileName = NULL;
|
||||
@ -745,7 +745,7 @@ int main(int argCount, const char* argv[])
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
@ -807,7 +807,7 @@ int main(int argCount, const char* argv[])
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
/* compression Level */
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
dictCLevel = cLevel = readU32FromChar(&argument);
|
||||
dictCLevel = cLevel = (int)readU32FromChar(&argument);
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
@ -856,7 +856,7 @@ int main(int argCount, const char* argv[])
|
||||
|
||||
/* destination file name */
|
||||
case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
|
||||
|
||||
|
||||
/* limit decompression memory */
|
||||
case 'M':
|
||||
argument++;
|
||||
@ -879,7 +879,7 @@ int main(int argCount, const char* argv[])
|
||||
case 'e':
|
||||
/* compression Level */
|
||||
argument++;
|
||||
cLevelLast = readU32FromChar(&argument);
|
||||
cLevelLast = (int)readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* Modify Nb Iterations (benchmark only) */
|
||||
@ -905,7 +905,7 @@ int main(int argCount, const char* argv[])
|
||||
/* nb of threads (hidden option) */
|
||||
case 'T':
|
||||
argument++;
|
||||
nbWorkers = readU32FromChar(&argument);
|
||||
nbWorkers = (int)readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* Dictionary Selection level */
|
||||
@ -1042,16 +1042,16 @@ int main(int argCount, const char* argv[])
|
||||
#ifndef ZSTD_NOBENCH
|
||||
benchParams.blockSize = blockSize;
|
||||
benchParams.nbWorkers = nbWorkers;
|
||||
benchParams.realTime = setRealTimePrio;
|
||||
benchParams.realTime = (unsigned)setRealTimePrio;
|
||||
benchParams.nbSeconds = bench_nbSeconds;
|
||||
benchParams.ldmFlag = ldmFlag;
|
||||
benchParams.ldmMinMatch = g_ldmMinMatch;
|
||||
benchParams.ldmHashLog = g_ldmHashLog;
|
||||
benchParams.ldmMinMatch = (int)g_ldmMinMatch;
|
||||
benchParams.ldmHashLog = (int)g_ldmHashLog;
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
|
||||
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
|
||||
benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
|
||||
}
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
|
||||
benchParams.ldmHashRateLog = g_ldmHashRateLog;
|
||||
benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
|
||||
}
|
||||
benchParams.literalCompressionMode = literalCompressionMode;
|
||||
|
||||
@ -1092,16 +1092,16 @@ int main(int argCount, const char* argv[])
|
||||
#ifndef ZSTD_NODICT
|
||||
ZDICT_params_t zParams;
|
||||
zParams.compressionLevel = dictCLevel;
|
||||
zParams.notificationLevel = g_displayLevel;
|
||||
zParams.notificationLevel = (unsigned)g_displayLevel;
|
||||
zParams.dictID = dictID;
|
||||
if (dict == cover) {
|
||||
int const optimize = !coverParams.k || !coverParams.d;
|
||||
coverParams.nbThreads = nbWorkers;
|
||||
coverParams.nbThreads = (unsigned)nbWorkers;
|
||||
coverParams.zParams = zParams;
|
||||
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
|
||||
} else if (dict == fastCover) {
|
||||
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
|
||||
fastCoverParams.nbThreads = nbWorkers;
|
||||
fastCoverParams.nbThreads = (unsigned)nbWorkers;
|
||||
fastCoverParams.zParams = zParams;
|
||||
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
|
||||
} else {
|
||||
@ -1120,7 +1120,7 @@ int main(int argCount, const char* argv[])
|
||||
}
|
||||
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
|
||||
if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
|
||||
#endif
|
||||
|
||||
/* No input filename ==> use stdin and stdout */
|
||||
@ -1156,14 +1156,14 @@ int main(int argCount, const char* argv[])
|
||||
if (operation==zom_compress) {
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
FIO_setNbWorkers(prefs, nbWorkers);
|
||||
FIO_setBlockSize(prefs, (U32)blockSize);
|
||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog);
|
||||
FIO_setLdmFlag(prefs, ldmFlag);
|
||||
FIO_setLdmHashLog(prefs, g_ldmHashLog);
|
||||
FIO_setLdmMinMatch(prefs, g_ldmMinMatch);
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog);
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog);
|
||||
FIO_setAdaptiveMode(prefs, adapt);
|
||||
FIO_setBlockSize(prefs, (int)blockSize);
|
||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
|
||||
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
|
||||
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
|
||||
FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
|
||||
FIO_setAdaptiveMode(prefs, (unsigned)adapt);
|
||||
FIO_setAdaptMin(prefs, adaptMin);
|
||||
FIO_setAdaptMax(prefs, adaptMax);
|
||||
FIO_setRsyncable(prefs, rsyncable);
|
||||
@ -1177,7 +1177,7 @@ int main(int argCount, const char* argv[])
|
||||
if ((filenameIdx==1) && outFileName)
|
||||
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
|
||||
else
|
||||
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
#else
|
||||
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
|
||||
DISPLAY("Compression not supported \n");
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTDGREP" "1" "September 2019" "zstd 1.4.4" "User Commands"
|
||||
.TH "ZSTDGREP" "1" "October 2019" "zstd 1.4.4" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTDLESS" "1" "September 2019" "zstd 1.4.4" "User Commands"
|
||||
.TH "ZSTDLESS" "1" "October 2019" "zstd 1.4.4" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdless\fR \- view zstandard\-compressed files
|
||||
|
||||
@ -250,7 +250,8 @@ clean:
|
||||
$(MAKE) -C $(ZSTDDIR) clean
|
||||
$(MAKE) -C $(PRGDIR) clean
|
||||
@$(RM) -fR $(TESTARTEFACT)
|
||||
@$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \
|
||||
@$(RM) -rf tmp* # some test directories are named tmp*
|
||||
@$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
|
||||
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fullbench-lib$(EXT) fullbench-dll$(EXT) \
|
||||
@ -264,7 +265,7 @@ clean:
|
||||
|
||||
|
||||
#----------------------------------------------------------------------------------
|
||||
#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets
|
||||
# valgrind tests are validated only for some posix platforms
|
||||
#----------------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||
HOST_OS = POSIX
|
||||
@ -286,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench
|
||||
endif
|
||||
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||
ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
|
||||
HOST_OS = MSYS
|
||||
endif
|
||||
|
||||
|
||||
@ -125,13 +125,14 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
|
||||
{
|
||||
size_t neededBufSize;
|
||||
|
||||
/* Give a random portion of src data to the producer, to use for
|
||||
parameter generation. The rest will be used for (de)compression */
|
||||
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
|
||||
size = FUZZ_dataProducer_reserveDataPrefix(producer);
|
||||
|
||||
size_t neededBufSize;
|
||||
neededBufSize = ZSTD_compressBound(size) * 5;
|
||||
neededBufSize = ZSTD_compressBound(size) * 15;
|
||||
|
||||
/* Allocate all buffers and contexts if not already allocated */
|
||||
if (neededBufSize > bufSize) {
|
||||
|
||||
@ -1387,7 +1387,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
|
||||
if (cdict==NULL) goto _output_error;
|
||||
@ -1395,7 +1395,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
|
||||
if (cdict!=NULL) goto _output_error;
|
||||
@ -1403,7 +1403,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
|
||||
{
|
||||
size_t ret;
|
||||
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
|
||||
@ -1417,7 +1417,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
|
||||
{
|
||||
size_t ret;
|
||||
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
|
||||
|
||||
@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs"
|
||||
TESTDIR="$SCRIPT_DIR/../tests"
|
||||
UNAME=$(uname)
|
||||
|
||||
isTerminal=false
|
||||
detectedTerminal=false
|
||||
if [ -t 0 ] && [ -t 1 ]
|
||||
then
|
||||
isTerminal=true
|
||||
detectedTerminal=true
|
||||
fi
|
||||
isTerminal=${isTerminal:-$detectedTerminal}
|
||||
|
||||
isWindows=false
|
||||
INTOVOID="/dev/null"
|
||||
@ -211,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t
|
||||
$ZSTD tmp -c --compress-literals -1 | $ZSTD -t
|
||||
$ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t
|
||||
$ZSTD tmp -c --compress-literals -19 | $ZSTD -t
|
||||
$ZSTD -b --fast=1 -i1e1 tmp --compress-literals
|
||||
$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals
|
||||
$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
|
||||
$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
|
||||
|
||||
println "test : file removal"
|
||||
$ZSTD -f --rm tmp
|
||||
@ -239,7 +240,7 @@ rm tmp # erase source file
|
||||
touch tmp.zst # create destination file
|
||||
$ZSTD -f tmp && die "attempt to compress a non existing file"
|
||||
test -f tmp.zst # destination file should still be present
|
||||
rm tmp*
|
||||
rm -rf tmp* # may also erase tmp* directory from previous failed run
|
||||
|
||||
println "\n===> decompression only tests "
|
||||
head -c 1048576 /dev/zero > tmp
|
||||
@ -283,7 +284,7 @@ test -f tmpOutDir/tmp1.zst
|
||||
test -f tmpOutDir/tmp2.zst
|
||||
println "test : decompress multiple files into an output directory, --output-dir-flat"
|
||||
mkdir tmpOutDirDecomp
|
||||
$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp
|
||||
$ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
|
||||
test -f tmpOutDirDecomp/tmp2
|
||||
test -f tmpOutDirDecomp/tmp1
|
||||
rm -rf tmp*
|
||||
@ -577,30 +578,27 @@ println "- Create dictionary with short dictID"
|
||||
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
|
||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||
println "- Create dictionaries with shrink-dict flag enabled"
|
||||
$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
|
||||
$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
|
||||
$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
|
||||
$ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
|
||||
$ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
|
||||
$ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
|
||||
println "- Create dictionary with size limit"
|
||||
$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
|
||||
println "- Compare size of dictionary from 90% training samples with 80% training samples"
|
||||
$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
|
||||
println "- Create dictionary using all samples for both training and testing"
|
||||
$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
println "- Create dictionary using f=16"
|
||||
$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
|
||||
$ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
|
||||
println "- Create dictionary using accel=2"
|
||||
$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
println "- Create dictionary using accel=10"
|
||||
$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
println "- Create dictionary with multithreading"
|
||||
$ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
println "- Test -o before --train-fastcover"
|
||||
rm -f tmpDict dictionary
|
||||
$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
test -f tmpDict
|
||||
$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
$ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
|
||||
test -f dictionary
|
||||
rm tmp* dictionary
|
||||
|
||||
@ -674,10 +672,10 @@ $ZSTD -i0b0e3 tmp1
|
||||
println "bench negative level"
|
||||
$ZSTD -bi0 --fast tmp1
|
||||
println "with recursive and quiet modes"
|
||||
$ZSTD -rqi1b1e2 tmp1
|
||||
$ZSTD -rqi0b1e2 tmp1
|
||||
println "benchmark decompression only"
|
||||
$ZSTD -f tmp1
|
||||
$ZSTD -b -d -i1 tmp1.zst
|
||||
$ZSTD -b -d -i0 tmp1.zst
|
||||
|
||||
println "\n===> zstd compatibility tests "
|
||||
|
||||
|
||||
@ -46,7 +46,7 @@ static int testOrder(size_t numThreads, size_t queueSize)
|
||||
POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
|
||||
ASSERT_TRUE(ctx);
|
||||
data.i = 0;
|
||||
(void)ZSTD_pthread_mutex_init(&data.mutex, NULL);
|
||||
ASSERT_FALSE(ZSTD_pthread_mutex_init(&data.mutex, NULL));
|
||||
{ size_t i;
|
||||
for (i = 0; i < 16; ++i) {
|
||||
POOL_add(ctx, &fn, &data);
|
||||
|
||||
@ -18,13 +18,15 @@ EXAMPLE_PATH = examples
|
||||
PROGRAMS_PATH = ../programs
|
||||
TEST_FILE = ../doc/zstd_compression_format.md
|
||||
|
||||
CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
|
||||
-I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
|
||||
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
|
||||
-Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
|
||||
-Wstrict-aliasing=1
|
||||
|
||||
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
|
||||
-I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
|
||||
STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \
|
||||
-DNO_snprintf -DNO_vsnprintf # strict ISO C90 is missing these prototypes
|
||||
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
|
||||
-Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
|
||||
-Wstrict-aliasing=1
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
@ -33,6 +35,11 @@ else
|
||||
EXT =
|
||||
endif
|
||||
|
||||
default : release
|
||||
|
||||
release : STDFLAGS =
|
||||
release : DEBUGFLAGS =
|
||||
release : all
|
||||
|
||||
all: fitblk example zwrapbench minigzip
|
||||
|
||||
@ -68,35 +75,34 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
|
||||
$(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
|
||||
|
||||
#.c.o:
|
||||
# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
|
||||
# $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
|
||||
|
||||
minigzip: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
minigzip: $(EXAMPLE_PATH)/minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
|
||||
|
||||
minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
|
||||
|
||||
example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
example: $(EXAMPLE_PATH)/example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
|
||||
|
||||
example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
example_zstd: $(EXAMPLE_PATH)/example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
|
||||
|
||||
fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
|
||||
|
||||
fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
|
||||
|
||||
zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
|
||||
zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
|
||||
|
||||
|
||||
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
|
||||
zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||
|
||||
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
|
||||
zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
|
||||
zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||
|
||||
$(ZSTDLIBDIR)/libzstd.a:
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
|
||||
|
||||
@ -159,7 +159,7 @@ int main(int argc, char **argv)
|
||||
if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
|
||||
|
||||
/* allocate memory for buffers and compression engine */
|
||||
blk = malloc(size + EXCESS);
|
||||
blk = (unsigned char*)malloc(size + EXCESS);
|
||||
def.zalloc = Z_NULL;
|
||||
def.zfree = Z_NULL;
|
||||
def.opaque = Z_NULL;
|
||||
@ -180,8 +180,8 @@ int main(int argc, char **argv)
|
||||
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
|
||||
/* write block to stdout */
|
||||
have = size + EXCESS - def.avail_out;
|
||||
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
// quit("error writing output");
|
||||
/* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
* quit("error writing output"); */
|
||||
|
||||
/* clean up and print results to stderr */
|
||||
ret = deflateEnd(&def);
|
||||
@ -200,7 +200,7 @@ int main(int argc, char **argv)
|
||||
inf.avail_in = 0;
|
||||
inf.next_in = Z_NULL;
|
||||
ret = inflateInit(&inf);
|
||||
tmp = malloc(size + EXCESS);
|
||||
tmp = (unsigned char*)malloc(size + EXCESS);
|
||||
if (ret != Z_OK || tmp == NULL)
|
||||
quit("out of memory");
|
||||
ret = deflateReset(&def);
|
||||
@ -237,8 +237,8 @@ int main(int argc, char **argv)
|
||||
|
||||
/* done -- write block to stdout */
|
||||
have = size - def.avail_out;
|
||||
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
// quit("error writing output");
|
||||
/* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
* quit("error writing output"); */
|
||||
|
||||
/* clean up and print results to stderr */
|
||||
free(tmp);
|
||||
|
||||
@ -311,14 +311,14 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
|
||||
ret = deflateReset(&def);
|
||||
if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
|
||||
if (useSetDict) {
|
||||
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
|
||||
ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
|
||||
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
|
||||
if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
|
||||
}
|
||||
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
|
||||
def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
|
||||
def.avail_in = (uInt)blockTable[blockNb].srcSize;
|
||||
def.total_in = 0;
|
||||
def.next_out = (void*) blockTable[blockNb].cPtr;
|
||||
def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
|
||||
def.avail_out = (uInt)blockTable[blockNb].cRoom;
|
||||
def.total_out = 0;
|
||||
ret = deflate(&def, Z_FINISH);
|
||||
@ -343,13 +343,13 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
|
||||
ret = deflateInit(&def, cLevel);
|
||||
if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
|
||||
if (dictBuffer) {
|
||||
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
|
||||
ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
|
||||
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
|
||||
}
|
||||
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
|
||||
def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
|
||||
def.avail_in = (uInt)blockTable[blockNb].srcSize;
|
||||
def.total_in = 0;
|
||||
def.next_out = (void*) blockTable[blockNb].cPtr;
|
||||
def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
|
||||
def.avail_out = (uInt)blockTable[blockNb].cRoom;
|
||||
def.total_out = 0;
|
||||
ret = deflate(&def, Z_FINISH);
|
||||
@ -451,15 +451,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
|
||||
else
|
||||
ret = inflateReset(&inf);
|
||||
if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
|
||||
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
|
||||
inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
|
||||
inf.avail_in = (uInt)blockTable[blockNb].cSize;
|
||||
inf.total_in = 0;
|
||||
inf.next_out = (void*) blockTable[blockNb].resPtr;
|
||||
inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
|
||||
inf.avail_out = (uInt)blockTable[blockNb].srcSize;
|
||||
inf.total_out = 0;
|
||||
ret = inflate(&inf, Z_FINISH);
|
||||
if (ret == Z_NEED_DICT) {
|
||||
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize);
|
||||
ret = inflateSetDictionary(&inf, (const z_Bytef*)dictBuffer, dictBufferSize);
|
||||
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
|
||||
ret = inflate(&inf, Z_FINISH);
|
||||
}
|
||||
@ -483,15 +483,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
|
||||
inf.opaque = Z_NULL;
|
||||
ret = inflateInit(&inf);
|
||||
if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
|
||||
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
|
||||
inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
|
||||
inf.avail_in = (uInt)blockTable[blockNb].cSize;
|
||||
inf.total_in = 0;
|
||||
inf.next_out = (void*) blockTable[blockNb].resPtr;
|
||||
inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
|
||||
inf.avail_out = (uInt)blockTable[blockNb].srcSize;
|
||||
inf.total_out = 0;
|
||||
ret = inflate(&inf, Z_FINISH);
|
||||
if (ret == Z_NEED_DICT) {
|
||||
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize);
|
||||
ret = inflateSetDictionary(&inf, (const z_Bytef*) dictBuffer, dictBufferSize);
|
||||
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
|
||||
ret = inflate(&inf, Z_FINISH);
|
||||
}
|
||||
|
||||
@ -19,7 +19,7 @@ int ZEXPORT gzclose(file)
|
||||
|
||||
if (file == NULL)
|
||||
return Z_STREAM_ERROR;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file);
|
||||
#else
|
||||
|
||||
@ -216,7 +216,7 @@ local gzFile gz_open(path, fd, mode)
|
||||
#if !defined(NO_snprintf) && !defined(NO_vsnprintf)
|
||||
(void)snprintf(state.state->path, len + 1, "%s", (const char *)path);
|
||||
#else
|
||||
strcpy(state.state->path, path);
|
||||
strcpy(state.state->path, (const char*)path);
|
||||
#endif
|
||||
|
||||
/* compute the flags for open() */
|
||||
@ -325,7 +325,7 @@ int ZEXPORT gzbuffer(file, size)
|
||||
/* get internal structure and check integrity */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
|
||||
return -1;
|
||||
|
||||
@ -351,7 +351,7 @@ int ZEXPORT gzrewind(file)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're reading and that there's no error */
|
||||
if (state.state->mode != GZ_READ ||
|
||||
@ -378,7 +378,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence)
|
||||
/* get internal structure and check integrity */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
|
||||
return -1;
|
||||
|
||||
@ -463,7 +463,7 @@ z_off64_t ZEXPORT gztell64(file)
|
||||
/* get internal structure and check integrity */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
|
||||
return -1;
|
||||
|
||||
@ -491,7 +491,7 @@ z_off64_t ZEXPORT gzoffset64(file)
|
||||
/* get internal structure and check integrity */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
|
||||
return -1;
|
||||
|
||||
@ -523,7 +523,7 @@ int ZEXPORT gzeof(file)
|
||||
/* get internal structure and check integrity */
|
||||
if (file == NULL)
|
||||
return 0;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
|
||||
return 0;
|
||||
|
||||
@ -541,7 +541,7 @@ const char * ZEXPORT gzerror(file, errnum)
|
||||
/* get internal structure and check integrity */
|
||||
if (file == NULL)
|
||||
return NULL;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
|
||||
return NULL;
|
||||
|
||||
@ -561,7 +561,7 @@ void ZEXPORT gzclearerr(file)
|
||||
/* get internal structure and check integrity */
|
||||
if (file == NULL)
|
||||
return;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
|
||||
return;
|
||||
|
||||
|
||||
@ -386,7 +386,7 @@ int ZEXPORT gzread(file, buf, len)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're reading and that there's no (serious) error */
|
||||
if (state.state->mode != GZ_READ ||
|
||||
@ -424,7 +424,7 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return 0;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're reading and that there's no (serious) error */
|
||||
if (state.state->mode != GZ_READ ||
|
||||
@ -470,7 +470,7 @@ int ZEXPORT gzgetc(file)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're reading and that there's no (serious) error */
|
||||
if (state.state->mode != GZ_READ ||
|
||||
@ -485,7 +485,7 @@ int ZEXPORT gzgetc(file)
|
||||
}
|
||||
|
||||
/* nothing there -- try gz_read() */
|
||||
ret = (unsigned)gz_read(state, buf, 1);
|
||||
ret = (int)gz_read(state, buf, 1);
|
||||
return ret < 1 ? -1 : buf[0];
|
||||
}
|
||||
|
||||
@ -505,7 +505,7 @@ int ZEXPORT gzungetc(c, file)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're reading and that there's no (serious) error */
|
||||
if (state.state->mode != GZ_READ ||
|
||||
@ -569,7 +569,7 @@ char * ZEXPORT gzgets(file, buf, len)
|
||||
/* check parameters and get internal structure */
|
||||
if (file == NULL || buf == NULL || len < 1)
|
||||
return NULL;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're reading and that there's no (serious) error */
|
||||
if (state.state->mode != GZ_READ ||
|
||||
@ -628,7 +628,7 @@ int ZEXPORT gzdirect(file)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return 0;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* if the state is not known, but we can find out, then do so (this is
|
||||
mainly for right after a gzopen() or gzdopen()) */
|
||||
@ -649,7 +649,7 @@ int ZEXPORT gzclose_r(file)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return Z_STREAM_ERROR;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're reading */
|
||||
if (state.state->mode != GZ_READ)
|
||||
|
||||
@ -258,7 +258,7 @@ int ZEXPORT gzwrite(file, buf, len)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return 0;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're writing and that there's no error */
|
||||
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
|
||||
@ -289,7 +289,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
|
||||
assert(size != 0);
|
||||
if (file == NULL)
|
||||
return 0;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're writing and that there's no error */
|
||||
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
|
||||
@ -319,7 +319,7 @@ int ZEXPORT gzputc(file, c)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
strm = &(state.state->strm);
|
||||
|
||||
/* check that we're writing and that there's no error */
|
||||
@ -366,7 +366,7 @@ int ZEXPORT gzputs(file, str)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return -1;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're writing and that there's no error */
|
||||
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
|
||||
@ -393,7 +393,7 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return Z_STREAM_ERROR;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
strm = &(state.state->strm);
|
||||
|
||||
/* check that we're writing and that there's no error */
|
||||
@ -565,7 +565,7 @@ int ZEXPORT gzflush(file, flush)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return Z_STREAM_ERROR;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're writing and that there's no error */
|
||||
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
|
||||
@ -599,7 +599,7 @@ int ZEXPORT gzsetparams(file, level, strategy)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return Z_STREAM_ERROR;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
strm = &(state.state->strm);
|
||||
|
||||
/* check that we're writing and that there's no error */
|
||||
@ -639,7 +639,7 @@ int ZEXPORT gzclose_w(file)
|
||||
/* get internal structure */
|
||||
if (file == NULL)
|
||||
return Z_STREAM_ERROR;
|
||||
state = (gz_statep)file;
|
||||
state.file = file;
|
||||
|
||||
/* check that we're writing */
|
||||
if (state.state->mode != GZ_WRITE)
|
||||
|
||||
@ -54,7 +54,7 @@ int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; }
|
||||
|
||||
static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO;
|
||||
|
||||
void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; };
|
||||
void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }
|
||||
|
||||
ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; }
|
||||
|
||||
@ -99,7 +99,7 @@ typedef struct {
|
||||
unsigned long long pledgedSrcSize;
|
||||
} ZWRAP_CCtx;
|
||||
|
||||
typedef ZWRAP_CCtx internal_state;
|
||||
/* typedef ZWRAP_CCtx internal_state; */
|
||||
|
||||
|
||||
|
||||
@ -121,8 +121,10 @@ static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
|
||||
if (zwc==NULL) return NULL;
|
||||
memset(zwc, 0, sizeof(ZWRAP_CCtx));
|
||||
memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
|
||||
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
|
||||
zwc->customMem = ZWRAP_customMem; }
|
||||
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
|
||||
ZWRAP_customMem.opaque = &zwc->allocFunc;
|
||||
zwc->customMem = ZWRAP_customMem;
|
||||
}
|
||||
} else {
|
||||
zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc));
|
||||
if (zwc==NULL) return NULL;
|
||||
@ -462,8 +464,10 @@ static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
|
||||
if (zwd==NULL) return NULL;
|
||||
memset(zwd, 0, sizeof(ZWRAP_DCtx));
|
||||
zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */
|
||||
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
|
||||
zwd->customMem = ZWRAP_customMem; }
|
||||
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
|
||||
ZWRAP_customMem.opaque = &zwd->allocFunc;
|
||||
zwd->customMem = ZWRAP_customMem;
|
||||
}
|
||||
} else {
|
||||
zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
|
||||
if (zwd==NULL) return NULL;
|
||||
@ -509,7 +513,7 @@ static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
|
||||
const char *version, int stream_size))
|
||||
const char* version, int stream_size))
|
||||
{
|
||||
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
|
||||
strm->reserved = ZWRAP_ZLIB_STREAM;
|
||||
@ -520,7 +524,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
|
||||
LOG_WRAPPERD("- inflateInit\n");
|
||||
if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
|
||||
|
||||
zwd->version = ZSTD_malloc(strlen(version)+1, zwd->customMem);
|
||||
zwd->version = (char*)ZSTD_malloc(strlen(version)+1, zwd->customMem);
|
||||
if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
|
||||
strcpy(zwd->version, version);
|
||||
|
||||
@ -1003,7 +1007,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm))
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); };
|
||||
ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); }
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user