This commit is contained in:
Sen Huang 2019-10-21 19:02:52 -04:00
commit 5b2f4ac1a8
44 changed files with 853 additions and 513 deletions

View File

@ -54,7 +54,10 @@ matrix:
- name: Trusty (clang-3.8 + MSan + Test Zstd) - name: Trusty (clang-3.8 + MSan + Test Zstd)
script: script:
- make clang38install - make clang38install
- CC=clang-3.8 make clean msan-test-zstd # External libraries must be turned off when using MSAN tests,
# because they are not msan-instrumented,
# so any data coming from these libraries is always considered "uninitialized"
- CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
- name: Trusty (Minimal Decompressor Macros) - name: Trusty (Minimal Decompressor Macros)
script: script:

View File

@ -1,3 +1,7 @@
# Following tests are run _only_ on master branch
# To reproduce these tests, it's possible to push into a branch `appveyorTest`
# or a branch `visual*`, they will intentionnally trigger `master` tests
- -
version: 1.0.{build} version: 1.0.{build}
branches: branches:
@ -176,6 +180,11 @@
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
) )
# The following tests are for regular pushes
# into `dev` or some feature branch
# There run less tests, for shorter feedback loop
- -
version: 1.0.{build} version: 1.0.{build}
environment: environment:
@ -249,3 +258,11 @@
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
) )
test_script:
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
- if [%HOST%]==[mingw] (
set "CC=%COMPILER%" &&
make check
)

2
doc/educational_decoder/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# Build artifacts
harness

View File

@ -7,8 +7,15 @@
# in the COPYING file in the root directory of this source tree). # in the COPYING file in the root directory of this source tree).
# ################################################################ # ################################################################
ZSTD ?= zstd # requires zstd installation on local system ZSTD ?= zstd # note: requires zstd installation on local system
UNAME?= $(shell uname)
ifeq ($(UNAME), SunOS)
DIFF ?= gdiff
else
DIFF ?= diff DIFF ?= diff
endif
HARNESS_FILES=*.c HARNESS_FILES=*.c
MULTITHREAD_LDFLAGS = -pthread MULTITHREAD_LDFLAGS = -pthread
@ -29,26 +36,27 @@ harness: $(HARNESS_FILES)
$(CC) $(FLAGS) $^ -o $@ $(CC) $(FLAGS) $^ -o $@
clean: clean:
@$(RM) -f harness @$(RM) harness
@$(RM) -rf harness.dSYM @$(RM) -rf harness.dSYM # MacOS specific
test: harness test: harness
# #
# Testing single-file decompression with educational decoder # Testing single-file decompression with educational decoder
# #
@$(ZSTD) README.md -o tmp.zst @$(ZSTD) -f README.md -o tmp.zst
@./harness tmp.zst tmp @./harness tmp.zst tmp
@$(DIFF) -s tmp README.md @$(DIFF) -s tmp README.md
@$(RM) -f tmp* @$(RM) tmp*
# #
# Testing dictionary decompression with education decoder # Testing dictionary decompression with education decoder
# #
# note : files are presented multiple for training, to reach minimum threshold # note : files are presented multiple for training, to reach minimum threshold
@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \ @$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md \ harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md harness.c zstd_decompress.c zstd_decompress.h README.md \
@$(ZSTD) -D dictionary README.md -o tmp.zst -o dictionary
@$(ZSTD) -f README.md -D dictionary -o tmp.zst
@./harness tmp.zst tmp dictionary @./harness tmp.zst tmp dictionary
@$(DIFF) -s tmp README.md @$(DIFF) -s tmp README.md
@$(RM) -f tmp* dictionary @$(RM) tmp* dictionary
@$(MAKE) clean @$(MAKE) clean

View File

@ -21,88 +21,90 @@ typedef unsigned char u8;
// Protect against allocating too much memory for output // Protect against allocating too much memory for output
#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024) #define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
u8 *input; static size_t read_file(const char *path, u8 **ptr)
u8 *output; {
u8 *dict; FILE* const f = fopen(path, "rb");
size_t read_file(const char *path, u8 **ptr) {
FILE *f = fopen(path, "rb");
if (!f) { if (!f) {
fprintf(stderr, "failed to open file %s\n", path); fprintf(stderr, "failed to open file %s \n", path);
exit(1); exit(1);
} }
fseek(f, 0L, SEEK_END); fseek(f, 0L, SEEK_END);
size_t size = (size_t)ftell(f); size_t const size = (size_t)ftell(f);
rewind(f); rewind(f);
*ptr = malloc(size); *ptr = malloc(size);
if (!ptr) { if (!ptr) {
fprintf(stderr, "failed to allocate memory to hold %s\n", path); fprintf(stderr, "failed to allocate memory to hold %s \n", path);
exit(1); exit(1);
} }
size_t pos = 0; size_t const read = fread(*ptr, 1, size, f);
while (!feof(f)) { if (read != size) { /* must read everything in one pass */
size_t read = fread(&(*ptr)[pos], 1, size, f); fprintf(stderr, "error while reading file %s \n", path);
if (ferror(f)) { exit(1);
fprintf(stderr, "error while reading file %s\n", path);
exit(1);
}
pos += read;
} }
fclose(f); fclose(f);
return pos; return read;
} }
void write_file(const char *path, const u8 *ptr, size_t size) { static void write_file(const char *path, const u8 *ptr, size_t size)
FILE *f = fopen(path, "wb"); {
FILE* const f = fopen(path, "wb");
if (!f) {
fprintf(stderr, "failed to open file %s \n", path);
exit(1);
}
size_t written = 0; size_t written = 0;
while (written < size) { while (written < size) {
written += fwrite(&ptr[written], 1, size, f); written += fwrite(ptr+written, 1, size, f);
if (ferror(f)) { if (ferror(f)) {
fprintf(stderr, "error while writing file %s\n", path); fprintf(stderr, "error while writing file %s\n", path);
exit(1); exit(1);
} } }
}
fclose(f); fclose(f);
} }
int main(int argc, char **argv) { int main(int argc, char **argv)
{
if (argc < 3) { if (argc < 3) {
fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary]\n", fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary] \n",
argv[0]); argv[0]);
return 1; return 1;
} }
size_t input_size = read_file(argv[1], &input); u8* input;
size_t const input_size = read_file(argv[1], &input);
u8* dict = NULL;
size_t dict_size = 0; size_t dict_size = 0;
if (argc >= 4) { if (argc >= 4) {
dict_size = read_file(argv[3], &dict); dict_size = read_file(argv[3], &dict);
} }
size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size); size_t out_capacity = ZSTD_get_decompressed_size(input, input_size);
if (decompressed_size == (size_t)-1) { if (out_capacity == (size_t)-1) {
decompressed_size = MAX_COMPRESSION_RATIO * input_size; out_capacity = MAX_COMPRESSION_RATIO * input_size;
fprintf(stderr, "WARNING: Compressed data does not contain " fprintf(stderr, "WARNING: Compressed data does not contain "
"decompressed size, going to assume the compression " "decompressed size, going to assume the compression "
"ratio is at most %d (decompressed size of at most " "ratio is at most %d (decompressed size of at most "
"%zu)\n", "%u) \n",
MAX_COMPRESSION_RATIO, decompressed_size); MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
} }
if (decompressed_size > MAX_OUTPUT_SIZE) { if (out_capacity > MAX_OUTPUT_SIZE) {
fprintf(stderr, fprintf(stderr,
"Required output size too large for this implementation\n"); "Required output size too large for this implementation \n");
return 1; return 1;
} }
output = malloc(decompressed_size);
u8* const output = malloc(out_capacity);
if (!output) { if (!output) {
fprintf(stderr, "failed to allocate memory\n"); fprintf(stderr, "failed to allocate memory \n");
return 1; return 1;
} }
@ -110,16 +112,17 @@ int main(int argc, char **argv) {
if (dict) { if (dict) {
parse_dictionary(parsed_dict, dict, dict_size); parse_dictionary(parsed_dict, dict, dict_size);
} }
size_t decompressed = size_t const decompressed_size =
ZSTD_decompress_with_dict(output, decompressed_size, ZSTD_decompress_with_dict(output, out_capacity,
input, input_size, parsed_dict); input, input_size,
parsed_dict);
free_dictionary(parsed_dict); free_dictionary(parsed_dict);
write_file(argv[2], output, decompressed); write_file(argv[2], output, decompressed_size);
free(input); free(input);
free(output); free(output);
free(dict); free(dict);
input = output = dict = NULL; return 0;
} }

View File

@ -856,8 +856,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx,
// Impossible // Impossible
IMPOSSIBLE(); IMPOSSIBLE();
} }
if (regenerated_size > MAX_LITERALS_SIZE || if (regenerated_size > MAX_LITERALS_SIZE) {
compressed_size >= regenerated_size) {
CORRUPTION(); CORRUPTION();
} }
@ -1530,7 +1529,7 @@ void free_dictionary(dictionary_t *const dict) {
/******* END DICTIONARY PARSING ***********************************************/ /******* END DICTIONARY PARSING ***********************************************/
/******* IO STREAM OPERATIONS *************************************************/ /******* IO STREAM OPERATIONS *************************************************/
#define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream")
/// Reads `num` bits from a bitstream, and updates the internal offset /// Reads `num` bits from a bitstream, and updates the internal offset
static inline u64 IO_read_bits(istream_t *const in, const int num_bits) { static inline u64 IO_read_bits(istream_t *const in, const int num_bits) {
if (num_bits > 64 || num_bits <= 0) { if (num_bits > 64 || num_bits <= 0) {
@ -1609,7 +1608,7 @@ static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len) {
INP_SIZE(); INP_SIZE();
} }
if (in->bit_offset != 0) { if (in->bit_offset != 0) {
UNALIGNED(); ERROR("Attempting to operate on a non-byte aligned stream");
} }
const u8 *const ptr = in->ptr; const u8 *const ptr = in->ptr;
in->ptr += len; in->ptr += len;
@ -1635,7 +1634,7 @@ static inline void IO_advance_input(istream_t *const in, size_t len) {
INP_SIZE(); INP_SIZE();
} }
if (in->bit_offset != 0) { if (in->bit_offset != 0) {
UNALIGNED(); ERROR("Attempting to operate on a non-byte aligned stream");
} }
in->ptr += len; in->ptr += len;

View File

@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version ### Version
0.3.3 (16/08/19) 0.3.4 (16/08/19)
Introduction Introduction
@ -1107,18 +1107,18 @@ It follows the following build rule :
The table has a size of `Table_Size = 1 << Accuracy_Log`. The table has a size of `Table_Size = 1 << Accuracy_Log`.
Each cell describes the symbol decoded, Each cell describes the symbol decoded,
and instructions to get the next state. and instructions to get the next state (`Number_of_Bits` and `Baseline`).
Symbols are scanned in their natural order for "less than 1" probabilities. Symbols are scanned in their natural order for "less than 1" probabilities.
Symbols with this probability are being attributed a single cell, Symbols with this probability are being attributed a single cell,
starting from the end of the table and retreating. starting from the end of the table and retreating.
These symbols define a full state reset, reading `Accuracy_Log` bits. These symbols define a full state reset, reading `Accuracy_Log` bits.
All remaining symbols are allocated in their natural order. Then, all remaining symbols, sorted in natural order, are allocated cells.
Starting from symbol `0` and table position `0`, Starting from symbol `0` (if it exists), and table position `0`,
each symbol gets allocated as many cells as its probability. each symbol gets allocated as many cells as its probability.
Cell allocation is spreaded, not linear : Cell allocation is spreaded, not linear :
each successor position follow this rule : each successor position follows this rule :
``` ```
position += (tableSize>>1) + (tableSize>>3) + 3; position += (tableSize>>1) + (tableSize>>3) + 3;
@ -1130,40 +1130,41 @@ A position is skipped if already occupied by a "less than 1" probability symbol.
each position in the table, switching to the next symbol when enough each position in the table, switching to the next symbol when enough
states have been allocated to the current one. states have been allocated to the current one.
The result is a list of state values. The process guarantees that the table is entirely filled.
Each state will decode the current symbol. Each cell corresponds to a state value, which contains the symbol being decoded.
To get the `Number_of_Bits` and `Baseline` required for next state, To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
it's first necessary to sort all states in their natural order. it's first necessary to sort all occurrences of each symbol in state order.
The lower states will need 1 more bit than higher ones. Lower states will need 1 more bit than higher ones.
The process is repeated for each symbol. The process is repeated for each symbol.
__Example__ : __Example__ :
Presuming a symbol has a probability of 5. Presuming a symbol has a probability of 5,
It receives 5 state values. States are sorted in natural order. it receives 5 cells, corresponding to 5 state values.
These state values are then sorted in natural order.
Next power of 2 is 8. Next power of 2 after 5 is 8.
Space of probabilities is divided into 8 equal parts. Space of probabilities must be divided into 8 equal parts.
Presuming the `Accuracy_Log` is 7, it defines 128 states. Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
Divided by 8, each share is 16 large. Divided by 8, each share is 16 large.
In order to reach 8, 8-5=3 lowest states will count "double", In order to reach 8 shares, 8-5=3 lowest states will count "double",
doubling the number of shares (32 in width), doubling their shares (32 in width), hence requiring one more bit.
requiring one more bit in the process.
Baseline is assigned starting from the higher states using fewer bits, Baseline is assigned starting from the higher states using fewer bits,
and proceeding naturally, then resuming at the first state, increasing at each state, then resuming at the first state,
each takes its allocated width from Baseline. each state takes its allocated width from Baseline.
| state order | 0 | 1 | 2 | 3 | 4 | | state value | 1 | 39 | 77 | 84 | 122 |
| ---------------- | ----- | ----- | ------ | ---- | ----- | | state order | 0 | 1 | 2 | 3 | 4 |
| width | 32 | 32 | 32 | 16 | 16 | | ---------------- | ----- | ----- | ------ | ---- | ------ |
| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 | | width | 32 | 32 | 32 | 16 | 16 |
| range number | 2 | 4 | 6 | 0 | 1 | | `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 |
| `Baseline` | 32 | 64 | 96 | 0 | 16 | | range number | 2 | 4 | 6 | 0 | 1 |
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 | | `Baseline` | 32 | 64 | 96 | 0 | 16 |
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
The next state is determined from current state During decoding, the next state value is determined from current state value,
by reading the required `Number_of_Bits`, and adding the specified `Baseline`. by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
See [Appendix A] for the results of this process applied to the default distributions. See [Appendix A] for the results of this process applied to the default distributions.
@ -1657,6 +1658,7 @@ or at least provide a meaningful error code explaining for which reason it canno
Version changes Version changes
--------------- ---------------
- 0.3.4 : clarifications for FSE decoding table
- 0.3.3 : clarifications for field Block_Size - 0.3.3 : clarifications for field Block_Size
- 0.3.2 : remove additional block size restriction on compressed blocks - 0.3.2 : remove additional block size restriction on compressed blocks
- 0.3.1 : minor clarification regarding offset history update rules - 0.3.1 : minor clarification regarding offset history update rules

View File

@ -796,7 +796,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
It's a CPU consuming operation, with non-negligible impact on latency. It's a CPU consuming operation, with non-negligible impact on latency.
If there is a need to use the same prefix multiple times, consider loadDictionary instead. If there is a need to use the same prefix multiple times, consider loadDictionary instead.
Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation.
</p></pre><BR> </p></pre><BR>
@ -840,7 +840,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Note 2 : Prefix buffer is referenced. It **must** outlive decompression. Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
Prefix buffer must remain unmodified up to the end of frame, Prefix buffer must remain unmodified up to the end of frame,
reached when ZSTD_decompressStream() returns 0. reached when ZSTD_decompressStream() returns 0.
Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
A full dictionary is more costly, as it requires building tables. A full dictionary is more costly, as it requires building tables.
@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
<BR></pre> <BR></pre>
<pre><b>typedef struct {
unsigned int matchPos; </b>/* Match pos in dst */<b>
</b>/* If seqDef.offset > 3, then this is seqDef.offset - 3<b>
* If seqDef.offset < 3, then this is the corresponding repeat offset
* But if seqDef.offset < 3 and litLength == 0, this is the
* repeat offset before the corresponding repeat offset
* And if seqDef.offset == 3 and litLength == 0, this is the
* most recent repeat offset - 1
*/
unsigned int offset;
unsigned int litLength; </b>/* Literal length */<b>
unsigned int matchLength; </b>/* Match length */<b>
</b>/* 0 when seq not rep and seqDef.offset otherwise<b>
* when litLength == 0 this will be <= 4, otherwise <= 3 like normal
*/
unsigned int rep;
} ZSTD_Sequence;
</b></pre><BR>
<pre><b>typedef struct { <pre><b>typedef struct {
unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b> unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b>
unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b> unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b>
@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
or an error code (if srcSize is too small) or an error code (if srcSize is too small)
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize);
</b><p> Extract sequences from the sequence store
zc can be used to insert custom compression params.
This function invokes ZSTD_compress2
@return : number of sequences extracted
</p></pre><BR>
<a name="Chapter16"></a><h2>Memory management</h2><pre></pre> <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel); <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
@ -1322,7 +1349,7 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigne
* *
* Creates of an internal CDict (incompatible with static CCtx), except if * Creates of an internal CDict (incompatible with static CCtx), except if
* dict == NULL or dictSize < 8, in which case no dict is used. * dict == NULL or dictSize < 8, in which case no dict is used.
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
*/ */
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@ -1337,7 +1364,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
* *
* pledgedSrcSize must be correct. If srcSize is not known at init time, use * pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
*/ */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTD_parameters params, unsigned long long pledgedSrcSize);

View File

@ -74,10 +74,50 @@ void __msan_poison(const volatile void *a, size_t size);
intptr_t __msan_test_shadow(const volatile void *x, size_t size); intptr_t __msan_test_shadow(const volatile void *x, size_t size);
#endif #endif
#if defined (MEMORY_SANITIZER) /* detects whether we are being compiled under asan */
# define MEM_SKIP_MSAN __attribute__((no_sanitize("memory"))) #if defined (__has_feature)
#else # if __has_feature(address_sanitizer)
# define MEM_SKIP_MSAN # define ADDRESS_SANITIZER 1
# endif
#elif defined(__SANITIZE_ADDRESS__)
# define ADDRESS_SANITIZER 1
#endif
#if defined (ADDRESS_SANITIZER)
/* Not all platforms that support asan provide sanitizers/asan_interface.h.
* We therefore declare the functions we need ourselves, rather than trying to
* include the header file... */
/**
* Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
*
* This memory must be previously allocated by your program. Instrumented
* code is forbidden from accessing addresses in this region until it is
* unpoisoned. This function is not guaranteed to poison the entire region -
* it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
* alignment restrictions.
*
* \note This function is not thread-safe because no two threads can poison or
* unpoison memory in the same memory region simultaneously.
*
* \param addr Start of memory region.
* \param size Size of memory region. */
void __asan_poison_memory_region(void const volatile *addr, size_t size);
/**
* Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
*
* This memory must be previously allocated by your program. Accessing
* addresses in this region is allowed until this region is poisoned again.
* This function could unpoison a super-region of <c>[addr, addr+size)</c> due
* to ASan alignment restrictions.
*
* \note This function is not thread-safe because no two threads can
* poison or unpoison memory in the same memory region simultaneously.
*
* \param addr Start of memory region.
* \param size Size of memory region. */
void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
#endif #endif

View File

@ -127,9 +127,13 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
ctx->queueTail = 0; ctx->queueTail = 0;
ctx->numThreadsBusy = 0; ctx->numThreadsBusy = 0;
ctx->queueEmpty = 1; ctx->queueEmpty = 1;
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); {
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); int error = 0;
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
if (error) { POOL_free(ctx); return NULL; }
}
ctx->shutdown = 0; ctx->shutdown = 0;
/* Allocate space for the thread handles */ /* Allocate space for the thread handles */
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem); ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);

View File

@ -14,6 +14,8 @@
* This file will hold wrapper for systems, which do not support pthreads * This file will hold wrapper for systems, which do not support pthreads
*/ */
#include "threading.h"
/* create fake symbol to avoid empty translation unit warning */ /* create fake symbol to avoid empty translation unit warning */
int g_ZSTD_threading_useless_symbol; int g_ZSTD_threading_useless_symbol;
@ -28,7 +30,6 @@ int g_ZSTD_threading_useless_symbol;
/* === Dependencies === */ /* === Dependencies === */
#include <process.h> #include <process.h>
#include <errno.h> #include <errno.h>
#include "threading.h"
/* === Implementation === */ /* === Implementation === */
@ -73,3 +74,47 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
} }
#endif /* ZSTD_MULTITHREAD */ #endif /* ZSTD_MULTITHREAD */
#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
#include <stdlib.h>
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
{
*mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t));
if (!*mutex)
return 1;
return pthread_mutex_init(*mutex, attr);
}
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
{
if (!*mutex)
return 0;
{
int const ret = pthread_mutex_destroy(*mutex);
free(*mutex);
return ret;
}
}
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
{
*cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t));
if (!*cond)
return 1;
return pthread_cond_init(*cond, attr);
}
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
{
if (!*cond)
return 0;
{
int const ret = pthread_cond_destroy(*cond);
free(*cond);
return ret;
}
}
#endif

View File

@ -13,6 +13,8 @@
#ifndef THREADING_H_938743 #ifndef THREADING_H_938743
#define THREADING_H_938743 #define THREADING_H_938743
#include "debug.h"
#if defined (__cplusplus) #if defined (__cplusplus)
extern "C" { extern "C" {
#endif #endif
@ -75,10 +77,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
*/ */
#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ #elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
/* === POSIX Systems === */ /* === POSIX Systems === */
# include <pthread.h> # include <pthread.h>
#if DEBUGLEVEL < 1
#define ZSTD_pthread_mutex_t pthread_mutex_t #define ZSTD_pthread_mutex_t pthread_mutex_t
#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) #define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) #define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
@ -96,6 +100,33 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) #define ZSTD_pthread_join(a, b) pthread_join((a),(b))
#else /* DEBUGLEVEL >= 1 */
/* Debug implementation of threading.
* In this implementation we use pointers for mutexes and condition variables.
* This way, if we forget to init/destroy them the program will crash or ASAN
* will report leaks.
*/
#define ZSTD_pthread_mutex_t pthread_mutex_t*
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr);
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex);
#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a))
#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a))
#define ZSTD_pthread_cond_t pthread_cond_t*
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr);
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b))
#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a))
#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a))
#define ZSTD_pthread_t pthread_t
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
#endif
#else /* ZSTD_MULTITHREAD not defined */ #else /* ZSTD_MULTITHREAD not defined */
/* No multithreading support */ /* No multithreading support */

View File

@ -202,7 +202,7 @@ static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); }
typedef enum { typedef enum {
ZSTD_no_overlap, ZSTD_no_overlap,
ZSTD_overlap_src_before_dst, ZSTD_overlap_src_before_dst
/* ZSTD_overlap_dst_before_src, */ /* ZSTD_overlap_dst_before_src, */
} ZSTD_overlap_e; } ZSTD_overlap_e;
@ -247,20 +247,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
} }
} }
/*! ZSTD_wildcopy8() :
* The same as ZSTD_wildcopy(), but it can only overwrite 8 bytes, and works for
* overlapping buffers that are at least 8 bytes apart.
*/
MEM_STATIC void ZSTD_wildcopy8(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = (BYTE*)op + length;
do {
COPY8(op, ip);
} while (op < oend);
}
/*-******************************************* /*-*******************************************
* Private declarations * Private declarations

View File

@ -131,15 +131,11 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
{ {
assert(cctx != NULL); assert(cctx != NULL);
assert(cctx->staticSize == 0); assert(cctx->staticSize == 0);
/* Only free workspace if cctx not in workspace, otherwise the workspace
* will be freed when the cctx itself is freed. */
if ((void*)cctx->workspace.workspace != (void*)cctx) {
ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
}
ZSTD_clearAllDicts(cctx); ZSTD_clearAllDicts(cctx);
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
#endif #endif
ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
} }
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
@ -147,8 +143,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
if (cctx==NULL) return 0; /* support free on NULL */ if (cctx==NULL) return 0; /* support free on NULL */
RETURN_ERROR_IF(cctx->staticSize, memory_allocation, RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"not compatible with static CCtx"); "not compatible with static CCtx");
ZSTD_freeCCtxContent(cctx); {
ZSTD_free(cctx, cctx->customMem); int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
ZSTD_freeCCtxContent(cctx);
if (!cctxInWorkspace) {
ZSTD_free(cctx, cctx->customMem);
}
}
return 0; return 0;
} }
@ -1078,10 +1079,19 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
size_t const hSize = ((size_t)1) << cParams->hashLog; size_t const hSize = ((size_t)1) << cParams->hashLog;
U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = ((size_t)1) << hashLog3; size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't
size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32) * surrounded by redzones in ASAN. */
+ (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t)); size_t const tableSpace = chainSize * sizeof(U32)
+ hSize * sizeof(U32)
+ h3Size * sizeof(U32);
size_t const optPotentialSpace =
ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((1<<Litbits) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
? optPotentialSpace ? optPotentialSpace
: 0; : 0;
@ -1098,20 +1108,23 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
U32 const divider = (cParams.minMatch==3) ? 3 : 4; U32 const divider = (cParams.minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider; size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
size_t const entropySpace = HUF_WORKSPACE_SIZE; + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1);
size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams);
size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq); size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq));
size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace + size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace +
matchStateSize + ldmSpace + ldmSeqSpace; matchStateSize + ldmSpace + ldmSeqSpace;
size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx));
DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace);
DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
return sizeof(ZSTD_CCtx) + neededSpace; return cctxSpace + neededSpace;
} }
} }
@ -1147,7 +1160,8 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
size_t const streamingSize = inBuffSize + outBuffSize; size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize)
+ ZSTD_cwksp_alloc_size(outBuffSize);
return CCtxSize + streamingSize; return CCtxSize + streamingSize;
} }
@ -1394,7 +1408,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; U32 const divider = (params.cParams.minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider; size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0;
size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0;
size_t const matchStateSize = ZSTD_sizeof_matchState(&params.cParams, /* forCCtx */ 1); size_t const matchStateSize = ZSTD_sizeof_matchState(&params.cParams, /* forCCtx */ 1);
@ -1409,12 +1425,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_cwksp_bump_oversized_duration(ws, 0); ZSTD_cwksp_bump_oversized_duration(ws, 0);
/* Check if workspace is large enough, alloc a new one if needed */ /* Check if workspace is large enough, alloc a new one if needed */
{ size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0; { size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
size_t const entropySpace = HUF_WORKSPACE_SIZE; size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const bufferSpace = buffInSize + buffOutSize; size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize);
size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams);
size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq));
size_t const neededSpace = size_t const neededSpace =
cctxSpace + cctxSpace +
@ -1650,12 +1666,13 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
/* copy tables */ /* copy tables */
{ size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog);
size_t const hSize = (size_t)1 << cdict_cParams->hashLog; size_t const hSize = (size_t)1 << cdict_cParams->hashLog;
size_t const tableSpace = (chainSize + hSize) * sizeof(U32);
assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ memcpy(cctx->blockState.matchState.hashTable,
assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize); cdict->matchState.hashTable,
assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */ hSize * sizeof(U32));
assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize); memcpy(cctx->blockState.matchState.chainTable,
memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace); /* presumes all tables follow each other */ cdict->matchState.chainTable,
chainSize * sizeof(U32));
} }
/* Zero the hashTable3, since the cdict never fills it */ /* Zero the hashTable3, since the cdict never fills it */
@ -1742,10 +1759,16 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
int const h3log = srcCCtx->blockState.matchState.hashLog3; int const h3log = srcCCtx->blockState.matchState.hashLog3;
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ memcpy(dstCCtx->blockState.matchState.hashTable,
assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize); srcCCtx->blockState.matchState.hashTable,
memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */ hSize * sizeof(U32));
memcpy(dstCCtx->blockState.matchState.chainTable,
srcCCtx->blockState.matchState.chainTable,
chainSize * sizeof(U32));
memcpy(dstCCtx->blockState.matchState.hashTable3,
srcCCtx->blockState.matchState.hashTable3,
h3Size * sizeof(U32));
} }
ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace);
@ -3139,8 +3162,11 @@ size_t ZSTD_estimateCDictSize_advanced(
ZSTD_dictLoadMethod_e dictLoadMethod) ZSTD_dictLoadMethod_e dictLoadMethod)
{ {
DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict));
return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void *))); + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
} }
size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
@ -3220,11 +3246,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ size_t const workspaceSize = { size_t const workspaceSize =
sizeof(ZSTD_CDict) + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
HUF_WORKSPACE_SIZE + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) +
(dictLoadMethod == ZSTD_dlm_byRef ? 0 (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_align(dictSize, sizeof(void*))); : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
void* const workspace = ZSTD_malloc(workspaceSize, customMem); void* const workspace = ZSTD_malloc(workspaceSize, customMem);
ZSTD_cwksp ws; ZSTD_cwksp ws;
ZSTD_CDict* cdict; ZSTD_CDict* cdict;
@ -3240,18 +3266,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
assert(cdict != NULL); assert(cdict != NULL);
ZSTD_cwksp_move(&cdict->workspace, &ws); ZSTD_cwksp_move(&cdict->workspace, &ws);
cdict->customMem = customMem; cdict->customMem = customMem;
<<<<<<< HEAD
<<<<<<< HEAD
=======
cdict->workspace = workspace;
cdict->workspaceSize = workspaceSize;
cdict->compressionLevel = 0; /* signals advanced API usage */ cdict->compressionLevel = 0; /* signals advanced API usage */
>>>>>>> Fix error
=======
cdict->compressionLevel = 0; /* signals advanced API usage */
>>>>>>> merge conflicts round 2
if (ZSTD_isError( ZSTD_initCDict_internal(cdict, if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dictBuffer, dictSize, dictBuffer, dictSize,
dictLoadMethod, dictContentType, dictLoadMethod, dictContentType,
@ -3287,12 +3303,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
{ {
if (cdict==NULL) return 0; /* support free on NULL */ if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = cdict->customMem; { ZSTD_customMem const cMem = cdict->customMem;
/* Only free workspace if cdict not in workspace, otherwise the int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict);
* workspace will be freed when the cdict itself is freed. */ ZSTD_cwksp_free(&cdict->workspace, cMem);
if ((void*)cdict->workspace.workspace != (void*)cdict) { if (!cdictInWorkspace) {
ZSTD_cwksp_free(&cdict->workspace, cMem); ZSTD_free(cdict, cMem);
} }
ZSTD_free(cdict, cMem);
return 0; return 0;
} }
} }
@ -3318,8 +3333,11 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_compressionParameters cParams) ZSTD_compressionParameters cParams)
{ {
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0);
size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void*))) size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ HUF_WORKSPACE_SIZE + matchStateSize; + (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ matchStateSize;
ZSTD_CDict* cdict; ZSTD_CDict* cdict;
if ((size_t)workspace & 7) return NULL; /* 8-aligned */ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
@ -3552,7 +3570,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
/* ZSTD_initCStream_advanced() : /* ZSTD_initCStream_advanced() :
* pledgedSrcSize must be exact. * pledgedSrcSize must be exact.
* if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
* dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pss) ZSTD_parameters params, unsigned long long pss)

View File

@ -34,6 +34,17 @@ extern "C" {
* In which case, resize it down to free some memory */ * In which case, resize it down to free some memory */
#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
/* Since the workspace is effectively its own little malloc implementation /
* arena, when we run under ASAN, we should similarly insert redzones between
* each internal element of the workspace, so ASAN will catch overruns that
* reach outside an object but that stay inside the workspace.
*
* This defines the size of that redzone.
*/
#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE
#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
#endif
/*-************************************* /*-*************************************
* Structures * Structures
***************************************/ ***************************************/
@ -166,6 +177,24 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
return (size + mask) & ~mask; return (size + mask) & ~mask;
} }
/**
* Use this to determine how much space in the workspace we will consume to
* allocate this object. (Normally it should be exactly the size of the object,
* but under special conditions, like ASAN, where we pad each object, it might
* be larger.)
*
* Since tables aren't currently redzoned, you don't need to call through this
* to figure out how much space you need for the matchState tables. Everything
* else is though.
*/
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#else
return size;
#endif
}
MEM_STATIC void ZSTD_cwksp_internal_advance_phase( MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) {
assert(phase >= ws->phase); assert(phase >= ws->phase);
@ -192,6 +221,13 @@ MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
} }
} }
/**
* Returns whether this object/buffer/etc was allocated in this workspace.
*/
MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) {
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
}
/** /**
* Internal function. Do not use directly. * Internal function. Do not use directly.
*/ */
@ -201,8 +237,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
void* bottom = ws->tableEnd; void* bottom = ws->tableEnd;
ZSTD_cwksp_internal_advance_phase(ws, phase); ZSTD_cwksp_internal_advance_phase(ws, phase);
alloc = (BYTE *)ws->allocStart - bytes; alloc = (BYTE *)ws->allocStart - bytes;
DEBUGLOG(5, "cwksp: reserving %zd bytes, %zd bytes remaining",
bytes, ZSTD_cwksp_available_space(ws) - bytes); #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* over-reserve space */
alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#endif
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
ZSTD_cwksp_assert_internal_consistency(ws); ZSTD_cwksp_assert_internal_consistency(ws);
assert(alloc >= bottom); assert(alloc >= bottom);
if (alloc < bottom) { if (alloc < bottom) {
@ -214,6 +256,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
ws->tableValidEnd = alloc; ws->tableValidEnd = alloc;
} }
ws->allocStart = alloc; ws->allocStart = alloc;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
* either size. */
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
__asan_unpoison_memory_region(alloc, bytes);
#endif
return alloc; return alloc;
} }
@ -242,8 +292,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
void* alloc = ws->tableEnd; void* alloc = ws->tableEnd;
void* end = (BYTE *)alloc + bytes; void* end = (BYTE *)alloc + bytes;
void* top = ws->allocStart; void* top = ws->allocStart;
DEBUGLOG(5, "cwksp: reserving table %zd bytes, %zd bytes remaining",
bytes, ZSTD_cwksp_available_space(ws) - bytes); DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
assert((bytes & (sizeof(U32)-1)) == 0); assert((bytes & (sizeof(U32)-1)) == 0);
ZSTD_cwksp_internal_advance_phase(ws, phase); ZSTD_cwksp_internal_advance_phase(ws, phase);
ZSTD_cwksp_assert_internal_consistency(ws); ZSTD_cwksp_assert_internal_consistency(ws);
@ -254,6 +305,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
return NULL; return NULL;
} }
ws->tableEnd = end; ws->tableEnd = end;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
__asan_unpoison_memory_region(alloc, bytes);
#endif
return alloc; return alloc;
} }
@ -262,12 +318,18 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
*/ */
MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
void* start = ws->objectEnd; void* alloc = ws->objectEnd;
void* end = (BYTE*)start + roundedBytes; void* end = (BYTE*)alloc + roundedBytes;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* over-reserve space */
end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#endif
DEBUGLOG(5, DEBUGLOG(5,
"cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining",
bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
assert(((size_t)start & (sizeof(void*)-1)) == 0); assert(((size_t)alloc & (sizeof(void*)-1)) == 0);
assert((bytes & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0);
ZSTD_cwksp_assert_internal_consistency(ws); ZSTD_cwksp_assert_internal_consistency(ws);
/* we must be in the first phase, no advance is possible */ /* we must be in the first phase, no advance is possible */
@ -279,7 +341,15 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
ws->objectEnd = end; ws->objectEnd = end;
ws->tableEnd = end; ws->tableEnd = end;
ws->tableValidEnd = end; ws->tableValidEnd = end;
return start;
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
* either size. */
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
__asan_unpoison_memory_region(alloc, bytes);
#endif
return alloc;
} }
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) {
@ -331,6 +401,14 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
*/ */
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
DEBUGLOG(4, "cwksp: clearing tables!"); DEBUGLOG(4, "cwksp: clearing tables!");
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
{
size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
__asan_poison_memory_region(ws->objectEnd, size);
}
#endif
ws->tableEnd = ws->objectEnd; ws->tableEnd = ws->objectEnd;
ZSTD_cwksp_assert_internal_consistency(ws); ZSTD_cwksp_assert_internal_consistency(ws);
} }
@ -353,6 +431,13 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
} }
#endif #endif
#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
{
size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
__asan_poison_memory_region(ws->objectEnd, size);
}
#endif
ws->tableEnd = ws->objectEnd; ws->tableEnd = ws->objectEnd;
ws->allocStart = ws->workspaceEnd; ws->allocStart = ws->workspaceEnd;
ws->allocFailed = 0; ws->allocFailed = 0;
@ -389,9 +474,10 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem
} }
MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
void *ptr = ws->workspace;
DEBUGLOG(4, "cwksp: freeing workspace"); DEBUGLOG(4, "cwksp: freeing workspace");
ZSTD_free(ws->workspace, customMem);
memset(ws, 0, sizeof(ZSTD_cwksp)); memset(ws, 0, sizeof(ZSTD_cwksp));
ZSTD_free(ptr, customMem);
} }
/** /**
@ -404,7 +490,7 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
} }
MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
} }
MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {

View File

@ -49,9 +49,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params)
{ {
size_t const ldmHSize = ((size_t)1) << params.hashLog; size_t const ldmHSize = ((size_t)1) << params.hashLog;
size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog);
size_t const ldmBucketSize = size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
((size_t)1) << (params.hashLog - ldmBucketSizeLog); size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize)
size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t); + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t));
return params.enableLdm ? totalSize : 0; return params.enableLdm ? totalSize : 0;
} }

View File

@ -927,12 +927,18 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
unsigned jobID; unsigned jobID;
DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
/* Copy the mutex/cond out */
ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex;
ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond;
DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start);
ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
mtctx->jobs[jobID].dstBuff = g_nullBuffer;
mtctx->jobs[jobID].cSize = 0; /* Clear the job description, but keep the mutex/cond */
memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID]));
mtctx->jobs[jobID].job_mutex = mutex;
mtctx->jobs[jobID].job_cond = cond;
} }
memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
mtctx->inBuff.buffer = g_nullBuffer; mtctx->inBuff.buffer = g_nullBuffer;
mtctx->inBuff.filled = 0; mtctx->inBuff.filled = 0;
mtctx->allJobsCompleted = 1; mtctx->allJobsCompleted = 1;

View File

@ -398,7 +398,7 @@ typedef enum {
ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003, ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004, ZSTD_c_experimentalParam7=1004
} ZSTD_cParameter; } ZSTD_cParameter;
typedef struct { typedef struct {
@ -928,7 +928,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
* It's a CPU consuming operation, with non-negligible impact on latency. * It's a CPU consuming operation, with non-negligible impact on latency.
* If there is a need to use the same prefix multiple times, consider loadDictionary instead. * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize); const void* prefix, size_t prefixSize);
@ -972,7 +972,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
* Note 2 : Prefix buffer is referenced. It **must** outlive decompression. * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
* Prefix buffer must remain unmodified up to the end of frame, * Prefix buffer must remain unmodified up to the end of frame,
* reached when ZSTD_decompressStream() returns 0. * reached when ZSTD_decompressStream() returns 0.
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
* A full dictionary is more costly, as it requires building tables. * A full dictionary is more costly, as it requires building tables.
@ -1124,7 +1124,7 @@ typedef enum {
typedef enum { typedef enum {
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
} ZSTD_dictLoadMethod_e; } ZSTD_dictLoadMethod_e;
typedef enum { typedef enum {
@ -1138,7 +1138,7 @@ typedef enum {
* This question could be kept for later, when there are actually multiple formats to support, * This question could be kept for later, when there are actually multiple formats to support,
* but there is also the question of pinning enum values, and pinning value `0` is especially important */ * but there is also the question of pinning enum values, and pinning value `0` is especially important */
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number. ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
* Useful to save 4 bytes per generated frame. * Useful to save 4 bytes per generated frame.
* Decoder cannot recognise automatically this format, requiring this instruction. */ * Decoder cannot recognise automatically this format, requiring this instruction. */
} ZSTD_format_e; } ZSTD_format_e;
@ -1188,7 +1188,7 @@ typedef enum {
* levels will be compressed. */ * levels will be compressed. */
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
* emitted if Huffman compression is not profitable. */ * emitted if Huffman compression is not profitable. */
ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */ ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
} ZSTD_literalCompressionMode_e; } ZSTD_literalCompressionMode_e;
@ -1677,7 +1677,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
* *
* Creates of an internal CDict (incompatible with static CCtx), except if * Creates of an internal CDict (incompatible with static CCtx), except if
* dict == NULL or dictSize < 8, in which case no dict is used. * dict == NULL or dictSize < 8, in which case no dict is used.
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
*/ */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@ -1692,7 +1692,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
* *
* pledgedSrcSize must be correct. If srcSize is not known at init time, use * pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
*/ */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTD_parameters params, unsigned long long pledgedSrcSize);

View File

@ -173,10 +173,13 @@ Benchmark arguments :
``` ```
#### Restricted usage of Environment Variables #### Restricted usage of Environment Variables
Using environment variables to set compression/decompression parameters has security implications. Therefore, Using environment variables to set parameters has security implications.
we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level. Therefore, this avenue is intentionally restricted.
Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message. If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
Note that command line options will override corresponding environment variable settings. `ZSTD_CLEVEL` just replaces the default compression level (`3`).
It can be overridden by corresponding command line arguments.
#### Long distance matching mode #### Long distance matching mode
The long distance matching mode, enabled with `--long`, is designed to improve The long distance matching mode, enabled with `--long`, is designed to improve

View File

@ -88,7 +88,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#endif #endif
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
#define EXM_THROW_INT(errorNum, ...) { \ #define RETURN_ERROR_INT(errorNum, ...) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", errorNum); \ DISPLAYLEVEL(1, "Error %i : ", errorNum); \
DISPLAYLEVEL(1, __VA_ARGS__); \ DISPLAYLEVEL(1, __VA_ARGS__); \
@ -401,9 +401,9 @@ BMK_benchMemAdvancedNoAlloc(
BMK_initCCtxArgs cctxprep; BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep; BMK_initDCtxArgs dctxprep;
cbp.benchFn = local_defaultCompress; cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */
cbp.benchPayload = cctx; cbp.benchPayload = cctx;
cbp.initFn = local_initCCtx; cbp.initFn = local_initCCtx; /* BMK_initCCtx */
cbp.initPayload = &cctxprep; cbp.initPayload = &cctxprep;
cbp.errorFn = ZSTD_isError; cbp.errorFn = ZSTD_isError;
cbp.blockCount = nbBlocks; cbp.blockCount = nbBlocks;
@ -534,8 +534,8 @@ BMK_benchMemAdvancedNoAlloc(
if (u==srcSize-1) { /* should never happen */ if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n"); DISPLAY("no difference detected\n");
} }
} } /* for (u=0; u<srcSize; u++) */
} } /* if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) */
} /* CRC Checking */ } /* CRC Checking */
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
@ -754,8 +754,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
size_t pos = 0, totalSize = 0; size_t pos = 0, totalSize = 0;
unsigned n; unsigned n;
for (n=0; n<nbFiles; n++) { for (n=0; n<nbFiles; n++) {
FILE* f; U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); /* last file may be shortened */
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
if (UTIL_isDirectory(fileNamesTable[n])) { if (UTIL_isDirectory(fileNamesTable[n])) {
DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]); DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
fileSizes[n] = 0; fileSizes[n] = 0;
@ -766,20 +765,20 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
fileSizes[n] = 0; fileSizes[n] = 0;
continue; continue;
} }
f = fopen(fileNamesTable[n], "rb"); { FILE* const f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW_INT(10, "impossible to open file %s", fileNamesTable[n]); if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]);
pos += readSize; pos += readSize;
} }
fileSizes[n] = (size_t)fileSize; fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize; totalSize += (size_t)fileSize;
fclose(f); fclose(f);
} } }
if (totalSize == 0) EXM_THROW_INT(12, "no data to bench"); if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench");
return 0; return 0;
} }

View File

@ -205,7 +205,6 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
#endif /* BENCH_ZSTD_H_3242387 */ #endif /* BENCH_ZSTD_H_3242387 */
#if defined (__cplusplus) #if defined (__cplusplus)

View File

@ -284,9 +284,10 @@ void FIO_addAbortHandler()
/*-************************************* /*-*************************************
* Parameters: Typedefs * Parameters: FIO_prefs_t
***************************************/ ***************************************/
/* typedef'd to FIO_prefs_t within fileio.h */
struct FIO_prefs_s { struct FIO_prefs_s {
/* Algorithm preferences */ /* Algorithm preferences */
@ -308,6 +309,7 @@ struct FIO_prefs_s {
size_t streamSrcSize; size_t streamSrcSize;
size_t targetCBlockSize; size_t targetCBlockSize;
int srcSizeHint; int srcSizeHint;
int testMode;
ZSTD_literalCompressionMode_e literalCompressionMode; ZSTD_literalCompressionMode_e literalCompressionMode;
/* IO preferences */ /* IO preferences */
@ -355,6 +357,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->streamSrcSize = 0; ret->streamSrcSize = 0;
ret->targetCBlockSize = 0; ret->targetCBlockSize = 0;
ret->srcSizeHint = 0; ret->srcSizeHint = 0;
ret->testMode = 0;
ret->literalCompressionMode = ZSTD_lcm_auto; ret->literalCompressionMode = ZSTD_lcm_auto;
return ret; return ret;
} }
@ -435,6 +438,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
} }
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
prefs->testMode = (testMode!=0);
}
void FIO_setLiteralCompressionMode( void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs, FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode) { ZSTD_literalCompressionMode_e mode) {
@ -529,8 +536,12 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
/** FIO_openDstFile() : /** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL. * condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */ * @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName) static FILE*
FIO_openDstFile(FIO_prefs_t* const prefs,
const char* srcFileName, const char* dstFileName)
{ {
if (prefs->testMode) return NULL; /* do not open file in test mode */
assert(dstFileName != NULL); assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) { if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output \n"); DISPLAYLEVEL(4,"Using stdout for output \n");
@ -555,10 +566,14 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
if (UTIL_isRegularFile(dstFileName)) { if (UTIL_isRegularFile(dstFileName)) {
/* Check if destination file already exists */ /* Check if destination file already exists */
FILE* const fCheck = fopen( dstFileName, "rb" ); FILE* const fCheck = fopen( dstFileName, "rb" );
#if !defined(_WIN32)
/* this test does not work on Windows :
* `NUL` and `nul` are detected as regular files */
if (!strcmp(dstFileName, nulmark)) { if (!strcmp(dstFileName, nulmark)) {
EXM_THROW(40, "%s is unexpectedly categorized as a regular file", EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
dstFileName); dstFileName);
} }
#endif
if (fCheck != NULL) { /* dst file exists, authorization prompt */ if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck); fclose(fCheck);
if (!prefs->overwrite) { if (!prefs->overwrite) {
@ -671,56 +686,50 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
return 0; return 0;
} }
static const char*
extractFilename(const char* path, char separator)
{
const char* search = strrchr(path, separator);
if (search == NULL) return path;
return search+1;
}
/* FIO_createFilename_fromOutDir() : /* FIO_createFilename_fromOutDir() :
* Takes a source file name and specified output directory, and * Takes a source file name and specified output directory, and
* allocates memory for and returns a pointer to final path. * allocates memory for and returns a pointer to final path.
* This function never returns an error (it may abort() in case of pb) * This function never returns an error (it may abort() in case of pb)
*/ */
static char* static char*
FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen) FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen)
{ {
const char* c, *filenameBegin; const char* filenameStart;
char* filename, *result; char separator;
size_t finalPathLen; char* result;
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
c = "\\"; separator = '\\';
#else #else
c = "/"; separator = '/';
#endif #endif
finalPathLen = strlen(outDirName); filenameStart = extractFilename(path, separator);
filenameBegin = strrchr(srcFilename, c[0]); #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
if (filenameBegin == NULL) { filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */
filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char)); #endif
if (!filename) {
EXM_THROW(30, "zstd: %s", strerror(errno));
}
strcpy(filename, srcFilename);
} else {
filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
if (!filename) {
EXM_THROW(30, "zstd: %s", strerror(errno));
}
strcpy(filename, filenameBegin+1);
}
finalPathLen += strlen(filename); result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1);
result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
if (!result) { if (!result) {
free(filename); EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno));
EXM_THROW(30, "zstd: %s", strerror(errno));
} }
strcpy(result, outDirName); memcpy(result, outDirName, strlen(outDirName));
if (outDirName[strlen(outDirName)-1] == c[0]) { if (outDirName[strlen(outDirName)-1] == separator) {
strcat(result, filename); memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart));
} else { } else {
strcat(result, c); memcpy(result + strlen(outDirName), &separator, 1);
strcat(result, filename); memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart));
} }
free(filename);
return result; return result;
} }
@ -825,13 +834,12 @@ static void FIO_freeCResources(cRess_t ress)
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS
static unsigned long long static unsigned long long
FIO_compressGzFrame(cRess_t* ress, FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */
const char* srcFileName, U64 const srcFileSize, const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize) int compressionLevel, U64* readsize)
{ {
unsigned long long inFileSize = 0, outFileSize = 0; unsigned long long inFileSize = 0, outFileSize = 0;
z_stream strm; z_stream strm;
int ret;
if (compressionLevel > Z_BEST_COMPRESSION) if (compressionLevel > Z_BEST_COMPRESSION)
compressionLevel = Z_BEST_COMPRESSION; compressionLevel = Z_BEST_COMPRESSION;
@ -840,11 +848,12 @@ FIO_compressGzFrame(cRess_t* ress,
strm.zfree = Z_NULL; strm.zfree = Z_NULL;
strm.opaque = Z_NULL; strm.opaque = Z_NULL;
ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, { int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
15 /* maxWindowLogSize */ + 16 /* gzip only */, 15 /* maxWindowLogSize */ + 16 /* gzip only */,
8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
if (ret != Z_OK) if (ret != Z_OK) {
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
} }
strm.next_in = 0; strm.next_in = 0;
strm.avail_in = 0; strm.avail_in = 0;
@ -852,6 +861,7 @@ FIO_compressGzFrame(cRess_t* ress,
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
while (1) { while (1) {
int ret;
if (strm.avail_in == 0) { if (strm.avail_in == 0) {
size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
if (inSize == 0) break; if (inSize == 0) break;
@ -862,32 +872,31 @@ FIO_compressGzFrame(cRess_t* ress,
ret = deflate(&strm, Z_NO_FLUSH); ret = deflate(&strm, Z_NO_FLUSH);
if (ret != Z_OK) if (ret != Z_OK)
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const cSize = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (cSize) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno)); EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno));
outFileSize += decompBytes; outFileSize += cSize;
strm.next_out = (Bytef*)ress->dstBuffer; strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
} } }
} if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100) (double)outFileSize/inFileSize*100)
else } else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100); (double)outFileSize/inFileSize*100);
} } }
while (1) { while (1) {
ret = deflate(&strm, Z_FINISH); int const ret = deflate(&strm, Z_FINISH);
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const cSize = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (cSize) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
EXM_THROW(75, "Write error : %s", strerror(errno)); EXM_THROW(75, "Write error : %s ", strerror(errno));
outFileSize += decompBytes; outFileSize += cSize;
strm.next_out = (Bytef*)ress->dstBuffer; strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
} } } }
@ -896,11 +905,11 @@ FIO_compressGzFrame(cRess_t* ress,
EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
} }
ret = deflateEnd(&strm); { int const ret = deflateEnd(&strm);
if (ret != Z_OK) if (ret != Z_OK) {
EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
} }
*readsize = inFileSize; *readsize = inFileSize;
return outFileSize; return outFileSize;
} }
#endif #endif
@ -923,14 +932,14 @@ FIO_compressLzmaFrame(cRess_t* ress,
if (plain_lzma) { if (plain_lzma) {
lzma_options_lzma opt_lzma; lzma_options_lzma opt_lzma;
if (lzma_lzma_preset(&opt_lzma, compressionLevel)) if (lzma_lzma_preset(&opt_lzma, compressionLevel))
EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName); EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName);
ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
if (ret != LZMA_OK) if (ret != LZMA_OK)
EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
} else { } else {
ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
if (ret != LZMA_OK) if (ret != LZMA_OK)
EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
} }
strm.next_in = 0; strm.next_in = 0;
@ -950,11 +959,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
ret = lzma_code(&strm, action); ret = lzma_code(&strm, action);
if (ret != LZMA_OK && ret != LZMA_STREAM_END) if (ret != LZMA_OK && ret != LZMA_STREAM_END)
EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
{ size_t const compBytes = ress->dstBufferSize - strm.avail_out; { size_t const compBytes = ress->dstBufferSize - strm.avail_out;
if (compBytes) { if (compBytes) {
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
EXM_THROW(73, "Write error : %s", strerror(errno)); EXM_THROW(85, "Write error : %s", strerror(errno));
outFileSize += compBytes; outFileSize += compBytes;
strm.next_out = (BYTE*)ress->dstBuffer; strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize; strm.avail_out = ress->dstBufferSize;
@ -1522,8 +1531,9 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
* or into one file each (outFileName == NULL, but suffix != NULL), * or into one file each (outFileName == NULL, but suffix != NULL),
* or into a destination folder (specified with -O) * or into a destination folder (specified with -O)
*/ */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char* outDirName, unsigned nbFiles, const char** inFileNamesTable, unsigned nbFiles,
const char* outDirName,
const char* outFileName, const char* suffix, const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams) ZSTD_compressionParameters comprParams)
@ -1617,7 +1627,11 @@ static void FIO_freeDResources(dRess_t ress)
/** FIO_fwriteSparse() : /** FIO_fwriteSparse() :
* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) static unsigned
FIO_fwriteSparse(const FIO_prefs_t* const prefs,
FILE* file,
const void* buffer, size_t bufferSize,
unsigned storedSkips)
{ {
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
size_t bufferSizeT = bufferSize / sizeof(size_t); size_t bufferSizeT = bufferSize / sizeof(size_t);
@ -1625,6 +1639,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
const size_t* ptrT = bufferT; const size_t* ptrT = bufferT;
static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
if (prefs->testMode) return 0; /* do not output anything in test mode */
if (!prefs->sparseFileSupport) { /* normal write */ if (!prefs->sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize) if (sizeCheck != bufferSize)
@ -1637,7 +1653,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (storedSkips > 1 GB) { if (storedSkips > 1 GB) {
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR); int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
if (seekResult != 0) if (seekResult != 0)
EXM_THROW(71, "1 GB skip error (sparse file support)"); EXM_THROW(91, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB; storedSkips -= 1 GB;
} }
@ -1653,13 +1669,13 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (nb0T != seg0SizeT) { /* not all 0s */ if (nb0T != seg0SizeT) { /* not all 0s */
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse");
storedSkips = 0; storedSkips = 0;
seg0SizeT -= nb0T; seg0SizeT -= nb0T;
ptrT += nb0T; ptrT += nb0T;
{ size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
if (sizeCheck != seg0SizeT) if (sizeCheck != seg0SizeT)
EXM_THROW(73, "Write error : cannot write decoded block : %s", EXM_THROW(93, "Write error : cannot write decoded block : %s",
strerror(errno)); strerror(errno));
} } } }
ptrT += seg0SizeT; ptrT += seg0SizeT;
@ -1677,11 +1693,11 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (restPtr != restEnd) { if (restPtr != restEnd) {
int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
if (seekResult) if (seekResult)
EXM_THROW(74, "Sparse skip error ; try --no-sparse"); EXM_THROW(94, "Sparse skip error ; try --no-sparse");
storedSkips = 0; storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
if (sizeCheck != (size_t)(restEnd - restPtr)) if (sizeCheck != (size_t)(restEnd - restPtr))
EXM_THROW(75, "Write error : cannot write decoded end of block : %s", EXM_THROW(95, "Write error : cannot write decoded end of block : %s",
strerror(errno)); strerror(errno));
} } } } } } } }
@ -1689,8 +1705,9 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
} }
static void static void
FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
{ {
if (prefs->testMode) assert(storedSkips == 0);
if (storedSkips>0) { if (storedSkips>0) {
assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
(void)prefs; /* assert can be disabled, in which case prefs becomes unused */ (void)prefs; /* assert can be disabled, in which case prefs becomes unused */
@ -1707,7 +1724,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */ @return : 0 (no error) */
static int FIO_passThrough(FIO_prefs_t* const prefs, static int FIO_passThrough(const FIO_prefs_t* const prefs,
FILE* foutput, FILE* finput, FILE* foutput, FILE* finput,
void* buffer, size_t bufferSize, void* buffer, size_t bufferSize,
size_t alreadyLoaded) size_t alreadyLoaded)
@ -1747,7 +1764,10 @@ static unsigned FIO_highbit64(unsigned long long v)
/* FIO_zstdErrorHelp() : /* FIO_zstdErrorHelp() :
* detailed error message when requested window size is too large */ * detailed error message when requested window size is too large */
static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName) static void
FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
const dRess_t* ress,
size_t err, const char* srcFileName)
{ {
ZSTD_frameHeader header; ZSTD_frameHeader header;
@ -1779,12 +1799,10 @@ static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t er
* @return : size of decoded zstd frame, or an error code * @return : size of decoded zstd frame, or an error code
*/ */
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
static unsigned long long FIO_decompressZstdFrame( static unsigned long long
FIO_prefs_t* const prefs, FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, dRess_t* ress, FILE* finput,
FILE* finput, const char* srcFileName, U64 alreadyDecoded)
const char* srcFileName,
U64 alreadyDecoded)
{ {
U64 frameSize = 0; U64 frameSize = 0;
U32 storedSkips = 0; U32 storedSkips = 0;
@ -1848,13 +1866,16 @@ static unsigned long long FIO_decompressZstdFrame(
#ifdef ZSTD_GZDECOMPRESS #ifdef ZSTD_GZDECOMPRESS
static unsigned long long FIO_decompressGzFrame(dRess_t* ress, static unsigned long long
FILE* srcFile, const char* srcFileName) FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName)
{ {
unsigned long long outFileSize = 0; unsigned long long outFileSize = 0;
z_stream strm; z_stream strm;
int flush = Z_NO_FLUSH; int flush = Z_NO_FLUSH;
int decodingError = 0; int decodingError = 0;
unsigned storedSkips = 0;
strm.zalloc = Z_NULL; strm.zalloc = Z_NULL;
strm.zfree = Z_NULL; strm.zfree = Z_NULL;
@ -1889,10 +1910,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
} }
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
decodingError = 1; break;
}
outFileSize += decompBytes; outFileSize += decompBytes;
strm.next_out = (Bytef*)ress->dstBuffer; strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
@ -1909,19 +1927,24 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
decodingError = 1; decodingError = 1;
} }
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
} }
#endif #endif
#ifdef ZSTD_LZMADECOMPRESS #ifdef ZSTD_LZMADECOMPRESS
static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma) static unsigned long long
FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName, int plain_lzma)
{ {
unsigned long long outFileSize = 0; unsigned long long outFileSize = 0;
lzma_stream strm = LZMA_STREAM_INIT; lzma_stream strm = LZMA_STREAM_INIT;
lzma_action action = LZMA_RUN; lzma_action action = LZMA_RUN;
lzma_ret initRet; lzma_ret initRet;
int decodingError = 0; int decodingError = 0;
unsigned storedSkips = 0;
strm.next_in = 0; strm.next_in = 0;
strm.avail_in = 0; strm.avail_in = 0;
@ -1964,10 +1987,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
} }
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
decodingError = 1; break;
}
outFileSize += decompBytes; outFileSize += decompBytes;
strm.next_out = (BYTE*)ress->dstBuffer; strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize; strm.avail_out = ress->dstBufferSize;
@ -1979,19 +1999,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
memmove(ress->srcBuffer, strm.next_in, strm.avail_in); memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
ress->srcBufferLoaded = strm.avail_in; ress->srcBufferLoaded = strm.avail_in;
lzma_end(&strm); lzma_end(&strm);
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
} }
#endif #endif
#ifdef ZSTD_LZ4DECOMPRESS #ifdef ZSTD_LZ4DECOMPRESS
static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, static unsigned long long
FILE* srcFile, const char* srcFileName) FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName)
{ {
unsigned long long filesize = 0; unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad; LZ4F_errorCode_t nextToLoad;
LZ4F_decompressionContext_t dCtx; LZ4F_decompressionContext_t dCtx;
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
int decodingError = 0; int decodingError = 0;
unsigned storedSkips = 0;
if (LZ4F_isError(errorCode)) { if (LZ4F_isError(errorCode)) {
DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
@ -2035,10 +2059,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
/* Write Block */ /* Write Block */
if (decodedBytes) { if (decodedBytes) {
if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) { storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips);
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
decodingError = 1; nextToLoad = 0; break;
}
filesize += decodedBytes; filesize += decodedBytes;
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
} }
@ -2059,6 +2080,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
LZ4F_freeDecompressionContext(dCtx); LZ4F_freeDecompressionContext(dCtx);
ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; return decodingError ? FIO_ERROR_FRAME_DECODING : filesize;
} }
@ -2072,8 +2094,9 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
* @return : 0 : OK * @return : 0 : OK
* 1 : error * 1 : error
*/ */
static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName) dRess_t ress, FILE* srcFile,
const char* dstFileName, const char* srcFileName)
{ {
unsigned readSomething = 0; unsigned readSomething = 0;
unsigned long long filesize = 0; unsigned long long filesize = 0;
@ -2105,7 +2128,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
filesize += frameSize; filesize += frameSize;
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
#ifdef ZSTD_GZDECOMPRESS #ifdef ZSTD_GZDECOMPRESS
unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize; filesize += frameSize;
#else #else
@ -2115,7 +2138,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
} else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
|| (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
#ifdef ZSTD_LZMADECOMPRESS #ifdef ZSTD_LZMADECOMPRESS
unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize; filesize += frameSize;
#else #else
@ -2124,7 +2147,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
#endif #endif
} else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
#ifdef ZSTD_LZ4DECOMPRESS #ifdef ZSTD_LZ4DECOMPRESS
unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize; filesize += frameSize;
#else #else
@ -2164,11 +2187,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
int transfer_permissions = 0; int transfer_permissions = 0;
int releaseDstFile = 0; int releaseDstFile = 0;
if (ress.dstFile == NULL) { if ((ress.dstFile == NULL) && (prefs->testMode==0)) {
releaseDstFile = 1; releaseDstFile = 1;
ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
if (ress.dstFile==0) return 1; if (ress.dstFile==NULL) return 1;
/* Must only be added after FIO_openDstFile() succeeds. /* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists, * Otherwise we may delete the destination file if it already exists,
@ -2181,7 +2204,6 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
transfer_permissions = 1; transfer_permissions = 1;
} }
result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName); result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
if (releaseDstFile) { if (releaseDstFile) {
@ -2357,11 +2379,13 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
if (outFileName) { if (outFileName) {
unsigned u; unsigned u;
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (!prefs->testMode) {
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
}
for (u=0; u<nbFiles; u++) for (u=0; u<nbFiles; u++)
error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]); error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]);
if (fclose(ress.dstFile)) if ((!prefs->testMode) && (fclose(ress.dstFile)))
EXM_THROW(72, "Write error : %s : cannot properly close output file", EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno)); strerror(errno));
} else { } else {

View File

@ -26,7 +26,7 @@ extern "C" {
#define stdinmark "/*stdin*\\" #define stdinmark "/*stdin*\\"
#define stdoutmark "/*stdout*\\" #define stdoutmark "/*stdout*\\"
#ifdef _WIN32 #ifdef _WIN32
# define nulmark "nul" # define nulmark "NUL"
#else #else
# define nulmark "/dev/null" # define nulmark "/dev/null"
#endif #endif
@ -74,6 +74,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize); void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize); void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint); void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
void FIO_setLiteralCompressionMode( void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs, FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode); ZSTD_literalCompressionMode_e mode);
@ -85,14 +86,14 @@ void FIO_setNotificationLevel(int level);
* Single File functions * Single File functions
***************************************/ ***************************************/
/** FIO_compressFilename() : /** FIO_compressFilename() :
@return : 0 == ok; 1 == pb with src file. */ * @return : 0 == ok; 1 == pb with src file. */
int FIO_compressFilename (FIO_prefs_t* const prefs, int FIO_compressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* outfilename, const char* infilename,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams); ZSTD_compressionParameters comprParams);
/** FIO_decompressFilename() : /** FIO_decompressFilename() :
@return : 0 == ok; 1 == pb with src file. */ * @return : 0 == ok; 1 == pb with src file. */
int FIO_decompressFilename (FIO_prefs_t* const prefs, int FIO_decompressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* dictFileName); const char* outfilename, const char* infilename, const char* dictFileName);
@ -103,15 +104,16 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
* Multiple File functions * Multiple File functions
***************************************/ ***************************************/
/** FIO_compressMultipleFilenames() : /** FIO_compressMultipleFilenames() :
@return : nb of missing files */ * @return : nb of missing files */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char* outDirName, unsigned nbFiles, const char** inFileNamesTable, unsigned nbFiles,
const char* outDirName,
const char* outFileName, const char* suffix, const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams); ZSTD_compressionParameters comprParams);
/** FIO_decompressMultipleFilenames() : /** FIO_decompressMultipleFilenames() :
@return : nb of missing or skipped files */ * @return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles, const char** srcNamesTable, unsigned nbFiles,
const char* outDirName, const char* outDirName,
@ -119,10 +121,12 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char* dictFileName); const char* dictFileName);
/* FIO_checkFilenameCollisions() : /* FIO_checkFilenameCollisions() :
* Checks for and warns if thereå are any files that would have the same output path * Checks for and warns if there are any files that would have the same output path
*/ */
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles); int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
/*-************************************* /*-*************************************
* Advanced stuff (should actually be hosted elsewhere) * Advanced stuff (should actually be hosted elsewhere)
***************************************/ ***************************************/

View File

@ -72,7 +72,8 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
#else #else
{ {
/* (atime, mtime) */ /* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim }; struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim;
res += utimensat(AT_FDCWD, filename, timebuf, 0); res += utimensat(AT_FDCWD, filename, timebuf, 0);
} }
#endif #endif
@ -105,20 +106,23 @@ int UTIL_compareStr(const void *p1, const void *p2) {
return strcmp(* (char * const *) p1, * (char * const *) p2); return strcmp(* (char * const *) p1, * (char * const *) p2);
} }
int UTIL_isSameFile(const char* file1, const char* file2) int UTIL_isSameFile(const char* fName1, const char* fName2)
{ {
#if defined(_MSC_VER) assert(fName1 != NULL); assert(fName2 != NULL);
#if defined(_MSC_VER) || defined(_WIN32)
/* note : Visual does not support file identification by inode. /* note : Visual does not support file identification by inode.
* inode does not work on Windows, even with a posix layer, like msys2.
* The following work-around is limited to detecting exact name repetition only, * The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */ * aka `filename` is considered different from `subdir/../filename` */
return !strcmp(file1, file2); return !strcmp(fName1, fName2);
#else #else
stat_t file1Stat; { stat_t file1Stat;
stat_t file2Stat; stat_t file2Stat;
return UTIL_getFileStat(file1, &file1Stat) return UTIL_getFileStat(fName1, &file1Stat)
&& UTIL_getFileStat(file2, &file2Stat) && UTIL_getFileStat(fName2, &file2Stat)
&& (file1Stat.st_dev == file2Stat.st_dev) && (file1Stat.st_dev == file2Stat.st_dev)
&& (file1Stat.st_ino == file2Stat.st_ino); && (file1Stat.st_ino == file2Stat.st_ino);
}
#endif #endif
} }
@ -239,19 +243,20 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
DIR *dir; DIR *dir;
struct dirent *entry; struct dirent *entry;
char* path; char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0; size_t dirLength, fnameLength, pathLength;
int nbFiles = 0;
if (!(dir = opendir(dirName))) { if (!(dir = opendir(dirName))) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0; return 0;
} }
dirLength = (int)strlen(dirName); dirLength = strlen(dirName);
errno = 0; errno = 0;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
if (strcmp (entry->d_name, "..") == 0 || if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue; strcmp (entry->d_name, ".") == 0) continue;
fnameLength = (int)strlen(entry->d_name); fnameLength = strlen(entry->d_name);
path = (char*) malloc(dirLength + fnameLength + 2); path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; } if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength); memcpy(path, dirName, dirLength);
@ -273,7 +278,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
} else { } else {
if (*bufStart + *pos + pathLength >= *bufEnd) { if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize); assert(newListSize >= 0);
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
*bufEnd = *bufStart + newListSize; *bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; } if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} }
@ -322,7 +328,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
unsigned i, nbFiles; unsigned i, nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE); char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE; char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
if (!buf) return NULL; if (!buf) return NULL;
@ -331,36 +336,37 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
size_t const len = strlen(inputNames[i]); size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) { if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize); assert(newListSize >= 0);
buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
bufend = buf + newListSize; bufend = buf + newListSize;
if (!buf) return NULL; if (!buf) return NULL;
} }
if (buf + pos + len < bufend) { if (buf + pos + len < bufend) {
memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */ memcpy(buf+pos, inputNames[i], len+1); /* including final \0 */
pos += len + 1; pos += len + 1;
nbFiles++; nbFiles++;
} }
} else { } else {
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks); nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL; if (buf == NULL) return NULL;
} } } }
if (nbFiles == 0) { free(buf); return NULL; } if (nbFiles == 0) { free(buf); return NULL; }
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); { const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
if (!fileTable) { free(buf); return NULL; } if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++) { for (i = 0, pos = 0; i < nbFiles; i++) {
fileTable[i] = buf + pos; fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1; if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
pos += strlen(fileTable[i]) + 1;
}
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
} }
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
} }
@ -393,8 +399,13 @@ int UTIL_countPhysicalCores(void)
DWORD returnLength = 0; DWORD returnLength = 0;
size_t byteOffset = 0; size_t byteOffset = 0;
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), #if defined(_MSC_VER)
"GetLogicalProcessorInformation"); /* Visual Studio does not like the following cast */
# pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */
# pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */
#endif
glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
"GetLogicalProcessorInformation");
if (glpi == NULL) { if (glpi == NULL) {
goto failed; goto failed;

Binary file not shown.

Binary file not shown.

View File

@ -1,5 +1,5 @@
. .
.TH "ZSTD" "1" "September 2019" "zstd 1.4.4" "User Commands" .TH "ZSTD" "1" "October 2019" "zstd 1.4.4" "User Commands"
. .
.SH "NAME" .SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@ -206,6 +206,9 @@ add integrity check computed from uncompressed data (default: enabled)
\fB\-\-\fR \fB\-\-\fR
All arguments after \fB\-\-\fR are treated as files All arguments after \fB\-\-\fR are treated as files
. .
.SS "Restricted usage of Environment Variables"
Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR is supported currently, for setting compression level\. \fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. It can be overridden by corresponding command line arguments\.
.
.SH "DICTIONARY BUILDER" .SH "DICTIONARY BUILDER"
\fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\. \fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\.
. .

View File

@ -214,6 +214,16 @@ the last one takes effect.
* `--`: * `--`:
All arguments after `--` are treated as files All arguments after `--` are treated as files
### Restricted usage of Environment Variables
Using environment variables to set parameters has security implications.
Therefore, this avenue is intentionally restricted.
Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
`ZSTD_CLEVEL` just replaces the default compression level (`3`).
It can be overridden by corresponding command line arguments.
DICTIONARY BUILDER DICTIONARY BUILDER
------------------ ------------------

View File

@ -287,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise. * @return 0 and doesn't modify *stringPtr otherwise.
*/ */
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) static int longCommandWArg(const char** stringPtr, const char* longCommand)
{ {
size_t const comSize = strlen(longCommand); size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize); int const result = !strncmp(*stringPtr, longCommand, comSize);
@ -430,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr) static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
{ {
for ( ; ;) { for ( ; ;) {
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
DISPLAYLEVEL(4, "invalid compression parameter \n"); DISPLAYLEVEL(4, "invalid compression parameter \n");
return 0; return 0;
} }
@ -526,7 +526,7 @@ static int init_cLevel(void) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env); DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
return ZSTDCLI_CLEVEL_DEFAULT; return ZSTDCLI_CLEVEL_DEFAULT;
} else if (*ptr == 0) { } else if (*ptr == 0) {
return sign * absLevel; return sign * (int)absLevel;
} }
} }
@ -584,7 +584,7 @@ int main(int argCount, const char* argv[])
int cLevelLast = -1000000000; int cLevelLast = -1000000000;
unsigned recursive = 0; unsigned recursive = 0;
unsigned memLimit = 0; unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0; unsigned filenameIdx = 0;
const char* programName = argv[0]; const char* programName = argv[0];
const char* outFileName = NULL; const char* outFileName = NULL;
@ -745,7 +745,7 @@ int main(int argCount, const char* argv[])
continue; continue;
} }
#endif #endif
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
@ -807,7 +807,7 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
/* compression Level */ /* compression Level */
if ((*argument>='0') && (*argument<='9')) { if ((*argument>='0') && (*argument<='9')) {
dictCLevel = cLevel = readU32FromChar(&argument); dictCLevel = cLevel = (int)readU32FromChar(&argument);
continue; continue;
} }
#endif #endif
@ -879,7 +879,7 @@ int main(int argCount, const char* argv[])
case 'e': case 'e':
/* compression Level */ /* compression Level */
argument++; argument++;
cLevelLast = readU32FromChar(&argument); cLevelLast = (int)readU32FromChar(&argument);
break; break;
/* Modify Nb Iterations (benchmark only) */ /* Modify Nb Iterations (benchmark only) */
@ -905,7 +905,7 @@ int main(int argCount, const char* argv[])
/* nb of threads (hidden option) */ /* nb of threads (hidden option) */
case 'T': case 'T':
argument++; argument++;
nbWorkers = readU32FromChar(&argument); nbWorkers = (int)readU32FromChar(&argument);
break; break;
/* Dictionary Selection level */ /* Dictionary Selection level */
@ -1042,16 +1042,16 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
benchParams.blockSize = blockSize; benchParams.blockSize = blockSize;
benchParams.nbWorkers = nbWorkers; benchParams.nbWorkers = nbWorkers;
benchParams.realTime = setRealTimePrio; benchParams.realTime = (unsigned)setRealTimePrio;
benchParams.nbSeconds = bench_nbSeconds; benchParams.nbSeconds = bench_nbSeconds;
benchParams.ldmFlag = ldmFlag; benchParams.ldmFlag = ldmFlag;
benchParams.ldmMinMatch = g_ldmMinMatch; benchParams.ldmMinMatch = (int)g_ldmMinMatch;
benchParams.ldmHashLog = g_ldmHashLog; benchParams.ldmHashLog = (int)g_ldmHashLog;
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
} }
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) { if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
benchParams.ldmHashRateLog = g_ldmHashRateLog; benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
} }
benchParams.literalCompressionMode = literalCompressionMode; benchParams.literalCompressionMode = literalCompressionMode;
@ -1092,16 +1092,16 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NODICT #ifndef ZSTD_NODICT
ZDICT_params_t zParams; ZDICT_params_t zParams;
zParams.compressionLevel = dictCLevel; zParams.compressionLevel = dictCLevel;
zParams.notificationLevel = g_displayLevel; zParams.notificationLevel = (unsigned)g_displayLevel;
zParams.dictID = dictID; zParams.dictID = dictID;
if (dict == cover) { if (dict == cover) {
int const optimize = !coverParams.k || !coverParams.d; int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = nbWorkers; coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams; coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize); operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
} else if (dict == fastCover) { } else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d; int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = nbWorkers; fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams; fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize); operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
} else { } else {
@ -1120,7 +1120,7 @@ int main(int argCount, const char* argv[])
} }
#ifndef ZSTD_NODECOMPRESS #ifndef ZSTD_NODECOMPRESS
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */ if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
#endif #endif
/* No input filename ==> use stdin and stdout */ /* No input filename ==> use stdin and stdout */
@ -1156,14 +1156,14 @@ int main(int argCount, const char* argv[])
if (operation==zom_compress) { if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
FIO_setNbWorkers(prefs, nbWorkers); FIO_setNbWorkers(prefs, nbWorkers);
FIO_setBlockSize(prefs, (U32)blockSize); FIO_setBlockSize(prefs, (int)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, ldmFlag); FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
FIO_setLdmHashLog(prefs, g_ldmHashLog); FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
FIO_setLdmMinMatch(prefs, g_ldmMinMatch); FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog); if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog); if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, adapt); FIO_setAdaptiveMode(prefs, (unsigned)adapt);
FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax); FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(prefs, rsyncable); FIO_setRsyncable(prefs, rsyncable);
@ -1177,7 +1177,7 @@ int main(int argCount, const char* argv[])
if ((filenameIdx==1) && outFileName) if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
else else
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else #else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n"); DISPLAY("Compression not supported \n");

View File

@ -1,5 +1,5 @@
. .
.TH "ZSTDGREP" "1" "September 2019" "zstd 1.4.4" "User Commands" .TH "ZSTDGREP" "1" "October 2019" "zstd 1.4.4" "User Commands"
. .
.SH "NAME" .SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files

View File

@ -1,5 +1,5 @@
. .
.TH "ZSTDLESS" "1" "September 2019" "zstd 1.4.4" "User Commands" .TH "ZSTDLESS" "1" "October 2019" "zstd 1.4.4" "User Commands"
. .
.SH "NAME" .SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files \fBzstdless\fR \- view zstandard\-compressed files

View File

@ -250,7 +250,8 @@ clean:
$(MAKE) -C $(ZSTDDIR) clean $(MAKE) -C $(ZSTDDIR) clean
$(MAKE) -C $(PRGDIR) clean $(MAKE) -C $(PRGDIR) clean
@$(RM) -fR $(TESTARTEFACT) @$(RM) -fR $(TESTARTEFACT)
@$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \ @$(RM) -rf tmp* # some test directories are named tmp*
@$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \
@ -264,7 +265,7 @@ clean:
#---------------------------------------------------------------------------------- #----------------------------------------------------------------------------------
#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets # valgrind tests are validated only for some posix platforms
#---------------------------------------------------------------------------------- #----------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX HOST_OS = POSIX
@ -286,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench
endif endif
ifneq (,$(filter MSYS%,$(shell uname))) ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
HOST_OS = MSYS HOST_OS = MSYS
endif endif

View File

@ -125,13 +125,14 @@ static size_t compress(uint8_t *dst, size_t capacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{ {
size_t neededBufSize;
/* Give a random portion of src data to the producer, to use for /* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */ parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
size = FUZZ_dataProducer_reserveDataPrefix(producer); size = FUZZ_dataProducer_reserveDataPrefix(producer);
size_t neededBufSize; neededBufSize = ZSTD_compressBound(size) * 15;
neededBufSize = ZSTD_compressBound(size) * 5;
/* Allocate all buffers and contexts if not already allocated */ /* Allocate all buffers and contexts if not already allocated */
if (neededBufSize > bufSize) { if (neededBufSize > bufSize) {

View File

@ -1387,7 +1387,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++); DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict==NULL) goto _output_error; if (cdict==NULL) goto _output_error;
@ -1395,7 +1395,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++); DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict!=NULL) goto _output_error; if (cdict!=NULL) goto _output_error;
@ -1403,7 +1403,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++); DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
{ {
size_t ret; size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@ -1417,7 +1417,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++); DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
{ {
size_t ret; size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);

View File

@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs"
TESTDIR="$SCRIPT_DIR/../tests" TESTDIR="$SCRIPT_DIR/../tests"
UNAME=$(uname) UNAME=$(uname)
isTerminal=false detectedTerminal=false
if [ -t 0 ] && [ -t 1 ] if [ -t 0 ] && [ -t 1 ]
then then
isTerminal=true detectedTerminal=true
fi fi
isTerminal=${isTerminal:-$detectedTerminal}
isWindows=false isWindows=false
INTOVOID="/dev/null" INTOVOID="/dev/null"
@ -211,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t
$ZSTD tmp -c --compress-literals -1 | $ZSTD -t $ZSTD tmp -c --compress-literals -1 | $ZSTD -t
$ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t $ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t
$ZSTD tmp -c --compress-literals -19 | $ZSTD -t $ZSTD tmp -c --compress-literals -19 | $ZSTD -t
$ZSTD -b --fast=1 -i1e1 tmp --compress-literals $ZSTD -b --fast=1 -i0e1 tmp --compress-literals
$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
println "test : file removal" println "test : file removal"
$ZSTD -f --rm tmp $ZSTD -f --rm tmp
@ -239,7 +240,7 @@ rm tmp # erase source file
touch tmp.zst # create destination file touch tmp.zst # create destination file
$ZSTD -f tmp && die "attempt to compress a non existing file" $ZSTD -f tmp && die "attempt to compress a non existing file"
test -f tmp.zst # destination file should still be present test -f tmp.zst # destination file should still be present
rm tmp* rm -rf tmp* # may also erase tmp* directory from previous failed run
println "\n===> decompression only tests " println "\n===> decompression only tests "
head -c 1048576 /dev/zero > tmp head -c 1048576 /dev/zero > tmp
@ -283,7 +284,7 @@ test -f tmpOutDir/tmp1.zst
test -f tmpOutDir/tmp2.zst test -f tmpOutDir/tmp2.zst
println "test : decompress multiple files into an output directory, --output-dir-flat" println "test : decompress multiple files into an output directory, --output-dir-flat"
mkdir tmpOutDirDecomp mkdir tmpOutDirDecomp
$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp $ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2 test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1 test -f tmpOutDirDecomp/tmp1
rm -rf tmp* rm -rf tmp*
@ -577,30 +578,27 @@ println "- Create dictionary with short dictID"
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionaries with shrink-dict flag enabled" println "- Create dictionaries with shrink-dict flag enabled"
$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict $ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 $ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 $ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
println "- Create dictionary with size limit" println "- Create dictionary with size limit"
$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K $ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Compare size of dictionary from 90% training samples with 80% training samples"
$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using all samples for both training and testing" println "- Create dictionary using all samples for both training and testing"
$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using f=16" println "- Create dictionary using f=16"
$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" $ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
println "- Create dictionary using accel=2" println "- Create dictionary using accel=2"
$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using accel=10" println "- Create dictionary using accel=10"
$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary with multithreading" println "- Create dictionary with multithreading"
$ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-fastcover" println "- Test -o before --train-fastcover"
rm -f tmpDict dictionary rm -f tmpDict dictionary
$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict test -f tmpDict
$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary test -f dictionary
rm tmp* dictionary rm tmp* dictionary
@ -674,10 +672,10 @@ $ZSTD -i0b0e3 tmp1
println "bench negative level" println "bench negative level"
$ZSTD -bi0 --fast tmp1 $ZSTD -bi0 --fast tmp1
println "with recursive and quiet modes" println "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1 $ZSTD -rqi0b1e2 tmp1
println "benchmark decompression only" println "benchmark decompression only"
$ZSTD -f tmp1 $ZSTD -f tmp1
$ZSTD -b -d -i1 tmp1.zst $ZSTD -b -d -i0 tmp1.zst
println "\n===> zstd compatibility tests " println "\n===> zstd compatibility tests "

View File

@ -46,7 +46,7 @@ static int testOrder(size_t numThreads, size_t queueSize)
POOL_ctx* const ctx = POOL_create(numThreads, queueSize); POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx); ASSERT_TRUE(ctx);
data.i = 0; data.i = 0;
(void)ZSTD_pthread_mutex_init(&data.mutex, NULL); ASSERT_FALSE(ZSTD_pthread_mutex_init(&data.mutex, NULL));
{ size_t i; { size_t i;
for (i = 0; i < 16; ++i) { for (i = 0; i < 16; ++i) {
POOL_add(ctx, &fn, &data); POOL_add(ctx, &fn, &data);

View File

@ -18,13 +18,15 @@ EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs PROGRAMS_PATH = ../programs
TEST_FILE = ../doc/zstd_compression_format.md TEST_FILE = ../doc/zstd_compression_format.md
CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
-I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99 STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -DNO_snprintf -DNO_vsnprintf # strict ISO C90 is missing these prototypes
-Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-Wstrict-aliasing=1 -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wstrict-aliasing=1
CFLAGS ?= -O3
CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
# Define *.exe as extension for Windows systems # Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS))) ifneq (,$(filter Windows%,$(OS)))
@ -33,6 +35,11 @@ else
EXT = EXT =
endif endif
default : release
release : STDFLAGS =
release : DEBUGFLAGS =
release : all
all: fitblk example zwrapbench minigzip all: fitblk example zwrapbench minigzip
@ -68,35 +75,34 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
$(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests $(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
#.c.o: #.c.o:
# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< # $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
minigzip: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) minigzip: $(EXAMPLE_PATH)/minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@
example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) example: $(EXAMPLE_PATH)/example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) example_zstd: $(EXAMPLE_PATH)/example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
$(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(ZSTDLIBDIR)/libzstd.a: $(ZSTDLIBDIR)/libzstd.a:
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a $(MAKE) -C $(ZSTDLIBDIR) libzstd.a

View File

@ -159,7 +159,7 @@ int main(int argc, char **argv)
if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
/* allocate memory for buffers and compression engine */ /* allocate memory for buffers and compression engine */
blk = malloc(size + EXCESS); blk = (unsigned char*)malloc(size + EXCESS);
def.zalloc = Z_NULL; def.zalloc = Z_NULL;
def.zfree = Z_NULL; def.zfree = Z_NULL;
def.opaque = Z_NULL; def.opaque = Z_NULL;
@ -180,8 +180,8 @@ int main(int argc, char **argv)
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
/* write block to stdout */ /* write block to stdout */
have = size + EXCESS - def.avail_out; have = size + EXCESS - def.avail_out;
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
// quit("error writing output"); * quit("error writing output"); */
/* clean up and print results to stderr */ /* clean up and print results to stderr */
ret = deflateEnd(&def); ret = deflateEnd(&def);
@ -200,7 +200,7 @@ int main(int argc, char **argv)
inf.avail_in = 0; inf.avail_in = 0;
inf.next_in = Z_NULL; inf.next_in = Z_NULL;
ret = inflateInit(&inf); ret = inflateInit(&inf);
tmp = malloc(size + EXCESS); tmp = (unsigned char*)malloc(size + EXCESS);
if (ret != Z_OK || tmp == NULL) if (ret != Z_OK || tmp == NULL)
quit("out of memory"); quit("out of memory");
ret = deflateReset(&def); ret = deflateReset(&def);
@ -237,8 +237,8 @@ int main(int argc, char **argv)
/* done -- write block to stdout */ /* done -- write block to stdout */
have = size - def.avail_out; have = size - def.avail_out;
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
// quit("error writing output"); * quit("error writing output"); */
/* clean up and print results to stderr */ /* clean up and print results to stderr */
free(tmp); free(tmp);

View File

@ -311,14 +311,14 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
ret = deflateReset(&def); ret = deflateReset(&def);
if (ret != Z_OK) EXM_THROW(1, "deflateReset failure"); if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
if (useSetDict) { if (useSetDict) {
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
} }
def.next_in = (z_const void*) blockTable[blockNb].srcPtr; def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
def.avail_in = (uInt)blockTable[blockNb].srcSize; def.avail_in = (uInt)blockTable[blockNb].srcSize;
def.total_in = 0; def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr; def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
def.avail_out = (uInt)blockTable[blockNb].cRoom; def.avail_out = (uInt)blockTable[blockNb].cRoom;
def.total_out = 0; def.total_out = 0;
ret = deflate(&def, Z_FINISH); ret = deflate(&def, Z_FINISH);
@ -343,13 +343,13 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
ret = deflateInit(&def, cLevel); ret = deflateInit(&def, cLevel);
if (ret != Z_OK) EXM_THROW(1, "deflateInit failure"); if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
if (dictBuffer) { if (dictBuffer) {
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
} }
def.next_in = (z_const void*) blockTable[blockNb].srcPtr; def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
def.avail_in = (uInt)blockTable[blockNb].srcSize; def.avail_in = (uInt)blockTable[blockNb].srcSize;
def.total_in = 0; def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr; def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
def.avail_out = (uInt)blockTable[blockNb].cRoom; def.avail_out = (uInt)blockTable[blockNb].cRoom;
def.total_out = 0; def.total_out = 0;
ret = deflate(&def, Z_FINISH); ret = deflate(&def, Z_FINISH);
@ -451,15 +451,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
else else
ret = inflateReset(&inf); ret = inflateReset(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
inf.next_in = (z_const void*) blockTable[blockNb].cPtr; inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.avail_in = (uInt)blockTable[blockNb].cSize;
inf.total_in = 0; inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr; inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.avail_out = (uInt)blockTable[blockNb].srcSize;
inf.total_out = 0; inf.total_out = 0;
ret = inflate(&inf, Z_FINISH); ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) { if (ret == Z_NEED_DICT) {
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize); ret = inflateSetDictionary(&inf, (const z_Bytef*)dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
ret = inflate(&inf, Z_FINISH); ret = inflate(&inf, Z_FINISH);
} }
@ -483,15 +483,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
inf.opaque = Z_NULL; inf.opaque = Z_NULL;
ret = inflateInit(&inf); ret = inflateInit(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
inf.next_in = (z_const void*) blockTable[blockNb].cPtr; inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.avail_in = (uInt)blockTable[blockNb].cSize;
inf.total_in = 0; inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr; inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.avail_out = (uInt)blockTable[blockNb].srcSize;
inf.total_out = 0; inf.total_out = 0;
ret = inflate(&inf, Z_FINISH); ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) { if (ret == Z_NEED_DICT) {
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize); ret = inflateSetDictionary(&inf, (const z_Bytef*) dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
ret = inflate(&inf, Z_FINISH); ret = inflate(&inf, Z_FINISH);
} }

View File

@ -19,7 +19,7 @@ int ZEXPORT gzclose(file)
if (file == NULL) if (file == NULL)
return Z_STREAM_ERROR; return Z_STREAM_ERROR;
state = (gz_statep)file; state.file = file;
return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file);
#else #else

View File

@ -216,7 +216,7 @@ local gzFile gz_open(path, fd, mode)
#if !defined(NO_snprintf) && !defined(NO_vsnprintf) #if !defined(NO_snprintf) && !defined(NO_vsnprintf)
(void)snprintf(state.state->path, len + 1, "%s", (const char *)path); (void)snprintf(state.state->path, len + 1, "%s", (const char *)path);
#else #else
strcpy(state.state->path, path); strcpy(state.state->path, (const char*)path);
#endif #endif
/* compute the flags for open() */ /* compute the flags for open() */
@ -325,7 +325,7 @@ int ZEXPORT gzbuffer(file, size)
/* get internal structure and check integrity */ /* get internal structure and check integrity */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1; return -1;
@ -351,7 +351,7 @@ int ZEXPORT gzrewind(file)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
/* check that we're reading and that there's no error */ /* check that we're reading and that there's no error */
if (state.state->mode != GZ_READ || if (state.state->mode != GZ_READ ||
@ -378,7 +378,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence)
/* get internal structure and check integrity */ /* get internal structure and check integrity */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1; return -1;
@ -463,7 +463,7 @@ z_off64_t ZEXPORT gztell64(file)
/* get internal structure and check integrity */ /* get internal structure and check integrity */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1; return -1;
@ -491,7 +491,7 @@ z_off64_t ZEXPORT gzoffset64(file)
/* get internal structure and check integrity */ /* get internal structure and check integrity */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return -1; return -1;
@ -523,7 +523,7 @@ int ZEXPORT gzeof(file)
/* get internal structure and check integrity */ /* get internal structure and check integrity */
if (file == NULL) if (file == NULL)
return 0; return 0;
state = (gz_statep)file; state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return 0; return 0;
@ -541,7 +541,7 @@ const char * ZEXPORT gzerror(file, errnum)
/* get internal structure and check integrity */ /* get internal structure and check integrity */
if (file == NULL) if (file == NULL)
return NULL; return NULL;
state = (gz_statep)file; state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return NULL; return NULL;
@ -561,7 +561,7 @@ void ZEXPORT gzclearerr(file)
/* get internal structure and check integrity */ /* get internal structure and check integrity */
if (file == NULL) if (file == NULL)
return; return;
state = (gz_statep)file; state.file = file;
if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE)
return; return;

View File

@ -386,7 +386,7 @@ int ZEXPORT gzread(file, buf, len)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
/* check that we're reading and that there's no (serious) error */ /* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ || if (state.state->mode != GZ_READ ||
@ -424,7 +424,7 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return 0; return 0;
state = (gz_statep)file; state.file = file;
/* check that we're reading and that there's no (serious) error */ /* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ || if (state.state->mode != GZ_READ ||
@ -470,7 +470,7 @@ int ZEXPORT gzgetc(file)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
/* check that we're reading and that there's no (serious) error */ /* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ || if (state.state->mode != GZ_READ ||
@ -485,7 +485,7 @@ int ZEXPORT gzgetc(file)
} }
/* nothing there -- try gz_read() */ /* nothing there -- try gz_read() */
ret = (unsigned)gz_read(state, buf, 1); ret = (int)gz_read(state, buf, 1);
return ret < 1 ? -1 : buf[0]; return ret < 1 ? -1 : buf[0];
} }
@ -505,7 +505,7 @@ int ZEXPORT gzungetc(c, file)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
/* check that we're reading and that there's no (serious) error */ /* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ || if (state.state->mode != GZ_READ ||
@ -569,7 +569,7 @@ char * ZEXPORT gzgets(file, buf, len)
/* check parameters and get internal structure */ /* check parameters and get internal structure */
if (file == NULL || buf == NULL || len < 1) if (file == NULL || buf == NULL || len < 1)
return NULL; return NULL;
state = (gz_statep)file; state.file = file;
/* check that we're reading and that there's no (serious) error */ /* check that we're reading and that there's no (serious) error */
if (state.state->mode != GZ_READ || if (state.state->mode != GZ_READ ||
@ -628,7 +628,7 @@ int ZEXPORT gzdirect(file)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return 0; return 0;
state = (gz_statep)file; state.file = file;
/* if the state is not known, but we can find out, then do so (this is /* if the state is not known, but we can find out, then do so (this is
mainly for right after a gzopen() or gzdopen()) */ mainly for right after a gzopen() or gzdopen()) */
@ -649,7 +649,7 @@ int ZEXPORT gzclose_r(file)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return Z_STREAM_ERROR; return Z_STREAM_ERROR;
state = (gz_statep)file; state.file = file;
/* check that we're reading */ /* check that we're reading */
if (state.state->mode != GZ_READ) if (state.state->mode != GZ_READ)

View File

@ -258,7 +258,7 @@ int ZEXPORT gzwrite(file, buf, len)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return 0; return 0;
state = (gz_statep)file; state.file = file;
/* check that we're writing and that there's no error */ /* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -289,7 +289,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
assert(size != 0); assert(size != 0);
if (file == NULL) if (file == NULL)
return 0; return 0;
state = (gz_statep)file; state.file = file;
/* check that we're writing and that there's no error */ /* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -319,7 +319,7 @@ int ZEXPORT gzputc(file, c)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
strm = &(state.state->strm); strm = &(state.state->strm);
/* check that we're writing and that there's no error */ /* check that we're writing and that there's no error */
@ -366,7 +366,7 @@ int ZEXPORT gzputs(file, str)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return -1; return -1;
state = (gz_statep)file; state.file = file;
/* check that we're writing and that there's no error */ /* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -393,7 +393,7 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return Z_STREAM_ERROR; return Z_STREAM_ERROR;
state = (gz_statep)file; state.file = file;
strm = &(state.state->strm); strm = &(state.state->strm);
/* check that we're writing and that there's no error */ /* check that we're writing and that there's no error */
@ -565,7 +565,7 @@ int ZEXPORT gzflush(file, flush)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return Z_STREAM_ERROR; return Z_STREAM_ERROR;
state = (gz_statep)file; state.file = file;
/* check that we're writing and that there's no error */ /* check that we're writing and that there's no error */
if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) if (state.state->mode != GZ_WRITE || state.state->err != Z_OK)
@ -599,7 +599,7 @@ int ZEXPORT gzsetparams(file, level, strategy)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return Z_STREAM_ERROR; return Z_STREAM_ERROR;
state = (gz_statep)file; state.file = file;
strm = &(state.state->strm); strm = &(state.state->strm);
/* check that we're writing and that there's no error */ /* check that we're writing and that there's no error */
@ -639,7 +639,7 @@ int ZEXPORT gzclose_w(file)
/* get internal structure */ /* get internal structure */
if (file == NULL) if (file == NULL)
return Z_STREAM_ERROR; return Z_STREAM_ERROR;
state = (gz_statep)file; state.file = file;
/* check that we're writing */ /* check that we're writing */
if (state.state->mode != GZ_WRITE) if (state.state->mode != GZ_WRITE)

View File

@ -54,7 +54,7 @@ int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; }
static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO; static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO;
void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }; void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }
ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; } ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; }
@ -99,7 +99,7 @@ typedef struct {
unsigned long long pledgedSrcSize; unsigned long long pledgedSrcSize;
} ZWRAP_CCtx; } ZWRAP_CCtx;
typedef ZWRAP_CCtx internal_state; /* typedef ZWRAP_CCtx internal_state; */
@ -121,8 +121,10 @@ static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
if (zwc==NULL) return NULL; if (zwc==NULL) return NULL;
memset(zwc, 0, sizeof(ZWRAP_CCtx)); memset(zwc, 0, sizeof(ZWRAP_CCtx));
memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
zwc->customMem = ZWRAP_customMem; } ZWRAP_customMem.opaque = &zwc->allocFunc;
zwc->customMem = ZWRAP_customMem;
}
} else { } else {
zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc)); zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc));
if (zwc==NULL) return NULL; if (zwc==NULL) return NULL;
@ -462,8 +464,10 @@ static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
if (zwd==NULL) return NULL; if (zwd==NULL) return NULL;
memset(zwd, 0, sizeof(ZWRAP_DCtx)); memset(zwd, 0, sizeof(ZWRAP_DCtx));
zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */ zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
zwd->customMem = ZWRAP_customMem; } ZWRAP_customMem.opaque = &zwd->allocFunc;
zwd->customMem = ZWRAP_customMem;
}
} else { } else {
zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd)); zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
if (zwd==NULL) return NULL; if (zwd==NULL) return NULL;
@ -509,7 +513,7 @@ static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
const char *version, int stream_size)) const char* version, int stream_size))
{ {
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
strm->reserved = ZWRAP_ZLIB_STREAM; strm->reserved = ZWRAP_ZLIB_STREAM;
@ -520,7 +524,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
LOG_WRAPPERD("- inflateInit\n"); LOG_WRAPPERD("- inflateInit\n");
if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
zwd->version = ZSTD_malloc(strlen(version)+1, zwd->customMem); zwd->version = (char*)ZSTD_malloc(strlen(version)+1, zwd->customMem);
if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
strcpy(zwd->version, version); strcpy(zwd->version, version);
@ -1003,7 +1007,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm))
} }
ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); }; ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); }