Merge pull request #15 from facebook/dev
Merging facebook/dev into bimbashrestha/dev
This commit is contained in:
Bimba Shrestha
GitHub
commit
cf62c31f82
@ -26,6 +26,12 @@ env:
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
- name: arm64
|
||||
os: linux
|
||||
arch: arm64
|
||||
script:
|
||||
- make check
|
||||
|
||||
- name: Trusty (Test All)
|
||||
script:
|
||||
- make test
|
||||
@ -222,5 +228,6 @@ matrix:
|
||||
- tree ./staging
|
||||
after_failure:
|
||||
- cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
|
||||
|
||||
allow_failures:
|
||||
- env: ALLOW_FAILURES=true
|
||||
|
||||
29
CHANGELOG
29
CHANGELOG
@ -1,8 +1,33 @@
|
||||
v1.4.4
|
||||
perf: Improved decompression speed, by > 10%, by @terrelln
|
||||
perf: Better compression speed when re-using a context, by @felixhandte
|
||||
perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
|
||||
perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
|
||||
perf: minor generic speed optimization, by @davidbolvansky
|
||||
api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
|
||||
api: fixed decoding of magic-less frames, by @terrelln
|
||||
api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
|
||||
cli: Named pipes support, by @bimbashrestha
|
||||
cli: short tar's extension support, by @stokito
|
||||
cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
|
||||
cli: commands --stream-size=# and --size-hint=#, by @nmagerko
|
||||
cli: command --exclude-compressed, by @shashank0791
|
||||
cli: faster `-t` test mode
|
||||
cli: improved some error messages, by @vangyzen
|
||||
cli: rare deadlock condition within dictionary builder, by @terrelln
|
||||
build: single-file decoder with emscripten compilation script, by @cwoffenden
|
||||
build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
|
||||
build: fixed deprecation warning for certain gcc version, reported by @jasonma163
|
||||
build: fix compilation on old gcc versions, by @cemeyer
|
||||
build: improved installation directories for cmake script, by Dmitri Shubin
|
||||
pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
|
||||
misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
|
||||
misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
|
||||
|
||||
v1.4.3
|
||||
bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
|
||||
bug: Fix Buffer Overflow in v0.3 Decompression by @felixhandte (#1722)
|
||||
bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
|
||||
build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
|
||||
misc: Add NULL pointer check in util.c by @leeyoung624 (#1706)
|
||||
|
||||
v1.4.2
|
||||
bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
|
||||
|
||||
@ -43,7 +43,7 @@ on the [Silesia compression corpus].
|
||||
|
||||
| Compressor name | Ratio | Compression| Decompress.|
|
||||
| --------------- | ------| -----------| ---------- |
|
||||
| **zstd 1.4.0 -1** | 2.884 | 530 MB/s | 1360 MB/s |
|
||||
| **zstd 1.4.4 -1** | 2.884 | 520 MB/s | 1600 MB/s |
|
||||
| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 440 MB/s |
|
||||
| brotli 1.0.7 -0 | 2.701 | 430 MB/s | 470 MB/s |
|
||||
| quicklz 1.5.0 -1 | 2.238 | 600 MB/s | 800 MB/s |
|
||||
|
||||
@ -173,8 +173,6 @@
|
||||
sh -e playTests.sh --test-large-data &&
|
||||
fullbench.exe -i1 &&
|
||||
fullbench.exe -i1 -P0 &&
|
||||
fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
|
||||
|
||||
@ -124,9 +124,13 @@ endif ()
|
||||
#-----------------------------------------------------------------------------
|
||||
add_subdirectory(lib)
|
||||
|
||||
option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
|
||||
|
||||
if (ZSTD_BUILD_PROGRAMS)
|
||||
if (NOT ZSTD_BUILD_STATIC)
|
||||
if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build static library to build zstd CLI")
|
||||
elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
|
||||
message(SEND_ERROR "You need to build shared library to build zstd CLI")
|
||||
endif ()
|
||||
|
||||
add_subdirectory(programs)
|
||||
|
||||
@ -5,9 +5,9 @@ use case sensitivity that matches modern (ie. cmake version 2.6 and above)
|
||||
conventions of using lower-case for commands, and upper-case for
|
||||
variables.
|
||||
|
||||
# How to build
|
||||
## How to build
|
||||
|
||||
As cmake doesn't support command like `cmake clean`, it's recommanded to perform a "out of source build".
|
||||
As cmake doesn't support command like `cmake clean`, it's recommended to perform a "out of source build".
|
||||
To do this, you can create a new directory and build in it:
|
||||
```sh
|
||||
cd build/cmake
|
||||
@ -16,7 +16,7 @@ cd builddir
|
||||
cmake ..
|
||||
make
|
||||
```
|
||||
Then you can clean all cmake caches by simpily delete the new directory:
|
||||
Then you can clean all cmake caches by simply delete the new directory:
|
||||
```sh
|
||||
rm -rf build/cmake/builddir
|
||||
```
|
||||
@ -34,19 +34,19 @@ cd build/cmake/builddir
|
||||
cmake -LH ..
|
||||
```
|
||||
|
||||
Bool options can be set to ON/OFF with -D\[option\]=\[ON/OFF\]. You can configure cmake options like this:
|
||||
Bool options can be set to `ON/OFF` with `-D[option]=[ON/OFF]`. You can configure cmake options like this:
|
||||
```sh
|
||||
cd build/cmake/builddir
|
||||
cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON ..
|
||||
make
|
||||
```
|
||||
|
||||
## referring
|
||||
### referring
|
||||
[Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
|
||||
|
||||
# CMake Style Recommendations
|
||||
## CMake Style Recommendations
|
||||
|
||||
## Indent all code correctly, i.e. the body of
|
||||
### Indent all code correctly, i.e. the body of
|
||||
|
||||
* if/else/endif
|
||||
* foreach/endforeach
|
||||
@ -57,7 +57,7 @@ make
|
||||
Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of
|
||||
spaces for indenting as is used in the rest of the file. Do not use tabs.
|
||||
|
||||
## Upper/lower casing
|
||||
### Upper/lower casing
|
||||
|
||||
Most important: use consistent upper- or lowercasing within one file !
|
||||
|
||||
@ -77,7 +77,7 @@ Add_Executable(hello hello.c)
|
||||
aDd_ExEcUtAbLe(blub blub.c)
|
||||
```
|
||||
|
||||
## End commands
|
||||
### End commands
|
||||
To make the code easier to read, use empty commands for endforeach(), endif(),
|
||||
endfunction(), endmacro() and endwhile(). Also, use empty else() commands.
|
||||
|
||||
@ -99,6 +99,6 @@ if(BARVAR)
|
||||
endif(BARVAR)
|
||||
```
|
||||
|
||||
## Other resources for best practices
|
||||
### Other resources for best practices
|
||||
|
||||
`https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules`
|
||||
https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules
|
||||
|
||||
@ -134,11 +134,10 @@ if (UNIX)
|
||||
# pkg-config
|
||||
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
|
||||
set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
|
||||
set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
|
||||
set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
|
||||
add_custom_target(libzstd.pc ALL
|
||||
${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
|
||||
-DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
|
||||
-DPREFIX="${PREFIX}" -DVERSION="${VERSION}"
|
||||
-P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake"
|
||||
COMMENT "Creating pkg-config file")
|
||||
|
||||
|
||||
@ -21,13 +21,19 @@ if (ZSTD_LEGACY_SUPPORT)
|
||||
include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
endif ()
|
||||
|
||||
if (ZSTD_PROGRAMS_LINK_SHARED)
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared)
|
||||
else ()
|
||||
set(PROGRAMS_ZSTD_LINK_TARGET libzstd_static)
|
||||
endif ()
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
|
||||
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
|
||||
endif ()
|
||||
|
||||
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
|
||||
target_link_libraries(zstd libzstd_static)
|
||||
target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
|
||||
target_link_libraries(zstd rt)
|
||||
endif ()
|
||||
@ -68,7 +74,7 @@ if (UNIX)
|
||||
DESTINATION "${MAN_INSTALL_DIR}")
|
||||
|
||||
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
|
||||
target_link_libraries(zstd-frugal libzstd_static)
|
||||
target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
|
||||
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||
endif ()
|
||||
|
||||
|
||||
@ -692,12 +692,17 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
||||
|
||||
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
|
||||
int compressionLevel);
|
||||
</b><p> When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
|
||||
ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
|
||||
</b><p> When compressing multiple messages or blocks using the same dictionary,
|
||||
it's recommended to digest the dictionary only once, since it's a costly operation.
|
||||
ZSTD_createCDict() will create a state from digesting a dictionary.
|
||||
The resulting state can be used for future compression operations with very limited startup cost.
|
||||
ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
||||
`dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
|
||||
Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
|
||||
Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data.
|
||||
@dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
|
||||
Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
|
||||
Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
|
||||
in which case the only thing that it transports is the @compressionLevel.
|
||||
This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
|
||||
expecting a ZSTD_CDict parameter with any data, including those without a known dictionary.
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
||||
@ -947,7 +952,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
* to evolve and should be considered only in the context of extremely
|
||||
* advanced performance tuning.
|
||||
*
|
||||
* Zstd currently supports the use of a CDict in two ways:
|
||||
* Zstd currently supports the use of a CDict in three ways:
|
||||
*
|
||||
* - The contents of the CDict can be copied into the working context. This
|
||||
* means that the compression can search both the dictionary and input
|
||||
@ -963,6 +968,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
* working context's tables can be reused). For small inputs, this can be
|
||||
* faster than copying the CDict's tables.
|
||||
*
|
||||
* - The CDict's tables are not used at all, and instead we use the working
|
||||
* context alone to reload the dictionary and use params based on the source
|
||||
* size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
|
||||
* This method is effective when the dictionary sizes are very small relative
|
||||
* to the input size, and the input size is fairly large to begin with.
|
||||
*
|
||||
* Zstd has a simple internal heuristic that selects which strategy to use
|
||||
* at the beginning of a compression. However, if experimentation shows that
|
||||
* Zstd is making poor choices, it is possible to override that choice with
|
||||
@ -970,7 +981,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
*/
|
||||
ZSTD_dictDefaultAttach = 0, </b>/* Use the default heuristic. */<b>
|
||||
ZSTD_dictForceAttach = 1, </b>/* Never copy the dictionary. */<b>
|
||||
ZSTD_dictForceCopy = 2 </b>/* Always copy the dictionary. */<b>
|
||||
ZSTD_dictForceCopy = 2, </b>/* Always copy the dictionary. */<b>
|
||||
ZSTD_dictForceLoad = 3 </b>/* Always reload the dictionary */<b>
|
||||
} ZSTD_dictAttachPref_e;
|
||||
</b></pre><BR>
|
||||
<pre><b>typedef enum {
|
||||
|
||||
@ -244,8 +244,6 @@ libzstd.pc:
|
||||
libzstd.pc: libzstd.pc.in
|
||||
@echo creating pkgconfig
|
||||
@sed -e 's|@PREFIX@|$(PREFIX)|' \
|
||||
-e 's|@LIBDIR@|$(LIBDIR)|' \
|
||||
-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
|
||||
-e 's|@VERSION@|$(VERSION)|' \
|
||||
$< >$@
|
||||
|
||||
|
||||
@ -27,10 +27,10 @@ Enabling multithreading requires 2 conditions :
|
||||
Both conditions are automatically applied when invoking `make lib-mt` target.
|
||||
|
||||
When linking a POSIX program with a multithreaded version of `libzstd`,
|
||||
note that it's necessary to request the `-pthread` flag during link stage.
|
||||
note that it's necessary to invoke the `-pthread` flag during link stage.
|
||||
|
||||
Multithreading capabilities are exposed
|
||||
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
|
||||
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).
|
||||
|
||||
|
||||
#### API
|
||||
@ -112,6 +112,17 @@ The file structure is designed to make this selection manually achievable for an
|
||||
will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
|
||||
the shared library, which is now hidden by default.
|
||||
|
||||
- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries
|
||||
which can detect at runtime the presence of BMI2 instructions, and use them only if present.
|
||||
These instructions contribute to better performance, notably on the decoder side.
|
||||
By default, this feature is automatically enabled on detecting
|
||||
the right instruction set (x64) and compiler (clang or gcc >= 5).
|
||||
It's obviously disabled for different cpus,
|
||||
or when BMI2 instruction set is _required_ by the compiler command line
|
||||
(in this case, only the BMI2 code path is generated).
|
||||
Setting this macro will either force to generate the BMI2 dispatcher (1)
|
||||
or prevent it (0). It overrides automatic detection.
|
||||
|
||||
|
||||
#### Windows : using MinGW+MSYS to create DLL
|
||||
|
||||
|
||||
@ -50,6 +50,7 @@ struct ZSTD_CDict_s {
|
||||
ZSTD_compressedBlockState_t cBlockState;
|
||||
ZSTD_customMem customMem;
|
||||
U32 dictID;
|
||||
int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */
|
||||
}; /* typedef'd to ZSTD_CDict within "zstd.h" */
|
||||
|
||||
ZSTD_CCtx* ZSTD_createCCtx(void)
|
||||
@ -387,7 +388,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
|
||||
case ZSTD_c_forceAttachDict:
|
||||
ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy);
|
||||
bounds.lowerBound = ZSTD_dictDefaultAttach;
|
||||
bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */
|
||||
bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */
|
||||
return bounds;
|
||||
|
||||
case ZSTD_c_literalCompressionMode:
|
||||
@ -2770,7 +2771,7 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym
|
||||
/*! ZSTD_loadZstdDictionary() :
|
||||
* @return : dictID, or an error code
|
||||
* assumptions : magic number supposed already checked
|
||||
* dictSize supposed > 8
|
||||
* dictSize supposed >= 8
|
||||
*/
|
||||
static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
|
||||
ZSTD_matchState_t* ms,
|
||||
@ -2787,7 +2788,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
|
||||
size_t dictID;
|
||||
|
||||
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
|
||||
assert(dictSize > 8);
|
||||
assert(dictSize >= 8);
|
||||
assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY);
|
||||
|
||||
dictPtr += 4; /* skip magic number */
|
||||
@ -2889,7 +2890,10 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
|
||||
void* workspace)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
|
||||
if ((dict==NULL) || (dictSize<=8)) return 0;
|
||||
if ((dict==NULL) || (dictSize<8)) {
|
||||
RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTD_reset_compressedBlockState(bs);
|
||||
|
||||
@ -2912,6 +2916,9 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
|
||||
bs, ms, ws, params, dict, dictSize, dtlm, workspace);
|
||||
}
|
||||
|
||||
#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
|
||||
#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6)
|
||||
|
||||
/*! ZSTD_compressBegin_internal() :
|
||||
* @return : 0, or an error code */
|
||||
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
|
||||
@ -2926,17 +2933,27 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
|
||||
/* params are supposed to be fully validated at this point */
|
||||
assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
|
||||
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
|
||||
|
||||
if (cdict && cdict->dictContentSize>0) {
|
||||
if ( (cdict)
|
||||
&& (cdict->dictContentSize > 0)
|
||||
&& ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
|
||||
|| pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
|
||||
|| pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
|
||||
|| cdict->compressionLevel == 0)
|
||||
&& (params->attachDictPref != ZSTD_dictForceLoad) ) {
|
||||
return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
|
||||
}
|
||||
|
||||
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize,
|
||||
ZSTDcrp_makeClean, zbuff) );
|
||||
{ size_t const dictID = ZSTD_compress_insertDictionary(
|
||||
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
|
||||
&cctx->workspace, params, dict, dictSize, dictContentType, dtlm,
|
||||
cctx->entropyWorkspace);
|
||||
{ size_t const dictID = cdict ?
|
||||
ZSTD_compress_insertDictionary(
|
||||
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
|
||||
&cctx->workspace, params, cdict->dictContent, cdict->dictContentSize,
|
||||
dictContentType, dtlm, cctx->entropyWorkspace)
|
||||
: ZSTD_compress_insertDictionary(
|
||||
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
|
||||
&cctx->workspace, params, dict, dictSize,
|
||||
dictContentType, dtlm, cctx->entropyWorkspace);
|
||||
FORWARD_IF_ERROR(dictID);
|
||||
assert(dictID <= UINT_MAX);
|
||||
cctx->dictID = (U32)dictID;
|
||||
@ -3205,7 +3222,7 @@ static size_t ZSTD_initCDict_internal(
|
||||
ZSTDirp_reset,
|
||||
ZSTD_resetTarget_CDict));
|
||||
/* (Maybe) load the dictionary
|
||||
* Skips loading the dictionary if it is <= 8 bytes.
|
||||
* Skips loading the dictionary if it is < 8 bytes.
|
||||
*/
|
||||
{ ZSTD_CCtx_params params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
@ -3254,6 +3271,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
|
||||
assert(cdict != NULL);
|
||||
ZSTD_cwksp_move(&cdict->workspace, &ws);
|
||||
cdict->customMem = customMem;
|
||||
cdict->compressionLevel = 0; /* signals advanced API usage */
|
||||
|
||||
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
|
||||
dictBuffer, dictSize,
|
||||
dictLoadMethod, dictContentType,
|
||||
@ -3269,9 +3288,12 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
|
||||
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
return ZSTD_createCDict_advanced(dict, dictSize,
|
||||
ZSTD_dlm_byCopy, ZSTD_dct_auto,
|
||||
cParams, ZSTD_defaultCMem);
|
||||
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize,
|
||||
ZSTD_dlm_byCopy, ZSTD_dct_auto,
|
||||
cParams, ZSTD_defaultCMem);
|
||||
if (cdict)
|
||||
cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
|
||||
return cdict;
|
||||
}
|
||||
|
||||
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
|
||||
@ -3361,7 +3383,15 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
|
||||
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
|
||||
RETURN_ERROR_IF(cdict==NULL, dictionary_wrong);
|
||||
{ ZSTD_CCtx_params params = cctx->requestedParams;
|
||||
params.cParams = ZSTD_getCParamsFromCDict(cdict);
|
||||
params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
|
||||
|| pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
|
||||
|| pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
|
||||
|| cdict->compressionLevel == 0 )
|
||||
&& (params.attachDictPref != ZSTD_dictForceLoad) ?
|
||||
ZSTD_getCParamsFromCDict(cdict)
|
||||
: ZSTD_getCParams(cdict->compressionLevel,
|
||||
pledgedSrcSize,
|
||||
cdict->dictContentSize);
|
||||
/* Increase window log to fit the entire dictionary and source if the
|
||||
* source size is known. Limit the increase to 19, which is the
|
||||
* window log for compression level 1 with the largest source size.
|
||||
|
||||
@ -1096,7 +1096,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
||||
size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
|
||||
for (i=0; i<3; i++) {
|
||||
U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
|
||||
RETURN_ERROR_IF(rep==0 || rep >= dictContentSize,
|
||||
RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
|
||||
dictionary_corrupted);
|
||||
entropy->rep[i] = rep;
|
||||
} }
|
||||
@ -1265,7 +1265,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
|
||||
{
|
||||
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
||||
ZSTD_clearDict(dctx);
|
||||
if (dict && dictSize >= 8) {
|
||||
if (dict && dictSize != 0) {
|
||||
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
|
||||
RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
|
||||
dctx->ddict = dctx->ddictLocal;
|
||||
|
||||
@ -617,7 +617,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_
|
||||
ptrdiff_t const diff = op - ip;
|
||||
BYTE* const oend = op + length;
|
||||
|
||||
assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8)) ||
|
||||
assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
|
||||
(ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
|
||||
|
||||
if (length < 8) {
|
||||
|
||||
@ -36,16 +36,17 @@ extern "C" {
|
||||
*****************************************************************/
|
||||
/* Deprecation warnings */
|
||||
/* Should these warnings be a problem,
|
||||
it is generally possible to disable them,
|
||||
typically with -Wno-deprecated-declarations for gcc
|
||||
or _CRT_SECURE_NO_WARNINGS in Visual.
|
||||
Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */
|
||||
* it is generally possible to disable them,
|
||||
* typically with -Wno-deprecated-declarations for gcc
|
||||
* or _CRT_SECURE_NO_WARNINGS in Visual.
|
||||
* Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS
|
||||
*/
|
||||
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
|
||||
#else
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
|
||||
# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
|
||||
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
|
||||
|
||||
@ -3,8 +3,9 @@
|
||||
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
prefix=@PREFIX@
|
||||
libdir=@LIBDIR@
|
||||
includedir=@INCLUDEDIR@
|
||||
exec_prefix=${prefix}
|
||||
includedir=${prefix}/include
|
||||
libdir=${exec_prefix}/lib
|
||||
|
||||
Name: zstd
|
||||
Description: fast lossless compression algorithm library
|
||||
|
||||
26
lib/zstd.h
26
lib/zstd.h
@ -808,12 +808,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
typedef struct ZSTD_CDict_s ZSTD_CDict;
|
||||
|
||||
/*! ZSTD_createCDict() :
|
||||
* When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
|
||||
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
|
||||
* When compressing multiple messages or blocks using the same dictionary,
|
||||
* it's recommended to digest the dictionary only once, since it's a costly operation.
|
||||
* ZSTD_createCDict() will create a state from digesting a dictionary.
|
||||
* The resulting state can be used for future compression operations with very limited startup cost.
|
||||
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
||||
* `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
|
||||
* Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
|
||||
* Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
|
||||
* @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
|
||||
* Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
|
||||
* Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
|
||||
* in which case the only thing that it transports is the @compressionLevel.
|
||||
* This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
|
||||
* expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
|
||||
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
|
||||
int compressionLevel);
|
||||
|
||||
@ -1152,7 +1157,7 @@ typedef enum {
|
||||
* to evolve and should be considered only in the context of extremely
|
||||
* advanced performance tuning.
|
||||
*
|
||||
* Zstd currently supports the use of a CDict in two ways:
|
||||
* Zstd currently supports the use of a CDict in three ways:
|
||||
*
|
||||
* - The contents of the CDict can be copied into the working context. This
|
||||
* means that the compression can search both the dictionary and input
|
||||
@ -1168,6 +1173,12 @@ typedef enum {
|
||||
* working context's tables can be reused). For small inputs, this can be
|
||||
* faster than copying the CDict's tables.
|
||||
*
|
||||
* - The CDict's tables are not used at all, and instead we use the working
|
||||
* context alone to reload the dictionary and use params based on the source
|
||||
* size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
|
||||
* This method is effective when the dictionary sizes are very small relative
|
||||
* to the input size, and the input size is fairly large to begin with.
|
||||
*
|
||||
* Zstd has a simple internal heuristic that selects which strategy to use
|
||||
* at the beginning of a compression. However, if experimentation shows that
|
||||
* Zstd is making poor choices, it is possible to override that choice with
|
||||
@ -1175,7 +1186,8 @@ typedef enum {
|
||||
*/
|
||||
ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
|
||||
ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
|
||||
ZSTD_dictForceCopy = 2 /* Always copy the dictionary. */
|
||||
ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
|
||||
ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
|
||||
} ZSTD_dictAttachPref_e;
|
||||
|
||||
typedef enum {
|
||||
|
||||
@ -319,6 +319,8 @@ struct FIO_prefs_s {
|
||||
/* Computation resources preferences */
|
||||
unsigned memLimit;
|
||||
int nbWorkers;
|
||||
|
||||
int excludeCompressedFiles;
|
||||
};
|
||||
|
||||
|
||||
@ -359,6 +361,7 @@ FIO_prefs_t* FIO_createPreferences(void)
|
||||
ret->srcSizeHint = 0;
|
||||
ret->testMode = 0;
|
||||
ret->literalCompressionMode = ZSTD_lcm_auto;
|
||||
ret->excludeCompressedFiles = 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -402,6 +405,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
|
||||
prefs->nbWorkers = nbWorkers;
|
||||
}
|
||||
|
||||
void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; }
|
||||
|
||||
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
|
||||
if (blockSize && prefs->nbWorkers==0)
|
||||
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
|
||||
@ -520,7 +525,11 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!UTIL_isRegularFile(srcFileName)) {
|
||||
if (!UTIL_isRegularFile(srcFileName)
|
||||
#ifndef _MSC_VER
|
||||
&& !UTIL_isFIFO(srcFileName)
|
||||
#endif /* _MSC_VER */
|
||||
) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
|
||||
srcFileName);
|
||||
return NULL;
|
||||
@ -1425,6 +1434,21 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
|
||||
return result;
|
||||
}
|
||||
|
||||
/* List used to compare file extensions (used with --exclude-compressed flag)
|
||||
* Different from the suffixList and should only apply to ZSTD compress operationResult
|
||||
*/
|
||||
static const char *compressedFileExtensions[] = {
|
||||
ZSTD_EXTENSION,
|
||||
TZSTD_EXTENSION,
|
||||
GZ_EXTENSION,
|
||||
TGZ_EXTENSION,
|
||||
LZMA_EXTENSION,
|
||||
XZ_EXTENSION,
|
||||
TXZ_EXTENSION,
|
||||
LZ4_EXTENSION,
|
||||
TLZ4_EXTENSION,
|
||||
NULL
|
||||
};
|
||||
|
||||
/*! FIO_compressFilename_srcFile() :
|
||||
* @return : 0 : compression completed correctly,
|
||||
@ -1451,6 +1475,15 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
|
||||
* YES => ZSTD will skip compression of the file and will return 0.
|
||||
* NO => ZSTD will resume with compress operation.
|
||||
*/
|
||||
if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
|
||||
DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ress.srcFile = FIO_openSrcFile(srcFileName);
|
||||
if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */
|
||||
|
||||
@ -1496,17 +1529,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
|
||||
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
|
||||
char* outDirFilename = NULL;
|
||||
size_t sfnSize = strlen(srcFileName);
|
||||
size_t const suffixSize = strlen(suffix);
|
||||
size_t const srcSuffixLen = strlen(suffix);
|
||||
if (outDirName) {
|
||||
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize);
|
||||
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen);
|
||||
sfnSize = strlen(outDirFilename);
|
||||
assert(outDirFilename != NULL);
|
||||
}
|
||||
|
||||
if (dfnbCapacity <= sfnSize+suffixSize+1) {
|
||||
if (dfnbCapacity <= sfnSize+srcSuffixLen+1) {
|
||||
/* resize buffer for dstName */
|
||||
free(dstFileNameBuffer);
|
||||
dfnbCapacity = sfnSize + suffixSize + 30;
|
||||
dfnbCapacity = sfnSize + srcSuffixLen + 30;
|
||||
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
|
||||
if (!dstFileNameBuffer) {
|
||||
EXM_THROW(30, "zstd: %s", strerror(errno));
|
||||
@ -1520,7 +1553,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
|
||||
} else {
|
||||
memcpy(dstFileNameBuffer, srcFileName, sfnSize);
|
||||
}
|
||||
memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
|
||||
memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */);
|
||||
return dstFileNameBuffer;
|
||||
}
|
||||
|
||||
@ -2287,6 +2320,37 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs,
|
||||
return decodingError;
|
||||
}
|
||||
|
||||
static const char *suffixList[] = {
|
||||
ZSTD_EXTENSION,
|
||||
TZSTD_EXTENSION,
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
GZ_EXTENSION,
|
||||
TGZ_EXTENSION,
|
||||
#endif
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
LZMA_EXTENSION,
|
||||
XZ_EXTENSION,
|
||||
TXZ_EXTENSION,
|
||||
#endif
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
LZ4_EXTENSION,
|
||||
TLZ4_EXTENSION,
|
||||
#endif
|
||||
NULL
|
||||
};
|
||||
|
||||
static const char *suffixListStr =
|
||||
ZSTD_EXTENSION "/" TZSTD_EXTENSION
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
"/" GZ_EXTENSION "/" TGZ_EXTENSION
|
||||
#endif
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
"/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION
|
||||
#endif
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
"/" LZ4_EXTENSION "/" TLZ4_EXTENSION
|
||||
#endif
|
||||
;
|
||||
|
||||
/* FIO_determineDstName() :
|
||||
* create a destination filename from a srcFileName.
|
||||
@ -2297,71 +2361,78 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
|
||||
{
|
||||
static size_t dfnbCapacity = 0;
|
||||
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
|
||||
size_t dstFileNameEndPos;
|
||||
char* outDirFilename = NULL;
|
||||
const char* dstSuffix = "";
|
||||
size_t dstSuffixLen = 0;
|
||||
|
||||
size_t sfnSize = strlen(srcFileName);
|
||||
size_t suffixSize;
|
||||
|
||||
const char* const suffixPtr = strrchr(srcFileName, '.');
|
||||
if (suffixPtr == NULL) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
|
||||
srcFileName);
|
||||
size_t srcSuffixLen;
|
||||
const char* const srcSuffix = strrchr(srcFileName, '.');
|
||||
if (srcSuffix == NULL) {
|
||||
DISPLAYLEVEL(1,
|
||||
"zstd: %s: unknown suffix (%s expected). "
|
||||
"Can't derive the output file name. "
|
||||
"Specify it with -o dstFileName. Ignoring.\n",
|
||||
srcFileName, suffixListStr);
|
||||
return NULL;
|
||||
}
|
||||
suffixSize = strlen(suffixPtr);
|
||||
srcSuffixLen = strlen(srcSuffix);
|
||||
|
||||
/* check suffix is authorized */
|
||||
if (sfnSize <= suffixSize
|
||||
|| ( strcmp(suffixPtr, ZSTD_EXTENSION)
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
&& strcmp(suffixPtr, GZ_EXTENSION)
|
||||
#endif
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
&& strcmp(suffixPtr, XZ_EXTENSION)
|
||||
&& strcmp(suffixPtr, LZMA_EXTENSION)
|
||||
#endif
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
&& strcmp(suffixPtr, LZ4_EXTENSION)
|
||||
#endif
|
||||
) ) {
|
||||
const char* suffixlist = ZSTD_EXTENSION
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
"/" GZ_EXTENSION
|
||||
#endif
|
||||
#ifdef ZSTD_LZMADECOMPRESS
|
||||
"/" XZ_EXTENSION "/" LZMA_EXTENSION
|
||||
#endif
|
||||
#ifdef ZSTD_LZ4DECOMPRESS
|
||||
"/" LZ4_EXTENSION
|
||||
#endif
|
||||
;
|
||||
DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n",
|
||||
srcFileName, suffixlist);
|
||||
return NULL;
|
||||
{
|
||||
const char** matchedSuffixPtr;
|
||||
for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) {
|
||||
if (!strcmp(*matchedSuffixPtr, srcSuffix)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* check suffix is authorized */
|
||||
if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) {
|
||||
DISPLAYLEVEL(1,
|
||||
"zstd: %s: unknown suffix (%s expected). "
|
||||
"Can't derive the output file name. "
|
||||
"Specify it with -o dstFileName. Ignoring.\n",
|
||||
srcFileName, suffixListStr);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if ((*matchedSuffixPtr)[1] == 't') {
|
||||
dstSuffix = ".tar";
|
||||
dstSuffixLen = strlen(dstSuffix);
|
||||
}
|
||||
}
|
||||
|
||||
if (outDirName) {
|
||||
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0);
|
||||
sfnSize = strlen(outDirFilename);
|
||||
assert(outDirFilename != NULL);
|
||||
}
|
||||
|
||||
if (dfnbCapacity+suffixSize <= sfnSize+1) {
|
||||
if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) {
|
||||
/* allocate enough space to write dstFilename into it */
|
||||
free(dstFileNameBuffer);
|
||||
dfnbCapacity = sfnSize + 20;
|
||||
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
|
||||
if (dstFileNameBuffer==NULL)
|
||||
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
|
||||
EXM_THROW(74, "%s : not enough memory for dstFileName",
|
||||
strerror(errno));
|
||||
}
|
||||
|
||||
/* return dst name == src name truncated from suffix */
|
||||
assert(dstFileNameBuffer != NULL);
|
||||
dstFileNameEndPos = sfnSize - srcSuffixLen;
|
||||
if (outDirFilename) {
|
||||
memcpy(dstFileNameBuffer, outDirFilename, sfnSize - suffixSize);
|
||||
memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos);
|
||||
free(outDirFilename);
|
||||
} else {
|
||||
memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
|
||||
memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos);
|
||||
}
|
||||
dstFileNameBuffer[sfnSize-suffixSize] = '\0';
|
||||
|
||||
/* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar"
|
||||
* extension on decompression. Also writes terminating null. */
|
||||
strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix);
|
||||
return dstFileNameBuffer;
|
||||
|
||||
/* note : dstFileNameBuffer memory is not going to be free */
|
||||
|
||||
@ -30,11 +30,23 @@ extern "C" {
|
||||
#else
|
||||
# define nulmark "/dev/null"
|
||||
#endif
|
||||
|
||||
/**
|
||||
* We test whether the extension we found starts with 't', and if so, we append
|
||||
* ".tar" to the end of the output name.
|
||||
*/
|
||||
#define LZMA_EXTENSION ".lzma"
|
||||
#define XZ_EXTENSION ".xz"
|
||||
#define TXZ_EXTENSION ".txz"
|
||||
|
||||
#define GZ_EXTENSION ".gz"
|
||||
#define TGZ_EXTENSION ".tgz"
|
||||
|
||||
#define ZSTD_EXTENSION ".zst"
|
||||
#define TZSTD_EXTENSION ".tzst"
|
||||
|
||||
#define LZ4_EXTENSION ".lz4"
|
||||
#define TLZ4_EXTENSION ".tlz4"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@ -81,6 +93,7 @@ void FIO_setLiteralCompressionMode(
|
||||
|
||||
void FIO_setNoProgress(unsigned noProgress);
|
||||
void FIO_setNotificationLevel(int level);
|
||||
void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
|
||||
|
||||
/*-*************************************
|
||||
* Single File functions
|
||||
|
||||
@ -126,6 +126,21 @@ int UTIL_isSameFile(const char* fName1, const char* fName2)
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef _MSC_VER
|
||||
/* Using this to distinguish named pipes */
|
||||
U32 UTIL_isFIFO(const char* infilename)
|
||||
{
|
||||
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
|
||||
#if PLATFORM_POSIX_VERSION >= 200112L
|
||||
stat_t statbuf;
|
||||
int r = UTIL_getFileStat(infilename, &statbuf);
|
||||
if (!r && S_ISFIFO(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
(void)infilename;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
U32 UTIL_isLink(const char* infilename)
|
||||
{
|
||||
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
|
||||
@ -313,6 +328,27 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
|
||||
|
||||
#endif /* #ifdef _WIN32 */
|
||||
|
||||
int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
|
||||
{
|
||||
const char* ext = UTIL_getFileExtension(inputName);
|
||||
while(*extensionList!=NULL)
|
||||
{
|
||||
const int isCompressedExtension = strcmp(ext,*extensionList);
|
||||
if(isCompressedExtension==0)
|
||||
return 1;
|
||||
++extensionList;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*Utility function to get file extension from file */
|
||||
const char* UTIL_getFileExtension(const char* infilename)
|
||||
{
|
||||
const char* extension = strrchr(infilename, '.');
|
||||
if(!extension || extension==infilename) return "";
|
||||
return extension;
|
||||
}
|
||||
|
||||
/*
|
||||
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
|
||||
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
|
||||
|
||||
@ -40,7 +40,6 @@ extern "C" {
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
|
||||
#include "mem.h" /* U32, U64 */
|
||||
|
||||
|
||||
/*-************************************************************
|
||||
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
|
||||
***************************************************************/
|
||||
@ -135,7 +134,12 @@ U32 UTIL_isDirectory(const char* infilename);
|
||||
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
|
||||
int UTIL_isSameFile(const char* file1, const char* file2);
|
||||
int UTIL_compareStr(const void *p1, const void *p2);
|
||||
int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
|
||||
const char* UTIL_getFileExtension(const char* infilename);
|
||||
|
||||
#ifndef _MSC_VER
|
||||
U32 UTIL_isFIFO(const char* infilename);
|
||||
#endif
|
||||
U32 UTIL_isLink(const char* infilename);
|
||||
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
|
||||
U64 UTIL_getFileSize(const char* infilename);
|
||||
|
||||
@ -136,6 +136,7 @@ static int usage_advanced(const char* programName)
|
||||
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
|
||||
DISPLAY( " -c : force write to standard output, even if it is the console\n");
|
||||
DISPLAY( " -l : print information about zstd compressed files \n");
|
||||
DISPLAY( "--exclude-compressed: only compress files that are not previously compressed \n");
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
|
||||
DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
|
||||
@ -708,7 +709,7 @@ int main(int argCount, const char* argv[])
|
||||
if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
|
||||
if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
|
||||
if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
|
||||
|
||||
if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
|
||||
/* long commands with arguments */
|
||||
#ifndef ZSTD_NODICT
|
||||
if (longCommandWArg(&argument, "--train-cover")) {
|
||||
@ -1004,7 +1005,11 @@ int main(int argCount, const char* argv[])
|
||||
if (!followLinks) {
|
||||
unsigned u;
|
||||
for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
|
||||
if (UTIL_isLink(filenameTable[u])) {
|
||||
if (UTIL_isLink(filenameTable[u])
|
||||
#ifndef _MSC_VER
|
||||
&& !UTIL_isFIFO(filenameTable[u])
|
||||
#endif /* _MSC_VER */
|
||||
) {
|
||||
DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
|
||||
} else {
|
||||
filenameTable[fileNamesNb++] = filenameTable[u];
|
||||
|
||||
@ -160,19 +160,13 @@ fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
|
||||
zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
|
||||
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
zbufftest zbufftest32 zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest zbufftest32 zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
|
||||
zbufftest32 : CFLAGS += -m32
|
||||
zbufftest zbufftest32 : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest-dll : zstd-dll
|
||||
zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
|
||||
zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
@ -73,7 +73,8 @@ FUZZ_TARGETS := \
|
||||
dictionary_round_trip \
|
||||
dictionary_decompress \
|
||||
zstd_frame_info \
|
||||
simple_compress
|
||||
simple_compress \
|
||||
dictionary_loader
|
||||
|
||||
all: $(FUZZ_TARGETS)
|
||||
|
||||
@ -110,6 +111,9 @@ simple_compress: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_compress.o
|
||||
zstd_frame_info: $(FUZZ_HEADERS) $(FUZZ_OBJ) zstd_frame_info.o
|
||||
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) zstd_frame_info.o $(LIB_FUZZING_ENGINE) -o $@
|
||||
|
||||
dictionary_loader: $(FUZZ_HEADERS) $(FUZZ_OBJ) dictionary_loader.o
|
||||
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) dictionary_loader.o $(LIB_FUZZING_ENGINE) -o $@
|
||||
|
||||
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_driver.o
|
||||
$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
|
||||
|
||||
|
||||
93
tests/fuzz/dictionary_loader.c
Normal file
93
tests/fuzz/dictionary_loader.c
Normal file
@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*/
|
||||
|
||||
/**
|
||||
* This fuzz target makes sure that whenever a compression dictionary can be
|
||||
* loaded, the data can be round tripped.
|
||||
*/
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "fuzz_helpers.h"
|
||||
#include "zstd_helpers.h"
|
||||
#include "fuzz_data_producer.h"
|
||||
|
||||
/**
|
||||
* Compresses the data and returns the compressed size or an error.
|
||||
*/
|
||||
static size_t compress(void* compressed, size_t compressedCapacity,
|
||||
void const* source, size_t sourceSize,
|
||||
void const* dict, size_t dictSize,
|
||||
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||
ZSTD_dictContentType_e dictContentType)
|
||||
{
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
|
||||
cctx, dict, dictSize, dictLoadMethod, dictContentType));
|
||||
size_t const compressedSize = ZSTD_compress2(
|
||||
cctx, compressed, compressedCapacity, source, sourceSize);
|
||||
ZSTD_freeCCtx(cctx);
|
||||
return compressedSize;
|
||||
}
|
||||
|
||||
static size_t decompress(void* result, size_t resultCapacity,
|
||||
void const* compressed, size_t compressedSize,
|
||||
void const* dict, size_t dictSize,
|
||||
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||
ZSTD_dictContentType_e dictContentType)
|
||||
{
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
|
||||
dctx, dict, dictSize, dictLoadMethod, dictContentType));
|
||||
size_t const resultSize = ZSTD_decompressDCtx(
|
||||
dctx, result, resultCapacity, compressed, compressedSize);
|
||||
FUZZ_ZASSERT(resultSize);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return resultSize;
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
|
||||
{
|
||||
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
|
||||
ZSTD_dictLoadMethod_e const dlm =
|
||||
size = FUZZ_dataProducer_uint32Range(producer, 0, 1);
|
||||
ZSTD_dictContentType_e const dct =
|
||||
FUZZ_dataProducer_uint32Range(producer, 0, 2);
|
||||
size = FUZZ_dataProducer_remainingBytes(producer);
|
||||
|
||||
DEBUGLOG(2, "Dict load method %d", dlm);
|
||||
DEBUGLOG(2, "Dict content type %d", dct);
|
||||
DEBUGLOG(2, "Dict size %u", (unsigned)size);
|
||||
|
||||
void* const rBuf = malloc(size);
|
||||
FUZZ_ASSERT(rBuf);
|
||||
size_t const cBufSize = ZSTD_compressBound(size);
|
||||
void* const cBuf = malloc(cBufSize);
|
||||
FUZZ_ASSERT(cBuf);
|
||||
|
||||
size_t const cSize =
|
||||
compress(cBuf, cBufSize, src, size, src, size, dlm, dct);
|
||||
/* compression failing is okay */
|
||||
if (ZSTD_isError(cSize)) {
|
||||
FUZZ_ASSERT_MSG(dct != ZSTD_dct_rawContent, "Raw must always succeed!");
|
||||
goto out;
|
||||
}
|
||||
size_t const rSize =
|
||||
decompress(rBuf, size, cBuf, cSize, src, size, dlm, dct);
|
||||
FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
|
||||
FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
|
||||
|
||||
out:
|
||||
free(cBuf);
|
||||
free(rBuf);
|
||||
FUZZ_dataProducer_free(producer);
|
||||
return 0;
|
||||
}
|
||||
@ -27,6 +27,7 @@ def abs_join(a, *p):
|
||||
class InputType(object):
|
||||
RAW_DATA = 1
|
||||
COMPRESSED_DATA = 2
|
||||
DICTIONARY_DATA = 3
|
||||
|
||||
|
||||
class FrameType(object):
|
||||
@ -54,6 +55,7 @@ TARGET_INFO = {
|
||||
'dictionary_decompress': TargetInfo(InputType.COMPRESSED_DATA),
|
||||
'zstd_frame_info': TargetInfo(InputType.COMPRESSED_DATA),
|
||||
'simple_compress': TargetInfo(InputType.RAW_DATA),
|
||||
'dictionary_loader': TargetInfo(InputType.DICTIONARY_DATA),
|
||||
}
|
||||
TARGETS = list(TARGET_INFO.keys())
|
||||
ALL_TARGETS = TARGETS + ['all']
|
||||
@ -73,6 +75,7 @@ LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a')
|
||||
AFL_FUZZ = os.environ.get('AFL_FUZZ', 'afl-fuzz')
|
||||
DECODECORPUS = os.environ.get('DECODECORPUS',
|
||||
abs_join(FUZZ_DIR, '..', 'decodecorpus'))
|
||||
ZSTD = os.environ.get('ZSTD', abs_join(FUZZ_DIR, '..', '..', 'zstd'))
|
||||
|
||||
# Sanitizer environment variables
|
||||
MSAN_EXTRA_CPPFLAGS = os.environ.get('MSAN_EXTRA_CPPFLAGS', '')
|
||||
@ -673,6 +676,11 @@ def gen_parser(args):
|
||||
default=DECODECORPUS,
|
||||
help="decodecorpus binary (default: $DECODECORPUS='{}')".format(
|
||||
DECODECORPUS))
|
||||
parser.add_argument(
|
||||
'--zstd',
|
||||
type=str,
|
||||
default=ZSTD,
|
||||
help="zstd binary (default: $ZSTD='{}')".format(ZSTD))
|
||||
parser.add_argument(
|
||||
'--fuzz-rng-seed-size',
|
||||
type=int,
|
||||
@ -707,46 +715,66 @@ def gen(args):
|
||||
return 1
|
||||
|
||||
seed = create(args.seed)
|
||||
with tmpdir() as compressed:
|
||||
with tmpdir() as decompressed:
|
||||
cmd = [
|
||||
args.decodecorpus,
|
||||
'-n{}'.format(args.number),
|
||||
'-p{}/'.format(compressed),
|
||||
'-o{}'.format(decompressed),
|
||||
with tmpdir() as compressed, tmpdir() as decompressed, tmpdir() as dict:
|
||||
info = TARGET_INFO[args.TARGET]
|
||||
|
||||
if info.input_type == InputType.DICTIONARY_DATA:
|
||||
number = max(args.number, 1000)
|
||||
else:
|
||||
number = args.number
|
||||
cmd = [
|
||||
args.decodecorpus,
|
||||
'-n{}'.format(args.number),
|
||||
'-p{}/'.format(compressed),
|
||||
'-o{}'.format(decompressed),
|
||||
]
|
||||
|
||||
if info.frame_type == FrameType.BLOCK:
|
||||
cmd += [
|
||||
'--gen-blocks',
|
||||
'--max-block-size-log={}'.format(min(args.max_size_log, 17))
|
||||
]
|
||||
else:
|
||||
cmd += ['--max-content-size-log={}'.format(args.max_size_log)]
|
||||
|
||||
info = TARGET_INFO[args.TARGET]
|
||||
if info.frame_type == FrameType.BLOCK:
|
||||
cmd += [
|
||||
'--gen-blocks',
|
||||
'--max-block-size-log={}'.format(min(args.max_size_log, 17))
|
||||
print(' '.join(cmd))
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
if info.input_type == InputType.RAW_DATA:
|
||||
print('using decompressed data in {}'.format(decompressed))
|
||||
samples = decompressed
|
||||
elif info.input_type == InputType.COMPRESSED_DATA:
|
||||
print('using compressed data in {}'.format(compressed))
|
||||
samples = compressed
|
||||
else:
|
||||
assert info.input_type == InputType.DICTIONARY_DATA
|
||||
print('making dictionary data from {}'.format(decompressed))
|
||||
samples = dict
|
||||
min_dict_size_log = 9
|
||||
max_dict_size_log = max(min_dict_size_log + 1, args.max_size_log)
|
||||
for dict_size_log in range(min_dict_size_log, max_dict_size_log):
|
||||
dict_size = 1 << dict_size_log
|
||||
cmd = [
|
||||
args.zstd,
|
||||
'--train',
|
||||
'-r', decompressed,
|
||||
'--maxdict={}'.format(dict_size),
|
||||
'-o', abs_join(dict, '{}.zstd-dict'.format(dict_size))
|
||||
]
|
||||
else:
|
||||
cmd += ['--max-content-size-log={}'.format(args.max_size_log)]
|
||||
print(' '.join(cmd))
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
print(' '.join(cmd))
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
if info.input_type == InputType.RAW_DATA:
|
||||
print('using decompressed data in {}'.format(decompressed))
|
||||
samples = decompressed
|
||||
else:
|
||||
assert info.input_type == InputType.COMPRESSED_DATA
|
||||
print('using compressed data in {}'.format(compressed))
|
||||
samples = compressed
|
||||
|
||||
# Copy the samples over and prepend the RNG seeds
|
||||
for name in os.listdir(samples):
|
||||
samplename = abs_join(samples, name)
|
||||
outname = abs_join(seed, name)
|
||||
with open(samplename, 'rb') as sample:
|
||||
with open(outname, 'wb') as out:
|
||||
CHUNK_SIZE = 131072
|
||||
# Copy the samples over and prepend the RNG seeds
|
||||
for name in os.listdir(samples):
|
||||
samplename = abs_join(samples, name)
|
||||
outname = abs_join(seed, name)
|
||||
with open(samplename, 'rb') as sample:
|
||||
with open(outname, 'wb') as out:
|
||||
CHUNK_SIZE = 131072
|
||||
chunk = sample.read(CHUNK_SIZE)
|
||||
while len(chunk) > 0:
|
||||
out.write(chunk)
|
||||
chunk = sample.read(CHUNK_SIZE)
|
||||
while len(chunk) > 0:
|
||||
out.write(chunk)
|
||||
chunk = sample.read(CHUNK_SIZE)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@ -215,6 +215,37 @@ $ZSTD tmp -c --compress-literals -19 | $ZSTD -t
|
||||
$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
|
||||
$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
|
||||
|
||||
println "test: --exclude-compressed flag"
|
||||
rm -rf precompressedFilterTestDir
|
||||
mkdir -p precompressedFilterTestDir
|
||||
./datagen $size > precompressedFilterTestDir/input.5
|
||||
./datagen $size > precompressedFilterTestDir/input.6
|
||||
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
|
||||
sleep 5
|
||||
./datagen $size > precompressedFilterTestDir/input.7
|
||||
./datagen $size > precompressedFilterTestDir/input.8
|
||||
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
|
||||
test ! -f precompressedFilterTestDir/input.5.zst.zst
|
||||
test ! -f precompressedFilterTestDir/input.6.zst.zst
|
||||
file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
|
||||
file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
|
||||
if [[ $file2timestamp -ge $file1timestamp ]]; then
|
||||
println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again."
|
||||
else
|
||||
println "Test is not successful"
|
||||
fi
|
||||
#File Extension check.
|
||||
./datagen $size > precompressedFilterTestDir/input.zstbar
|
||||
$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
|
||||
#ZSTD should compress input.zstbar
|
||||
test -f precompressedFilterTestDir/input.zstbar.zst
|
||||
#Check without the --exclude-compressed flag
|
||||
$ZSTD --long --rm -r precompressedFilterTestDir
|
||||
#Files should get compressed again without the --exclude-compressed flag.
|
||||
test -f precompressedFilterTestDir/input.5.zst.zst
|
||||
test -f precompressedFilterTestDir/input.6.zst.zst
|
||||
println "Test completed"
|
||||
|
||||
println "test : file removal"
|
||||
$ZSTD -f --rm tmp
|
||||
test ! -f tmp # tmp should no longer be present
|
||||
@ -845,6 +876,46 @@ if [ $LZ4MODE -ne 1 ]; then
|
||||
grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed"
|
||||
fi
|
||||
|
||||
println "\n===> tar extension tests "
|
||||
|
||||
rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4
|
||||
|
||||
./datagen > tmp
|
||||
tar cf tmp.tar tmp
|
||||
$ZSTD tmp.tar -o tmp.tzst
|
||||
rm tmp.tar
|
||||
$ZSTD -d tmp.tzst
|
||||
[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!"
|
||||
rm -f tmp.tar tmp.tzst
|
||||
|
||||
if [ $GZIPMODE -eq 1 ]; then
|
||||
tar czf tmp.tgz tmp
|
||||
$ZSTD -d tmp.tgz
|
||||
[ -e tmp.tar ] || die ".tgz failed to decompress to .tar!"
|
||||
rm -f tmp.tar tmp.tgz
|
||||
fi
|
||||
|
||||
if [ $LZMAMODE -eq 1 ]; then
|
||||
tar c tmp | $ZSTD --format=xz > tmp.txz
|
||||
$ZSTD -d tmp.txz
|
||||
[ -e tmp.tar ] || die ".txz failed to decompress to .tar!"
|
||||
rm -f tmp.tar tmp.txz
|
||||
fi
|
||||
|
||||
if [ $LZ4MODE -eq 1 ]; then
|
||||
tar c tmp | $ZSTD --format=lz4 > tmp.tlz4
|
||||
$ZSTD -d tmp.tlz4
|
||||
[ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!"
|
||||
rm -f tmp.tar tmp.tlz4
|
||||
fi
|
||||
|
||||
touch tmp.t tmp.tz tmp.tzs
|
||||
! $ZSTD -d tmp.t
|
||||
! $ZSTD -d tmp.tz
|
||||
! $ZSTD -d tmp.tzs
|
||||
|
||||
exit
|
||||
|
||||
println "\n===> zstd round-trip tests "
|
||||
|
||||
roundTripTest
|
||||
@ -1099,4 +1170,18 @@ test -f dictionary
|
||||
rm -f tmp* dictionary
|
||||
|
||||
|
||||
if [ "$isWindows" = false ] ; then
|
||||
|
||||
println "\n===> zstd fifo named pipe test "
|
||||
head -c 10 /dev/zero > tmp_original
|
||||
mkfifo named_pipe
|
||||
head -c 10 /dev/zero > named_pipe &
|
||||
$ZSTD named_pipe -o tmp_compressed
|
||||
$ZSTD -d -o tmp_decompressed tmp_compressed
|
||||
$DIFF -s tmp_original tmp_decompressed
|
||||
rm -rf tmp*
|
||||
rm -rf named_pipe
|
||||
|
||||
fi
|
||||
|
||||
rm -f tmp*
|
||||
|
||||
@ -444,6 +444,8 @@ static int init_cstream(
|
||||
ZSTD_parameters const params = config_get_zstd_params(config, 0, 0);
|
||||
ZSTD_CDict* dict = NULL;
|
||||
if (cdict) {
|
||||
if (!config->use_dictionary)
|
||||
return 1;
|
||||
*cdict = ZSTD_createCDict_advanced(
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
@ -459,14 +461,18 @@ static int init_cstream(
|
||||
} else {
|
||||
zret = ZSTD_initCStream_advanced(
|
||||
zcs,
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
config->use_dictionary ? state->dictionary.data : NULL,
|
||||
config->use_dictionary ? state->dictionary.size : 0,
|
||||
params,
|
||||
ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
}
|
||||
} else {
|
||||
int const level = config_get_level(config);
|
||||
if (level == CONFIG_NO_LEVEL)
|
||||
return 1;
|
||||
if (cdict) {
|
||||
if (!config->use_dictionary)
|
||||
return 1;
|
||||
*cdict = ZSTD_createCDict(
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
@ -477,7 +483,10 @@ static int init_cstream(
|
||||
zret = ZSTD_initCStream_usingCDict(zcs, *cdict);
|
||||
} else if (config->use_dictionary) {
|
||||
zret = ZSTD_initCStream_usingDict(
|
||||
zcs, state->dictionary.data, state->dictionary.size, level);
|
||||
zcs,
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
level);
|
||||
} else {
|
||||
zret = ZSTD_initCStream(zcs, level);
|
||||
}
|
||||
@ -506,9 +515,17 @@ static result_t old_streaming_compress_internal(
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
if (!advanced && config_get_level(config) == CONFIG_NO_LEVEL) {
|
||||
result = result_error(result_error_skip);
|
||||
goto out;
|
||||
}
|
||||
if (cdict && !config->use_dictionary) {
|
||||
result = result_error(result_error_skip);
|
||||
goto out;
|
||||
}
|
||||
if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
result_data_t data = {.total_size = 0};
|
||||
@ -629,21 +646,21 @@ method_t const old_streaming = {
|
||||
method_t const old_streaming_advanced = {
|
||||
.name = "old streaming advanced",
|
||||
.create = buffer_state_create,
|
||||
.compress = old_streaming_compress,
|
||||
.compress = old_streaming_compress_advanced,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const old_streaming_cdict = {
|
||||
.name = "old streaming cdcit",
|
||||
.create = buffer_state_create,
|
||||
.compress = old_streaming_compress,
|
||||
.compress = old_streaming_compress_cdict,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const old_streaming_advanced_cdict = {
|
||||
.name = "old streaming advanced cdict",
|
||||
.create = buffer_state_create,
|
||||
.compress = old_streaming_compress,
|
||||
.compress = old_streaming_compress_cdict_advanced,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
|
||||
@ -461,17 +461,9 @@ silesia, level 13, old stre
|
||||
silesia, level 16, old streaming, 4377389
|
||||
silesia, level 19, old streaming, 4293262
|
||||
silesia, no source size, old streaming, 4849455
|
||||
silesia, long distance mode, old streaming, 12000408
|
||||
silesia, multithreaded, old streaming, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming, 12000408
|
||||
silesia, small window log, old streaming, 12000408
|
||||
silesia, small hash log, old streaming, 12000408
|
||||
silesia, small chain log, old streaming, 12000408
|
||||
silesia, explicit params, old streaming, 12000408
|
||||
silesia, uncompressed literals, old streaming, 4849491
|
||||
silesia, uncompressed literals optimal, old streaming, 4293262
|
||||
silesia, huffman literals, old streaming, 6183385
|
||||
silesia, multithreaded with advanced params, old streaming, 12000408
|
||||
silesia.tar, level -5, old streaming, 6982738
|
||||
silesia.tar, level -3, old streaming, 6641264
|
||||
silesia.tar, level -1, old streaming, 6190789
|
||||
@ -487,17 +479,9 @@ silesia.tar, level 13, old stre
|
||||
silesia.tar, level 16, old streaming, 4381284
|
||||
silesia.tar, level 19, old streaming, 4281511
|
||||
silesia.tar, no source size, old streaming, 4861372
|
||||
silesia.tar, long distance mode, old streaming, 12022046
|
||||
silesia.tar, multithreaded, old streaming, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming, 12022046
|
||||
silesia.tar, small window log, old streaming, 12022046
|
||||
silesia.tar, small hash log, old streaming, 12022046
|
||||
silesia.tar, small chain log, old streaming, 12022046
|
||||
silesia.tar, explicit params, old streaming, 12022046
|
||||
silesia.tar, uncompressed literals, old streaming, 4861376
|
||||
silesia.tar, uncompressed literals optimal, old streaming, 4281511
|
||||
silesia.tar, huffman literals, old streaming, 6190789
|
||||
silesia.tar, multithreaded with advanced params, old streaming, 12022046
|
||||
github, level -5, old streaming, 205285
|
||||
github, level -5 with dict, old streaming, 46718
|
||||
github, level -3, old streaming, 190643
|
||||
@ -527,17 +511,9 @@ github, level 16 with dict, old stre
|
||||
github, level 19, old streaming, 133717
|
||||
github, level 19 with dict, old streaming, 37576
|
||||
github, no source size, old streaming, 140631
|
||||
github, long distance mode, old streaming, 412933
|
||||
github, multithreaded, old streaming, 412933
|
||||
github, multithreaded long distance mode, old streaming, 412933
|
||||
github, small window log, old streaming, 412933
|
||||
github, small hash log, old streaming, 412933
|
||||
github, small chain log, old streaming, 412933
|
||||
github, explicit params, old streaming, 412933
|
||||
github, uncompressed literals, old streaming, 136311
|
||||
github, uncompressed literals optimal, old streaming, 133717
|
||||
github, huffman literals, old streaming, 175568
|
||||
github, multithreaded with advanced params, old streaming, 412933
|
||||
silesia, level -5, old streaming advanced, 6882466
|
||||
silesia, level -3, old streaming advanced, 6568358
|
||||
silesia, level -1, old streaming advanced, 6183385
|
||||
@ -553,17 +529,17 @@ silesia, level 13, old stre
|
||||
silesia, level 16, old streaming advanced, 4377389
|
||||
silesia, level 19, old streaming advanced, 4293262
|
||||
silesia, no source size, old streaming advanced, 4849455
|
||||
silesia, long distance mode, old streaming advanced, 12000408
|
||||
silesia, multithreaded, old streaming advanced, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming advanced, 12000408
|
||||
silesia, small window log, old streaming advanced, 12000408
|
||||
silesia, small hash log, old streaming advanced, 12000408
|
||||
silesia, small chain log, old streaming advanced, 12000408
|
||||
silesia, explicit params, old streaming advanced, 12000408
|
||||
silesia, long distance mode, old streaming advanced, 4849491
|
||||
silesia, multithreaded, old streaming advanced, 4849491
|
||||
silesia, multithreaded long distance mode, old streaming advanced, 4849491
|
||||
silesia, small window log, old streaming advanced, 7123534
|
||||
silesia, small hash log, old streaming advanced, 6554898
|
||||
silesia, small chain log, old streaming advanced, 4931093
|
||||
silesia, explicit params, old streaming advanced, 4797048
|
||||
silesia, uncompressed literals, old streaming advanced, 4849491
|
||||
silesia, uncompressed literals optimal, old streaming advanced, 4293262
|
||||
silesia, huffman literals, old streaming advanced, 6183385
|
||||
silesia, multithreaded with advanced params, old streaming advanced, 12000408
|
||||
silesia, multithreaded with advanced params, old streaming advanced, 4849491
|
||||
silesia.tar, level -5, old streaming advanced, 6982738
|
||||
silesia.tar, level -3, old streaming advanced, 6641264
|
||||
silesia.tar, level -1, old streaming advanced, 6190789
|
||||
@ -579,238 +555,82 @@ silesia.tar, level 13, old stre
|
||||
silesia.tar, level 16, old streaming advanced, 4381284
|
||||
silesia.tar, level 19, old streaming advanced, 4281511
|
||||
silesia.tar, no source size, old streaming advanced, 4861372
|
||||
silesia.tar, long distance mode, old streaming advanced, 12022046
|
||||
silesia.tar, multithreaded, old streaming advanced, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming advanced, 12022046
|
||||
silesia.tar, small window log, old streaming advanced, 12022046
|
||||
silesia.tar, small hash log, old streaming advanced, 12022046
|
||||
silesia.tar, small chain log, old streaming advanced, 12022046
|
||||
silesia.tar, explicit params, old streaming advanced, 12022046
|
||||
silesia.tar, long distance mode, old streaming advanced, 4861376
|
||||
silesia.tar, multithreaded, old streaming advanced, 4861376
|
||||
silesia.tar, multithreaded long distance mode, old streaming advanced, 4861376
|
||||
silesia.tar, small window log, old streaming advanced, 7127552
|
||||
silesia.tar, small hash log, old streaming advanced, 6587834
|
||||
silesia.tar, small chain log, old streaming advanced, 4943271
|
||||
silesia.tar, explicit params, old streaming advanced, 4808570
|
||||
silesia.tar, uncompressed literals, old streaming advanced, 4861376
|
||||
silesia.tar, uncompressed literals optimal, old streaming advanced, 4281511
|
||||
silesia.tar, huffman literals, old streaming advanced, 6190789
|
||||
silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046
|
||||
github, level -5, old streaming advanced, 205285
|
||||
github, level -5 with dict, old streaming advanced, 46718
|
||||
github, level -3, old streaming advanced, 190643
|
||||
github, level -3 with dict, old streaming advanced, 45395
|
||||
github, level -1, old streaming advanced, 175568
|
||||
github, level -1 with dict, old streaming advanced, 43170
|
||||
github, level 0, old streaming advanced, 136311
|
||||
github, level 0 with dict, old streaming advanced, 41148
|
||||
github, level 1, old streaming advanced, 142450
|
||||
github, level 1 with dict, old streaming advanced, 41682
|
||||
github, level 3, old streaming advanced, 136311
|
||||
github, level 3 with dict, old streaming advanced, 41148
|
||||
github, level 4, old streaming advanced, 136144
|
||||
github, level 4 with dict, old streaming advanced, 41251
|
||||
github, level 5, old streaming advanced, 135106
|
||||
github, level 5 with dict, old streaming advanced, 38938
|
||||
github, level 6, old streaming advanced, 135108
|
||||
github, level 6 with dict, old streaming advanced, 38632
|
||||
github, level 7, old streaming advanced, 135108
|
||||
github, level 7 with dict, old streaming advanced, 38766
|
||||
github, level 9, old streaming advanced, 135108
|
||||
github, level 9 with dict, old streaming advanced, 39326
|
||||
github, level 13, old streaming advanced, 133717
|
||||
github, level 13 with dict, old streaming advanced, 39716
|
||||
github, level 16, old streaming advanced, 133717
|
||||
github, level 16 with dict, old streaming advanced, 37577
|
||||
silesia.tar, multithreaded with advanced params, old streaming advanced, 4861376
|
||||
github, level -5, old streaming advanced, 216734
|
||||
github, level -5 with dict, old streaming advanced, 49562
|
||||
github, level -3, old streaming advanced, 192160
|
||||
github, level -3 with dict, old streaming advanced, 44956
|
||||
github, level -1, old streaming advanced, 181108
|
||||
github, level -1 with dict, old streaming advanced, 42383
|
||||
github, level 0, old streaming advanced, 141090
|
||||
github, level 0 with dict, old streaming advanced, 41113
|
||||
github, level 1, old streaming advanced, 143682
|
||||
github, level 1 with dict, old streaming advanced, 42430
|
||||
github, level 3, old streaming advanced, 141090
|
||||
github, level 3 with dict, old streaming advanced, 41113
|
||||
github, level 4, old streaming advanced, 141090
|
||||
github, level 4 with dict, old streaming advanced, 41084
|
||||
github, level 5, old streaming advanced, 139391
|
||||
github, level 5 with dict, old streaming advanced, 39159
|
||||
github, level 6, old streaming advanced, 139394
|
||||
github, level 6 with dict, old streaming advanced, 38749
|
||||
github, level 7, old streaming advanced, 138675
|
||||
github, level 7 with dict, old streaming advanced, 38746
|
||||
github, level 9, old streaming advanced, 138675
|
||||
github, level 9 with dict, old streaming advanced, 38987
|
||||
github, level 13, old streaming advanced, 138675
|
||||
github, level 13 with dict, old streaming advanced, 39724
|
||||
github, level 16, old streaming advanced, 138675
|
||||
github, level 16 with dict, old streaming advanced, 40771
|
||||
github, level 19, old streaming advanced, 133717
|
||||
github, level 19 with dict, old streaming advanced, 37576
|
||||
github, no source size, old streaming advanced, 140631
|
||||
github, long distance mode, old streaming advanced, 412933
|
||||
github, multithreaded, old streaming advanced, 412933
|
||||
github, multithreaded long distance mode, old streaming advanced, 412933
|
||||
github, small window log, old streaming advanced, 412933
|
||||
github, small hash log, old streaming advanced, 412933
|
||||
github, small chain log, old streaming advanced, 412933
|
||||
github, explicit params, old streaming advanced, 412933
|
||||
github, uncompressed literals, old streaming advanced, 136311
|
||||
github, long distance mode, old streaming advanced, 141090
|
||||
github, multithreaded, old streaming advanced, 141090
|
||||
github, multithreaded long distance mode, old streaming advanced, 141090
|
||||
github, small window log, old streaming advanced, 141090
|
||||
github, small hash log, old streaming advanced, 141578
|
||||
github, small chain log, old streaming advanced, 139258
|
||||
github, explicit params, old streaming advanced, 140930
|
||||
github, uncompressed literals, old streaming advanced, 141090
|
||||
github, uncompressed literals optimal, old streaming advanced, 133717
|
||||
github, huffman literals, old streaming advanced, 175568
|
||||
github, multithreaded with advanced params, old streaming advanced, 412933
|
||||
silesia, level -5, old streaming cdcit, 6882466
|
||||
silesia, level -3, old streaming cdcit, 6568358
|
||||
silesia, level -1, old streaming cdcit, 6183385
|
||||
silesia, level 0, old streaming cdcit, 4849491
|
||||
silesia, level 1, old streaming cdcit, 5314109
|
||||
silesia, level 3, old streaming cdcit, 4849491
|
||||
silesia, level 4, old streaming cdcit, 4786913
|
||||
silesia, level 5, old streaming cdcit, 4710178
|
||||
silesia, level 6, old streaming cdcit, 4659996
|
||||
silesia, level 7, old streaming cdcit, 4596234
|
||||
silesia, level 9, old streaming cdcit, 4543862
|
||||
silesia, level 13, old streaming cdcit, 4482073
|
||||
silesia, level 16, old streaming cdcit, 4377389
|
||||
silesia, level 19, old streaming cdcit, 4293262
|
||||
silesia, no source size, old streaming cdcit, 4849455
|
||||
silesia, long distance mode, old streaming cdcit, 12000408
|
||||
silesia, multithreaded, old streaming cdcit, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming cdcit, 12000408
|
||||
silesia, small window log, old streaming cdcit, 12000408
|
||||
silesia, small hash log, old streaming cdcit, 12000408
|
||||
silesia, small chain log, old streaming cdcit, 12000408
|
||||
silesia, explicit params, old streaming cdcit, 12000408
|
||||
silesia, uncompressed literals, old streaming cdcit, 4849491
|
||||
silesia, uncompressed literals optimal, old streaming cdcit, 4293262
|
||||
silesia, huffman literals, old streaming cdcit, 6183385
|
||||
silesia, multithreaded with advanced params, old streaming cdcit, 12000408
|
||||
silesia.tar, level -5, old streaming cdcit, 6982738
|
||||
silesia.tar, level -3, old streaming cdcit, 6641264
|
||||
silesia.tar, level -1, old streaming cdcit, 6190789
|
||||
silesia.tar, level 0, old streaming cdcit, 4861376
|
||||
silesia.tar, level 1, old streaming cdcit, 5336879
|
||||
silesia.tar, level 3, old streaming cdcit, 4861376
|
||||
silesia.tar, level 4, old streaming cdcit, 4799583
|
||||
silesia.tar, level 5, old streaming cdcit, 4722276
|
||||
silesia.tar, level 6, old streaming cdcit, 4672240
|
||||
silesia.tar, level 7, old streaming cdcit, 4606657
|
||||
silesia.tar, level 9, old streaming cdcit, 4554106
|
||||
silesia.tar, level 13, old streaming cdcit, 4491707
|
||||
silesia.tar, level 16, old streaming cdcit, 4381284
|
||||
silesia.tar, level 19, old streaming cdcit, 4281511
|
||||
silesia.tar, no source size, old streaming cdcit, 4861372
|
||||
silesia.tar, long distance mode, old streaming cdcit, 12022046
|
||||
silesia.tar, multithreaded, old streaming cdcit, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming cdcit, 12022046
|
||||
silesia.tar, small window log, old streaming cdcit, 12022046
|
||||
silesia.tar, small hash log, old streaming cdcit, 12022046
|
||||
silesia.tar, small chain log, old streaming cdcit, 12022046
|
||||
silesia.tar, explicit params, old streaming cdcit, 12022046
|
||||
silesia.tar, uncompressed literals, old streaming cdcit, 4861376
|
||||
silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281511
|
||||
silesia.tar, huffman literals, old streaming cdcit, 6190789
|
||||
silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046
|
||||
github, level -5, old streaming cdcit, 205285
|
||||
github, huffman literals, old streaming advanced, 181108
|
||||
github, multithreaded with advanced params, old streaming advanced, 141090
|
||||
github, level -5 with dict, old streaming cdcit, 46718
|
||||
github, level -3, old streaming cdcit, 190643
|
||||
github, level -3 with dict, old streaming cdcit, 45395
|
||||
github, level -1, old streaming cdcit, 175568
|
||||
github, level -1 with dict, old streaming cdcit, 43170
|
||||
github, level 0, old streaming cdcit, 136311
|
||||
github, level 0 with dict, old streaming cdcit, 41148
|
||||
github, level 1, old streaming cdcit, 142450
|
||||
github, level 1 with dict, old streaming cdcit, 41682
|
||||
github, level 3, old streaming cdcit, 136311
|
||||
github, level 3 with dict, old streaming cdcit, 41148
|
||||
github, level 4, old streaming cdcit, 136144
|
||||
github, level 4 with dict, old streaming cdcit, 41251
|
||||
github, level 5, old streaming cdcit, 135106
|
||||
github, level 5 with dict, old streaming cdcit, 38938
|
||||
github, level 6, old streaming cdcit, 135108
|
||||
github, level 6 with dict, old streaming cdcit, 38632
|
||||
github, level 7, old streaming cdcit, 135108
|
||||
github, level 7 with dict, old streaming cdcit, 38766
|
||||
github, level 9, old streaming cdcit, 135108
|
||||
github, level 9 with dict, old streaming cdcit, 39326
|
||||
github, level 13, old streaming cdcit, 133717
|
||||
github, level 13 with dict, old streaming cdcit, 39716
|
||||
github, level 16, old streaming cdcit, 133717
|
||||
github, level 16 with dict, old streaming cdcit, 37577
|
||||
github, level 19, old streaming cdcit, 133717
|
||||
github, level 19 with dict, old streaming cdcit, 37576
|
||||
github, no source size, old streaming cdcit, 140631
|
||||
github, long distance mode, old streaming cdcit, 412933
|
||||
github, multithreaded, old streaming cdcit, 412933
|
||||
github, multithreaded long distance mode, old streaming cdcit, 412933
|
||||
github, small window log, old streaming cdcit, 412933
|
||||
github, small hash log, old streaming cdcit, 412933
|
||||
github, small chain log, old streaming cdcit, 412933
|
||||
github, explicit params, old streaming cdcit, 412933
|
||||
github, uncompressed literals, old streaming cdcit, 136311
|
||||
github, uncompressed literals optimal, old streaming cdcit, 133717
|
||||
github, huffman literals, old streaming cdcit, 175568
|
||||
github, multithreaded with advanced params, old streaming cdcit, 412933
|
||||
silesia, level -5, old streaming advanced cdict, 6882466
|
||||
silesia, level -3, old streaming advanced cdict, 6568358
|
||||
silesia, level -1, old streaming advanced cdict, 6183385
|
||||
silesia, level 0, old streaming advanced cdict, 4849491
|
||||
silesia, level 1, old streaming advanced cdict, 5314109
|
||||
silesia, level 3, old streaming advanced cdict, 4849491
|
||||
silesia, level 4, old streaming advanced cdict, 4786913
|
||||
silesia, level 5, old streaming advanced cdict, 4710178
|
||||
silesia, level 6, old streaming advanced cdict, 4659996
|
||||
silesia, level 7, old streaming advanced cdict, 4596234
|
||||
silesia, level 9, old streaming advanced cdict, 4543862
|
||||
silesia, level 13, old streaming advanced cdict, 4482073
|
||||
silesia, level 16, old streaming advanced cdict, 4377389
|
||||
silesia, level 19, old streaming advanced cdict, 4293262
|
||||
silesia, no source size, old streaming advanced cdict, 4849455
|
||||
silesia, long distance mode, old streaming advanced cdict, 12000408
|
||||
silesia, multithreaded, old streaming advanced cdict, 12000408
|
||||
silesia, multithreaded long distance mode, old streaming advanced cdict, 12000408
|
||||
silesia, small window log, old streaming advanced cdict, 12000408
|
||||
silesia, small hash log, old streaming advanced cdict, 12000408
|
||||
silesia, small chain log, old streaming advanced cdict, 12000408
|
||||
silesia, explicit params, old streaming advanced cdict, 12000408
|
||||
silesia, uncompressed literals, old streaming advanced cdict, 4849491
|
||||
silesia, uncompressed literals optimal, old streaming advanced cdict, 4293262
|
||||
silesia, huffman literals, old streaming advanced cdict, 6183385
|
||||
silesia, multithreaded with advanced params, old streaming advanced cdict, 12000408
|
||||
silesia.tar, level -5, old streaming advanced cdict, 6982738
|
||||
silesia.tar, level -3, old streaming advanced cdict, 6641264
|
||||
silesia.tar, level -1, old streaming advanced cdict, 6190789
|
||||
silesia.tar, level 0, old streaming advanced cdict, 4861376
|
||||
silesia.tar, level 1, old streaming advanced cdict, 5336879
|
||||
silesia.tar, level 3, old streaming advanced cdict, 4861376
|
||||
silesia.tar, level 4, old streaming advanced cdict, 4799583
|
||||
silesia.tar, level 5, old streaming advanced cdict, 4722276
|
||||
silesia.tar, level 6, old streaming advanced cdict, 4672240
|
||||
silesia.tar, level 7, old streaming advanced cdict, 4606657
|
||||
silesia.tar, level 9, old streaming advanced cdict, 4554106
|
||||
silesia.tar, level 13, old streaming advanced cdict, 4491707
|
||||
silesia.tar, level 16, old streaming advanced cdict, 4381284
|
||||
silesia.tar, level 19, old streaming advanced cdict, 4281511
|
||||
silesia.tar, no source size, old streaming advanced cdict, 4861372
|
||||
silesia.tar, long distance mode, old streaming advanced cdict, 12022046
|
||||
silesia.tar, multithreaded, old streaming advanced cdict, 12022046
|
||||
silesia.tar, multithreaded long distance mode, old streaming advanced cdict, 12022046
|
||||
silesia.tar, small window log, old streaming advanced cdict, 12022046
|
||||
silesia.tar, small hash log, old streaming advanced cdict, 12022046
|
||||
silesia.tar, small chain log, old streaming advanced cdict, 12022046
|
||||
silesia.tar, explicit params, old streaming advanced cdict, 12022046
|
||||
silesia.tar, uncompressed literals, old streaming advanced cdict, 4861376
|
||||
silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281511
|
||||
silesia.tar, huffman literals, old streaming advanced cdict, 6190789
|
||||
silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046
|
||||
github, level -5, old streaming advanced cdict, 205285
|
||||
github, level -5 with dict, old streaming advanced cdict, 46718
|
||||
github, level -3, old streaming advanced cdict, 190643
|
||||
github, level -3 with dict, old streaming advanced cdict, 45395
|
||||
github, level -1, old streaming advanced cdict, 175568
|
||||
github, level -1 with dict, old streaming advanced cdict, 43170
|
||||
github, level 0, old streaming advanced cdict, 136311
|
||||
github, level 0 with dict, old streaming advanced cdict, 41148
|
||||
github, level 1, old streaming advanced cdict, 142450
|
||||
github, level 1 with dict, old streaming advanced cdict, 41682
|
||||
github, level 3, old streaming advanced cdict, 136311
|
||||
github, level 3 with dict, old streaming advanced cdict, 41148
|
||||
github, level 4, old streaming advanced cdict, 136144
|
||||
github, level 4 with dict, old streaming advanced cdict, 41251
|
||||
github, level 5, old streaming advanced cdict, 135106
|
||||
github, level 5 with dict, old streaming advanced cdict, 38938
|
||||
github, level 6, old streaming advanced cdict, 135108
|
||||
github, level 6 with dict, old streaming advanced cdict, 38632
|
||||
github, level 7, old streaming advanced cdict, 135108
|
||||
github, level 7 with dict, old streaming advanced cdict, 38766
|
||||
github, level 9, old streaming advanced cdict, 135108
|
||||
github, level 9 with dict, old streaming advanced cdict, 39326
|
||||
github, level 13, old streaming advanced cdict, 133717
|
||||
github, level 13 with dict, old streaming advanced cdict, 39716
|
||||
github, level 16, old streaming advanced cdict, 133717
|
||||
github, level 16 with dict, old streaming advanced cdict, 37577
|
||||
github, level 19, old streaming advanced cdict, 133717
|
||||
github, level -5 with dict, old streaming advanced cdict, 49562
|
||||
github, level -3 with dict, old streaming advanced cdict, 44956
|
||||
github, level -1 with dict, old streaming advanced cdict, 42383
|
||||
github, level 0 with dict, old streaming advanced cdict, 41113
|
||||
github, level 1 with dict, old streaming advanced cdict, 42430
|
||||
github, level 3 with dict, old streaming advanced cdict, 41113
|
||||
github, level 4 with dict, old streaming advanced cdict, 41084
|
||||
github, level 5 with dict, old streaming advanced cdict, 39158
|
||||
github, level 6 with dict, old streaming advanced cdict, 38748
|
||||
github, level 7 with dict, old streaming advanced cdict, 38744
|
||||
github, level 9 with dict, old streaming advanced cdict, 38986
|
||||
github, level 13 with dict, old streaming advanced cdict, 39724
|
||||
github, level 16 with dict, old streaming advanced cdict, 40771
|
||||
github, level 19 with dict, old streaming advanced cdict, 37576
|
||||
github, no source size, old streaming advanced cdict, 140631
|
||||
github, long distance mode, old streaming advanced cdict, 412933
|
||||
github, multithreaded, old streaming advanced cdict, 412933
|
||||
github, multithreaded long distance mode, old streaming advanced cdict, 412933
|
||||
github, small window log, old streaming advanced cdict, 412933
|
||||
github, small hash log, old streaming advanced cdict, 412933
|
||||
github, small chain log, old streaming advanced cdict, 412933
|
||||
github, explicit params, old streaming advanced cdict, 412933
|
||||
github, uncompressed literals, old streaming advanced cdict, 136311
|
||||
github, uncompressed literals optimal, old streaming advanced cdict, 133717
|
||||
github, huffman literals, old streaming advanced cdict, 175568
|
||||
github, multithreaded with advanced params, old streaming advanced cdict, 412933
|
||||
|
||||
|
@ -81,7 +81,7 @@ static U64 g_clockTime = 0;
|
||||
@return : a 27 bits random value, from a 32-bits `seed`.
|
||||
`seed` is also modified */
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
static unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
{
|
||||
U32 rand32 = *seedPtr;
|
||||
rand32 *= prime1;
|
||||
@ -467,7 +467,7 @@ _output_error:
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
int FUZ_usage(const char* programName)
|
||||
static int FUZ_usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
|
||||
@ -103,6 +103,7 @@ zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/z
|
||||
|
||||
zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
|
||||
zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $< -c -o $@
|
||||
|
||||
$(ZSTDLIBDIR)/libzstd.a:
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
|
||||
|
||||
@ -8,6 +8,14 @@
|
||||
|
||||
#include "gzguts.h"
|
||||
|
||||
/* fix for Visual Studio, which doesn't support ssize_t type.
|
||||
* see https://github.com/facebook/zstd/issues/1800#issuecomment-545945050 */
|
||||
#if defined(_MSC_VER) && !defined(ssize_t)
|
||||
# include <BaseTsd.h>
|
||||
typedef SSIZE_T ssize_t;
|
||||
#endif
|
||||
|
||||
|
||||
/* Local functions */
|
||||
local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *));
|
||||
local int gz_avail OF((gz_statep));
|
||||
|
||||
Loading…
Reference in New Issue
Block a user