From ccd2d426fca01d4dd75c4650d65e2515ff71e3ff Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Oct 2018 17:25:49 -0700 Subject: [PATCH 1/7] separate DDict logic into its own module created zstd_ddict.c within lib/decompress --- build/VS2008/fullbench/fullbench.vcproj | 4 + build/VS2008/fuzzer/fuzzer.vcproj | 4 + build/VS2008/zstd/zstd.vcproj | 4 + build/VS2008/zstdlib/zstdlib.vcproj | 4 + build/VS2010/fullbench/fullbench.vcxproj | 2 + build/VS2010/fuzzer/fuzzer.vcxproj | 2 + build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 2 + build/VS2010/libzstd/libzstd.vcxproj | 2 + build/VS2010/zstd/zstd.vcxproj | 2 + build/cmake/contrib/pzstd/CMakeLists.txt | 2 +- build/cmake/lib/CMakeLists.txt | 3 + build/cmake/tests/CMakeLists.txt | 6 +- lib/common/zstd_common.c | 4 +- lib/common/zstd_internal.h | 3 + lib/decompress/zstd_ddict.c | 241 ++++++++++++++ lib/decompress/zstd_ddict.h | 25 ++ lib/decompress/zstd_decompress.c | 317 +------------------ lib/decompress/zstd_decompress_internal.h | 117 +++++++ 18 files changed, 438 insertions(+), 306 deletions(-) create mode 100644 lib/decompress/zstd_ddict.c create mode 100644 lib/decompress/zstd_ddict.h create mode 100644 lib/decompress/zstd_decompress_internal.h diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index c9547dfe..6a6494d1 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -380,6 +380,10 @@ RelativePath="..\..\..\lib\decompress\zstd_decompress.c" > + + diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index a1f5ce07..dcc75052 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -412,6 +412,10 @@ RelativePath="..\..\..\lib\decompress\zstd_decompress.c" > + + diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index ec8e9486..352682e8 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -420,6 +420,10 @@ RelativePath="..\..\..\lib\decompress\zstd_decompress.c" > + + diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index 5f89ebf8..9737d515 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -420,6 +420,10 @@ RelativePath="..\..\..\lib\decompress\zstd_decompress.c" > + + diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index b7d754e1..9986c0b5 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -176,6 +176,7 @@ + @@ -196,6 +197,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 946cd33b..ba8152f2 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -175,6 +175,7 @@ + @@ -199,6 +200,7 @@ + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index e7e906e5..a598bb3a 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -39,6 +39,7 @@ + @@ -82,6 +83,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index 7c2af2b7..b1bac7d1 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -39,6 +39,7 @@ + @@ -82,6 +83,7 @@ + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index c9a7a1aa..a8f4eb9c 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -39,6 +39,7 @@ + @@ -76,6 +77,7 @@ + diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt index e6db4719..2edebf84 100644 --- a/build/cmake/contrib/pzstd/CMakeLists.txt +++ b/build/cmake/contrib/pzstd/CMakeLists.txt @@ -17,7 +17,7 @@ SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs) SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd) INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR}) -ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp) +ADD_EXECUTABLE(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp) SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG") SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow") diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index ffc196dd..bc5ac432 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -46,6 +46,7 @@ SET(Sources ${LIBRARY_DIR}/compress/zstd_ldm.c ${LIBRARY_DIR}/decompress/huf_decompress.c ${LIBRARY_DIR}/decompress/zstd_decompress.c + ${LIBRARY_DIR}/decompress/zstd_ddict.c ${LIBRARY_DIR}/dictBuilder/cover.c ${LIBRARY_DIR}/dictBuilder/fastcover.c ${LIBRARY_DIR}/dictBuilder/divsufsort.c @@ -74,6 +75,8 @@ SET(Headers ${LIBRARY_DIR}/compress/zstd_opt.h ${LIBRARY_DIR}/compress/zstd_ldm.h ${LIBRARY_DIR}/compress/zstdmt_compress.h + ${LIBRARY_DIR}/decompress/zstd_ddict.h + ${LIBRARY_DIR}/decompress/zstd_decompress_internal.h ${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/cover.h ${LIBRARY_DIR}/deprecated/zbuff.h) diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index 9f4c64cf..4c198996 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -43,13 +43,13 @@ INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/c ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c) TARGET_LINK_LIBRARIES(datagen libzstd_static) -ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c) +ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c) TARGET_LINK_LIBRARIES(fullbench libzstd_static) -ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c) +ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/fuzzer.c) TARGET_LINK_LIBRARIES(fuzzer libzstd_static) IF (UNIX) - ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c) + ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/paramgrill.c) TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library ENDIF (UNIX) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 6f05d240..667f4a27 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -30,8 +30,10 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } /*-**************************************** * ZSTD Error Management ******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ /*! ZSTD_isError() : - * tells if a return value is an error code */ + * tells if a return value is an error code + * symbol is required for external callers */ unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } /*! ZSTD_getErrorName() : diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index e75adfa6..202caacf 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -41,6 +41,9 @@ extern "C" { /* ---- static assert (debug) --- */ #define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError /*-************************************* diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c new file mode 100644 index 00000000..6569eeed --- /dev/null +++ b/lib/decompress/zstd_ddict.c @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include /* memcpy, memmove, memset */ +#include "compiler.h" /* prefetch */ +#include "cpu.h" /* bmi2 */ +#include "mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + CHECK_E( ZSTD_loadDEntropy(&ddict->entropy, + ddict->dictContent, ddict->dictSize), + dictionary_corrupted ); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) ); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_free(ddict->dictBuffer, cMem); + ZSTD_free(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); +} diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h new file mode 100644 index 00000000..01449d29 --- /dev/null +++ b/lib/decompress/zstd_ddict.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-******************************************************* + * Dependencies + *********************************************************/ +#include /* size_t */ +#include "zstd.h" /* ZSTD_DDict */ + + +/*-******************************************************* +* Interface +*********************************************************/ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 4a4930f0..fffcc34d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -64,22 +64,13 @@ #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) # include "zstd_legacy.h" #endif -static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict); -static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict); - - -/*-************************************* -* Errors -***************************************/ -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - /*_******************************************************* * Memory operations @@ -90,94 +81,11 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } /*-************************************************************* * Context management ***************************************************************/ -typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, - ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, - ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; - -typedef enum { zdss_init=0, zdss_loadHeader, - zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; - - typedef struct { U32 fastMode; U32 tableLog; } ZSTD_seqSymbol_header; -typedef struct { - U16 nextState; - BYTE nbAdditionalBits; - BYTE nbBits; - U32 baseValue; -} ZSTD_seqSymbol; - -#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) - -typedef struct { - ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ - ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ - ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ - U32 rep[ZSTD_REP_NUM]; -} ZSTD_entropyDTables_t; - -struct ZSTD_DCtx_s -{ - const ZSTD_seqSymbol* LLTptr; - const ZSTD_seqSymbol* MLTptr; - const ZSTD_seqSymbol* OFTptr; - const HUF_DTable* HUFptr; - ZSTD_entropyDTables_t entropy; - U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ - const void* previousDstEnd; /* detect continuity */ - const void* prefixStart; /* start of current segment */ - const void* virtualStart; /* virtual start of previous segment if it was just before current one */ - const void* dictEnd; /* end of previous segment */ - size_t expected; - ZSTD_frameHeader fParams; - U64 decodedSize; - blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ - ZSTD_dStage stage; - U32 litEntropy; - U32 fseEntropy; - XXH64_state_t xxhState; - size_t headerSize; - ZSTD_format_e format; - const BYTE* litPtr; - ZSTD_customMem customMem; - size_t litSize; - size_t rleSize; - size_t staticSize; - int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ - - /* dictionary */ - ZSTD_DDict* ddictLocal; - const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ - U32 dictID; - int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ - - /* streaming */ - ZSTD_dStreamStage streamStage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - size_t maxWindowSize; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t lhSize; - void* legacyContext; - U32 previousLegacyVersion; - U32 legacyVersion; - U32 hostageByte; - int noForwardProgress; - - /* workspace */ - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; -}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ - size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; /* support sizeof NULL */ @@ -1946,8 +1854,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ if (ddict) { - dict = ZSTD_DDictDictContent(ddict); - dictSize = ZSTD_DDictDictSize(ddict); + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); } while (srcSize >= ZSTD_frameHeaderSize_prefix) { @@ -2238,11 +2146,12 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict return 0; } -/*! ZSTD_loadEntropy() : +/*! ZSTD_loadDEntropy() : * dict : must point at beginning of a valid zstd dictionary. * @return : size of entropy tables read */ -static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, - const void* const dict, size_t const dictSize) +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; @@ -2324,7 +2233,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); /* load entropy tables */ - { size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); dict = (const char*)dict + eSize; dictSize -= eSize; @@ -2368,209 +2277,25 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t /* ====== ZSTD_DDict ====== */ -struct ZSTD_DDict_s { - void* dictBuffer; - const void* dictContent; - size_t dictSize; - ZSTD_entropyDTables_t entropy; - U32 dictID; - U32 entropyPresent; - ZSTD_customMem cMem; -}; /* typedef'd to ZSTD_DDict within "zstd.h" */ - -static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) -{ - assert(ddict != NULL); - return ddict->dictContent; -} - -static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) -{ - assert(ddict != NULL); - return ddict->dictSize; -} - size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); assert(dctx != NULL); if (ddict) { - dctx->ddictIsCold = (dctx->dictEnd != (const char*)ddict->dictContent + ddict->dictSize); + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); DEBUGLOG(4, "DDict is %s", dctx->ddictIsCold ? "~cold~" : "hot!"); } CHECK_F( ZSTD_decompressBegin(dctx) ); if (ddict) { /* NULL ddict is equivalent to no dictionary */ - dctx->dictID = ddict->dictID; - dctx->prefixStart = ddict->dictContent; - dctx->virtualStart = ddict->dictContent; - dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; - dctx->previousDstEnd = dctx->dictEnd; - if (ddict->entropyPresent) { - dctx->litEntropy = 1; - dctx->fseEntropy = 1; - dctx->LLTptr = ddict->entropy.LLTable; - dctx->MLTptr = ddict->entropy.MLTable; - dctx->OFTptr = ddict->entropy.OFTable; - dctx->HUFptr = ddict->entropy.hufTable; - dctx->entropy.rep[0] = ddict->entropy.rep[0]; - dctx->entropy.rep[1] = ddict->entropy.rep[1]; - dctx->entropy.rep[2] = ddict->entropy.rep[2]; - } else { - dctx->litEntropy = 0; - dctx->fseEntropy = 0; - } + ZSTD_copyDDictParameters(dctx, ddict); } return 0; } -static size_t -ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, - ZSTD_dictContentType_e dictContentType) -{ - ddict->dictID = 0; - ddict->entropyPresent = 0; - if (dictContentType == ZSTD_dct_rawContent) return 0; - - if (ddict->dictSize < 8) { - if (dictContentType == ZSTD_dct_fullDict) - return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ - return 0; /* pure content mode */ - } - { U32 const magic = MEM_readLE32(ddict->dictContent); - if (magic != ZSTD_MAGIC_DICTIONARY) { - if (dictContentType == ZSTD_dct_fullDict) - return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ - return 0; /* pure content mode */ - } - } - ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); - - /* load entropy tables */ - CHECK_E( ZSTD_loadEntropy(&ddict->entropy, - ddict->dictContent, ddict->dictSize), - dictionary_corrupted ); - ddict->entropyPresent = 1; - return 0; -} - - -static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType) -{ - if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { - ddict->dictBuffer = NULL; - ddict->dictContent = dict; - if (!dict) dictSize = 0; - } else { - void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); - ddict->dictBuffer = internalBuffer; - ddict->dictContent = internalBuffer; - if (!internalBuffer) return ERROR(memory_allocation); - memcpy(internalBuffer, dict, dictSize); - } - ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ - - /* parse dictionary content */ - CHECK_F( ZSTD_loadEntropy_inDDict(ddict, dictContentType) ); - - return 0; -} - -ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_customMem customMem) -{ - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - - { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); - if (ddict == NULL) return NULL; - ddict->cMem = customMem; - { size_t const initResult = ZSTD_initDDict_internal(ddict, - dict, dictSize, - dictLoadMethod, dictContentType); - if (ZSTD_isError(initResult)) { - ZSTD_freeDDict(ddict); - return NULL; - } } - return ddict; - } -} - -/*! ZSTD_createDDict() : -* Create a digested dictionary, to start decompression without startup delay. -* `dict` content is copied inside DDict. -* Consequently, `dict` can be released after `ZSTD_DDict` creation */ -ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) -{ - ZSTD_customMem const allocator = { NULL, NULL, NULL }; - return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); -} - -/*! ZSTD_createDDict_byReference() : - * Create a digested dictionary, to start decompression without startup delay. - * Dictionary content is simply referenced, it will be accessed during decompression. - * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ -ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) -{ - ZSTD_customMem const allocator = { NULL, NULL, NULL }; - return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); -} - - -const ZSTD_DDict* ZSTD_initStaticDDict( - void* sBuffer, size_t sBufferSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType) -{ - size_t const neededSpace = sizeof(ZSTD_DDict) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); - ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; - assert(sBuffer != NULL); - assert(dict != NULL); - if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ - if (sBufferSize < neededSpace) return NULL; - if (dictLoadMethod == ZSTD_dlm_byCopy) { - memcpy(ddict+1, dict, dictSize); /* local copy */ - dict = ddict+1; - } - if (ZSTD_isError( ZSTD_initDDict_internal(ddict, - dict, dictSize, - ZSTD_dlm_byRef, dictContentType) )) - return NULL; - return ddict; -} - - -size_t ZSTD_freeDDict(ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; /* support free on NULL */ - { ZSTD_customMem const cMem = ddict->cMem; - ZSTD_free(ddict->dictBuffer, cMem); - ZSTD_free(ddict, cMem); - return 0; - } -} - -/*! ZSTD_estimateDDictSize() : - * Estimate amount of memory that will be needed to create a dictionary for decompression. - * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ -size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) -{ - return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); -} - -size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; -} - /*! ZSTD_getDictID_fromDict() : * Provides the dictID stored within dictionary. * if @return == 0, the dictionary is not conformant with Zstandard specification. @@ -2582,16 +2307,6 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); } -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); -} - /*! ZSTD_getDictID_fromFrame() : * Provides the dictID required to decompresse frame stored within `src`. * If @return == 0, the dictID could not be decoded. @@ -2872,8 +2587,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { - const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL; - size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; + const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL; + size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0; DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); /* legacy support is incompatible with static dctx */ if (zds->staticSize) return ERROR(memory_allocation); diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h new file mode 100644 index 00000000..f450b3dc --- /dev/null +++ b/lib/decompress/zstd_decompress_internal.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "mem.h" /* BYTE, U16, U32 */ +#include "zstd_internal.h" /* ZSTD_seqSymbol */ + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; + U32 hostageByte; + int noForwardProgress; + + /* workspace */ + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + + /*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ + size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); From 1ae38772dfc32b31421ee7d49f0205220a126ce4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Oct 2018 17:40:15 -0700 Subject: [PATCH 2/7] restored cmake build tests in Travis for some reason, these tests were no longer played on Travis. As a consequence, I discovered some left-over from the util.c change while testing. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a61b8287..b01a8059 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,8 @@ matrix: - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' + - env: Cmd='make cmakebuild' + - env: Cmd='make staticAnalyze' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' From 2f7e1c8737fa8c8d766965f343a54073486b6161 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Oct 2018 17:44:43 -0700 Subject: [PATCH 3/7] fixed VS2010 build script --- build/VS2010/fullbench/fullbench.vcxproj | 2 +- build/VS2010/fuzzer/fuzzer.vcxproj | 2 +- build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 2 +- build/VS2010/libzstd/libzstd.vcxproj | 2 +- build/VS2010/zstd/zstd.vcxproj | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 9986c0b5..23f696b0 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -197,7 +197,7 @@ - + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index ba8152f2..a0aba109 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -200,7 +200,7 @@ - + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index a598bb3a..6bb42c37 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -83,7 +83,7 @@ - + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index b1bac7d1..a394a459 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -83,7 +83,7 @@ - + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index a8f4eb9c..6de9aaa2 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -77,7 +77,7 @@ - + From cc3612e1c5e53d45be2291012ac1eb77d3a24ec0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Oct 2018 17:55:23 -0700 Subject: [PATCH 4/7] added simple guard macros in case of accidental multi-includes --- lib/decompress/zstd_ddict.h | 13 ++++++++++--- lib/decompress/zstd_decompress_internal.h | 6 ++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h index 01449d29..87bbbca6 100644 --- a/lib/decompress/zstd_ddict.h +++ b/lib/decompress/zstd_ddict.h @@ -8,7 +8,11 @@ * You may select, at your option, one of the above-listed licenses. */ - /*-******************************************************* + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* * Dependencies *********************************************************/ #include /* size_t */ @@ -16,10 +20,13 @@ /*-******************************************************* -* Interface -*********************************************************/ + * Interface + *********************************************************/ const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + +#endif /* ZSTD_DDICT_H */ diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h index f450b3dc..3de8e4e7 100644 --- a/lib/decompress/zstd_decompress_internal.h +++ b/lib/decompress/zstd_decompress_internal.h @@ -12,6 +12,9 @@ /* zstd_decompress_internal: * objects and definitions shared within lib/decompress modules */ + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + /*-******************************************************* * Dependencies @@ -115,3 +118,6 @@ struct ZSTD_DCtx_s * @return : size of entropy tables read */ size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ From debff3929b285818b925711189d2a6ed2ffd0af1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 24 Oct 2018 10:36:06 -0700 Subject: [PATCH 5/7] fixed warnings in testpools --- lib/decompress/zstd_decompress.c | 18 +++++++++--------- tests/poolTests.c | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index fffcc34d..54b9bf70 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -994,6 +994,13 @@ typedef struct { } seqState_t; +/* ZSTD_execSequenceLast7(): + * exceptional case : decompress a match starting within last 7 bytes of output buffer. + * requires more careful checks, to ensure there is no overflow. + * performance does not matter though. + * note : this case is supposed to be never generated "naturally" by reference encoder, + * since in most cases it needs at least 8 bytes to look for a match. + * but it's allowed by the specification. */ FORCE_NOINLINE size_t ZSTD_execSequenceLast7(BYTE* op, BYTE* const oend, seq_t sequence, @@ -1003,21 +1010,14 @@ size_t ZSTD_execSequenceLast7(BYTE* op, BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; /* check */ - if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must fit within dstBuffer */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* try to read beyond literal buffer */ /* copy literals */ - if (op < oend_w) { - ZSTD_wildcopy(op, *litPtr, oend_w - op); - *litPtr += oend_w - op; - op = oend_w; - } while (op < oLitEnd) *op++ = *(*litPtr)++; /* copy Match */ diff --git a/tests/poolTests.c b/tests/poolTests.c index 9661b529..8b9a4700 100644 --- a/tests/poolTests.c +++ b/tests/poolTests.c @@ -30,7 +30,7 @@ struct data { size_t i; }; -void fn(void *opaque) { +static void fn(void *opaque) { struct data *data = (struct data *)opaque; ZSTD_pthread_mutex_lock(&data->mutex); data->data[data->i] = data->i; @@ -38,7 +38,7 @@ void fn(void *opaque) { ZSTD_pthread_mutex_unlock(&data->mutex); } -int testOrder(size_t numThreads, size_t queueSize) { +static int testOrder(size_t numThreads, size_t queueSize) { struct data data; POOL_ctx *ctx = POOL_create(numThreads, queueSize); ASSERT_TRUE(ctx); @@ -63,13 +63,13 @@ int testOrder(size_t numThreads, size_t queueSize) { /* --- test deadlocks --- */ -void waitFn(void *opaque) { +static void waitFn(void *opaque) { (void)opaque; UTIL_sleepMilli(1); } /* Tests for deadlock */ -int testWait(size_t numThreads, size_t queueSize) { +static int testWait(size_t numThreads, size_t queueSize) { struct data data; POOL_ctx *ctx = POOL_create(numThreads, queueSize); ASSERT_TRUE(ctx); @@ -92,7 +92,7 @@ typedef struct { ZSTD_pthread_cond_t cond; } poolTest_t; -void waitLongFn(void *opaque) { +static void waitLongFn(void *opaque) { poolTest_t* test = (poolTest_t*) opaque; UTIL_sleepMilli(10); ZSTD_pthread_mutex_lock(&test->mut); @@ -167,7 +167,7 @@ typedef struct { int val; } abruptEndCanary_t; -void waitIncFn(void *opaque) { +static void waitIncFn(void *opaque) { abruptEndCanary_t* test = (abruptEndCanary_t*) opaque; UTIL_sleepMilli(10); ZSTD_pthread_mutex_lock(&test->mut); From b7421f8e6994fc8317b50993a0b5ccf5a843395a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 24 Oct 2018 11:32:09 -0700 Subject: [PATCH 6/7] fix and refactored libzstd_partial_build.sh make the test intention clearer (easier to maintain) also resist the existence of zstd_ddict.o within libzstd.a --- tests/libzstd_partial_builds.sh | 89 ++++++++++++++++++++++++++------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/tests/libzstd_partial_builds.sh b/tests/libzstd_partial_builds.sh index 34d8ea55..b1c1e3b1 100755 --- a/tests/libzstd_partial_builds.sh +++ b/tests/libzstd_partial_builds.sh @@ -1,36 +1,89 @@ #!/bin/sh -e +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +ECHO=echo +RM="rm -f" +GREP="grep" +INTOVOID="/dev/null" + die() { $ECHO "$@" 1>&2 exit 1 } -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +isPresent() { + $GREP $@ tmplog || die "$@" "should be present" +} -INTOVOID="/dev/null" -case "$OS" in - Windows*) - INTOVOID="NUL" - ;; -esac +mustBeAbsent() { + $GREP $@ tmplog && die "$@ should not be there !!" + $ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??) +} +# default compilation : all features enabled +make clean > /dev/null +$ECHO "testing default library compilation" +CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +isPresent "zdict.o" +isPresent "zstd_v07.o" +isPresent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog + +# compression disabled => also disable zdict and zbuff +$ECHO "testing with compression disabled" ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -! grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Compression macro failed" - +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +mustBeAbsent "zstd_compress.o" +isPresent "zstd_decompress.o" +mustBeAbsent "zdict.o" +isPresent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# decompression disabled => also disable legacy and zbuff +$ECHO "testing with decompression disabled" ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Decompression macro failed" +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +mustBeAbsent "zstd_decompress.o" +isPresent "zdict.o" +mustBeAbsent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# deprecated function disabled => only remove zbuff +$ECHO "testing with deprecated functions disabled" ZSTD_LIB_DEPRECATED=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Deprecated macro failed" +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +isPresent "zdict.o" +isPresent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# dictionary builder disabled => only remove zdict +$ECHO "testing with dictionary builder disabled" ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Dictbuilder macro failed" +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +mustBeAbsent "zdict.o" +isPresent "zstd_v07.o" +isPresent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# both decompression and dictionary builder disabled => only compression remains +$ECHO "testing with both decompression and dictionary builder disabled (only compression remains)" ZSTD_LIB_DECOMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Multi-macro failed" \ No newline at end of file +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +mustBeAbsent "zstd_decompress.o" +mustBeAbsent "zdict.o" +mustBeAbsent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog From cb320a9fc0ff512a5630b9c67f924c88ad59db87 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 24 Oct 2018 16:50:03 -0700 Subject: [PATCH 7/7] added comment on public ddict functions --- lib/decompress/zstd_ddict.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_ddict.h b/lib/decompress/zstd_ddict.h index 87bbbca6..0479d11b 100644 --- a/lib/decompress/zstd_ddict.h +++ b/lib/decompress/zstd_ddict.h @@ -16,17 +16,29 @@ * Dependencies *********************************************************/ #include /* size_t */ -#include "zstd.h" /* ZSTD_DDict */ +#include "zstd.h" /* ZSTD_DDict, and several public functions */ /*-******************************************************* * Interface *********************************************************/ +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + #endif /* ZSTD_DDICT_H */