AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #1388 from facebook/ddict

Browse Source 
separate DDict logic into its own module
This commit is contained in:
Yann Collet 2018-10-25 10:20:40 -07:00 committed by GitHub
commit e26126caca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 551 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -20,6 +20,8 @@ matrix:
- env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
- env: Cmd='make cmakebuild'
- env: Cmd='make staticAnalyze'
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'

4
build/VS2008/fullbench/fullbench.vcproj
View File

@ -380,6 +380,10 @@
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>

4
build/VS2008/fuzzer/fuzzer.vcproj
View File

@ -412,6 +412,10 @@
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_fast.c"
>

4
build/VS2008/zstd/zstd.vcproj
View File

@ -420,6 +420,10 @@
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
>

4
build/VS2008/zstdlib/zstdlib.vcproj
View File

@ -420,6 +420,10 @@
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_ddict.c"
>
</File>
<File
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
>

2
build/VS2010/fullbench/fullbench.vcxproj
View File

@ -176,6 +176,7 @@
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\bench.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />
@ -196,6 +197,7 @@
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\util.h" />

2
build/VS2010/fuzzer/fuzzer.vcxproj
View File

@ -175,6 +175,7 @@
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@ -199,6 +200,7 @@
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />

2
build/VS2010/libzstd-dll/libzstd-dll.vcxproj
View File

@ -39,6 +39,7 @@
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
@ -82,6 +83,7 @@
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="libzstd-dll.rc" />

2
build/VS2010/libzstd/libzstd.vcxproj
View File

@ -39,6 +39,7 @@
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
@ -82,6 +83,7 @@
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>

2
build/VS2010/zstd/zstd.vcxproj
View File

@ -39,6 +39,7 @@
<ClCompile Include="..\..\..\lib\compress\zstd_ldm.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@ -76,6 +77,7 @@
<ClInclude Include="..\..\..\lib\compress\zstd_lazy.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_ldm.h" />
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />

2
build/cmake/contrib/pzstd/CMakeLists.txt
View File

@ -17,7 +17,7 @@ SET(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
ADD_EXECUTABLE(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")

3
build/cmake/lib/CMakeLists.txt
View File

@ -46,6 +46,7 @@ SET(Sources
${LIBRARY_DIR}/compress/zstd_ldm.c
${LIBRARY_DIR}/decompress/huf_decompress.c
${LIBRARY_DIR}/decompress/zstd_decompress.c
${LIBRARY_DIR}/decompress/zstd_ddict.c
${LIBRARY_DIR}/dictBuilder/cover.c
${LIBRARY_DIR}/dictBuilder/fastcover.c
${LIBRARY_DIR}/dictBuilder/divsufsort.c
@ -74,6 +75,8 @@ SET(Headers
${LIBRARY_DIR}/compress/zstd_opt.h
${LIBRARY_DIR}/compress/zstd_ldm.h
${LIBRARY_DIR}/compress/zstdmt_compress.h
${LIBRARY_DIR}/decompress/zstd_ddict.h
${LIBRARY_DIR}/decompress/zstd_decompress_internal.h
${LIBRARY_DIR}/dictBuilder/zdict.h
${LIBRARY_DIR}/dictBuilder/cover.h
${LIBRARY_DIR}/deprecated/zbuff.h)

6
build/cmake/tests/CMakeLists.txt
View File

@ -43,13 +43,13 @@ INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/c
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
TARGET_LINK_LIBRARIES(datagen libzstd_static)
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c)
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/bench.c ${TESTS_DIR}/fullbench.c)
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/fuzzer.c)
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
IF (UNIX)
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/paramgrill.c)
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
ENDIF (UNIX)

4
lib/common/zstd_common.c
View File

@ -30,8 +30,10 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
/*-****************************************
* ZSTD Error Management
******************************************/
#undef ZSTD_isError /* defined within zstd_internal.h */
/*! ZSTD_isError() :
* tells if a return value is an error code */
* tells if a return value is an error code
* symbol is required for external callers */
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
/*! ZSTD_getErrorName() :

3
lib/common/zstd_internal.h
View File

@ -41,6 +41,9 @@ extern "C" {
/* ---- static assert (debug) --- */
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
#define ZSTD_isError ERR_isError /* for inlining */
#define FSE_isError ERR_isError
#define HUF_isError ERR_isError
/*-*************************************

241
lib/decompress/zstd_ddict.c Normal file
View File

@ -0,0 +1,241 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* zstd_ddict.c :
* concentrates all logic that needs to know the internals of ZSTD_DDict object */
/*-*******************************************************
* Dependencies
*********************************************************/
#include <string.h> /* memcpy, memmove, memset */
#include "compiler.h" /* prefetch */
#include "cpu.h" /* bmi2 */
#include "mem.h" /* low level memory routines */
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "zstd_decompress_internal.h"
#include "zstd_ddict.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
# include "zstd_legacy.h"
#endif
/*-*******************************************************
* Types
*********************************************************/
struct ZSTD_DDict_s {
void* dictBuffer;
const void* dictContent;
size_t dictSize;
ZSTD_entropyDTables_t entropy;
U32 dictID;
U32 entropyPresent;
ZSTD_customMem cMem;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictContent;
}
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictSize;
}
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
DEBUGLOG(4, "ZSTD_copyDDictParameters");
assert(dctx != NULL);
assert(ddict != NULL);
dctx->dictID = ddict->dictID;
dctx->prefixStart = ddict->dictContent;
dctx->virtualStart = ddict->dictContent;
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
dctx->previousDstEnd = dctx->dictEnd;
if (ddict->entropyPresent) {
dctx->litEntropy = 1;
dctx->fseEntropy = 1;
dctx->LLTptr = ddict->entropy.LLTable;
dctx->MLTptr = ddict->entropy.MLTable;
dctx->OFTptr = ddict->entropy.OFTable;
dctx->HUFptr = ddict->entropy.hufTable;
dctx->entropy.rep[0] = ddict->entropy.rep[0];
dctx->entropy.rep[1] = ddict->entropy.rep[1];
dctx->entropy.rep[2] = ddict->entropy.rep[2];
} else {
dctx->litEntropy = 0;
dctx->fseEntropy = 0;
}
}
static size_t
ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
ZSTD_dictContentType_e dictContentType)
{
ddict->dictID = 0;
ddict->entropyPresent = 0;
if (dictContentType == ZSTD_dct_rawContent) return 0;
if (ddict->dictSize < 8) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
{ U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_MAGIC_DICTIONARY) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
}
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
/* load entropy tables */
CHECK_E( ZSTD_loadDEntropy(&ddict->entropy,
ddict->dictContent, ddict->dictSize),
dictionary_corrupted );
ddict->entropyPresent = 1;
return 0;
}
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL;
ddict->dictContent = dict;
if (!dict) dictSize = 0;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
if (!internalBuffer) return ERROR(memory_allocation);
memcpy(internalBuffer, dict, dictSize);
}
ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */
CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
return 0;
}
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_customMem customMem)
{
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (ddict == NULL) return NULL;
ddict->cMem = customMem;
{ size_t const initResult = ZSTD_initDDict_internal(ddict,
dict, dictSize,
dictLoadMethod, dictContentType);
if (ZSTD_isError(initResult)) {
ZSTD_freeDDict(ddict);
return NULL;
} }
return ddict;
}
}
/*! ZSTD_createDDict() :
* Create a digested dictionary, to start decompression without startup delay.
* `dict` content is copied inside DDict.
* Consequently, `dict` can be released after `ZSTD_DDict` creation */
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
}
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, to start decompression without startup delay.
* Dictionary content is simply referenced, it will be accessed during decompression.
* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
}
const ZSTD_DDict* ZSTD_initStaticDDict(
void* sBuffer, size_t sBufferSize,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
size_t const neededSpace = sizeof(ZSTD_DDict)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
assert(sBuffer != NULL);
assert(dict != NULL);
if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
if (sBufferSize < neededSpace) return NULL;
if (dictLoadMethod == ZSTD_dlm_byCopy) {
memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1;
}
if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
dict, dictSize,
ZSTD_dlm_byRef, dictContentType) ))
return NULL;
return ddict;
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->cMem;
ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem);
return 0;
}
}
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
{
return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
}
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
}

44
lib/decompress/zstd_ddict.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_DDICT_H
#define ZSTD_DDICT_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include <stddef.h> /* size_t */
#include "zstd.h" /* ZSTD_DDict, and several public functions */
/*-*******************************************************
* Interface
*********************************************************/
/* note: several prototypes are already published in `zstd.h` :
* ZSTD_createDDict()
* ZSTD_createDDict_byReference()
* ZSTD_createDDict_advanced()
* ZSTD_freeDDict()
* ZSTD_initStaticDDict()
* ZSTD_sizeof_DDict()
* ZSTD_estimateDDictSize()
* ZSTD_getDictID_fromDict()
*/
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
#endif /* ZSTD_DDICT_H */

335
lib/decompress/zstd_decompress.c
View File

@ -64,22 +64,13 @@
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "zstd_internal.h"
#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
# include "zstd_legacy.h"
#endif
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict);
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
/*-*************************************
* Errors
***************************************/
#define ZSTD_isError ERR_isError /* for inlining */
#define FSE_isError ERR_isError
#define HUF_isError ERR_isError
/*_*******************************************************
* Memory operations
@ -90,94 +81,11 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
/*-*************************************************************
* Context management
***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
typedef enum { zdss_init=0, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
typedef struct {
U32 fastMode;
U32 tableLog;
} ZSTD_seqSymbol_header;
typedef struct {
U16 nextState;
BYTE nbAdditionalBits;
BYTE nbBits;
U32 baseValue;
} ZSTD_seqSymbol;
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
typedef struct {
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyDTables_t;
struct ZSTD_DCtx_s
{
const ZSTD_seqSymbol* LLTptr;
const ZSTD_seqSymbol* MLTptr;
const ZSTD_seqSymbol* OFTptr;
const HUF_DTable* HUFptr;
ZSTD_entropyDTables_t entropy;
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
const void* previousDstEnd; /* detect continuity */
const void* prefixStart; /* start of current segment */
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
ZSTD_frameHeader fParams;
U64 decodedSize;
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
ZSTD_dStage stage;
U32 litEntropy;
U32 fseEntropy;
XXH64_state_t xxhState;
size_t headerSize;
ZSTD_format_e format;
const BYTE* litPtr;
ZSTD_customMem customMem;
size_t litSize;
size_t rleSize;
size_t staticSize;
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
/* dictionary */
ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
U32 dictID;
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
/* streaming */
ZSTD_dStreamStage streamStage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
size_t maxWindowSize;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t lhSize;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
U32 hostageByte;
int noForwardProgress;
/* workspace */
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
{
if (dctx==NULL) return 0; /* support sizeof NULL */
@ -1086,6 +994,13 @@ typedef struct {
} seqState_t;
/* ZSTD_execSequenceLast7():
* exceptional case : decompress a match starting within last 7 bytes of output buffer.
* requires more careful checks, to ensure there is no overflow.
* performance does not matter though.
* note : this case is supposed to be never generated "naturally" by reference encoder,
* since in most cases it needs at least 8 bytes to look for a match.
* but it's allowed by the specification. */
FORCE_NOINLINE
size_t ZSTD_execSequenceLast7(BYTE* op,
BYTE* const oend, seq_t sequence,
@ -1095,21 +1010,14 @@ size_t ZSTD_execSequenceLast7(BYTE* op,
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
/* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must fit within dstBuffer */
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* try to read beyond literal buffer */
/* copy literals */
if (op < oend_w) {
ZSTD_wildcopy(op, *litPtr, oend_w - op);
*litPtr += oend_w - op;
op = oend_w;
}
while (op < oLitEnd) *op++ = *(*litPtr)++;
/* copy Match */
@ -1946,8 +1854,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */
if (ddict) {
dict = ZSTD_DDictDictContent(ddict);
dictSize = ZSTD_DDictDictSize(ddict);
dict = ZSTD_DDict_dictContent(ddict);
dictSize = ZSTD_DDict_dictSize(ddict);
}
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
@ -2238,11 +2146,12 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
return 0;
}
/*! ZSTD_loadEntropy() :
/*! ZSTD_loadDEntropy() :
* dict : must point at beginning of a valid zstd dictionary.
* @return : size of entropy tables read */
static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy,
const void* const dict, size_t const dictSize)
size_t
ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
const void* const dict, size_t const dictSize)
{
const BYTE* dictPtr = (const BYTE*)dict;
const BYTE* const dictEnd = dictPtr + dictSize;
@ -2324,7 +2233,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
/* load entropy tables */
{ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
{ size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + eSize;
dictSize -= eSize;
@ -2368,209 +2277,25 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
/* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s {
void* dictBuffer;
const void* dictContent;
size_t dictSize;
ZSTD_entropyDTables_t entropy;
U32 dictID;
U32 entropyPresent;
ZSTD_customMem cMem;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictContent;
}
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictSize;
}
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");
assert(dctx != NULL);
if (ddict) {
dctx->ddictIsCold = (dctx->dictEnd != (const char*)ddict->dictContent + ddict->dictSize);
const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict);
size_t const dictSize = ZSTD_DDict_dictSize(ddict);
const void* const dictEnd = dictStart + dictSize;
dctx->ddictIsCold = (dctx->dictEnd != dictEnd);
DEBUGLOG(4, "DDict is %s",
dctx->ddictIsCold ? "~cold~" : "hot!");
}
CHECK_F( ZSTD_decompressBegin(dctx) );
if (ddict) { /* NULL ddict is equivalent to no dictionary */
dctx->dictID = ddict->dictID;
dctx->prefixStart = ddict->dictContent;
dctx->virtualStart = ddict->dictContent;
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
dctx->previousDstEnd = dctx->dictEnd;
if (ddict->entropyPresent) {
dctx->litEntropy = 1;
dctx->fseEntropy = 1;
dctx->LLTptr = ddict->entropy.LLTable;
dctx->MLTptr = ddict->entropy.MLTable;
dctx->OFTptr = ddict->entropy.OFTable;
dctx->HUFptr = ddict->entropy.hufTable;
dctx->entropy.rep[0] = ddict->entropy.rep[0];
dctx->entropy.rep[1] = ddict->entropy.rep[1];
dctx->entropy.rep[2] = ddict->entropy.rep[2];
} else {
dctx->litEntropy = 0;
dctx->fseEntropy = 0;
}
ZSTD_copyDDictParameters(dctx, ddict);
}
return 0;
}
static size_t
ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict,
ZSTD_dictContentType_e dictContentType)
{
ddict->dictID = 0;
ddict->entropyPresent = 0;
if (dictContentType == ZSTD_dct_rawContent) return 0;
if (ddict->dictSize < 8) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
{ U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_MAGIC_DICTIONARY) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
}
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
/* load entropy tables */
CHECK_E( ZSTD_loadEntropy(&ddict->entropy,
ddict->dictContent, ddict->dictSize),
dictionary_corrupted );
ddict->entropyPresent = 1;
return 0;
}
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL;
ddict->dictContent = dict;
if (!dict) dictSize = 0;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
if (!internalBuffer) return ERROR(memory_allocation);
memcpy(internalBuffer, dict, dictSize);
}
ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */
CHECK_F( ZSTD_loadEntropy_inDDict(ddict, dictContentType) );
return 0;
}
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_customMem customMem)
{
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (ddict == NULL) return NULL;
ddict->cMem = customMem;
{ size_t const initResult = ZSTD_initDDict_internal(ddict,
dict, dictSize,
dictLoadMethod, dictContentType);
if (ZSTD_isError(initResult)) {
ZSTD_freeDDict(ddict);
return NULL;
} }
return ddict;
}
}
/*! ZSTD_createDDict() :
* Create a digested dictionary, to start decompression without startup delay.
* `dict` content is copied inside DDict.
* Consequently, `dict` can be released after `ZSTD_DDict` creation */
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
}
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, to start decompression without startup delay.
* Dictionary content is simply referenced, it will be accessed during decompression.
* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
}
const ZSTD_DDict* ZSTD_initStaticDDict(
void* sBuffer, size_t sBufferSize,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
size_t const neededSpace = sizeof(ZSTD_DDict)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
assert(sBuffer != NULL);
assert(dict != NULL);
if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
if (sBufferSize < neededSpace) return NULL;
if (dictLoadMethod == ZSTD_dlm_byCopy) {
memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1;
}
if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
dict, dictSize,
ZSTD_dlm_byRef, dictContentType) ))
return NULL;
return ddict;
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->cMem;
ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem);
return 0;
}
}
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
{
return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
}
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
@ -2582,16 +2307,6 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
}
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
}
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompresse frame stored within `src`.
* If @return == 0, the dictID could not be decoded.
@ -2872,8 +2587,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) {
const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL;
size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0;
DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
/* legacy support is incompatible with static dctx */
if (zds->staticSize) return ERROR(memory_allocation);

123
lib/decompress/zstd_decompress_internal.h Normal file
View File

@ -0,0 +1,123 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* zstd_decompress_internal:
* objects and definitions shared within lib/decompress modules */
#ifndef ZSTD_DECOMPRESS_INTERNAL_H
#define ZSTD_DECOMPRESS_INTERNAL_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include "mem.h" /* BYTE, U16, U32 */
#include "zstd_internal.h" /* ZSTD_seqSymbol */
/*-*******************************************************
* Decompression types
*********************************************************/
typedef struct {
U16 nextState;
BYTE nbAdditionalBits;
BYTE nbBits;
U32 baseValue;
} ZSTD_seqSymbol;
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
typedef struct {
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyDTables_t;
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
typedef enum { zdss_init=0, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
struct ZSTD_DCtx_s
{
const ZSTD_seqSymbol* LLTptr;
const ZSTD_seqSymbol* MLTptr;
const ZSTD_seqSymbol* OFTptr;
const HUF_DTable* HUFptr;
ZSTD_entropyDTables_t entropy;
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
const void* previousDstEnd; /* detect continuity */
const void* prefixStart; /* start of current segment */
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
ZSTD_frameHeader fParams;
U64 decodedSize;
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
ZSTD_dStage stage;
U32 litEntropy;
U32 fseEntropy;
XXH64_state_t xxhState;
size_t headerSize;
ZSTD_format_e format;
const BYTE* litPtr;
ZSTD_customMem customMem;
size_t litSize;
size_t rleSize;
size_t staticSize;
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
/* dictionary */
ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
U32 dictID;
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
/* streaming */
ZSTD_dStreamStage streamStage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
size_t maxWindowSize;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t lhSize;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
U32 hostageByte;
int noForwardProgress;
/* workspace */
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
/*-*******************************************************
* Shared internal functions
*********************************************************/
/*! ZSTD_loadDEntropy() :
* dict : must point at beginning of a valid zstd dictionary.
* @return : size of entropy tables read */
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
const void* const dict, size_t const dictSize);
#endif /* ZSTD_DECOMPRESS_INTERNAL_H */

89
tests/libzstd_partial_builds.sh
View File

@ -1,36 +1,89 @@
#!/bin/sh -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ECHO=echo
RM="rm -f"
GREP="grep"
INTOVOID="/dev/null"
die() {
$ECHO "$@" 1>&2
exit 1
}
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
isPresent() {
$GREP $@ tmplog || die "$@" "should be present"
}
INTOVOID="/dev/null"
case "$OS" in
Windows*)
INTOVOID="NUL"
;;
esac
mustBeAbsent() {
$GREP $@ tmplog && die "$@ should not be there !!"
$ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??)
}
# default compilation : all features enabled
make clean > /dev/null
$ECHO "testing default library compilation"
CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# compression disabled => also disable zdict and zbuff
$ECHO "testing with compression disabled"
ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
! grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Compression macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
mustBeAbsent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# decompression disabled => also disable legacy and zbuff
$ECHO "testing with decompression disabled"
ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Decompression macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
mustBeAbsent "zstd_decompress.o"
isPresent "zdict.o"
mustBeAbsent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# deprecated function disabled => only remove zbuff
$ECHO "testing with deprecated functions disabled"
ZSTD_LIB_DEPRECATED=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Deprecated macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# dictionary builder disabled => only remove zdict
$ECHO "testing with dictionary builder disabled"
ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Dictbuilder macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# both decompression and dictionary builder disabled => only compression remains
$ECHO "testing with both decompression and dictionary builder disabled (only compression remains)"
ZSTD_LIB_DECOMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Multi-macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
mustBeAbsent "zstd_decompress.o"
mustBeAbsent "zdict.o"
mustBeAbsent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog

12
tests/poolTests.c
View File

@ -30,7 +30,7 @@ struct data {
size_t i;
};
void fn(void *opaque) {
static void fn(void *opaque) {
struct data *data = (struct data *)opaque;
ZSTD_pthread_mutex_lock(&data->mutex);
data->data[data->i] = data->i;
@ -38,7 +38,7 @@ void fn(void *opaque) {
ZSTD_pthread_mutex_unlock(&data->mutex);
}
int testOrder(size_t numThreads, size_t queueSize) {
static int testOrder(size_t numThreads, size_t queueSize) {
struct data data;
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
@ -63,13 +63,13 @@ int testOrder(size_t numThreads, size_t queueSize) {
/* --- test deadlocks --- */
void waitFn(void *opaque) {
static void waitFn(void *opaque) {
(void)opaque;
UTIL_sleepMilli(1);
}
/* Tests for deadlock */
int testWait(size_t numThreads, size_t queueSize) {
static int testWait(size_t numThreads, size_t queueSize) {
struct data data;
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
@ -92,7 +92,7 @@ typedef struct {
ZSTD_pthread_cond_t cond;
} poolTest_t;
void waitLongFn(void *opaque) {
static void waitLongFn(void *opaque) {
poolTest_t* test = (poolTest_t*) opaque;
UTIL_sleepMilli(10);
ZSTD_pthread_mutex_lock(&test->mut);
@ -167,7 +167,7 @@ typedef struct {
int val;
} abruptEndCanary_t;
void waitIncFn(void *opaque) {
static void waitIncFn(void *opaque) {
abruptEndCanary_t* test = (abruptEndCanary_t*) opaque;
UTIL_sleepMilli(10);
ZSTD_pthread_mutex_lock(&test->mut);