AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-12-21 17:02:06 +01:00
commit 0d1ee9d1bb
24 changed files with 399 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -30,6 +30,8 @@ all:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ zstd32
$(MAKE) -C $(TESTDIR) $@ all32
$(MAKE) -C $(ZWRAPDIR) $@
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@
.PHONY: lib
lib:
@ -54,6 +56,7 @@ clean:
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
@$(MAKE) -C examples/ $@ > $(VOID)
@$(RM) zstd$(EXT) tmp*
@echo Cleaning completed

1
NEWS
View File

@ -1,6 +1,7 @@
v1.1.3
cli : new : commands for advanced parameters, by Przemyslaw Skibinski
API : fix : all symbols properly exposed in libzstd, by Nick Terrell
API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
v1.1.2
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init

9
appveyor.yml
View File

@ -2,10 +2,10 @@ version: 1.0.{build}
environment:
matrix:
- COMPILER: "gcc"
MAKE_PARAMS: "test"
MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
PLATFORM: "mingw64"
- COMPILER: "gcc"
MAKE_PARAMS: "test"
MAKE_PARAMS: "make test"
PLATFORM: "mingw32"
- COMPILER: "visual"
CONFIGURATION: "Debug"
@ -69,9 +69,8 @@ build_script:
ECHO *** &&
make -v &&
cc -v &&
ECHO make %MAKE_PARAMS% &&
make %MAKE_PARAMS% &&
make -C tests test-symbols
ECHO %MAKE_PARAMS% &&
sh -c %MAKE_PARAMS%
)
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
COPY programs\zstd.exe bin\zstd.exe &&

2
build/VS2010/fullbench-dll/fullbench-dll.vcxproj
View File

@ -99,6 +99,7 @@
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@ -138,6 +139,7 @@
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

2
build/cmake/lib/CMakeLists.txt
View File

@ -115,7 +115,7 @@ IF (MSVC)
ENDIF (MSVC)
# Split project to static and shared libraries build
ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources})
ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers})
ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
# Add specific compile definitions for MSVC project

51
lib/compress/zstd_compress.c
View File

@ -2557,9 +2557,9 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] == 0 || cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] == 0 || cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] == 0 || cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
{ U32 offcodeMax = MaxOff;
@ -2733,7 +2733,8 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
/* ===== Dictionary API ===== */
struct ZSTD_CDict_s {
void* dictContent;
void* dictBuffer;
const void* dictContent;
size_t dictContentSize;
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
@ -2744,36 +2745,42 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize;
}
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
if (!dictContent || !cdict || !cctx) {
ZSTD_free(dictContent, customMem);
if (!cdict || !cctx) {
ZSTD_free(cdict, customMem);
ZSTD_free(cctx, customMem);
return NULL;
}
if (dictSize) {
memcpy(dictContent, dict, dictSize);
if ((byReference) || (!dictBuffer) || (!dictSize)) {
cdict->dictBuffer = NULL;
cdict->dictContent = dictBuffer;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; }
memcpy(internalBuffer, dictBuffer, dictSize);
cdict->dictBuffer = internalBuffer;
cdict->dictContent = internalBuffer;
}
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
ZSTD_free(cdict, customMem);
ZSTD_free(cdict->dictBuffer, customMem);
ZSTD_free(cctx, customMem);
ZSTD_free(cdict, customMem);
return NULL;
} }
cdict->dictContent = dictContent;
cdict->dictContentSize = dictSize;
cdict->refContext = cctx;
cdict->dictContentSize = dictSize;
return cdict;
}
}
@ -2783,7 +2790,15 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator);
}
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator);
}
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
@ -2791,7 +2806,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext);
ZSTD_free(cdict->dictContent, cMem);
ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem);
return 0;
}
@ -2939,7 +2954,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (dict) {
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal;
} else zcs->cdict = NULL;

41
lib/decompress/zstd_decompress.c
View File

@ -1671,9 +1671,9 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
dctx->litEntropy = dctx->fseEntropy = 1;
@ -1713,39 +1713,44 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
/* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s {
void* dict;
void* dictBuffer;
const void* dictContent;
size_t dictSize;
ZSTD_DCtx* refContext;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
if (!dictContent || !ddict || !dctx) {
ZSTD_free(dictContent, customMem);
if (!ddict || !dctx) {
ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem);
return NULL;
}
if (dictSize) {
memcpy(dictContent, dict, dictSize);
if ((byReference) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL;
ddict->dictContent = dict;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_free(dctx, customMem); ZSTD_free(ddict, customMem); return NULL; }
memcpy(internalBuffer, dict, dictSize);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
}
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
ZSTD_free(ddict->dictBuffer, customMem);
ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem);
return NULL;
} }
ddict->dict = dictContent;
ddict->dictSize = dictSize;
ddict->refContext = dctx;
return ddict;
@ -1758,7 +1763,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, allocator);
return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
@ -1766,7 +1771,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->refContext->customMem;
ZSTD_freeDCtx(ddict->refContext);
ZSTD_free(ddict->dict, cMem);
ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem);
return 0;
}
@ -1796,7 +1801,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize);
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
}
/*! ZSTD_getDictID_fromFrame() :
@ -1827,7 +1832,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict)
{
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize);
#endif
ZSTD_refDCtx(dctx, ddict->refContext);
ZSTD_checkContinuity(dctx, dst);
@ -2007,7 +2012,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) {
const void* const dict = zds->ddict ? zds->ddict->dict : NULL;
const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
dict, dictSize));

2
lib/dll/example/fullbench-dll.vcxproj
View File

@ -100,6 +100,7 @@
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@ -141,6 +142,7 @@
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

6
lib/legacy/zstd_v07.c
View File

@ -4134,9 +4134,9 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
dctx->litEntropy = dctx->fseEntropy = 1;

27
lib/zstd.h
View File

@ -173,8 +173,8 @@ typedef struct ZSTD_CDict_s ZSTD_CDict;
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
* `dict` can be released after ZSTD_CDict creation. */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
* `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_freeCDict() :
* Function frees memory allocated by ZSTD_createCDict(). */
@ -194,8 +194,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation. */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
* dictBuffer can be released after DDict creation, as its content is copied inside DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
/*! ZSTD_freeDDict() :
* Function frees memory allocated with ZSTD_createDDict() */
@ -328,7 +328,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
* ***************************************************************************************/
/* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX_32 25
@ -400,9 +400,15 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
* Gives the amount of memory used by a given ZSTD_CCtx */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
/*! ZSTD_createCDict_byReference() :
* Create a digested dictionary for compression
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem);
/*! ZSTD_sizeof_CDict() :
@ -458,6 +464,15 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
* Gives the amount of memory used by a given ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
/*! ZSTD_sizeof_DDict() :
* Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);

18
programs/bench.c
View File

@ -8,10 +8,20 @@
*/
/* **************************************
* Compiler Warnings
****************************************/
#ifdef _MSC_VER
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/* *************************************
* Includes
***************************************/
#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
#include "platform.h" /* Large Files support */
#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
@ -236,7 +246,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_customMem const cmem = { NULL, NULL, NULL };
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0;
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
@ -452,7 +462,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams)
{
void* srcBuffer;
@ -523,7 +533,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams)
{
double const compressibility = (double)g_compressibilityDefault / 100;

21
programs/datagen.c
View File

@ -9,35 +9,16 @@
/* *************************************
* Compiler Options
***************************************/
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
# define _CRT_SECURE_NO_DEPRECATE /* removes VS2005 warning on strerror() */
#endif
/*-************************************
* Dependencies
**************************************/
#include "platform.h" /* SET_BINARY_MODE */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE, fwrite, fprintf */
#include <string.h> /* memcpy */
#include "mem.h" /* U32 */
/*-************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# define SET_BINARY_MODE(file) {int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
#else
# define SET_BINARY_MODE(file)
#endif
/*-************************************
* Macros
**************************************/

11
programs/dibio.c
View File

@ -9,10 +9,19 @@
/* **************************************
* Compiler Warnings
****************************************/
#ifdef _MSC_VER
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/*-*************************************
* Includes
***************************************/
#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_getTotalFileSize */
#include "platform.h" /* Large Files support */
#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */

18
programs/fileio.c
View File

@ -11,7 +11,7 @@
* Compiler Options
***************************************/
#ifdef _MSC_VER /* Visual */
# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4204) /* non-constant aggregate initializer */
#endif
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
@ -22,7 +22,8 @@
/*-*************************************
* Includes
***************************************/
#include "util.h" /* Compiler options, UTIL_GetFileSize, _LARGEFILE64_SOURCE */
#include "platform.h" /* Large Files support, SET_BINARY_MODE */
#include "util.h" /* UTIL_getFileSize */
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
@ -41,19 +42,6 @@
#endif
/*-*************************************
* OS-specific Includes
***************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); }
#else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
#endif
/*-*************************************
* Constants
***************************************/

135
programs/platform.h Normal file
View File

@ -0,0 +1,135 @@
/**
* platform.h - compiler and OS detection
*
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#ifndef PLATFORM_H_MODULE
#define PLATFORM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* **************************************
* Compiler Options
****************************************/
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# endif
#endif
/* **************************************
* Detect 64-bit OS
* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
****************************************/
#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \
|| defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \
|| (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \
|| defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \
|| defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \
|| (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \
|| defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \
|| (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */
# if !defined(__64BIT__)
# define __64BIT__ 1
# endif
#endif
/* *********************************************************
* Turn on Large Files support (>4GB) for 32-bit Linux/Unix
***********************************************************/
#if !defined(__64BIT__) /* No point defining Large file for 64 bit */
# if !defined(_FILE_OFFSET_BITS)
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
# endif
# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */
# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */
# endif
# if defined(_AIX) || defined(__hpux)
# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */
# endif
#endif
/* ************************************************************
* Detect POSIX version
* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
***************************************************************/
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|| defined(__midipix__) || defined(__VMS))
# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */
# define PLATFORM_POSIX_VERSION 200112L
# else
# if defined(__linux__) || defined(__linux)
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
# endif
# include <unistd.h> /* declares _POSIX_VERSION */
# if defined(_POSIX_VERSION) /* POSIX compliant */
# define PLATFORM_POSIX_VERSION _POSIX_VERSION
# else
# define PLATFORM_POSIX_VERSION 0
# endif
# endif
#endif
#if !defined(PLATFORM_POSIX_VERSION)
# define PLATFORM_POSIX_VERSION -1
#endif
/*-*********************************************
* Detect if isatty() and fileno() are available
************************************************/
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
# define IS_CONSOLE(stdStream) 0
#endif
/******************************
* OS-specific Includes
******************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
# if !defined(__DJGPP__)
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <winioctl.h> /* FSCTL_SET_SPARSE */
# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
# else
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
# define SET_SPARSE_FILE_MODE(file)
# endif
#else
# define SET_BINARY_MODE(file)
# define SET_SPARSE_FILE_MODE(file)
#endif
#if defined (__cplusplus)
}
#endif
#endif /* PLATFORM_H_MODULE */

116
programs/util.h
View File

@ -1,4 +1,6 @@
/**
* util.h - utility functions
*
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
@ -14,50 +16,57 @@
extern "C" {
#endif
/* **************************************
* Compiler Options
****************************************/
#if defined(__INTEL_COMPILER)
# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */
#endif
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
#endif
#endif
/* Unix Large Files support (>4GB) */
#if !defined(__LP64__) /* No point defining Large file for 64 bit */
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */
# define _LARGEFILE_SOURCE /* fseeko, ftello */
# elif !defined(_LARGEFILE64_SOURCE)
# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */
# endif
#endif
/*-****************************************
* Dependencies
******************************************/
#include <stdlib.h> /* features.h with _POSIX_C_SOURCE, malloc */
#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#include "platform.h" /* PLATFORM_POSIX_VERSION */
#include <stdlib.h> /* malloc */
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#if defined(_MSC_VER)
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
#else
# include <unistd.h> /* chown, stat */
# include <utime.h> /* utime */
#endif
#include <time.h> /* time */
#include <time.h> /* time */
#include <errno.h>
#include "mem.h" /* U32, U64 */
#include "mem.h" /* U32, U64 */
/*-****************************************
* Sleep functions: Windows - Posix - others
******************************************/
#if defined(_WIN32)
# include <windows.h>
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define UTIL_sleep(s) Sleep(1000*s)
# define UTIL_sleepMilli(milli) Sleep(milli)
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
# include <unistd.h>
# include <sys/resource.h> /* setpriority */
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
# if defined(PRIO_PROCESS)
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# else
# define SET_HIGH_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
# else
# define UTIL_sleepMilli(milli) /* disabled */
# endif
#else
# define SET_HIGH_PRIORITY /* disabled */
# define UTIL_sleep(s) /* disabled */
# define UTIL_sleepMilli(milli) /* disabled */
#endif
/* *************************************
@ -69,6 +78,9 @@ extern "C" {
/*-****************************************
* Compiler specifics
******************************************/
#if defined(__INTEL_COMPILER)
# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */
#endif
#if defined(__GNUC__)
# define UTIL_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@ -80,36 +92,6 @@ extern "C" {
#endif
/*-****************************************
* Sleep functions: Windows - Posix - others
******************************************/
#if defined(_WIN32)
# include <windows.h>
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define UTIL_sleep(s) Sleep(1000*s)
# define UTIL_sleepMilli(milli) Sleep(milli)
#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__)))
# include <unistd.h>
# include <sys/resource.h> /* setpriority */
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
# if defined(PRIO_PROCESS)
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# else
# define SET_HIGH_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L))
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
# else
# define UTIL_sleepMilli(milli) /* disabled */
# endif
#else
# define SET_HIGH_PRIORITY /* disabled */
# define UTIL_sleep(s) /* disabled */
# define UTIL_sleepMilli(milli) /* disabled */
#endif
/*-****************************************
* Time functions
******************************************/
@ -325,12 +307,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
return nbFiles;
}
#elif (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) || \
(defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) || \
defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
(defined(__linux__) && defined(_POSIX_C_SOURCE)) /* opendir */
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# define UTIL_HAS_CREATEFILELIST
# include <dirent.h> /* opendir, readdir */
# include <string.h> /* strerror, memcpy */
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
{
@ -392,7 +372,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
{
(void)bufStart; (void)bufEnd; (void)pos;
fprintf(stderr, "Directory %s ignored (zstd compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
return 0;
}

BIN
programs/windres/zstd32.res
View File

Binary file not shown.

BIN
programs/windres/zstd64.res
View File

Binary file not shown.

27
programs/zstdcli.c
View File

@ -23,7 +23,8 @@
/*-************************************
* Dependencies
**************************************/
#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
#include <string.h> /* strcmp, strlen */
#include <errno.h> /* errno */
#include "fileio.h"
@ -37,21 +38,6 @@
#include "zstd.h" /* ZSTD_VERSION_STRING */
/*-************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#else
# define IS_CONSOLE(stdStream) 0
#endif
/*-************************************
* Constants
**************************************/
@ -499,6 +485,15 @@ int main(int argCount, const char* argv[])
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef _POSIX_C_SOURCE
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
#endif
#ifdef _POSIX_VERSION
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
#endif
#ifdef PLATFORM_POSIX_VERSION
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
#endif
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */

1
tests/.gitignore vendored
View File

@ -15,6 +15,7 @@ paramgrill32
roundTripCrash
longmatch
symbols
invalidDictionaries
# Tmp test directory
zstdtest

10
tests/Makefile
View File

@ -146,6 +146,9 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
longmatch : $(ZSTD_FILES) longmatch.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
invalidDictionaries : $(ZSTD_FILES) invalidDictionaries.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
symbols : symbols.c
$(MAKE) -C $(ZSTDDIR) libzstd
ifneq (,$(filter Windows%,$(OS)))
@ -173,7 +176,7 @@ clean:
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
zstreamtest$(EXT) zstreamtest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
symbols$(EXT)
symbols$(EXT) invalidDictionaries$(EXT)
@echo Cleaning completed
@ -213,7 +216,7 @@ zstd-playTests: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch
test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
@ -273,6 +276,9 @@ test-zstream32: zstreamtest32
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
test-invalidDictionaries: invalidDictionaries
$(QEMU_SYS) ./invalidDictionaries
test-symbols: symbols
$(QEMU_SYS) ./symbols

23
tests/fullbench.c
View File

@ -179,6 +179,23 @@ size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, c
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
}
#define FIRST_BLOCK_SIZE 8
size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
(void)buff2;
memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
ZSTD_compressBegin(g_zcc, 1);
{ size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE);
if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; }
dst = (BYTE*)dst + compressResult;
dstCapacity -= compressResult;
}
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE);
}
size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
size_t regeneratedSize = 0;
@ -229,6 +246,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
benchFunction = local_ZSTD_compressContinue; benchName = "ZSTD_compressContinue";
break;
case 12:
benchFunction = local_ZSTD_compressContinue_extDict; benchName = "ZSTD_compressContinue_extDict";
break;
case 13:
benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue";
break;
case 31:
@ -268,6 +288,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
break;
case 12 :
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
break;
case 13 :
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;

51
tests/invalidDictionaries.c Normal file
View File

@ -0,0 +1,51 @@
#include <stddef.h>
#include "zstd.h"
static const char invalidRepCode[] = {
0x37, 0xa4, 0x30, 0xec, 0x2a, 0x00, 0x00, 0x00, 0x39, 0x10, 0xc0, 0xc2,
0xa6, 0x00, 0x0c, 0x30, 0xc0, 0x00, 0x03, 0x0c, 0x30, 0x20, 0x72, 0xf8,
0xb4, 0x6d, 0x4b, 0x9f, 0xfc, 0x97, 0x29, 0x49, 0xb2, 0xdf, 0x4b, 0x29,
0x7d, 0x4a, 0xfc, 0x83, 0x18, 0x22, 0x75, 0x23, 0x24, 0x44, 0x4d, 0x02,
0xb7, 0x97, 0x96, 0xf6, 0xcb, 0xd1, 0xcf, 0xe8, 0x22, 0xea, 0x27, 0x36,
0xb7, 0x2c, 0x40, 0x46, 0x01, 0x08, 0x23, 0x01, 0x00, 0x00, 0x06, 0x1e,
0x3c, 0x83, 0x81, 0xd6, 0x18, 0xd4, 0x12, 0x3a, 0x04, 0x00, 0x80, 0x03,
0x08, 0x0e, 0x12, 0x1c, 0x12, 0x11, 0x0d, 0x0e, 0x0a, 0x0b, 0x0a, 0x09,
0x10, 0x0c, 0x09, 0x05, 0x04, 0x03, 0x06, 0x06, 0x06, 0x02, 0x00, 0x03,
0x00, 0x00, 0x02, 0x02, 0x00, 0x04, 0x06, 0x03, 0x06, 0x08, 0x24, 0x6b,
0x0d, 0x01, 0x10, 0x04, 0x81, 0x07, 0x00, 0x00, 0x04, 0xb9, 0x58, 0x18,
0x06, 0x59, 0x92, 0x43, 0xce, 0x28, 0xa5, 0x08, 0x88, 0xc0, 0x80, 0x88,
0x8c, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x08, 0x00, 0x00, 0x00
};
typedef struct dictionary_s {
const char *data;
size_t size;
} dictionary;
static const dictionary dictionaries[] = {
{invalidRepCode, sizeof(invalidRepCode)},
{NULL, 0},
};
int main(int argc, const char** argv) {
const dictionary *dict;
for (dict = dictionaries; dict->data != NULL; ++dict) {
ZSTD_CDict *cdict;
ZSTD_DDict *ddict;
cdict = ZSTD_createCDict(dict->data, dict->size, 1);
if (cdict) {
ZSTD_freeCDict(cdict);
return 1;
}
ddict = ZSTD_createDDict(dict->data, dict->size);
if (ddict) {
ZSTD_freeDDict(ddict);
return 2;
}
}
(void)argc;
(void)argv;
return 0;
}

2
zlibWrapper/examples/zwrapbench.c
View File

@ -234,7 +234,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
if (compressor == BMK_ZSTD) {
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
do {