diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bfe02194..40b6625f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -217,7 +217,7 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned #define ZSTD_CLEVEL_CUSTOM 999 -static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* params, size_t srcSize) +static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* params, U64 srcSize) { if (params->compressionLevel == ZSTD_CLEVEL_CUSTOM) return; params->cParams = ZSTD_getCParams(params->compressionLevel, srcSize, 0); diff --git a/tests/Makefile b/tests/Makefile index 228f4cdd..3734f773 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -169,11 +169,6 @@ datagen : $(PRGDIR)/datagen.c datagencli.c roundTripCrash : $(ZSTD_FILES) roundTripCrash.c $(CC) $(FLAGS) $^ -o $@$(EXT) -cctxParamRoundTrip : LDFLAGS += $(MULTITHREAD_CPP) -cctxParamRoundTrip : LDFLAGS += $(MULTITHREAD_LD) -cctxParamRoundTrip : $(ZSTD_FILES) cctxParamRoundTrip.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - longmatch : $(ZSTD_FILES) longmatch.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -217,7 +212,7 @@ clean: fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\ zstreamtest$(EXT) zstreamtest32$(EXT) \ - datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) cctxParamRoundTrip$(EXT) longmatch$(EXT) \ + datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \ decodecorpus$(EXT) @echo Cleaning completed diff --git a/tests/cctxParamRoundTrip.c b/tests/cctxParamRoundTrip.c deleted file mode 100644 index 5ce47d4d..00000000 --- a/tests/cctxParamRoundTrip.c +++ /dev/null @@ -1,205 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -/* - This program takes a file in input, - performs a zstd round-trip test (compression - decompress) - compares the result with original - and generates a crash (double free) on corruption detection. -*/ - -/*=========================================== -* Dependencies -*==========================================*/ -#include /* size_t */ -#include /* malloc, free, exit */ -#include /* fprintf */ -#include /* stat */ -#include /* stat */ -#include "xxhash.h" - -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" - -/*=========================================== -* Macros -*==========================================*/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static void crash(int errorCode){ - /* abort if AFL/libfuzzer, exit otherwise */ - #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ - abort(); - #else - exit(errorCode); - #endif -} - -#define CHECK_Z(f) { \ - size_t const err = f; \ - if (ZSTD_isError(err)) { \ - fprintf(stderr, \ - "Error=> %s: %s", \ - #f, ZSTD_getErrorName(err)); \ - crash(1); \ -} } - -/** roundTripTest() : -* Compresses `srcBuff` into `compressedBuff`, -* then decompresses `compressedBuff` into `resultBuff`. -* -* Parameters are currently set manually. -* -* @return : result of decompression, which should be == `srcSize` -* or an error code if either compression or decompression fails. -* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)` -* for compression to be guaranteed to work */ -static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, - void* compressedBuff, size_t compressedBuffCapacity, - const void* srcBuff, size_t srcBuffSize) -{ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); - ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 }; - ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 }; - - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, 1) ); - CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbThreads, 3) ); - CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionStrategy, ZSTD_lazy) ); - CHECK_Z( ZSTD_CCtx_applyCCtxParams(cctx, cctxParams) ); - - CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); - - ZSTD_freeCCtxParams(cctxParams); - ZSTD_freeCCtx(cctx); - - return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos); -} - - -static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) -{ - const char* ip1 = (const char*)buff1; - const char* ip2 = (const char*)buff2; - size_t pos; - - for (pos=0; pos= `fileSize` */ -static void loadFile(void* buffer, const char* fileName, size_t fileSize) -{ - FILE* const f = fopen(fileName, "rb"); - if (isDirectory(fileName)) { - fprintf(stderr, "Ignoring %s directory \n", fileName); - exit(2); - } - if (f==NULL) { - fprintf(stderr, "Impossible to open %s \n", fileName); - exit(3); - } - { size_t const readSize = fread(buffer, 1, fileSize, f); - if (readSize != fileSize) { - fprintf(stderr, "Error reading %s \n", fileName); - exit(5); - } } - fclose(f); -} - - -static void fileCheck(const char* fileName) -{ - size_t const fileSize = getFileSize(fileName); - void* buffer = malloc(fileSize); - if (!buffer) { - fprintf(stderr, "not enough memory \n"); - exit(4); - } - loadFile(buffer, fileName, fileSize); - roundTripCheck(buffer, fileSize); - free (buffer); -} - -int main(int argCount, const char** argv) { - if (argCount < 2) { - fprintf(stderr, "Error : no argument : need input file \n"); - exit(9); - } - fileCheck(argv[1]); - fprintf(stderr, "no pb detected\n"); - return 0; -} diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c index 77c6737e..fb14fa87 100644 --- a/tests/roundTripCrash.c +++ b/tests/roundTripCrash.c @@ -20,9 +20,12 @@ #include /* size_t */ #include /* malloc, free, exit */ #include /* fprintf */ +#include /* strcmp */ #include /* stat */ #include /* stat */ #include "xxhash.h" + +#define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" /*=========================================== @@ -30,6 +33,24 @@ *==========================================*/ #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +static void crash(int errorCode){ + /* abort if AFL/libfuzzer, exit otherwise */ + #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ + abort(); + #else + exit(errorCode); + #endif +} + +#define CHECK_Z(f) { \ + size_t const err = f; \ + if (ZSTD_isError(err)) { \ + fprintf(stderr, \ + "Error=> %s: %s", \ + #f, ZSTD_getErrorName(err)); \ + crash(1); \ +} } + /** roundTripTest() : * Compresses `srcBuff` into `compressedBuff`, * then decompresses `compressedBuff` into `resultBuff`. @@ -54,6 +75,35 @@ static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize); } +/** cctxParamRoundTripTest() : + * Same as roundTripTest() except allows experimenting with ZSTD_CCtx_params. */ +static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcBuffSize) +{ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); + ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 }; + ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 }; + + static const int maxClevel = 19; + size_t const hashLength = MIN(128, srcBuffSize); + unsigned const h32 = XXH32(srcBuff, hashLength, 0); + int const cLevel = h32 % maxClevel; + + /* Set parameters */ + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, cLevel) ); + + /* Apply parameters */ + CHECK_Z( ZSTD_CCtx_applyCCtxParams(cctx, cctxParams) ); + + CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); + + ZSTD_freeCCtxParams(cctxParams); + ZSTD_freeCCtx(cctx); + + return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos); +} static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) { @@ -68,16 +118,7 @@ static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize return pos; } -static void crash(int errorCode){ - /* abort if AFL/libfuzzer, exit otherwise */ - #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ - abort(); - #else - exit(errorCode); - #endif -} - -static void roundTripCheck(const void* srcBuff, size_t srcBuffSize) +static void roundTripCheck(const void* srcBuff, size_t srcBuffSize, int testCCtxParams) { size_t const cBuffSize = ZSTD_compressBound(srcBuffSize); void* cBuff = malloc(cBuffSize); @@ -88,7 +129,9 @@ static void roundTripCheck(const void* srcBuff, size_t srcBuffSize) exit (1); } - { size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize); + { size_t const result = testCCtxParams ? + cctxParamRoundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize) + : roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize); if (ZSTD_isError(result)) { fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result)); crash(1); @@ -162,7 +205,7 @@ static void loadFile(void* buffer, const char* fileName, size_t fileSize) } -static void fileCheck(const char* fileName) +static void fileCheck(const char* fileName, int testCCtxParams) { size_t const fileSize = getFileSize(fileName); void* buffer = malloc(fileSize); @@ -171,16 +214,24 @@ static void fileCheck(const char* fileName) exit(4); } loadFile(buffer, fileName, fileSize); - roundTripCheck(buffer, fileSize); + roundTripCheck(buffer, fileSize, testCCtxParams); free (buffer); } int main(int argCount, const char** argv) { + int argNb = 1; + int testCCtxParams = 0; if (argCount < 2) { fprintf(stderr, "Error : no argument : need input file \n"); exit(9); } - fileCheck(argv[1]); + + if (!strcmp(argv[argNb], "--cctxParams")) { + testCCtxParams = 1; + argNb++; + } + + fileCheck(argv[argNb], testCCtxParams); fprintf(stderr, "no pb detected\n"); return 0; }