AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into m32

Browse Source
This commit is contained in:
Sean Purcell 2017-03-02 15:17:56 -08:00
commit 3d95925a59
26 changed files with 210 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.travis.yml
View File

@ -4,7 +4,7 @@ dist: trusty
matrix:
fast_finish: true
include:
# other feature branches => short tests
# Ubuntu 14.04
- env: Cmd="make libc6install && make -C tests test32"
- env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
@ -25,10 +25,11 @@ script:
# cron & master => full tests, as this is the final step towards a Release
# pull requests => normal tests (job numbers 1-3)
# other feature branches => short tests (job numbers 1-2)
- echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
- if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
else
if [ "$TRAVIS_PULL_REQUEST" == "true" ] && [ $JOB_NUMBER -lt 4 ]; then
if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ] && [ $JOB_NUMBER -lt 4 ]; then
sh -c "$Cmd" || travis_terminate 1;
else
if [ $JOB_NUMBER -lt 3 ]; then

6
Makefile
View File

@ -63,6 +63,10 @@ zstdmt:
zlibwrapper:
$(MAKE) -C $(ZWRAPDIR) test
.PHONY: shortest
shortest:
$(MAKE) -C $(TESTDIR) $@
.PHONY: test
test:
$(MAKE) -C $(TESTDIR) $@
@ -173,7 +177,7 @@ ppcinstall:
arminstall:
APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
valgrindinstall:
valgrindinstall:
APT_PACKAGES="valgrind" $(MAKE) apt-install
libc6install:

6
NEWS
View File

@ -3,9 +3,11 @@ cli : new : can compress in *.gz format, using --format=gzip command, by Przemys
cli : new : advanced benchmark command --priority=rt
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
cli : fix : --rm remains silent when input is stdin
speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
build:new : meson build system in contrib/meson, by Dima Krasner
build: new: meson build system in contrib/meson, by Dima Krasner
build: improved cmake script, by @Majlen
doc : new : educational decoder, by Sean Purcell
v1.1.3

5
build/cmake/CMakeLists.txt
View File

@ -11,6 +11,8 @@ PROJECT(zstd)
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
IF (ZSTD_LEGACY_SUPPORT)
MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
@ -23,6 +25,9 @@ ENDIF (ZSTD_LEGACY_SUPPORT)
ADD_SUBDIRECTORY(lib)
ADD_SUBDIRECTORY(programs)
ADD_SUBDIRECTORY(tests)
IF (ZSTD_BUILD_CONTRIB)
ADD_SUBDIRECTORY(contrib)
ENDIF (ZSTD_BUILD_CONTRIB)
#-----------------------------------------------------------------------------
# Add extra compilation flags

2
build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
View File

@ -132,10 +132,8 @@ MACRO(ADD_EXTRA_COMPILATION_FLAGS)
endif (ACTIVATE_WARNING_CAST_ALIGN)
if (ACTIVATE_WARNING_STRICT_PROTOTYPES)
list(APPEND CMAKE_CXX_FLAGS ${WARNING_STRICT_PROTOTYPES})
list(APPEND CMAKE_C_FLAGS ${WARNING_STRICT_PROTOTYPES})
else ()
string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
endif (ACTIVATE_WARNING_STRICT_PROTOTYPES)

16
build/cmake/contrib/CMakeLists.txt Normal file
View File

@ -0,0 +1,16 @@
# ################################################################
# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
# * All rights reserved.
# *
# * This source code is licensed under the BSD-style license found in the
# * LICENSE file in the root directory of this source tree. An additional grant
# * of patent rights can be found in the PATENTS file in the same directory.
#
# You can contact the author at :
# - zstd homepage : http://www.zstd.net/
# ################################################################
PROJECT(contrib)
ADD_SUBDIRECTORY(pzstd)

30
build/cmake/contrib/pzstd/CMakeLists.txt Normal file
View File

@ -0,0 +1,30 @@
# ################################################################
# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
# * All rights reserved.
# *
# * This source code is licensed under the BSD-style license found in the
# * LICENSE file in the root directory of this source tree. An additional grant
# * of patent rights can be found in the PATENTS file in the same directory.
#
# You can contact the author at :
# - zstd homepage : http://www.zstd.net/
# ################################################################
PROJECT(pzstd)
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
# Define project root directory
SET(ROOT_DIR ../../../..)
# Define programs directory, where sources and header files are located
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
SET(PZSTD_DIR ${ROOT_DIR}/contrib/pzstd)
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread)
SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG")
SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow")

22
build/cmake/lib/CMakeLists.txt
View File

@ -166,23 +166,17 @@ IF (UNIX)
SET(SHARED_LIBRARY_SYMLINK1_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK1})
SET(SHARED_LIBRARY_SYMLINK2_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK2})
if (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
FILE(REMOVE ${SHARED_LIBRARY_SYMLINK1_PATH})
endif (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
if (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
FILE(REMOVE ${SHARED_LIBRARY_SYMLINK2_PATH})
endif (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
DEPENDS ${SHARED_LIBRARY_LINK_PATH}
COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK1}")
ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
DEPENDS ${SHARED_LIBRARY_LINK_PATH}
COMMENT "Generating symbolic link")
COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
DEPENDS ${SHARED_LIBRARY_LINK_PATH}
COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK2}")
ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
DEPENDS ${SHARED_LIBRARY_LINK_PATH}
COMMENT "Generating symbolic link")
SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${SHARED_LIBRARY_SYMLINK1};${SHARED_LIBRARY_SYMLINK2}")
INSTALL(FILES ${SHARED_LIBRARY_SYMLINK1_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})
INSTALL(FILES ${SHARED_LIBRARY_SYMLINK2_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})

10
build/cmake/programs/CMakeLists.txt
View File

@ -34,9 +34,17 @@ ENDIF (MSVC)
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
TARGET_LINK_LIBRARIES(zstd libzstd_static)
IF (UNIX)
ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
ENDIF (UNIX)
IF (ZSTD_MULTITHREAD_SUPPORT)
ADD_EXECUTABLE(zstdmt ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
SET_TARGET_PROPERTIES(zstdmt PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
TARGET_LINK_LIBRARIES(zstdmt libzstd_static)
IF (UNIX)
TARGET_LINK_LIBRARIES(zstdmt pthread)
ENDIF (UNIX)
ENDIF (ZSTD_MULTITHREAD_SUPPORT)

4
lib/common/xxhash.c
View File

@ -104,7 +104,9 @@ static void XXH_free (void* p) { free(p); }
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
#define XXH_STATIC_LINKING_ONLY
#ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY
#endif
#include "xxhash.h"

26
lib/common/xxhash.h
View File

@ -64,16 +64,12 @@ XXH64 13.8 GB/s 1.9 GB/s
XXH32 6.8 GB/s 6.0 GB/s
*/
#ifndef XXHASH_H_5627135585666179
#define XXHASH_H_5627135585666179 1
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef XXH_NAMESPACE
# define XXH_NAMESPACE ZSTD_ /* Zstandard specific */
#endif
#ifndef XXHASH_H_5627135585666179
#define XXHASH_H_5627135585666179 1
/* ****************************
@ -242,6 +238,11 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH
/* **************************
* Canonical representation
****************************/
/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
*/
typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
@ -251,14 +252,9 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
*/
#endif /* XXHASH_H_5627135585666179 */
#ifdef XXH_STATIC_LINKING_ONLY
/* ================================================================================================
This section contains definitions which are not guaranteed to remain stable.
@ -266,6 +262,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
They shall only be used with static linking.
Never use these definitions in association with dynamic linking !
=================================================================================================== */
#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345)
#define XXH_STATIC_H_3543687687345
/* These definitions are only meant to allow allocation of XXH state
statically, on stack, or in a struct for example.
@ -299,11 +297,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
# include "xxhash.c" /* include xxhash functions as `static`, for inlining */
# endif
#endif /* XXH_STATIC_LINKING_ONLY */
#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */
#if defined (__cplusplus)
}
#endif
#endif /* XXHASH_H_5627135585666179 */

4
lib/common/zstd_internal.h
View File

@ -49,6 +49,10 @@
#include "error_private.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#endif
#include "xxhash.h" /* XXH_reset, update, digest */
/*-*************************************

2
lib/compress/zstd_compress.c
View File

@ -13,8 +13,6 @@
***************************************/
#include <string.h> /* memset */
#include "mem.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH_reset, update, digest */
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY

2
lib/compress/zstdmt_compress.c
View File

@ -25,8 +25,6 @@
#include "threading.h" /* mutex */
#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
#include "zstdmt_compress.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h"
/* ====== Debug ====== */

51
lib/decompress/zstd_decompress.c
View File

@ -43,8 +43,6 @@
*********************************************************/
#include <string.h> /* memcpy, memmove, memset */
#include "mem.h" /* low level memory routines */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64_* */
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
@ -1485,8 +1483,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
if (ZSTD_isError(headerSize)) return headerSize;
/* Frame Header */
{
size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
{ size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
if (ZSTD_isError(ret)) return ret;
if (ret > 0) return ERROR(srcSize_wrong);
}
@ -1519,7 +1516,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
}
/*! ZSTD_decompressFrame() :
* `dctx` must be properly initialized */
* @dctx must be properly initialized */
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void** srcPtr, size_t *srcSizePtr)
@ -1586,12 +1583,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
remainingSize -= 4;
}
// Allow caller to get size read
/* Allow caller to get size read */
*srcPtr = ip;
*srcSizePtr = remainingSize;
return op-ostart;
}
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict);
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -1599,6 +1599,17 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict)
{
void* const dststart = dst;
if (ddict) {
if (dict) {
/* programmer error, these two cases should be mutually exclusive */
return ERROR(GENERIC);
}
dict = ZSTD_DDictDictContent(ddict);
dictSize = ZSTD_DDictDictSize(ddict);
}
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
U32 magicNumber;
@ -1954,6 +1965,16 @@ struct ZSTD_DDict_s {
ZSTD_customMem cMem;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict)
{
return ddict->dictContent;
}
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
{
return ddict->dictSize;
}
static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
{
ZSTD_decompressBegin(dstDCtx); /* init */
@ -1982,6 +2003,7 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
{
ddict->dictID = 0;
ddict->entropyPresent = 0;
if (ddict->dictSize < 8) return 0;
{ U32 const magic = MEM_readLE32(ddict->dictContent);
@ -2116,7 +2138,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
{
/* pass content and size in case legacy frames are encountered */
return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
ddict->dictContent, ddict->dictSize,
NULL, 0,
ddict);
}
@ -2317,6 +2339,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
break;
} }
/* check for single-pass mode opportunity */
if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
if (cSize <= (size_t)(iend-istart)) {
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict);
if (ZSTD_isError(decompressedSize)) return decompressedSize;
ip = istart + cSize;
op += decompressedSize;
zds->dctx->expected = 0;
zds->stage = zdss_init;
someMoreWork = 0;
break;
} }
/* Consume header */
ZSTD_refDDict(zds->dctx, zds->ddict);
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */

13
lib/legacy/zstd_v07.c
View File

@ -13,12 +13,14 @@
#include <string.h> /* memcpy */
#include <stdlib.h> /* malloc, free, qsort */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64_* */
#ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#endif
#include "xxhash.h" /* XXH64_* */
#include "zstd_v07.h"
#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
#define ZSTDv07_STATIC_LINKING_ONLY
#include "error_private.h"
@ -4536,7 +4538,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
// break; /* ZBUFFds_flush follows */
/* break; */
/* pass-through */
} }
case ZBUFFds_flush:

4
programs/Makefile
View File

@ -24,7 +24,9 @@ else
ALIGN_LOOP =
endif
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder \
-DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library
CFLAGS ?= -O3
DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \

1
programs/bench.c
View File

@ -581,6 +581,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
{
double const compressibility = (double)g_compressibilityDefault / 100;
if (cLevel < 1) cLevel = 1; /* minimum compression level */
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;

2
programs/fileio.c
View File

@ -954,7 +954,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
/* Close file */
if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */
if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); };
if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); };
return 0;
}

4
programs/zstdcli.c
View File

@ -370,10 +370,12 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
#ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
#endif
/* long commands with arguments */
#ifndef ZSTD_NODICT
#ifndef ZSTD_NODICT
if (longCommandWArg(&argument, "--cover=")) {
cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName));
continue;

3
tests/Makefile
View File

@ -253,6 +253,9 @@ zstd-playTests: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
shortest: ZSTDRTTEST=
shortest: test-zstd
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
ifeq ($(QEMU_SYS),)
test: test-pool

20
tests/fuzzer.c
View File

@ -108,6 +108,7 @@ static int basicUnitTests(U32 seed, double compressibility)
void* const CNBuffer = malloc(CNBuffSize);
void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
void* const decodedBuffer = malloc(CNBuffSize);
ZSTD_DCtx* dctx = ZSTD_createDCtx();
int testResult = 0;
U32 testNb=0;
size_t cSize;
@ -155,6 +156,16 @@ static int basicUnitTests(U32 seed, double compressibility)
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with null DDict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
if (!ZSTD_isError(r)) goto _output_error;
@ -210,7 +221,6 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Dictionary and CCtx Duplication tests */
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
static const size_t dictSize = 551;
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
@ -283,12 +293,10 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_freeCCtx(ctxOrig);
ZSTD_freeCCtx(ctxDuplicated);
ZSTD_freeDCtx(dctx);
}
/* Dictionary and dictBuilder tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t dictSize = 16 KB;
void* dictBuffer = malloc(dictSize);
size_t const totalSampleSize = 1 MB;
@ -370,14 +378,12 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
free(dictBuffer);
free(samplesSizes);
}
/* COVER dictionary builder tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t dictSize = 16 KB;
size_t optDictSize = dictSize;
void* dictBuffer = malloc(dictSize);
@ -414,7 +420,7 @@ static int basicUnitTests(U32 seed, double compressibility)
memset(&params, 0, sizeof(params));
params.steps = 4;
optDictSize = COVER_optimizeTrainFromBuffer(dictBuffer, optDictSize,
CNBuffer, samplesSizes, nbSamples,
CNBuffer, samplesSizes, nbSamples / 4,
&params);
if (ZDICT_isError(optDictSize)) goto _output_error;
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize);
@ -425,7 +431,6 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK : %u \n", dictID);
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
free(dictBuffer);
free(samplesSizes);
}
@ -445,7 +450,6 @@ static int basicUnitTests(U32 seed, double compressibility)
/* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
static const size_t dictSize = 65 KB;
static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
size_t cSize2;

7
tests/gzip/Makefile
View File

@ -12,7 +12,10 @@ VOID = /dev/null
export PATH := .:$(PATH)
.PHONY: all
all: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix
#all: test-gzip-env
all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed
all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid
all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix
@echo Testing completed
.PHONY: zstd
@ -35,7 +38,7 @@ clean:
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
test-%: zstd
@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
# || echo ignoring error
endif

7
tests/gzip/gzip-env.sh
View File

@ -17,13 +17,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# limit so don't run it by default.
echo XXX=$PATH
gzip --version
. "${srcdir=.}/init.sh"; path_prepend_ .
echo XXX=$PATH
gzip --version
#echo PATH=$PATH
#gzip --version
echo a >exp || framework_failure_
gzip <exp >in || framework_failure_

36
tests/playTests.sh
View File

@ -304,11 +304,33 @@ $ECHO "\n**** benchmark mode tests **** "
$ECHO "bench one file"
./datagen > tmp1
$ZSTD -bi1 tmp1
$ZSTD -bi0 tmp1
$ECHO "bench multiple levels"
$ZSTD -i1b1e3 tmp1
$ZSTD -i0b0e3 tmp1
$ECHO "with recursive and quiet modes"
$ZSTD -rqi1b1e3 tmp1
$ZSTD -rqi1b1e2 tmp1
$ECHO "\n**** gzip compatibility tests **** "
GZIPMODE=1
$ZSTD --format=gzip -V || GZIPMODE=0
if [ $GZIPMODE -eq 1 ]; then
$ECHO "gzip support detected"
GZIPEXE=1
gzip -V || GZIPEXE=0
if [ $GZIPEXE -eq 1 ]; then
./datagen > tmp
$ZSTD --format=gzip -f tmp
gzip -t -v tmp.gz
gzip -f tmp
$ZSTD -d -f -v tmp.gz
else
$ECHO "gzip binary not detected"
fi
else
$ECHO "gzip mode not supported"
fi
$ECHO "\n**** zstd round-trip tests **** "
@ -317,10 +339,10 @@ roundTripTest
roundTripTest -g15K # TableID==3
roundTripTest -g127K # TableID==2
roundTripTest -g255K # TableID==1
roundTripTest -g513K # TableID==0
roundTripTest -g512K 6 # greedy, hash chain
roundTripTest -g512K 16 # btlazy2
roundTripTest -g512K 19 # btopt
roundTripTest -g522K # TableID==0
roundTripTest -g519K 6 # greedy, hash chain
roundTripTest -g517K 16 # btlazy2
roundTripTest -g516K 19 # btopt
rm tmp*

2
tests/zstreamtest.c
View File

@ -218,7 +218,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; }
if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */
if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
DISPLAYLEVEL(3, "OK \n");
/* Basic decompression test */