AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into zstdlib_api

Browse Source
This commit is contained in:
Christophe Chevalier 2015-12-09 15:45:57 +01:00
commit 844baf4fc5
4 changed files with 113 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/zstd_compress.c
View File

@ -65,6 +65,7 @@
/* *************************************
* Constants
***************************************/
unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const U32 g_searchStrength = 8;
@ -599,7 +600,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
@ -612,7 +613,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
# if defined(_MSC_VER)
unsigned long r=0;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
@ -2097,7 +2098,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint)
{
ZSTD_parameters result;
int tableID = ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
int tableID = ((srcSizeHint-1) <= 256 KB) + ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
result = ZSTD_defaultParameters[tableID][compressionLevel];

29
lib/zstd_static.h
View File

@ -120,7 +120,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz
Use ZSTD_compressBegin().
You may also prefer the advanced derivative ZSTD_compressBegin_advanced(), for finer parameter control.
It's then possible to add a dictionary with ZSTD_compressDictionary()
It's then possible to add a dictionary with ZSTD_compress_insertDictionary()
Note that dictionary presence is a "hidden" information,
the decoder needs to be aware that it is required for proper decoding, or decoding will fail.
@ -180,7 +180,8 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma
* Pre-defined compression levels
***************************************/
#define ZSTD_MAX_CLEVEL 20
static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = {
unsigned ZSTD_maxCLevel (void);
static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ /* "default" */
/* W, C, H, S, L, strat */
{ 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
@ -205,6 +206,30 @@ static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = {
{ 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
{ 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */
},
{ /* for srcSize <= 256 KB */
/* W, C, H, S, L, strat */
{ 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */
{ 0, 18, 16, 15, 1, 7, ZSTD_fast }, /* level 1 */
{ 0, 18, 16, 16, 1, 7, ZSTD_fast }, /* level 2 */
{ 0, 18, 18, 18, 1, 7, ZSTD_fast }, /* level 3 */
{ 0, 18, 14, 15, 4, 6, ZSTD_greedy }, /* level 4 */
{ 0, 18, 16, 16, 1, 6, ZSTD_lazy }, /* level 5 */
{ 0, 18, 15, 15, 3, 6, ZSTD_lazy }, /* level 6 */
{ 0, 18, 15, 15, 4, 6, ZSTD_lazy }, /* level 7 */
{ 0, 18, 16, 18, 4, 6, ZSTD_lazy }, /* level 8 */
{ 0, 18, 18, 18, 4, 6, ZSTD_lazy }, /* level 9 */
{ 0, 18, 18, 18, 5, 6, ZSTD_lazy }, /* level 10 */
{ 0, 18, 18, 19, 6, 6, ZSTD_lazy }, /* level 11 */
{ 0, 18, 18, 19, 7, 6, ZSTD_lazy }, /* level 12 */
{ 0, 18, 19, 15, 7, 5, ZSTD_btlazy2 }, /* level 13 */
{ 0, 18, 19, 16, 8, 5, ZSTD_btlazy2 }, /* level 14 */
{ 0, 18, 19, 17, 9, 5, ZSTD_btlazy2 }, /* level 15 */
{ 0, 18, 19, 17, 10, 5, ZSTD_btlazy2 }, /* level 16 */
{ 0, 18, 19, 17, 11, 5, ZSTD_btlazy2 }, /* level 17 */
{ 0, 18, 19, 17, 12, 5, ZSTD_btlazy2 }, /* level 18 */
{ 0, 18, 19, 17, 13, 5, ZSTD_btlazy2 }, /* level 19 */
{ 0, 18, 19, 17, 14, 5, ZSTD_btlazy2 }, /* level 20 */
},
{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, strat */
{ 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */

72
programs/Makefile
View File

@ -169,77 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
test-all: test test32 valgrindTest
zstd-playTests: datagen
@echo "\n**** frame concatenation **** "
@echo "hello " > hello.tmp
@echo "world!" > world.tmp
@cat hello.tmp world.tmp > helloworld.tmp
$(ZSTD) hello.tmp > hello.zstd
$(ZSTD) world.tmp > world.zstd
@cat hello.zstd world.zstd > helloworld.zstd
$(ZSTD) -df helloworld.zstd > result.tmp
cat result.tmp
sdiff helloworld.tmp result.tmp
@rm *.tmp *.zstd
@echo frame concatenation test completed
@echo "**** flush write error test **** "
echo foo | $(ZSTD) > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
echo foo | $(ZSTD) | $(ZSTD) -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
@echo "**** zstd round-trip tests **** "
@./datagen | md5sum > tmp1
./datagen | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen | $(ZSTD) -6 -v | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
@./datagen -g270000000 | md5sum > tmp1
./datagen -g270000000 | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g270000000 | $(ZSTD) -v2 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g270000000 | $(ZSTD) -v3 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
@./datagen -g140000000 -P60| md5sum > tmp1
./datagen -g140000000 -P60 | $(ZSTD) -v4 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g140000000 -P60 | $(ZSTD) -v5 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g140000000 -P60 | $(ZSTD) -v6 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
@./datagen -g70000000 -P70 | md5sum > tmp1
./datagen -g70000000 -P70 | $(ZSTD) -v7 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g70000000 -P70 | $(ZSTD) -v8 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g70000000 -P70 | $(ZSTD) -v9 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
@./datagen -g35000000 -P75 | md5sum > tmp1
./datagen -g35000000 -P75 | $(ZSTD) -v10 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g35000000 -P75 | $(ZSTD) -v11 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g35000000 -P75 | $(ZSTD) -v12 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
@./datagen -g18000000 -P80 | md5sum > tmp1
./datagen -g18000000 -P80 | $(ZSTD) -v13 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g18000000 -P80 | $(ZSTD) -v14 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g18000000 -P80 | $(ZSTD) -v15 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g18000000 -P80 | $(ZSTD) -v16 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g18000000 -P80 | $(ZSTD) -v17 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
@./datagen -g50000000 -P94 | md5sum > tmp1
./datagen -g50000000 -P94 | $(ZSTD) -v18 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g50000000 -P94 | $(ZSTD) -v19 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
@./datagen -g99000000 -P99 | md5sum > tmp1
./datagen -g99000000 -P99 | $(ZSTD) -v20 | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
./datagen -g6000000000 -P99| md5sum > tmp1
./datagen -g6000000000 -P99| $(ZSTD) -vq | $(ZSTD) -d | md5sum > tmp2
@diff tmp1 tmp2
ZSTD=$(ZSTD) ./playTests.sh --test-large-data
test-zstd: ZSTD = ./zstd
test-zstd: zstd zstd-playTests

81
programs/playTests.sh Executable file
View File

@ -0,0 +1,81 @@
#!/bin/sh -e
die() {
echo "$@" 1>&2
exit 1
}
roundTripTest() {
if [ -n "$3" ]; then
local c="$3"
local p="$2"
else
local c="$2"
fi
rm -f tmp1 tmp2
echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
./datagen $1 $p | md5sum > tmp1
./datagen $1 $p | $ZSTD -v$c | $ZSTD -d | md5sum > tmp2
diff -q tmp1 tmp2
}
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
printf "\n**** frame concatenation **** "
echo "hello " > hello.tmp
echo "world!" > world.tmp
cat hello.tmp world.tmp > helloworld.tmp
$ZSTD hello.tmp > hello.zstd
$ZSTD world.tmp > world.zstd
cat hello.zstd world.zstd > helloworld.zstd
$ZSTD -df helloworld.zstd > result.tmp
cat result.tmp
sdiff helloworld.tmp result.tmp
rm ./*.tmp ./*.zstd
echo frame concatenation test completed
echo "**** flush write error test **** "
echo foo | $ZSTD > /dev/full && die "write error not detected!"
echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
echo "**** zstd round-trip tests **** "
roundTripTest
roundTripTest '' 6
if [ "$1" != "--test-large-data" ]; then
echo "Skipping large data tests"
exit 0
fi
roundTripTest -g270000000 1
roundTripTest -g270000000 2
roundTripTest -g270000000 3
roundTripTest -g140000000 -P60 4
roundTripTest -g140000000 -P60 5
roundTripTest -g140000000 -P60 6
roundTripTest -g70000000 -P70 7
roundTripTest -g70000000 -P70 8
roundTripTest -g70000000 -P70 9
roundTripTest -g35000000 -P75 10
roundTripTest -g35000000 -P75 11
roundTripTest -g35000000 -P75 12
roundTripTest -g18000000 -P80 13
roundTripTest -g18000000 -P80 14
roundTripTest -g18000000 -P80 15
roundTripTest -g18000000 -P80 16
roundTripTest -g18000000 -P80 17
roundTripTest -g50000000 -P94 18
roundTripTest -g50000000 -P94 19
roundTripTest -g99000000 -P99 20
roundTripTest -g6000000000 -P99 q