AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #43 from Cyan4973/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet 2015-08-27 00:59:17 +02:00
commit 77dce3eba1
12 changed files with 99 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml
View File

@ -1,6 +1,6 @@
language: c
compiler: gcc
script: make test-travis
script: make $ZSTD_TRAVIS_CI_ENV
before_install:
- sudo apt-get update -qq
- sudo apt-get install -qq gcc-arm-linux-gnueabi
@ -11,11 +11,13 @@ before_install:
env:
- ZSTD_TRAVIS_CI_ENV=travis-install
- ZSTD_TRAVIS_CI_ENV=test-all
- ZSTD_TRAVIS_CI_ENV=clangtest
- ZSTD_TRAVIS_CI_ENV=gpptest
- ZSTD_TRAVIS_CI_ENV=armtest
- ZSTD_TRAVIS_CI_ENV=test
- ZSTD_TRAVIS_CI_ENV="-C programs test32"
- ZSTD_TRAVIS_CI_ENV=sanitize
- ZSTD_TRAVIS_CI_ENV="-C programs memtest"
matrix:
fast_finish: true

50
Makefile
View File

@ -32,40 +32,25 @@
# ################################################################
# Version number
export VERSION=0.1.0
export RELEASE=r$(VERSION)
export VERSION=0.1.1
DESTDIR?=
PREFIX ?= /usr
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
PRGDIR = programs
ZSTDDIR = lib
# Select test target for Travis CI's Build Matrix
ifneq (,$(filter test-%,$(ZSTD_TRAVIS_CI_ENV)))
TRAVIS_TARGET=prg-travis
else
TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV)
endif
.PHONY: clean
default: zstdprograms
default: zstdprogram
all:
@cd $(ZSTDDIR); $(MAKE) -e all
@cd $(PRGDIR); $(MAKE) -e all
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@
zstdprograms:
@cd $(PRGDIR); $(MAKE) -e
zstdprogram:
$(MAKE) -C $(PRGDIR)
clean:
@cd $(PRGDIR); $(MAKE) clean
@cd $(ZSTDDIR); $(MAKE) clean
# @cd examples; $(MAKE) clean
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@
@echo Cleaning completed
@ -74,23 +59,18 @@ clean:
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
install:
@cd $(ZSTDDIR); $(MAKE) -e install
@cd $(PRGDIR); $(MAKE) -e install
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@
uninstall:
@cd $(ZSTDDIR); $(MAKE) uninstall
@cd $(PRGDIR); $(MAKE) uninstall
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@
travis-install:
sudo $(MAKE) install
test:
@cd $(PRGDIR); $(MAKE) -e test
test-travis: $(TRAVIS_TARGET)
prg-travis:
@cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV)
$(MAKE) -C $(PRGDIR) $@
clangtest: clean
clang -v
@ -100,8 +80,8 @@ gpptest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
armtest: clean
cd $(ZSTDDIR); $(MAKE) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
$(MAKE) -C $(ZSTDDIR) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
$(MAKE) -C $(PRGDIR) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
sanitize: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"

4
lib/Makefile
View File

@ -32,14 +32,14 @@
# ################################################################
# Version numbers
VERSION?= 0
VERSION?= 0.1.1
LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
DESTDIR?=
PREFIX ?= /usr
PREFIX ?= /usr/local
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
LDFLAGS = -I.

23
lib/fse.c
View File

@ -135,7 +135,7 @@ typedef signed long long S64;
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets which generate assembly depending on alignment.
* It can generate buggy code on targets generating assembly depending on alignment.
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
@ -1398,6 +1398,7 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize);
if (FSE_isError(errorCode)) return errorCode;
if (errorCode == srcSize) return 1;
if (errorCode == 1) return 0; /* each symbol only present once */
if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
@ -1529,12 +1530,14 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz
*/
static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits)
{
return ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD->bitContainer)*8)-1))) >> 1) >> (((sizeof(bitD->bitContainer)*8)-1)-nbBits);
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
}
static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */
{
return (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD->bitContainer)*8)-nbBits);
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
}
static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits)
@ -1904,11 +1907,17 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
if (rankLast[nBitsToDecrease-1] == noOne)
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; // now there is one elt
huffNode[rankLast[nBitsToDecrease]].nbBits ++;
rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
rankLast[nBitsToDecrease] = noOne; // rank list emptied
if (rankLast[nBitsToDecrease] == 0)
rankLast[nBitsToDecrease] = noOne;
else
{
rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
rankLast[nBitsToDecrease] = noOne; // rank list emptied
}
}
while (totalCost < 0) // Sometimes, cost correction overshoot
while (totalCost < 0) /* Sometimes, cost correction overshoot */
{
if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
{

2
lib/zstd.c
View File

@ -1389,7 +1389,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
nbBits = offsetCode - 1;
if (offsetCode==0) nbBits = 0; /* cmove */
offset = ((size_t)1 << nbBits) + FSE_readBits(&(seqState->DStream), nbBits);
offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + FSE_readBits(&(seqState->DStream), nbBits);
if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
if (offsetCode==0) offset = prevOffset;
}

30
programs/Makefile
View File

@ -30,21 +30,18 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
RELEASE?= v0.1.0
VERSION?= v0.1.1
DESTDIR?=
PREFIX ?= /usr
PREFIX ?= /usr/local
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\"
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\"
LDFLAGS = -I../lib
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
BINDIR=$(PREFIX)/bin
MANDIR=$(PREFIX)/share/man/man1
ZSTDDIR=../lib
TEST_FILES = COPYING
TEST_TARGETS=test-native
BINDIR = $(PREFIX)/bin
MANDIR = $(PREFIX)/share/man/man1
ZSTDDIR = ../lib
# Define *.exe as extension for Windows systems
@ -120,9 +117,12 @@ test: test-zstd test-fullbench test-fuzzer
test32: test-zstd32 test-fullbench32 test-fuzzer32
test-all: test test32 test-mem
test-all: test test32 memtest
test-zstd: zstd datagen
@echo "*** zstd cli write error test ***"
echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
@echo "*** zstd round-trip tests *** "
./datagen | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)
@ -146,15 +146,15 @@ test-fuzzer: fuzzer
test-fuzzer32: fuzzer32
./fuzzer32
test-mem: zstd datagen fuzzer fullbench
memtest: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
./datagen -g16KB > tmp
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
./datagen -g64MB > tmp
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
@rm tmp
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
endif

11
programs/fileio.c
View File

@ -52,6 +52,7 @@
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <errno.h> /* errno */
#include "fileio.h"
#include "zstd_static.h"
@ -254,7 +255,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header");
if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
compressedfilesize += cSize;
/* Main compression loop */
@ -276,7 +277,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* Write cBlock */
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block");
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
compressedfilesize += cSize;
inSlot += inSize;
@ -288,7 +289,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end");
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
compressedfilesize += cSize;
/* Status */
@ -299,9 +300,9 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* clean */
free(inBuff);
free(outBuff);
fclose(finput);
fclose(foutput);
ZSTD_freeCCtx(ctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
return compressedfilesize;
}

2
programs/fullbench.c
View File

@ -616,7 +616,7 @@ int main(int argc, char** argv)
/* Modify Nb Iterations */
case 'i':
if ((argument[1] >='1') && (argument[1] <='9'))
if ((argument[1] >='0') && (argument[1] <='9'))
{
int iters = argument[1] - '0';
BMK_SetNbIterations(iters);

31
programs/unzstd.1
View File

@ -1,31 +0,0 @@
\"
\" unzstd.1: This is a manual page for 'unzstd' program. This file is part of
\" the zstd <https://github.com/Cyan4973/zstd/> project.
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH unzstd "1" "2014-06-20" "unzstd" "User Commands"
.SH NAME
\fBunzstd\fR - Utility based on zstd
.SH SYNOPSIS
.TP 5
\fBunzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
.SH DESCRIPTION
.PP
\fBunzstd\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm.
\fBunzstd\fR decompress input file, it is equivalent to \fBzstd -d\fR,
Available options are the same as \fBzstd\fR ones (man zstd).
.SH BUGS
Report bugs at:- https://github.com/Cyan4973/zstd/
.SH AUTHOR
Yann Collet

1
programs/unzstd.1 Symbolic link
View File

@ -0,0 +1 @@
zstd.1

31
programs/zstd.1
View File

@ -1,26 +1,35 @@
\"
\" zstd.1: This is a manual page for 'zstd' program. This file is part of the
\" zstd <https://github.com/Cyan4973/zstd> project.
\" zstd <http://www.zstd.net/> project.
\" Author: Yann Collet
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH zstd "1" "2015-01-22" "zstd" "User Commands"
.TH zstd "1" "2015-08-22" "zstd" "User Commands"
.SH NAME
\fBzstd\fR - standard compression algorithm
\fBzstd, unzstd, zstdcat\fR - Compress or decompress .zst files
.SH SYNOPSIS
.TP 5
\fBzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
.PP
.B unzstd
is equivalent to
.BR "zstd \-d"
.br
.B zstdcat
is equivalent to
.BR "zstd \-dc"
.br
.SH DESCRIPTION
.PP
\fBzstd\fR is a fast lossless compression algorithm
with highly parametrable compression strength and memory usage.
It is based on the \fBLZ77\fR family, coupled with FSE entropy stage.
zstd offers compression speed of 200 MB/s per core.
\fBzstd\fR is a fast lossless compression algorithm.
It is based on the \fBLZ77\fR family, with FSE & huff0 entropy stage.
zstd offers compression speed > 200 MB/s per core.
It also features a fast decoder, with speed > 500 MB/s per core.
\fBzstd\fR supports the following options :
@ -35,13 +44,13 @@ It also features a fast decoder, with speed > 500 MB/s per core.
.B \-f
overwrite output without prompting
.TP
.B \-h/\-H
.BR \-h/\-H ", " --help
display help/long help and exit
.TP
.B \-V
.BR \-V ", " --version
display Version number and exit
.TP
.B \-v
.BR \-v ", " --verbose
verbose mode
.TP
.B \-q
@ -63,7 +72,7 @@ It also features a fast decoder, with speed > 500 MB/s per core.
iteration loops [1-9](default : 3), benchmark mode only
.SH BUGS
Report bugs at:- https://github.com/Cyan4973/zstd
Report bugs at:- https://github.com/Cyan4973/zstd/issues
.SH AUTHOR
Yann Collet

32
programs/zstdcat.1
View File

@ -1,32 +0,0 @@
\"
\" zstdcat.1: This is a manual page for 'zstdcat' program. This file is part of
\" the zstd <https://github.com/Cyan4973/zstd/> project.
\"
\" No hyphenation
.hy 0
.nr HY 0
.TH zstdcat "1" "2014-06-20" "zstdcat" "User Commands"
.SH NAME
\fBzstdcat\fR - Utility based on zstd
.SH SYNOPSIS
.TP 5
\fBzstdcat\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
.SH DESCRIPTION
.PP
\fBzstdcat\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm.
\fBzstdcat\fR decompress input file or stream, redirecting its output to the console.
It is equivalent to \fBzstd -cd\fR,
Available options are the same as \fBzstd\fR ones (man zstd).
.SH BUGS
Report bugs at:- https://github.com/Cyan4973/zstd/
.SH AUTHOR
Yann Collet

1
programs/zstdcat.1 Symbolic link
View File

@ -0,0 +1 @@
zstd.1

30
programs/zstdcli.c
View File

@ -86,8 +86,9 @@
/**************************************
* Display Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static FILE* displayOut;
static unsigned displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information
@ -137,7 +138,7 @@ static int usage_advanced(const char* programName)
//DISPLAY( " -t : test compressed file integrity\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b : benchmark file(s)\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3)\n");
return 0;
}
@ -173,30 +174,32 @@ int main(int argc, char** argv)
char* dynNameSpace = NULL;
char extension[] = ZSTD_EXTENSION;
displayOut = stderr;
/* Pick out basename component. Don't rely on stdlib because of conflicting behaviour. */
for (i = (int)strlen(programName); i > 0; i--)
{
if (programName[i] == '/')
{
i++;
break;
}
if (programName[i] == '/') { i++; break; }
}
programName += i;
/* zstdcat behavior */
/* zstdcat preset behavior */
if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
/* unzstd behavior */
/* unzstd preset behavior */
if (!strcmp(programName, ZSTD_UNZSTD))
decode=1;
// command switches
/* command switches */
for(i=1; i<argc; i++)
{
char* argument = argv[i];
if(!argument) continue; // Protection if argument empty
if(!argument) continue; /* Protection if argument empty */
/* long commands (--long-word) */
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--help")) { displayOut=stdout; return usage_advanced(programName); }
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
/* Decode commands (note : aggregated commands are allowed) */
if (argument[0]=='-')
@ -206,6 +209,7 @@ int main(int argc, char** argv)
{
if (!inFileName) inFileName=stdinmark;
else outFileName=stdoutmark;
continue;
}
argument++;
@ -215,9 +219,9 @@ int main(int argc, char** argv)
switch(argument[0])
{
/* Display help */
case 'V': DISPLAY(WELCOME_MESSAGE); return 0; /* Version Only */
case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; /* Version Only */
case 'H':
case 'h': return usage_advanced(programName);
case 'h': displayOut=stdout; return usage_advanced(programName);
// Compression (default)
//case 'z': forceCompress = 1; break;