diff --git a/.travis.yml b/.travis.yml index a9c1db52..4959e97f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test-zstd' + - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' @@ -26,8 +26,7 @@ matrix: - env: Cmd='make arminstall && make aarch64fuzz' - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest' - - env: Cmd='make -j msanregressiontest' + - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - env: Cmd='make lz4install && make -C tests test-lz4' diff --git a/Makefile b/Makefile index 5756e630..4f515bb9 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ endif default: lib-release zstd-release .PHONY: all -all: | allmost examples manual contrib +all: allmost examples manual contrib .PHONY: allmost allmost: allzstd @@ -35,8 +35,7 @@ allmost: allzstd #skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd -allzstd: - $(MAKE) -C $(ZSTDDIR) all +allzstd: lib $(MAKE) -C $(PRGDIR) all $(MAKE) -C $(TESTDIR) all @@ -45,24 +44,15 @@ all32: $(MAKE) -C $(PRGDIR) zstd32 $(MAKE) -C $(TESTDIR) all32 -.PHONY: lib -lib: +.PHONY: lib lib-release +lib lib-release: @$(MAKE) -C $(ZSTDDIR) $@ -.PHONY: lib-release -lib-release: - @$(MAKE) -C $(ZSTDDIR) - -.PHONY: zstd -zstd: +.PHONY: zstd zstd-release +zstd zstd-release: @$(MAKE) -C $(PRGDIR) $@ cp $(PRGDIR)/zstd$(EXT) . -.PHONY: zstd-release -zstd-release: - @$(MAKE) -C $(PRGDIR) - cp $(PRGDIR)/zstd$(EXT) . - .PHONY: zstdmt zstdmt: @$(MAKE) -C $(PRGDIR) $@ @@ -85,7 +75,7 @@ shortest: check: shortest .PHONY: examples -examples: +examples: lib CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all .PHONY: manual diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile index 1847aa7e..6d9562df 100644 --- a/contrib/seekable_format/examples/Makefile +++ b/contrib/seekable_format/examples/Makefile @@ -9,13 +9,16 @@ # This Makefile presumes libzstd is built, using `make` in / or /lib/ -LDFLAGS += ../../../lib/libzstd.a +ZSTDLIB_PATH = ../../../lib +ZSTDLIB_NAME = libzstd.a +ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) + CPPFLAGS += -I../ -I../../../lib -I../../../lib/common CFLAGS ?= -O3 CFLAGS += -g -SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c +SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB) .PHONY: default all clean test @@ -23,6 +26,9 @@ default: all all: seekable_compression seekable_decompression parallel_processing +$(ZSTDLIB): + make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME) + seekable_compression : seekable_compression.c $(SEEKABLE_OBJS) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ diff --git a/contrib/seekable_format/zstd_seekable.h b/contrib/seekable_format/zstd_seekable.h index 438ac201..7ffd1ba0 100644 --- a/contrib/seekable_format/zstd_seekable.h +++ b/contrib/seekable_format/zstd_seekable.h @@ -6,8 +6,10 @@ extern "C" { #endif #include +#include "zstd.h" /* ZSTDLIB_API */ -static const unsigned ZSTD_seekTableFooterSize = 9; + +#define ZSTD_seekTableFooterSize 9 #define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1 diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index c6765333..966617a8 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -88,7 +88,7 @@ static int ZSTD_seekable_read_FILE(void* opaque, void* buffer, size_t n) return 0; } -static int ZSTD_seekable_seek_FILE(void* opaque, S64 offset, int origin) +static int ZSTD_seekable_seek_FILE(void* opaque, long long offset, int origin) { int const ret = LONG_SEEK((FILE*)opaque, offset, origin); if (ret) return ret; @@ -110,7 +110,7 @@ static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) return 0; } -static int ZSTD_seekable_seek_buff(void* opaque, S64 offset, int origin) +static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin) { buffWrapper_t* buff = (buffWrapper_t*) opaque; unsigned long long newOffset; @@ -197,7 +197,7 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs) * Performs a binary search to find the last frame with a decompressed offset * <= pos * @return : the frame's index */ -U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, U64 pos) +U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos) { U32 lo = 0; U32 hi = zs->seekTable.tableLen; @@ -222,13 +222,13 @@ U32 ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs) return zs->seekTable.tableLen; } -U64 ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex) +unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex) { if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; return zs->seekTable.entries[frameIndex].cOffset; } -U64 ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex) +unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex) { if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; return zs->seekTable.entries[frameIndex].dOffset; @@ -294,7 +294,6 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs) { /* Allocate an extra entry at the end so that we can do size * computations on the last element without special case */ seekEntry_t* entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1)); - const BYTE* tableBase = zs->inBuff + ZSTD_skippableHeaderSize; U32 idx = 0; U32 pos = 8; @@ -372,7 +371,7 @@ size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile sr return 0; } -size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, U64 offset) +size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsigned long long offset) { U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset); do { diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index aae6d0f3..3d107746 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -82,7 +82,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);

`src` should point to the start of a ZSTD encoded frame. `srcSize` must be at least as large as the frame header. hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - @return : - decompressed size of the frame in `src`, if known + @return : - decompressed size of `src` frame content, if known - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) note 1 : a 0 return value means the frame is valid but "empty". @@ -92,7 +92,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); Optionally, application can rely on some implicit limit, as ZSTD_decompress() only needs an upper bound of decompressed size. (For example, data could be necessarily cut into blocks <= 16 KB). - note 3 : decompressed size is always present when compression is done with ZSTD_compress() + note 3 : decompressed size is always present when compression is completed using single-pass functions, + such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). note 4 : decompressed size can be very large (64-bits value), potentially larger than what local system can handle as a single memory segment. In which case, it's necessary to use streaming mode to decompress data. @@ -107,8 +108,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); Both functions work the same way, but ZSTD_getDecompressedSize() blends "empty", "unknown" and "error" results to the same return value (0), while ZSTD_getFrameContentSize() gives them separate return values. - `src` is the start of a zstd compressed frame. - @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. + @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise.


Helper functions

#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */