AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-12-22 17:59:22 +01:00
commit 724bb2ed57
10 changed files with 72 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Makefile
View File

@ -26,13 +26,21 @@ endif
default: lib zstd
.PHONY: all
all:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ zstd32
$(MAKE) -C $(TESTDIR) $@ all32
$(MAKE) -C $(ZWRAPDIR) $@
all: allmost
CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@
.PHONY: allmost
allmost:
$(MAKE) -C $(ZSTDDIR) all
$(MAKE) -C $(PRGDIR) all
$(MAKE) -C $(TESTDIR) all
$(MAKE) -C $(ZWRAPDIR) all
.PHONY: all32
all32:
$(MAKE) -C $(PRGDIR) zstd32
$(MAKE) -C $(TESTDIR) all32
.PHONY: lib
lib:
@$(MAKE) -C $(ZSTDDIR)
@ -151,13 +159,13 @@ gnu90test: clean
CFLAGS="-std=gnu90" $(MAKE) all
c99test: clean
CFLAGS="-std=c99" $(MAKE) all
CFLAGS="-std=c99" $(MAKE) allmost
gnu99test: clean
CFLAGS="-std=gnu99" $(MAKE) all
c11test: clean
CFLAGS="-std=c11" $(MAKE) all
CFLAGS="-std=c11" $(MAKE) allmost
bmix64test: clean
CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test

2
lib/common/mem.h
View File

@ -39,7 +39,7 @@ extern "C" {
#endif
/* code only tested on 32 and 64 bits systems */
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }

12
lib/decompress/zstd_decompress.c
View File

@ -1766,6 +1766,18 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
}
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, 1, allocator);
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */

14
lib/dictBuilder/zdict.c
View File

@ -306,13 +306,13 @@ static dictItem ZDICT_analyzePos(
} while (length >=MINMATCHLENGTH);
/* look backward */
length = MINMATCHLENGTH;
while ((length >= MINMATCHLENGTH) & (start > 0)) {
length = ZDICT_count(b + pos, b + suffix[start - 1]);
if (length >= LLIMIT) length = LLIMIT - 1;
lengthList[length]++;
if (length >= MINMATCHLENGTH) start--;
}
length = MINMATCHLENGTH;
while ((length >= MINMATCHLENGTH) & (start > 0)) {
length = ZDICT_count(b + pos, b + suffix[start - 1]);
if (length >= LLIMIT) length = LLIMIT - 1;
lengthList[length]++;
if (length >= MINMATCHLENGTH) start--;
}
/* largest useful length */
memset(cumulLength, 0, sizeof(cumulLength));

3
lib/zstd.h
View File

@ -348,8 +348,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_TARGETLENGTH_MAX 999
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
#define ZSTD_FRAMEHEADERSIZE_MIN 6
static const size_t ZSTD_frameHeaderSize_prefix = 5;
static const size_t ZSTD_frameHeaderSize_min = 6;
static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */

4
zlibWrapper/examples/fitblk.c
View File

@ -82,7 +82,7 @@ local int partcompress(FILE *in, z_streamp def)
flush = Z_SYNC_FLUSH;
do {
def->avail_in = fread(raw, 1, RAWLEN, in);
def->avail_in = (uInt)fread(raw, 1, RAWLEN, in);
if (ferror(in))
return Z_ERRNO;
def->next_in = raw;
@ -148,7 +148,7 @@ int main(int argc, char **argv)
/* get requested output size */
if (argc != 2)
quit("need one argument: size of output block");
ret = strtol(argv[1], argv + 1, 10);
ret = (int)strtol(argv[1], argv + 1, 10);
if (argv[1][0] != 0)
quit("argument must be a number");
if (ret < 8) /* 8 is minimum zlib stream size */

16
zlibWrapper/examples/zwrapbench.c
View File

@ -315,10 +315,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
}
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
def.avail_in = blockTable[blockNb].srcSize;
def.avail_in = (uInt)blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
def.avail_out = blockTable[blockNb].cRoom;
def.avail_out = (uInt)blockTable[blockNb].cRoom;
def.total_out = 0;
ret = deflate(&def, Z_FINISH);
if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure ret=%d srcSize=%d" , ret, (int)blockTable[blockNb].srcSize);
@ -346,10 +346,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
}
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
def.avail_in = blockTable[blockNb].srcSize;
def.avail_in = (uInt)blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
def.avail_out = blockTable[blockNb].cRoom;
def.avail_out = (uInt)blockTable[blockNb].cRoom;
def.total_out = 0;
ret = deflate(&def, Z_FINISH);
if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure");
@ -451,10 +451,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
ret = inflateReset(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
inf.avail_in = blockTable[blockNb].cSize;
inf.avail_in = (uInt)blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;
inf.avail_out = blockTable[blockNb].srcSize;
inf.avail_out = (uInt)blockTable[blockNb].srcSize;
inf.total_out = 0;
ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) {
@ -483,10 +483,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
ret = inflateInit(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
inf.avail_in = blockTable[blockNb].cSize;
inf.avail_in = (uInt)blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;
inf.avail_out = blockTable[blockNb].srcSize;
inf.avail_out = (uInt)blockTable[blockNb].srcSize;
inf.total_out = 0;
ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) {

6
zlibWrapper/gzread.c
View File

@ -30,7 +30,7 @@ local int gz_load(state, buf, len, have)
*have = 0;
do {
ret = read(state.state->fd, buf + *have, len - *have);
ret = (int)read(state.state->fd, buf + *have, len - *have);
if (ret <= 0)
break;
*have += ret;
@ -469,7 +469,7 @@ int ZEXPORT gzungetc(c, file)
if (state.state->x.have == 0) {
state.state->x.have = 1;
state.state->x.next = state.state->out + (state.state->size << 1) - 1;
state.state->x.next[0] = c;
state.state->x.next[0] = (unsigned char)c;
state.state->x.pos--;
state.state->past = 0;
return c;
@ -491,7 +491,7 @@ int ZEXPORT gzungetc(c, file)
}
state.state->x.have++;
state.state->x.next--;
state.state->x.next[0] = c;
state.state->x.next[0] = (unsigned char)c;
state.state->x.pos--;
state.state->past = 0;
return c;

8
zlibWrapper/gzwrite.c
View File

@ -84,7 +84,7 @@ local int gz_comp(state, flush)
/* write directly if requested */
if (state.state->direct) {
got = write(state.state->fd, strm->next_in, strm->avail_in);
got = (int)write(state.state->fd, strm->next_in, strm->avail_in);
if (got < 0 || (unsigned)got != strm->avail_in) {
gz_error(state, Z_ERRNO, zstrerror());
return -1;
@ -101,7 +101,7 @@ local int gz_comp(state, flush)
if (strm->avail_out == 0 || (flush != Z_NO_FLUSH &&
(flush != Z_FINISH || ret == Z_STREAM_END))) {
have = (unsigned)(strm->next_out - state.state->x.next);
if (have && ((got = write(state.state->fd, state.state->x.next, have)) < 0 ||
if (have && ((got = (int)write(state.state->fd, state.state->x.next, have)) < 0 ||
(unsigned)got != have)) {
gz_error(state, Z_ERRNO, zstrerror());
return -1;
@ -278,7 +278,7 @@ int ZEXPORT gzputc(file, c)
strm->next_in = state.state->in;
have = (unsigned)((strm->next_in + strm->avail_in) - state.state->in);
if (have < state.state->size) {
state.state->in[have] = c;
state.state->in[have] = (unsigned char)c;
strm->avail_in++;
state.state->x.pos++;
return c & 0xff;
@ -286,7 +286,7 @@ int ZEXPORT gzputc(file, c)
}
/* no room in buffer or not initialized, use gz_write() */
buf[0] = c;
buf[0] = (unsigned char)c;
if (gzwrite(file, buf, 1) != 1)
return -1;
return c & 0xff;

36
zlibWrapper/zstd_zlibwrapper.c
View File

@ -61,7 +61,7 @@ ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); }
static void* ZWRAP_allocFunction(void* opaque, size_t size)
{
z_streamp strm = (z_streamp) opaque;
void* address = strm->zalloc(strm->opaque, 1, size);
void* address = strm->zalloc(strm->opaque, 1, (uInt)size);
/* printf("ZWRAP alloc %p, %d \n", address, (int)size); */
return address;
}
@ -130,7 +130,7 @@ int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize,
{
LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc);
if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR;
if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize;
{ ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
size_t errorCode;
@ -156,7 +156,7 @@ int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message)
ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
strm->msg = message;
if (zwc == NULL) return Z_STREAM_ERROR;
return ZWRAPC_finishWithError(zwc, strm, 0);
}
@ -165,7 +165,7 @@ int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize)
{
ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
if (zwc == NULL) return Z_STREAM_ERROR;
zwc->pledgedSrcSize = pledgedSrcSize;
zwc->comprState = ZWRAP_useInit;
return Z_OK;
@ -232,7 +232,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
LOG_WRAPPERC("- deflateReset\n");
if (!g_ZWRAP_useZSTDcompression)
return deflateReset(strm);
ZWRAP_deflateReset_keepDict(strm);
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
@ -284,7 +284,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
if (zwc->zbc == NULL) {
int res;
zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
@ -321,9 +321,9 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
strm->avail_in -= zwc->inBuffer.pos;
}
if (flush == Z_FULL_FLUSH
if (flush == Z_FULL_FLUSH
#if ZLIB_VERNUM >= 0x1240
|| flush == Z_TREES
|| flush == Z_TREES
#endif
|| flush == Z_BLOCK)
return ZWRAPC_finishWithErrorMsg(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!");
@ -424,7 +424,7 @@ typedef struct {
} ZWRAP_DCtx;
int ZWRAP_isUsingZSTDdecompression(z_streamp strm)
int ZWRAP_isUsingZSTDdecompression(z_streamp strm)
{
if (strm == NULL) return 0;
return (strm->reserved == ZWRAP_ZSTD_STREAM);
@ -458,7 +458,7 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
}
MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_frameHeaderSize_min); /* if compilation fails here, assertion is false */
MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN); /* if compilation fails here, assertion is false */
ZWRAP_initDCtx(zwd);
return zwd;
}
@ -488,7 +488,7 @@ int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
strm->msg = message;
if (zwd == NULL) return Z_STREAM_ERROR;
return ZWRAPD_finishWithError(zwd, strm, 0);
}
@ -528,7 +528,7 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits,
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
return inflateInit2_(strm, windowBits, version, stream_size);
}
{
int ret = z_inflateInit_ (strm, version, stream_size);
LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits);
@ -552,7 +552,7 @@ int ZWRAP_inflateReset_keepDict(z_streamp strm)
ZWRAP_initDCtx(zwd);
zwd->decompState = ZWRAP_useReset;
}
strm->total_in = 0;
strm->total_out = 0;
return Z_OK;
@ -569,7 +569,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
if (ret != Z_OK) return ret; }
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
if (zwd == NULL) return Z_STREAM_ERROR;
zwd->decompState = ZWRAP_useInit; }
return Z_OK;
@ -608,7 +608,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
zwd->decompState = ZWRAP_useReset;
zwd->decompState = ZWRAP_useReset;
if (strm->total_in == ZSTD_HEADERSIZE) {
zwd->inBuffer.src = zwd->headerBuf;
@ -787,9 +787,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
strm->total_in += zwd->inBuffer.pos;
strm->next_in += zwd->inBuffer.pos;
strm->avail_in -= zwd->inBuffer.pos;
if (errorCode == 0) {
LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
zwd->decompState = ZWRAP_streamEnd;
if (errorCode == 0) {
LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
zwd->decompState = ZWRAP_streamEnd;
return Z_STREAM_END;
}
}