AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Address @terrelln's comments

Browse Source
This commit is contained in:
Sean Purcell 2017-04-13 17:47:55 -07:00
parent 2785b28e05
commit 9626cf1ac6
4 changed files with 109 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/seekable_format/examples/Makefile
View File

@ -12,7 +12,7 @@
LDFLAGS += -L../../../lib/ -lzstd LDFLAGS += -L../../../lib/ -lzstd
CPPFLAGS += -I../ -I../../../lib -I../../../lib/common CPPFLAGS += -I../ -I../../../lib -I../../../lib/common
CFLAGS = -O3 CFLAGS ?= -O3
CFLAGS += -g CFLAGS += -g
SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c

9
contrib/seekable_format/zstd_seekable.h
View File

@ -38,8 +38,8 @@ typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream;
* *
* Data streamed to the seekable compressor will automatically be split into * Data streamed to the seekable compressor will automatically be split into
* frames of size `maxFrameSize` (provided in ZSTD_seekable_initCStream()), * frames of size `maxFrameSize` (provided in ZSTD_seekable_initCStream()),
* or if none is provided, will be cut off whenver ZSTD_endFrame() is called * or if none is provided, will be cut off whenever ZSTD_seekable_endFrame() is
* or when the default maximum frame size is reached (approximately 4GB). * called or when the default maximum frame size (2GB) is reached.
* *
* Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object * Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
* for a new compression operation. * for a new compression operation.
@ -59,7 +59,6 @@ typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream;
* ZSTD_isError(). * ZSTD_isError().
* Note 1 : it's just a hint, to help latency a little, any other * Note 1 : it's just a hint, to help latency a little, any other
* value will work fine. * value will work fine.
* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
* *
* At any time, call ZSTD_seekable_endFrame() to end the current frame and * At any time, call ZSTD_seekable_endFrame() to end the current frame and
* start a new one. * start a new one.
@ -98,8 +97,8 @@ ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outB
* small, a size hint for how much data to provide. * small, a size hint for how much data to provide.
* An error code may also be returned, checkable with ZSTD_isError() * An error code may also be returned, checkable with ZSTD_isError()
* *
* Use ZSTD_initDStream to prepare for a new decompression operation using the * Use ZSTD_seekable_initDStream to prepare for a new decompression operation
* seektable loaded with ZSTD_seekable_loadSeekTable(). * using the seektable loaded with ZSTD_seekable_loadSeekTable().
* Data in the range [rangeStart, rangeEnd) will be decompressed. * Data in the range [rangeStart, rangeEnd) will be decompressed.
* *
* Call ZSTD_seekable_decompressStream() repetitively to consume input stream. * Call ZSTD_seekable_decompressStream() repetitively to consume input stream.

129
contrib/seekable_format/zstdseek_compress.c
View File

@ -13,9 +13,22 @@
#define XXH_NAMESPACE ZSTD_ #define XXH_NAMESPACE ZSTD_
#include "xxhash.h" #include "xxhash.h"
#include "zstd_internal.h" /* includes zstd.h */ #define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "zstd_errors.h"
#include "mem.h"
#include "zstd_seekable.h" #include "zstd_seekable.h"
#define CHECK_Z(f) { size_t const ret = (f); if (ret != 0) return ret; }
#undef ERROR
#define ERROR(name) ((size_t)-ZSTD_error_##name)
#undef MIN
#undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#define MAX(a, b) ((a) > (b) ? (a) : (b))
typedef struct { typedef struct {
U32 cSize; U32 cSize;
U32 dSize; U32 dSize;
@ -42,6 +55,8 @@ struct ZSTD_seekable_CStream_s {
int checksumFlag; int checksumFlag;
int writingSeekTable; int writingSeekTable;
U32 seekTablePos;
U32 seekTableIndex;
}; };
ZSTD_seekable_CStream* ZSTD_seekable_createCStream() ZSTD_seekable_CStream* ZSTD_seekable_createCStream()
@ -57,8 +72,8 @@ ZSTD_seekable_CStream* ZSTD_seekable_createCStream()
/* allocate some initial space */ /* allocate some initial space */
{ size_t const FRAMELOG_STARTING_CAPACITY = 16; { size_t const FRAMELOG_STARTING_CAPACITY = 16;
zcs->framelog.entries = zcs->framelog.entries = (framelogEntry_t*)malloc(
malloc(sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY); sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY);
if (zcs->framelog.entries == NULL) goto failed2; if (zcs->framelog.entries == NULL) goto failed2;
zcs->framelog.capacity = FRAMELOG_STARTING_CAPACITY; zcs->framelog.capacity = FRAMELOG_STARTING_CAPACITY;
} }
@ -105,6 +120,8 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
XXH64_reset(&zcs->xxhState, 0); XXH64_reset(&zcs->xxhState, 0);
} }
zcs->seekTablePos = 0;
zcs->seekTableIndex = 0;
zcs->writingSeekTable = 0; zcs->writingSeekTable = 0;
return ZSTD_initCStream(zcs->cstream, compressionLevel); return ZSTD_initCStream(zcs->cstream, compressionLevel);
@ -115,17 +132,6 @@ static size_t ZSTD_seekable_logFrame(ZSTD_seekable_CStream* zcs)
if (zcs->framelog.size == ZSTD_SEEKABLE_MAXFRAMES) if (zcs->framelog.size == ZSTD_SEEKABLE_MAXFRAMES)
return ERROR(frameIndex_tooLarge); return ERROR(frameIndex_tooLarge);
zcs->framelog.entries[zcs->framelog.size] = (framelogEntry_t)
{
.cSize = zcs->frameCSize,
.dSize = zcs->frameDSize,
};
if (zcs->checksumFlag)
zcs->framelog.entries[zcs->framelog.size].checksum =
/* take lower 32 bits of digest */
XXH64_digest(&zcs->xxhState) & 0xFFFFFFFFU;
zcs->framelog.size++;
/* grow the buffer if required */ /* grow the buffer if required */
if (zcs->framelog.size == zcs->framelog.capacity) { if (zcs->framelog.size == zcs->framelog.capacity) {
/* exponential size increase for constant amortized runtime */ /* exponential size increase for constant amortized runtime */
@ -139,6 +145,15 @@ static size_t ZSTD_seekable_logFrame(ZSTD_seekable_CStream* zcs)
zcs->framelog.capacity = newCapacity; zcs->framelog.capacity = newCapacity;
} }
zcs->framelog.entries[zcs->framelog.size] = (framelogEntry_t){
zcs->frameCSize, zcs->frameDSize,
};
if (zcs->checksumFlag)
zcs->framelog.entries[zcs->framelog.size].checksum =
XXH64_digest(&zcs->xxhState) & 0xFFFFFFFFU; /* take lower 32 bits of digest */
zcs->framelog.size++;
return 0; return 0;
} }
@ -208,7 +223,7 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
return (size_t)(zcs->maxFrameSize - zcs->frameDSize); return (size_t)(zcs->maxFrameSize - zcs->frameDSize);
} }
static size_t ZSTD_seekable_seekTableSize(ZSTD_seekable_CStream* zcs) static inline size_t ZSTD_seekable_seekTableSize(ZSTD_seekable_CStream const* zcs)
{ {
size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0); size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0);
size_t const seekTableLen = ZSTD_skippableHeaderSize + size_t const seekTableLen = ZSTD_skippableHeaderSize +
@ -218,14 +233,29 @@ static size_t ZSTD_seekable_seekTableSize(ZSTD_seekable_CStream* zcs)
return seekTableLen; return seekTableLen;
} }
static inline size_t ZSTD_stwrite32(ZSTD_seekable_CStream* zcs,
ZSTD_outBuffer* output, U32 const value,
U32 const offset)
{
if (zcs->seekTablePos < offset + 4) {
BYTE tmp[4]; /* so that we can work with buffers too small to write a whole word to */
size_t const lenWrite =
MIN(output->size - output->pos, offset + 4 - zcs->seekTablePos);
MEM_writeLE32(tmp, value);
memcpy((BYTE*)output->dst + output->pos,
tmp + (zcs->seekTablePos - offset), lenWrite);
output->pos += lenWrite;
zcs->seekTablePos += lenWrite;
if (lenWrite < 4) return ZSTD_seekable_seekTableSize(zcs) - zcs->seekTablePos;
}
return 0;
}
static size_t ZSTD_seekable_writeSeekTable(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) static size_t ZSTD_seekable_writeSeekTable(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)
{ {
BYTE* op = (BYTE*) output->dst; /* seekTableIndex: the current index in the table and
BYTE tmp[4]; /* so that we can work with buffers too small to write a whole word to */ * seekTableSize: the amount of the table written so far
/* repurpose
* zcs->frameDSize: the current index in the table and
* zcs->frameCSize: the amount of the table written so far
* *
* This function is written this way so that if it has to return early * This function is written this way so that if it has to return early
* because of a small buffer, it can keep going where it left off. * because of a small buffer, it can keep going where it left off.
@ -234,53 +264,48 @@ static size_t ZSTD_seekable_writeSeekTable(ZSTD_seekable_CStream* zcs, ZSTD_outB
size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0); size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0);
size_t const seekTableLen = ZSTD_seekable_seekTableSize(zcs); size_t const seekTableLen = ZSTD_seekable_seekTableSize(zcs);
#define st_write32(x, o) \ CHECK_Z(ZSTD_stwrite32(zcs, output, ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0));
do { \ CHECK_Z(ZSTD_stwrite32(zcs, output, seekTableLen - ZSTD_skippableHeaderSize,
if (zcs->frameCSize < (o) + 4) { \ 4));
size_t const lenWrite = MIN(output->size - output->pos, \
(o) + 4 - zcs->frameCSize); \
MEM_writeLE32(tmp, (x)); \
memcpy(op + output->pos, tmp + (zcs->frameCSize - (o)), lenWrite); \
zcs->frameCSize += lenWrite; \
output->pos += lenWrite; \
if (lenWrite < 4) return seekTableLen - zcs->frameCSize; \
} \
} while (0)
st_write32(ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0); while (zcs->seekTableIndex < zcs->framelog.size) {
st_write32(seekTableLen - ZSTD_skippableHeaderSize, 4); CHECK_Z(ZSTD_stwrite32(
zcs, output, zcs->framelog.entries[zcs->seekTableIndex].cSize,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->seekTableIndex));
CHECK_Z(ZSTD_stwrite32(
zcs, output, zcs->framelog.entries[zcs->seekTableIndex].dSize,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->seekTableIndex + 4));
while (zcs->frameDSize < zcs->framelog.size) {
st_write32(zcs->framelog.entries[zcs->frameDSize].cSize,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize);
st_write32(zcs->framelog.entries[zcs->frameDSize].dSize,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize + 4);
if (zcs->checksumFlag) { if (zcs->checksumFlag) {
st_write32(zcs->framelog.entries[zcs->frameDSize].checksum, CHECK_Z(ZSTD_stwrite32(
ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize + 8); zcs, output,
zcs->framelog.entries[zcs->seekTableIndex].checksum,
ZSTD_skippableHeaderSize + sizePerFrame * zcs->seekTableIndex +
8));
} }
zcs->frameDSize++; zcs->seekTableIndex++;
} }
st_write32(zcs->framelog.size, seekTableLen - ZSTD_seekTableFooterSize); CHECK_Z(ZSTD_stwrite32(zcs, output, zcs->framelog.size,
seekTableLen - ZSTD_seekTableFooterSize));
if (output->size - output->pos < 1) return seekTableLen - zcs->frameCSize; if (output->size - output->pos < 1) return seekTableLen - zcs->seekTablePos;
if (zcs->frameCSize < seekTableLen - 4) { if (zcs->seekTablePos < seekTableLen - 4) {
BYTE sfd = 0; BYTE sfd = 0;
sfd |= (zcs->checksumFlag) << 7; sfd |= (zcs->checksumFlag) << 7;
op[output->pos] = sfd; ((BYTE*)output->dst)[output->pos] = sfd;
output->pos++; output->pos++;
zcs->frameCSize++; zcs->seekTablePos++;
} }
st_write32(ZSTD_SEEKABLE_MAGICNUMBER, seekTableLen - 4); CHECK_Z(ZSTD_stwrite32(zcs, output, ZSTD_SEEKABLE_MAGICNUMBER,
seekTableLen - 4));
if (zcs->frameCSize != seekTableLen) return ERROR(GENERIC); if (zcs->seekTablePos != seekTableLen) return ERROR(GENERIC);
return 0; return 0;
#undef st_write32
} }
size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output)

49
contrib/seekable_format/zstdseek_decompress.c
View File

@ -13,9 +13,20 @@
#define XXH_NAMESPACE ZSTD_ #define XXH_NAMESPACE ZSTD_
#include "xxhash.h" #include "xxhash.h"
#include "zstd_internal.h" /* includes zstd.h */ #define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "zstd_errors.h"
#include "mem.h" /* includes zstd.h */
#include "zstd_seekable.h" #include "zstd_seekable.h"
#undef ERROR
#define ERROR(name) ((size_t)-ZSTD_error_##name)
#undef MIN
#undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#define MAX(a, b) ((a) > (b) ? (a) : (b))
typedef struct { typedef struct {
U64 cOffset; U64 cOffset;
U64 dOffset; U64 dOffset;
@ -107,11 +118,8 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
{ {
const BYTE* ip = (const BYTE*)src + srcSize; const BYTE* ip = (const BYTE*)src + srcSize;
U32 numFrames;
int checksumFlag; int checksumFlag;
U32 sizePerEntry;
/* footer is fixed size */ /* footer is fixed size */
if (srcSize < ZSTD_seekTableFooterSize) if (srcSize < ZSTD_seekTableFooterSize)
return ZSTD_seekTableFooterSize; return ZSTD_seekTableFooterSize;
@ -129,10 +137,9 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
} }
} }
numFrames = MEM_readLE32(ip-9); { U32 const numFrames = MEM_readLE32(ip-9);
sizePerEntry = 8 + (checksumFlag?4:0); U32 const sizePerEntry = 8 + (checksumFlag?4:0);
U32 const tableSize = sizePerEntry * numFrames;
{ U32 const tableSize = sizePerEntry * numFrames;
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize; U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize;
const BYTE* base = ip - frameSize; const BYTE* base = ip - frameSize;
@ -148,7 +155,8 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
{ /* Allocate an extra entry at the end so that we can do size { /* Allocate an extra entry at the end so that we can do size
* computations on the last element without special case */ * computations on the last element without special case */
seekEntry_t* entries = malloc(sizeof(seekEntry_t) * (numFrames + 1)); seekEntry_t* entries =
(seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1));
const BYTE* tableBase = base + ZSTD_skippableHeaderSize; const BYTE* tableBase = base + ZSTD_skippableHeaderSize;
U32 idx; U32 idx;
@ -167,8 +175,10 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
entries[idx].cOffset = cOffset; entries[idx].cOffset = cOffset;
entries[idx].dOffset = dOffset; entries[idx].dOffset = dOffset;
cOffset += MEM_readLE32(tableBase + pos); pos += 4; cOffset += MEM_readLE32(tableBase + pos);
dOffset += MEM_readLE32(tableBase + pos); pos += 4; pos += 4;
dOffset += MEM_readLE32(tableBase + pos);
pos += 4;
if (checksumFlag) { if (checksumFlag) {
entries[idx].checksum = MEM_readLE32(tableBase + pos); entries[idx].checksum = MEM_readLE32(tableBase + pos);
pos += 4; pos += 4;
@ -188,18 +198,17 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src,
size_t ZSTD_seekable_initDStream(ZSTD_seekable_DStream* zds, U64 rangeStart, U64 rangeEnd) size_t ZSTD_seekable_initDStream(ZSTD_seekable_DStream* zds, U64 rangeStart, U64 rangeEnd)
{ {
/* restrict range to the end of the file, of non-negative size */ /* restrict range to the end of the file, of non-negative size */
rangeStart = MIN(rangeStart, zds->seekTable.entries[zds->seekTable.tableLen].dOffset);
rangeEnd = MIN(rangeEnd, zds->seekTable.entries[zds->seekTable.tableLen].dOffset); rangeEnd = MIN(rangeEnd, zds->seekTable.entries[zds->seekTable.tableLen].dOffset);
rangeEnd = MAX(rangeEnd, rangeStart); rangeStart = MIN(rangeStart, rangeEnd);
zds->targetStart = rangeStart; zds->targetStart = rangeStart;
zds->targetEnd = rangeEnd; zds->targetEnd = rangeEnd;
zds->stage = zsds_seek; zds->stage = zsds_seek;
/* force a seek first */ /* force a seek first */
zds->curFrame = (U32) -1; zds->curFrame = (U32)-1;
zds->compressedOffset = (U64) -1; zds->compressedOffset = (U64)-1;
zds->decompressedOffset = (U64) -1; zds->decompressedOffset = (U64)-1;
if (zds->seekTable.checksumFlag) { if (zds->seekTable.checksumFlag) {
XXH64_reset(&zds->xxhState, 0); XXH64_reset(&zds->xxhState, 0);
@ -247,9 +256,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer
size_t const prevInputPos = input->pos; size_t const prevInputPos = input->pos;
ZSTD_outBuffer outTmp = { ZSTD_outBuffer outTmp = {
.dst = outBase, outBase, (size_t)MIN((U64)outLen, toDecompress), 0};
.size = (size_t)MIN((U64)outLen, toDecompress),
.pos = 0};
size_t const ret = size_t const ret =
ZSTD_decompressStream(zds->dstream, &outTmp, input); ZSTD_decompressStream(zds->dstream, &outTmp, input);
@ -286,9 +293,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer
size_t const prevInputPos = input->pos; size_t const prevInputPos = input->pos;
ZSTD_outBuffer outTmp = { ZSTD_outBuffer outTmp = {
.dst = outBase, outBase, (size_t)MIN((U64)outLen, toDecompress), 0};
.size = (size_t)MIN((U64)outLen, toDecompress),
.pos = 0};
size_t const ret = size_t const ret =
ZSTD_decompressStream(zds->dstream, &outTmp, input); ZSTD_decompressStream(zds->dstream, &outTmp, input);