From 5ee1135f306a94df1c65889e13b3c70d4d4e6ef7 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Wed, 12 Apr 2017 11:06:00 -0700 Subject: [PATCH] s/chunk/frame/ --- .../examples/seekable_compression.c | 10 +- contrib/seekable_format/zstd_seekable.h | 36 ++-- .../zstd_seekable_compression_format.md | 20 +-- contrib/seekable_format/zstdseek_compress.c | 160 +++++++++--------- contrib/seekable_format/zstdseek_decompress.c | 54 +++--- lib/common/error_private.c | 2 +- lib/common/zstd_errors.h | 2 +- 7 files changed, 142 insertions(+), 142 deletions(-) diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c index 8d3fe857..a33952d9 100644 --- a/contrib/seekable_format/examples/seekable_compression.c +++ b/contrib/seekable_format/examples/seekable_compression.c @@ -59,7 +59,7 @@ static size_t fclose_orDie(FILE* file) exit(6); } -static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned chunkSize) +static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize) { FILE* const fin = fopen_orDie(fname, "rb"); FILE* const fout = fopen_orDie(outName, "wb"); @@ -70,7 +70,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_seekable_CStream* const cstream = ZSTD_seekable_createCStream(); if (cstream==NULL) { fprintf(stderr, "ZSTD_seekable_createCStream() error \n"); exit(10); } - size_t const initResult = ZSTD_seekable_initCStream(cstream, cLevel, 1, chunkSize); + size_t const initResult = ZSTD_seekable_initCStream(cstream, cLevel, 1, frameSize); if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } size_t read, toRead = buffInSize; @@ -116,15 +116,15 @@ int main(int argc, const char** argv) { if (argc!=3) { printf("wrong arguments\n"); printf("usage:\n"); - printf("%s FILE CHUNK_SIZE\n", exeName); + printf("%s FILE FRAME_SIZE\n", exeName); return 1; } { const char* const inFileName = argv[1]; - unsigned const chunkSize = (unsigned)atoi(argv[2]); + unsigned const frameSize = (unsigned)atoi(argv[2]); const char* const outFileName = createOutFilename_orDie(inFileName); - compressFile_orDie(inFileName, outFileName, 5, chunkSize); + compressFile_orDie(inFileName, outFileName, 5, frameSize); } return 0; diff --git a/contrib/seekable_format/zstd_seekable.h b/contrib/seekable_format/zstd_seekable.h index 959b1bd0..f389ec97 100644 --- a/contrib/seekable_format/zstd_seekable.h +++ b/contrib/seekable_format/zstd_seekable.h @@ -9,17 +9,17 @@ static const unsigned ZSTD_seekTableFooterSize = 9; #define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1 -#define ZSTD_SEEKABLE_MAXCHUNKS 0x8000000U +#define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U /* 0xFE03F607 is the largest number x such that ZSTD_compressBound(x) fits in a 32-bit integer */ -#define ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE 0xFE03F607 +#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0xFE03F607 /*-**************************************************************************** * Seekable Format * -* The seekable format splits the compressed data into a series of "chunks", +* The seekable format splits the compressed data into a series of "frames", * each compressed individually so that decompression of a section in the -* middle of an archive only requires zstd to decompress at most a chunk's +* middle of an archive only requires zstd to decompress at most a frame's * worth of extra data, instead of the entire archive. ******************************************************************************/ @@ -37,15 +37,15 @@ typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream; * compressor. * * Data streamed to the seekable compressor will automatically be split into -* chunks of size `maxChunkSize` (provided in ZSTD_seekable_initCStream()), -* or if none is provided, will be cut off whenver ZSTD_endChunk() is called -* or when the default maximum chunk size is reached (approximately 4GB). +* frames of size `maxFrameSize` (provided in ZSTD_seekable_initCStream()), +* or if none is provided, will be cut off whenver ZSTD_endFrame() is called +* or when the default maximum frame size is reached (approximately 4GB). * * Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object * for a new compression operation. -* `maxChunkSize` indicates the size at which to automatically start a new -* seekable frame. `maxChunkSize == 0` implies the default maximum size. -* `checksumFlag` indicates whether or not the seek table should include chunk +* `maxFrameSize` indicates the size at which to automatically start a new +* seekable frame. `maxFrameSize == 0` implies the default maximum size. +* `checksumFlag` indicates whether or not the seek table should include frame * checksums on the uncompressed data for verification. * @return : a size hint for input to provide for compression, or an error code * checkable with ZSTD_isError() @@ -61,14 +61,14 @@ typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream; * value will work fine. * Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() * -* At any time, call ZSTD_seekable_endChunk() to end the current chunk and +* At any time, call ZSTD_seekable_endFrame() to end the current frame and * start a new one. * -* ZSTD_endStream() will end the current chunk, and then write the seek table -* so that decompressors can efficiently find compressed chunks. -* ZSTD_endStream() may return a number > 0 if it was unable to flush all the -* necessary data to `output`. In this case, it should be called again until -* all remaining data is flushed out and 0 is returned. +* ZSTD_seekable_endStream() will end the current frame, and then write the seek +* table so that decompressors can efficiently find compressed frames. +* ZSTD_seekable_endStream() may return a number > 0 if it was unable to flush +* all the necessary data to `output`. In this case, it should be called again +* until all remaining data is flushed out and 0 is returned. ******************************************************************************/ /*===== Seekable compressor management =====*/ @@ -76,9 +76,9 @@ ZSTDLIB_API ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void); ZSTDLIB_API size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs); /*===== Seekable compression functions =====*/ -ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxChunkSize); +ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxFrameSize); ZSTDLIB_API size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -ZSTDLIB_API size_t ZSTD_seekable_endChunk(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output); /*-**************************************************************************** diff --git a/contrib/seekable_format/zstd_seekable_compression_format.md b/contrib/seekable_format/zstd_seekable_compression_format.md index cc98150c..bf3080f7 100644 --- a/contrib/seekable_format/zstd_seekable_compression_format.md +++ b/contrib/seekable_format/zstd_seekable_compression_format.md @@ -18,7 +18,7 @@ Distribution of this document is unlimited. ## Introduction This document defines a format for compressed data to be stored so that subranges of the data can be efficiently decompressed without requiring the entire document to be decompressed. -This is done by splitting up the input data into chunks, +This is done by splitting up the input data into frames, each of which are compressed independently, and so can be decompressed independently. Decompression then takes advantage of a provided 'seek table', which allows the decompressor to immediately jump to the desired data. This is done in a way that is compatible with the original Zstandard format by placing the seek table in a Zstandard skippable frame. @@ -31,7 +31,7 @@ In this document: ## Format -The format consists of a number of chunks (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table. +The format consists of a number of frames (Zstandard compressed frames and skippable frames), followed by a final skippable frame at the end containing the seek table. ### Seek Table Format The structure of the seek table frame is as follows: @@ -58,7 +58,7 @@ This is for compatibility with [Zstandard skippable frames]. #### `Seek_Table_Footer` The seek table footer format is as follows: -|`Number_Of_Chunks`|`Seek_Table_Descriptor`|`Seekable_Magic_Number`| +|`Number_Of_Frames`|`Seek_Table_Descriptor`|`Seekable_Magic_Number`| |------------------|-----------------------|-----------------------| | 4 bytes | 1 byte | 4 bytes | @@ -68,9 +68,9 @@ Value : 0x8F92EAB1. This value must be the last bytes present in the compressed file so that decoders can efficiently find it and determine if there is an actual seek table present. -__`Number_Of_Chunks`__ +__`Number_Of_Frames`__ -The number of stored chunks in the data. +The number of stored frames in the data. __`Seek_Table_Descriptor`__ @@ -87,13 +87,13 @@ for example the addition of inline dictionaries. __`Checksum_Flag`__ -If the checksum flag is set, each of the seek table entries contains a 4 byte checksum of the uncompressed data contained in its chunk. +If the checksum flag is set, each of the seek table entries contains a 4 byte checksum of the uncompressed data contained in its frame. `Reserved_Bits` are not currently used but may be used in the future for breaking changes, so a compliant decoder should ensure they are set to 0. `Unused_Bits` may be used in the future for non-breaking changes, so a compliant decoder should not interpret these bits. #### __`Seek_Table_Entries`__ -`Seek_Table_Entries` consists of `Number_Of_Chunks` (one for each chunk in the data, not including the seek table frame) entries of the following form, in sequence: +`Seek_Table_Entries` consists of `Number_Of_Frames` (one for each frame in the data, not including the seek table frame) entries of the following form, in sequence: |`Compressed_Size`|`Decompressed_Size`|`[Checksum]`| |-----------------|-------------------|------------| @@ -101,12 +101,12 @@ If the checksum flag is set, each of the seek table entries contains a 4 byte ch __`Compressed_Size`__ -The compressed size of the chunk. -The cumulative sum of the `Compressed_Size` fields of chunks `0` to `i` gives the offset in the compressed file of chunk `i+1`. +The compressed size of the frame. +The cumulative sum of the `Compressed_Size` fields of frames `0` to `i` gives the offset in the compressed file of frame `i+1`. __`Decompressed_Size`__ -The size of the decompressed data contained in the chunk. For skippable or otherwise empty frames, this value is 0. +The size of the decompressed data contained in the frame. For skippable or otherwise empty frames, this value is 0. __`Checksum`__ diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c index 44dd4afc..df4feff2 100644 --- a/contrib/seekable_format/zstdseek_compress.c +++ b/contrib/seekable_format/zstdseek_compress.c @@ -20,24 +20,24 @@ typedef struct { U32 cSize; U32 dSize; U32 checksum; -} chunklogEntry_t; +} framelogEntry_t; typedef struct { - chunklogEntry_t* entries; + framelogEntry_t* entries; U32 size; U32 capacity; -} chunklog_t; +} framelog_t; struct ZSTD_seekable_CStream_s { ZSTD_CStream* cstream; - chunklog_t chunklog; + framelog_t framelog; - U32 chunkCSize; - U32 chunkDSize; + U32 frameCSize; + U32 frameDSize; XXH64_state_t xxhState; - U32 maxChunkSize; + U32 maxFrameSize; int checksumFlag; @@ -56,11 +56,11 @@ ZSTD_seekable_CStream* ZSTD_seekable_createCStream() if (zcs->cstream == NULL) goto failed1; /* allocate some initial space */ - { size_t const CHUNKLOG_STARTING_CAPACITY = 16; - zcs->chunklog.entries = - malloc(sizeof(chunklogEntry_t) * CHUNKLOG_STARTING_CAPACITY); - if (zcs->chunklog.entries == NULL) goto failed2; - zcs->chunklog.capacity = CHUNKLOG_STARTING_CAPACITY; + { size_t const FRAMELOG_STARTING_CAPACITY = 16; + zcs->framelog.entries = + malloc(sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY); + if (zcs->framelog.entries == NULL) goto failed2; + zcs->framelog.capacity = FRAMELOG_STARTING_CAPACITY; } return zcs; @@ -76,7 +76,7 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs) { if (zcs == NULL) return 0; /* support free on null */ ZSTD_freeCStream(zcs->cstream); - free(zcs->chunklog.entries); + free(zcs->framelog.entries); free(zcs); return 0; @@ -85,20 +85,20 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs) size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, - U32 maxChunkSize) + U32 maxFrameSize) { - zcs->chunklog.size = 0; - zcs->chunkCSize = 0; - zcs->chunkDSize = 0; + zcs->framelog.size = 0; + zcs->frameCSize = 0; + zcs->frameDSize = 0; - /* make sure maxChunkSize has a reasonable value */ - if (maxChunkSize > ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE) { + /* make sure maxFrameSize has a reasonable value */ + if (maxFrameSize > ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE) { return ERROR(compressionParameter_unsupported); } - zcs->maxChunkSize = maxChunkSize - ? maxChunkSize - : ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE; + zcs->maxFrameSize = maxFrameSize + ? maxFrameSize + : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE; zcs->checksumFlag = checksumFlag; if (zcs->checksumFlag) { @@ -110,58 +110,58 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, return ZSTD_initCStream(zcs->cstream, compressionLevel); } -static size_t ZSTD_seekable_logChunk(ZSTD_seekable_CStream* zcs) +static size_t ZSTD_seekable_logFrame(ZSTD_seekable_CStream* zcs) { - if (zcs->chunklog.size == ZSTD_SEEKABLE_MAXCHUNKS) - return ERROR(chunkIndex_tooLarge); + if (zcs->framelog.size == ZSTD_SEEKABLE_MAXFRAMES) + return ERROR(frameIndex_tooLarge); - zcs->chunklog.entries[zcs->chunklog.size] = (chunklogEntry_t) + zcs->framelog.entries[zcs->framelog.size] = (framelogEntry_t) { - .cSize = zcs->chunkCSize, - .dSize = zcs->chunkDSize, + .cSize = zcs->frameCSize, + .dSize = zcs->frameDSize, }; if (zcs->checksumFlag) - zcs->chunklog.entries[zcs->chunklog.size].checksum = + zcs->framelog.entries[zcs->framelog.size].checksum = /* take lower 32 bits of digest */ XXH64_digest(&zcs->xxhState) & 0xFFFFFFFFU; - zcs->chunklog.size++; + zcs->framelog.size++; /* grow the buffer if required */ - if (zcs->chunklog.size == zcs->chunklog.capacity) { + if (zcs->framelog.size == zcs->framelog.capacity) { /* exponential size increase for constant amortized runtime */ - size_t const newCapacity = zcs->chunklog.capacity * 2; - chunklogEntry_t* const newEntries = realloc(zcs->chunklog.entries, - sizeof(chunklogEntry_t) * newCapacity); + size_t const newCapacity = zcs->framelog.capacity * 2; + framelogEntry_t* const newEntries = realloc(zcs->framelog.entries, + sizeof(framelogEntry_t) * newCapacity); if (newEntries == NULL) return ERROR(memory_allocation); - zcs->chunklog.entries = newEntries; - zcs->chunklog.capacity = newCapacity; + zcs->framelog.entries = newEntries; + zcs->framelog.capacity = newCapacity; } return 0; } -size_t ZSTD_seekable_endChunk(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) +size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) { size_t const prevOutPos = output->pos; /* end the frame */ size_t ret = ZSTD_endStream(zcs->cstream, output); - zcs->chunkCSize += output->pos - prevOutPos; + zcs->frameCSize += output->pos - prevOutPos; /* need to flush before doing the rest */ if (ret) return ret; /* frame done */ - /* store the chunk data for later */ - ret = ZSTD_seekable_logChunk(zcs); + /* store the frame data for later */ + ret = ZSTD_seekable_logFrame(zcs); if (ret) return ret; - /* reset for the next chunk */ - zcs->chunkCSize = 0; - zcs->chunkDSize = 0; + /* reset for the next frame */ + zcs->frameCSize = 0; + zcs->frameDSize = 0; ZSTD_resetCStream(zcs->cstream, 0); if (zcs->checksumFlag) @@ -175,9 +175,9 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* const BYTE* const inBase = (const BYTE*) input->src + input->pos; size_t inLen = input->size - input->pos; - inLen = MIN(inLen, (size_t)(zcs->maxChunkSize - zcs->chunkDSize)); + inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize)); - /* if we haven't finished flushing the last chunk, don't start writing a new one */ + /* if we haven't finished flushing the last frame, don't start writing a new one */ if (inLen > 0) { ZSTD_inBuffer inTmp = { inBase, inLen, 0 }; size_t const prevOutPos = output->pos; @@ -188,31 +188,31 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* XXH64_update(&zcs->xxhState, inBase, inTmp.pos); } - zcs->chunkCSize += output->pos - prevOutPos; - zcs->chunkDSize += inTmp.pos; + zcs->frameCSize += output->pos - prevOutPos; + zcs->frameDSize += inTmp.pos; input->pos += inTmp.pos; if (ZSTD_isError(ret)) return ret; } - if (zcs->maxChunkSize == zcs->chunkDSize) { - /* log the chunk and start over */ - size_t const ret = ZSTD_seekable_endChunk(zcs, output); + if (zcs->maxFrameSize == zcs->frameDSize) { + /* log the frame and start over */ + size_t const ret = ZSTD_seekable_endFrame(zcs, output); if (ZSTD_isError(ret)) return ret; - /* get the client ready for the next chunk */ - return (size_t)zcs->maxChunkSize; + /* get the client ready for the next frame */ + return (size_t)zcs->maxFrameSize; } - return (size_t)(zcs->maxChunkSize - zcs->chunkDSize); + return (size_t)(zcs->maxFrameSize - zcs->frameDSize); } static size_t ZSTD_seekable_seekTableSize(ZSTD_seekable_CStream* zcs) { - size_t const sizePerChunk = 8 + (zcs->checksumFlag?4:0); + size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0); size_t const seekTableLen = ZSTD_skippableHeaderSize + - sizePerChunk * zcs->chunklog.size + + sizePerFrame * zcs->framelog.size + ZSTD_seekTableFooterSize; return seekTableLen; @@ -224,60 +224,60 @@ static size_t ZSTD_seekable_writeSeekTable(ZSTD_seekable_CStream* zcs, ZSTD_outB BYTE tmp[4]; /* so that we can work with buffers too small to write a whole word to */ /* repurpose - * zcs->chunkDSize: the current index in the table and - * zcs->chunkCSize: the amount of the table written so far + * zcs->frameDSize: the current index in the table and + * zcs->frameCSize: the amount of the table written so far * * This function is written this way so that if it has to return early * because of a small buffer, it can keep going where it left off. */ - size_t const sizePerChunk = 8 + (zcs->checksumFlag?4:0); + size_t const sizePerFrame = 8 + (zcs->checksumFlag?4:0); size_t const seekTableLen = ZSTD_seekable_seekTableSize(zcs); #define st_write32(x, o) \ do { \ - if (zcs->chunkCSize < (o) + 4) { \ + if (zcs->frameCSize < (o) + 4) { \ size_t const lenWrite = MIN(output->size - output->pos, \ - (o) + 4 - zcs->chunkCSize); \ + (o) + 4 - zcs->frameCSize); \ MEM_writeLE32(tmp, (x)); \ - memcpy(op + output->pos, tmp + (zcs->chunkCSize - (o)), lenWrite); \ - zcs->chunkCSize += lenWrite; \ + memcpy(op + output->pos, tmp + (zcs->frameCSize - (o)), lenWrite); \ + zcs->frameCSize += lenWrite; \ output->pos += lenWrite; \ - if (lenWrite < 4) return seekTableLen - zcs->chunkCSize; \ + if (lenWrite < 4) return seekTableLen - zcs->frameCSize; \ } \ } while (0) st_write32(ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0); st_write32(seekTableLen - ZSTD_skippableHeaderSize, 4); - while (zcs->chunkDSize < zcs->chunklog.size) { - st_write32(zcs->chunklog.entries[zcs->chunkDSize].cSize, - ZSTD_skippableHeaderSize + sizePerChunk * zcs->chunkDSize); - st_write32(zcs->chunklog.entries[zcs->chunkDSize].dSize, - ZSTD_skippableHeaderSize + sizePerChunk * zcs->chunkDSize + 4); + while (zcs->frameDSize < zcs->framelog.size) { + st_write32(zcs->framelog.entries[zcs->frameDSize].cSize, + ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize); + st_write32(zcs->framelog.entries[zcs->frameDSize].dSize, + ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize + 4); if (zcs->checksumFlag) { - st_write32(zcs->chunklog.entries[zcs->chunkDSize].checksum, - ZSTD_skippableHeaderSize + sizePerChunk * zcs->chunkDSize + 8); + st_write32(zcs->framelog.entries[zcs->frameDSize].checksum, + ZSTD_skippableHeaderSize + sizePerFrame * zcs->frameDSize + 8); } - zcs->chunkDSize++; + zcs->frameDSize++; } - st_write32(zcs->chunklog.size, seekTableLen - ZSTD_seekTableFooterSize); + st_write32(zcs->framelog.size, seekTableLen - ZSTD_seekTableFooterSize); - if (output->size - output->pos < 1) return seekTableLen - zcs->chunkCSize; - if (zcs->chunkCSize < seekTableLen - 4) { + if (output->size - output->pos < 1) return seekTableLen - zcs->frameCSize; + if (zcs->frameCSize < seekTableLen - 4) { BYTE sfd = 0; sfd |= (zcs->checksumFlag) << 7; op[output->pos] = sfd; output->pos++; - zcs->chunkCSize++; + zcs->frameCSize++; } st_write32(ZSTD_SEEKABLE_MAGICNUMBER, seekTableLen - 4); - if (zcs->chunkCSize != seekTableLen) return ERROR(GENERIC); + if (zcs->frameCSize != seekTableLen) return ERROR(GENERIC); return 0; #undef st_write32 @@ -285,11 +285,11 @@ static size_t ZSTD_seekable_writeSeekTable(ZSTD_seekable_CStream* zcs, ZSTD_outB size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) { - if (!zcs->writingSeekTable && zcs->chunkDSize) { - const size_t endChunk = ZSTD_seekable_endChunk(zcs, output); - if (ZSTD_isError(endChunk)) return endChunk; + if (!zcs->writingSeekTable && zcs->frameDSize) { + const size_t endFrame = ZSTD_seekable_endFrame(zcs, output); + if (ZSTD_isError(endFrame)) return endFrame; /* return an accurate size hint */ - if (endChunk) return endChunk + ZSTD_seekable_seekTableSize(zcs); + if (endFrame) return endFrame + ZSTD_seekable_seekTableSize(zcs); } zcs->writingSeekTable = 1; diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index 2b97480f..add62c76 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -29,11 +29,11 @@ typedef struct { int checksumFlag; } seekTable_t; -/** ZSTD_seekable_offsetToChunk() : - * Performs a binary search to find the last chunk with a decompressed offset +/** ZSTD_seekable_offsetToFrame() : + * Performs a binary search to find the last frame with a decompressed offset * <= pos - * @return : the chunk's index */ -static U32 ZSTD_seekable_offsetToChunk(const seekTable_t* table, U64 pos) + * @return : the frame's index */ +static U32 ZSTD_seekable_offsetToFrame(const seekTable_t* table, U64 pos) { U32 lo = 0; U32 hi = table->tableLen; @@ -61,7 +61,7 @@ struct ZSTD_seekable_DStream_s { ZSTD_DStream* dstream; seekTable_t seekTable; - U32 curChunk; + U32 curFrame; U64 compressedOffset; U64 decompressedOffset; @@ -107,7 +107,7 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, { const BYTE* ip = (const BYTE*)src + srcSize; - U32 numChunks; + U32 numFrames; int checksumFlag; U32 sizePerEntry; @@ -129,10 +129,10 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, } } - numChunks = MEM_readLE32(ip-9); + numFrames = MEM_readLE32(ip-9); sizePerEntry = 8 + (checksumFlag?4:0); - { U32 const tableSize = sizePerEntry * numChunks; + { U32 const tableSize = sizePerEntry * numFrames; U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize; const BYTE* base = ip - frameSize; @@ -148,7 +148,7 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, { /* Allocate an extra entry at the end so that we can do size * computations on the last element without special case */ - seekEntry_t* entries = malloc(sizeof(seekEntry_t) * (numChunks + 1)); + seekEntry_t* entries = malloc(sizeof(seekEntry_t) * (numFrames + 1)); const BYTE* tableBase = base + ZSTD_skippableHeaderSize; U32 idx; @@ -163,7 +163,7 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, } /* compute cumulative positions */ - for (idx = 0, pos = 0; idx < numChunks; idx++) { + for (idx = 0, pos = 0; idx < numFrames; idx++) { entries[idx].cOffset = cOffset; entries[idx].dOffset = dOffset; @@ -174,11 +174,11 @@ size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, pos += 4; } } - entries[numChunks].cOffset = cOffset; - entries[numChunks].dOffset = dOffset; + entries[numFrames].cOffset = cOffset; + entries[numFrames].dOffset = dOffset; zds->seekTable.entries = entries; - zds->seekTable.tableLen = numChunks; + zds->seekTable.tableLen = numFrames; zds->seekTable.checksumFlag = checksumFlag; return 0; } @@ -197,7 +197,7 @@ size_t ZSTD_seekable_initDStream(ZSTD_seekable_DStream* zds, U64 rangeStart, U64 zds->stage = zsds_seek; /* force a seek first */ - zds->curChunk = (U32) -1; + zds->curFrame = (U32) -1; zds->compressedOffset = (U64) -1; zds->decompressedOffset = (U64) -1; @@ -271,7 +271,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer /* need more input */ return MIN( ZSTD_DStreamInSize(), - (size_t)(jt->entries[zds->curChunk + 1] + (size_t)(jt->entries[zds->curFrame + 1] .cOffset - zds->compressedOffset)); } @@ -281,7 +281,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer { U64 const toDecompress = MIN(zds->targetEnd, - jt->entries[zds->curChunk + 1].dOffset) - + jt->entries[zds->curFrame + 1].dOffset) - zds->decompressedOffset; size_t const prevInputPos = input->pos; @@ -305,7 +305,7 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer if (ret == 0) { /* verify the checksum */ U32 const digest = XXH64_digest(&zds->xxhState) & 0xFFFFFFFFU; - if (digest != jt->entries[zds->curChunk].checksum) { + if (digest != jt->entries[zds->curFrame].checksum) { return ERROR(checksum_wrong); } @@ -323,9 +323,9 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer /* frame is done */ /* make sure this lines up with the expected frame border */ if (zds->decompressedOffset != - jt->entries[zds->curChunk + 1].dOffset || + jt->entries[zds->curFrame + 1].dOffset || zds->compressedOffset != - jt->entries[zds->curChunk + 1].cOffset) + jt->entries[zds->curFrame + 1].cOffset) return ERROR(corruption_detected); ZSTD_resetDStream(zds->dstream); zds->stage = zsds_seek; @@ -334,29 +334,29 @@ size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer /* need more input */ return MIN(ZSTD_DStreamInSize(), (size_t)( - jt->entries[zds->curChunk + 1].cOffset - + jt->entries[zds->curFrame + 1].cOffset - zds->compressedOffset)); } } case zsds_seek: { - U32 targetChunk; + U32 targetFrame; if (zds->decompressedOffset < zds->targetStart || zds->decompressedOffset >= zds->targetEnd) { /* haven't started yet */ - targetChunk = ZSTD_seekable_offsetToChunk(jt, zds->targetStart); + targetFrame = ZSTD_seekable_offsetToFrame(jt, zds->targetStart); } else { - targetChunk = ZSTD_seekable_offsetToChunk(jt, zds->decompressedOffset); + targetFrame = ZSTD_seekable_offsetToFrame(jt, zds->decompressedOffset); } - zds->curChunk = targetChunk; + zds->curFrame = targetFrame; - if (zds->compressedOffset == jt->entries[targetChunk].cOffset) { + if (zds->compressedOffset == jt->entries[targetFrame].cOffset) { zds->stage = zsds_decompress; break; } - zds->nextSeek = jt->entries[targetChunk].cOffset; - zds->decompressedOffset = jt->entries[targetChunk].dOffset; + zds->nextSeek = jt->entries[targetFrame].cOffset; + zds->decompressedOffset = jt->entries[targetFrame].dOffset; /* signal to user that a seek is required */ return ERROR(needSeek); } diff --git a/lib/common/error_private.c b/lib/common/error_private.c index 4084edca..f32c6abd 100644 --- a/lib/common/error_private.c +++ b/lib/common/error_private.c @@ -38,7 +38,7 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; - case PREFIX(chunkIndex_tooLarge): return "Chunk index is too large"; + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(needSeek): return "Wrong file position, a seek is required to continue"; case PREFIX(maxCode): default: return notErrorCode; diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h index 13a5608e..d11c1ba2 100644 --- a/lib/common/zstd_errors.h +++ b/lib/common/zstd_errors.h @@ -58,7 +58,7 @@ typedef enum { ZSTD_error_dictionary_corrupted, ZSTD_error_dictionary_wrong, ZSTD_error_dictionaryCreation_failed, - ZSTD_error_chunkIndex_tooLarge, + ZSTD_error_frameIndex_tooLarge, ZSTD_error_needSeek, ZSTD_error_maxCode } ZSTD_ErrorCode;