Move seekable format content to /contrib
This commit is contained in:
Sean Purcell
parent
b13da709e8
commit
d048fefef7
2
contrib/seekable_format/examples/.gitignore
vendored
Normal file
2
contrib/seekable_format/examples/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
seekable_compression
|
||||
seekable_decompression
|
||||
35
contrib/seekable_format/examples/Makefile
Normal file
35
contrib/seekable_format/examples/Makefile
Normal file
@ -0,0 +1,35 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2017-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ################################################################
|
||||
|
||||
# This Makefile presumes libzstd is built, using `make` in / or /lib/
|
||||
|
||||
LDFLAGS += -L../../../lib/ -lzstd
|
||||
CPPFLAGS += -I../ -I../../../lib -I../../../lib/common
|
||||
|
||||
CFLAGS = -O3
|
||||
CFLAGS += -g
|
||||
|
||||
SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c
|
||||
|
||||
.PHONY: default all clean test
|
||||
|
||||
default: all
|
||||
|
||||
all: seekable_compression seekable_decompression
|
||||
|
||||
seekable_compression : seekable_compression.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(SEEKABLE_OBJS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
seekable_decompression : seekable_decompression.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(SEEKABLE_OBJS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp* result* *.zst \
|
||||
seekable_compression seekable_decompression
|
||||
@echo Cleaning completed
|
||||
BIN
contrib/seekable_format/examples/seekable_compression
Executable file
BIN
contrib/seekable_format/examples/seekable_compression
Executable file
Binary file not shown.
@ -12,6 +12,8 @@
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
|
||||
#include "zstd_seekable.h"
|
||||
|
||||
static void* malloc_orDie(size_t size)
|
||||
{
|
||||
void* const buff = malloc(size);
|
||||
@ -15,6 +15,8 @@
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include <zstd_errors.h>
|
||||
|
||||
#include "zstd_seekable.h"
|
||||
|
||||
|
||||
static void* malloc_orDie(size_t size)
|
||||
{
|
||||
140
contrib/seekable_format/zstd_seekable.h
Normal file
140
contrib/seekable_format/zstd_seekable.h
Normal file
@ -0,0 +1,140 @@
|
||||
#ifndef SEEKABLE_H
|
||||
#define SEEKABLE_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static const unsigned ZSTD_seekTableFooterSize = 9;
|
||||
|
||||
#define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1
|
||||
|
||||
#define ZSTD_SEEKABLE_MAXCHUNKS 0x8000000U
|
||||
|
||||
/* 0xFE03F607 is the largest number x such that ZSTD_compressBound(x) fits in a 32-bit integer */
|
||||
#define ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE 0xFE03F607
|
||||
|
||||
/*-****************************************************************************
|
||||
* Seekable Format
|
||||
*
|
||||
* The seekable format splits the compressed data into a series of "chunks",
|
||||
* each compressed individually so that decompression of a section in the
|
||||
* middle of an archive only requires zstd to decompress at most a chunk's
|
||||
* worth of extra data, instead of the entire archive.
|
||||
******************************************************************************/
|
||||
|
||||
typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream;
|
||||
typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream;
|
||||
|
||||
/*-****************************************************************************
|
||||
* Seekable compression - HowTo
|
||||
* A ZSTD_seekable_CStream object is required to tracking streaming operation.
|
||||
* Use ZSTD_seekable_createCStream() and ZSTD_seekable_freeCStream() to create/
|
||||
* release resources.
|
||||
*
|
||||
* Streaming objects are reusable to avoid allocation and deallocation,
|
||||
* to start a new compression operation call ZSTD_seekable_initCStream() on the
|
||||
* compressor.
|
||||
*
|
||||
* Data streamed to the seekable compressor will automatically be split into
|
||||
* chunks of size `maxChunkSize` (provided in ZSTD_seekable_initCStream()),
|
||||
* or if none is provided, will be cut off whenver ZSTD_endChunk() is called
|
||||
* or when the default maximum chunk size is reached (approximately 4GB).
|
||||
*
|
||||
* Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
|
||||
* for a new compression operation.
|
||||
* `maxChunkSize` indicates the size at which to automatically start a new
|
||||
* seekable frame. `maxChunkSize == 0` implies the default maximum size.
|
||||
* `checksumFlag` indicates whether or not the seek table should include chunk
|
||||
* checksums on the uncompressed data for verification.
|
||||
* @return : a size hint for input to provide for compression, or an error code
|
||||
* checkable with ZSTD_isError()
|
||||
*
|
||||
* Use ZSTD_seekable_compressStream() repetitively to consume input stream.
|
||||
* The function will automatically update both `pos` fields.
|
||||
* Note that it may not consume the entire input, in which case `pos < size`,
|
||||
* and it's up to the caller to present again remaining data.
|
||||
* @return : a size hint, preferred nb of bytes to use as input for next
|
||||
* function call or an error code, which can be tested using
|
||||
* ZSTD_isError().
|
||||
* Note 1 : it's just a hint, to help latency a little, any other
|
||||
* value will work fine.
|
||||
* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
|
||||
*
|
||||
* At any time, call ZSTD_seekable_endChunk() to end the current chunk and
|
||||
* start a new one.
|
||||
*
|
||||
* ZSTD_endStream() will end the current chunk, and then write the seek table
|
||||
* so that decompressors can efficiently find compressed chunks.
|
||||
* ZSTD_endStream() may return a number > 0 if it was unable to flush all the
|
||||
* necessary data to `output`. In this case, it should be called again until
|
||||
* all remaining data is flushed out and 0 is returned.
|
||||
******************************************************************************/
|
||||
|
||||
/*===== Seekable compressor management =====*/
|
||||
ZSTDLIB_API ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs);
|
||||
|
||||
/*===== Seekable compression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxChunkSize);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_endChunk(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
|
||||
|
||||
/*-****************************************************************************
|
||||
* Seekable decompression - HowTo
|
||||
* A ZSTD_seekable_DStream object is required to tracking streaming operation.
|
||||
* Use ZSTD_seekable_createDStream() and ZSTD_seekable_freeDStream() to create/
|
||||
* release resources.
|
||||
*
|
||||
* Streaming objects are reusable to avoid allocation and deallocation,
|
||||
* to start a new compression operation call ZSTD_seekable_initDStream() on the
|
||||
* compressor.
|
||||
*
|
||||
* Use ZSTD_seekable_loadSeekTable() to load the seek table from a file.
|
||||
* `src` should point to a block of data read from the end of the file,
|
||||
* i.e. `src + srcSize` should always be the end of the file.
|
||||
* @return : 0 if the table was loaded successfully, or if `srcSize` was too
|
||||
* small, a size hint for how much data to provide.
|
||||
* An error code may also be returned, checkable with ZSTD_isError()
|
||||
*
|
||||
* Use ZSTD_initDStream to prepare for a new decompression operation using the
|
||||
* seektable loaded with ZSTD_seekable_loadSeekTable().
|
||||
* Data in the range [rangeStart, rangeEnd) will be decompressed.
|
||||
*
|
||||
* Call ZSTD_seekable_decompressStream() repetitively to consume input stream.
|
||||
* @return : There are a number of possible return codes for this function
|
||||
* - 0, the decompression operation has completed.
|
||||
* - An error code checkable with ZSTD_isError
|
||||
* + If this error code is ZSTD_error_needSeek, the user should seek
|
||||
* to the file position provided by ZSTD_seekable_getSeekOffset()
|
||||
* and indicate this to the stream with
|
||||
* ZSTD_seekable_updateOffset(), before resuming decompression
|
||||
* + Otherwise, this is a regular decompression error and the input
|
||||
* file is likely corrupted or the API was incorrectly used.
|
||||
* - A size hint, the preferred nb of bytes to provide as input to the
|
||||
* next function call to improve latency.
|
||||
*
|
||||
* ZSTD_seekable_getSeekOffset() and ZSTD_seekable_updateOffset() are helper
|
||||
* functions to indicate where the user should seek their file stream to, when
|
||||
* a different position is required to continue decompression.
|
||||
* Note that ZSTD_seekable_updateOffset will error if given an offset other
|
||||
* than the one requested from ZSTD_seekable_getSeekOffset().
|
||||
******************************************************************************/
|
||||
|
||||
/*===== Seekable decompressor management =====*/
|
||||
ZSTDLIB_API ZSTD_seekable_DStream* ZSTD_seekable_createDStream(void);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_freeDStream(ZSTD_seekable_DStream* zds);
|
||||
|
||||
/*===== Seekable decompression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_initDStream(ZSTD_seekable_DStream* zds, unsigned long long rangeStart, unsigned long long rangeEnd);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
ZSTDLIB_API unsigned long long ZSTD_seekable_getSeekOffset(ZSTD_seekable_DStream* zds);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_updateOffset(ZSTD_seekable_DStream* zds, unsigned long long offset);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@ -10,10 +10,11 @@
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#define XXH_NAMESPACE ZSTD_
|
||||
#include "xxhash.h"
|
||||
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "seekable.h"
|
||||
#include "zstd_seekable.h"
|
||||
|
||||
typedef struct {
|
||||
U32 cSize;
|
||||
@ -10,10 +10,11 @@
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#define XXH_NAMESPACE ZSTD_
|
||||
#include "xxhash.h"
|
||||
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "seekable.h"
|
||||
#include "zstd_seekable.h"
|
||||
|
||||
typedef struct {
|
||||
U64 cOffset;
|
||||
@ -17,6 +17,3 @@ __`zstd_manual.html`__ : Documentation on the functions found in `zstd.h`.
|
||||
See [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html) for
|
||||
the manual released with the latest official `zstd` release.
|
||||
|
||||
__`zstd_seekable_compression_format.md`__ : This document defines the Zstandard
|
||||
format for seekable compression.
|
||||
|
||||
|
||||
@ -28,9 +28,6 @@
|
||||
<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter20">Block functions</a></li>
|
||||
<li><a href="#Chapter21">Seekable Format</a></li>
|
||||
<li><a href="#Chapter22">Seekable compression - HowTo</a></li>
|
||||
<li><a href="#Chapter23">Seekable decompression - HowTo</a></li>
|
||||
</ol>
|
||||
<hr>
|
||||
<a name="Chapter1"></a><h2>Introduction</h2><pre>
|
||||
@ -671,111 +668,5 @@ size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, cons
|
||||
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter21"></a><h2>Seekable Format</h2><pre>
|
||||
The seekable format splits the compressed data into a series of "chunks",
|
||||
each compressed individually so that decompression of a section in the
|
||||
middle of an archive only requires zstd to decompress at most a chunk's
|
||||
worth of extra data, instead of the entire archive.
|
||||
<BR></pre>
|
||||
|
||||
<a name="Chapter22"></a><h2>Seekable compression - HowTo</h2><pre> A ZSTD_seekable_CStream object is required to tracking streaming operation.
|
||||
Use ZSTD_seekable_createCStream() and ZSTD_seekable_freeCStream() to create/
|
||||
release resources.
|
||||
|
||||
Streaming objects are reusable to avoid allocation and deallocation,
|
||||
to start a new compression operation call ZSTD_seekable_initCStream() on the
|
||||
compressor.
|
||||
|
||||
Data streamed to the seekable compressor will automatically be split into
|
||||
chunks of size `maxChunkSize` (provided in ZSTD_seekable_initCStream()),
|
||||
or if none is provided, will be cut off whenver ZSTD_endChunk() is called
|
||||
or when the default maximum chunk size is reached (approximately 4GB).
|
||||
|
||||
Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
|
||||
for a new compression operation.
|
||||
`maxChunkSize` indicates the size at which to automatically start a new
|
||||
seekable frame. `maxChunkSize == 0` implies the default maximum size.
|
||||
`checksumFlag` indicates whether or not the seek table should include chunk
|
||||
checksums on the uncompressed data for verification.
|
||||
@return : a size hint for input to provide for compression, or an error code
|
||||
checkable with ZSTD_isError()
|
||||
|
||||
Use ZSTD_seekable_compressStream() repetitively to consume input stream.
|
||||
The function will automatically update both `pos` fields.
|
||||
Note that it may not consume the entire input, in which case `pos < size`,
|
||||
and it's up to the caller to present again remaining data.
|
||||
@return : a size hint, preferred nb of bytes to use as input for next
|
||||
function call or an error code, which can be tested using
|
||||
ZSTD_isError().
|
||||
Note 1 : it's just a hint, to help latency a little, any other
|
||||
value will work fine.
|
||||
Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
|
||||
|
||||
At any time, call ZSTD_seekable_endChunk() to end the current chunk and
|
||||
start a new one.
|
||||
|
||||
ZSTD_endStream() will end the current chunk, and then write the seek table
|
||||
so that decompressors can efficiently find compressed chunks.
|
||||
ZSTD_endStream() may return a number > 0 if it was unable to flush all the
|
||||
necessary data to `output`. In this case, it should be called again until
|
||||
all remaining data is flushed out and 0 is returned.
|
||||
<BR></pre>
|
||||
|
||||
<h3>Seekable compressor management</h3><pre></pre><b><pre>ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void);
|
||||
size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs);
|
||||
</pre></b><BR>
|
||||
<h3>Seekable compression functions</h3><pre></pre><b><pre>size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxChunkSize);
|
||||
size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
size_t ZSTD_seekable_endChunk(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
|
||||
size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
|
||||
</pre></b><BR>
|
||||
<a name="Chapter23"></a><h2>Seekable decompression - HowTo</h2><pre> A ZSTD_seekable_DStream object is required to tracking streaming operation.
|
||||
Use ZSTD_seekable_createDStream() and ZSTD_seekable_freeDStream() to create/
|
||||
release resources.
|
||||
|
||||
Streaming objects are reusable to avoid allocation and deallocation,
|
||||
to start a new compression operation call ZSTD_seekable_initDStream() on the
|
||||
compressor.
|
||||
|
||||
Use ZSTD_seekable_loadSeekTable() to load the seek table from a file.
|
||||
`src` should point to a block of data read from the end of the file,
|
||||
i.e. `src + srcSize` should always be the end of the file.
|
||||
@return : 0 if the table was loaded successfully, or if `srcSize` was too
|
||||
small, a size hint for how much data to provide.
|
||||
An error code may also be returned, checkable with ZSTD_isError()
|
||||
|
||||
Use ZSTD_initDStream to prepare for a new decompression operation using the
|
||||
seektable loaded with ZSTD_seekable_loadSeekTable().
|
||||
Data in the range [rangeStart, rangeEnd) will be decompressed.
|
||||
|
||||
Call ZSTD_seekable_decompressStream() repetitively to consume input stream.
|
||||
@return : There are a number of possible return codes for this function
|
||||
- 0, the decompression operation has completed.
|
||||
- An error code checkable with ZSTD_isError
|
||||
+ If this error code is ZSTD_error_needSeek, the user should seek
|
||||
to the file position provided by ZSTD_seekable_getSeekOffset()
|
||||
and indicate this to the stream with
|
||||
ZSTD_seekable_updateOffset(), before resuming decompression
|
||||
+ Otherwise, this is a regular decompression error and the input
|
||||
file is likely corrupted or the API was incorrectly used.
|
||||
- A size hint, the preferred nb of bytes to provide as input to the
|
||||
next function call to improve latency.
|
||||
|
||||
ZSTD_seekable_getSeekOffset() and ZSTD_seekable_updateOffset() are helper
|
||||
functions to indicate where the user should seek their file stream to, when
|
||||
a different position is required to continue decompression.
|
||||
Note that ZSTD_seekable_updateOffset will error if given an offset other
|
||||
than the one requested from ZSTD_seekable_getSeekOffset().
|
||||
<BR></pre>
|
||||
|
||||
<h3>Seekable decompressor management</h3><pre></pre><b><pre>ZSTD_seekable_DStream* ZSTD_seekable_createDStream(void);
|
||||
size_t ZSTD_seekable_freeDStream(ZSTD_seekable_DStream* zds);
|
||||
</pre></b><BR>
|
||||
<h3>Seekable decompression functions</h3><pre></pre><b><pre>size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, size_t srcSize);
|
||||
size_t ZSTD_seekable_initDStream(ZSTD_seekable_DStream* zds, unsigned long long rangeStart, unsigned long long rangeEnd);
|
||||
size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
unsigned long long ZSTD_seekable_getSeekOffset(ZSTD_seekable_DStream* zds);
|
||||
size_t ZSTD_seekable_updateOffset(ZSTD_seekable_DStream* zds, unsigned long long offset);
|
||||
</pre></b><BR>
|
||||
</html>
|
||||
</body>
|
||||
|
||||
2
examples/.gitignore
vendored
2
examples/.gitignore
vendored
@ -6,8 +6,6 @@ dictionary_decompression
|
||||
streaming_compression
|
||||
streaming_decompression
|
||||
multiple_streaming_compression
|
||||
seekable_compression
|
||||
seekable_decompression
|
||||
|
||||
#test artefact
|
||||
tmp*
|
||||
|
||||
@ -18,8 +18,7 @@ default: all
|
||||
all: simple_compression simple_decompression \
|
||||
dictionary_compression dictionary_decompression \
|
||||
streaming_compression streaming_decompression \
|
||||
multiple_streaming_compression \
|
||||
seekable_compression seekable_decompression
|
||||
multiple_streaming_compression
|
||||
|
||||
simple_compression : simple_compression.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
@ -42,12 +41,6 @@ multiple_streaming_compression : multiple_streaming_compression.c
|
||||
streaming_decompression : streaming_decompression.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
seekable_compression : seekable_compression.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
seekable_decompression : seekable_decompression.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp* result* *.zst \
|
||||
simple_compression simple_decompression \
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
#ifndef SEEKABLE_H
|
||||
#define SEEKABLE_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_internal.h"
|
||||
|
||||
static const unsigned ZSTD_seekTableFooterSize = 9;
|
||||
|
||||
#define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1
|
||||
|
||||
#define ZSTD_SEEKABLE_MAXCHUNKS 0x8000000U
|
||||
|
||||
/* 0xFE03F607 is the largest number x such that ZSTD_compressBound(x) fits in a 32-bit integer */
|
||||
#define ZSTD_SEEKABLE_MAX_CHUNK_DECOMPRESSED_SIZE 0xFE03F607
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
119
lib/zstd.h
119
lib/zstd.h
@ -776,125 +776,6 @@ ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCa
|
||||
ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
|
||||
|
||||
|
||||
/*-****************************************************************************
|
||||
* Seekable Format
|
||||
*
|
||||
* The seekable format splits the compressed data into a series of "chunks",
|
||||
* each compressed individually so that decompression of a section in the
|
||||
* middle of an archive only requires zstd to decompress at most a chunk's
|
||||
* worth of extra data, instead of the entire archive.
|
||||
******************************************************************************/
|
||||
|
||||
typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream;
|
||||
typedef struct ZSTD_seekable_DStream_s ZSTD_seekable_DStream;
|
||||
|
||||
/*-****************************************************************************
|
||||
* Seekable compression - HowTo
|
||||
* A ZSTD_seekable_CStream object is required to tracking streaming operation.
|
||||
* Use ZSTD_seekable_createCStream() and ZSTD_seekable_freeCStream() to create/
|
||||
* release resources.
|
||||
*
|
||||
* Streaming objects are reusable to avoid allocation and deallocation,
|
||||
* to start a new compression operation call ZSTD_seekable_initCStream() on the
|
||||
* compressor.
|
||||
*
|
||||
* Data streamed to the seekable compressor will automatically be split into
|
||||
* chunks of size `maxChunkSize` (provided in ZSTD_seekable_initCStream()),
|
||||
* or if none is provided, will be cut off whenver ZSTD_endChunk() is called
|
||||
* or when the default maximum chunk size is reached (approximately 4GB).
|
||||
*
|
||||
* Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
|
||||
* for a new compression operation.
|
||||
* `maxChunkSize` indicates the size at which to automatically start a new
|
||||
* seekable frame. `maxChunkSize == 0` implies the default maximum size.
|
||||
* `checksumFlag` indicates whether or not the seek table should include chunk
|
||||
* checksums on the uncompressed data for verification.
|
||||
* @return : a size hint for input to provide for compression, or an error code
|
||||
* checkable with ZSTD_isError()
|
||||
*
|
||||
* Use ZSTD_seekable_compressStream() repetitively to consume input stream.
|
||||
* The function will automatically update both `pos` fields.
|
||||
* Note that it may not consume the entire input, in which case `pos < size`,
|
||||
* and it's up to the caller to present again remaining data.
|
||||
* @return : a size hint, preferred nb of bytes to use as input for next
|
||||
* function call or an error code, which can be tested using
|
||||
* ZSTD_isError().
|
||||
* Note 1 : it's just a hint, to help latency a little, any other
|
||||
* value will work fine.
|
||||
* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
|
||||
*
|
||||
* At any time, call ZSTD_seekable_endChunk() to end the current chunk and
|
||||
* start a new one.
|
||||
*
|
||||
* ZSTD_endStream() will end the current chunk, and then write the seek table
|
||||
* so that decompressors can efficiently find compressed chunks.
|
||||
* ZSTD_endStream() may return a number > 0 if it was unable to flush all the
|
||||
* necessary data to `output`. In this case, it should be called again until
|
||||
* all remaining data is flushed out and 0 is returned.
|
||||
******************************************************************************/
|
||||
|
||||
/*===== Seekable compressor management =====*/
|
||||
ZSTDLIB_API ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs);
|
||||
|
||||
/*===== Seekable compression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxChunkSize);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_endChunk(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output);
|
||||
|
||||
/*-****************************************************************************
|
||||
* Seekable decompression - HowTo
|
||||
* A ZSTD_seekable_DStream object is required to tracking streaming operation.
|
||||
* Use ZSTD_seekable_createDStream() and ZSTD_seekable_freeDStream() to create/
|
||||
* release resources.
|
||||
*
|
||||
* Streaming objects are reusable to avoid allocation and deallocation,
|
||||
* to start a new compression operation call ZSTD_seekable_initDStream() on the
|
||||
* compressor.
|
||||
*
|
||||
* Use ZSTD_seekable_loadSeekTable() to load the seek table from a file.
|
||||
* `src` should point to a block of data read from the end of the file,
|
||||
* i.e. `src + srcSize` should always be the end of the file.
|
||||
* @return : 0 if the table was loaded successfully, or if `srcSize` was too
|
||||
* small, a size hint for how much data to provide.
|
||||
* An error code may also be returned, checkable with ZSTD_isError()
|
||||
*
|
||||
* Use ZSTD_initDStream to prepare for a new decompression operation using the
|
||||
* seektable loaded with ZSTD_seekable_loadSeekTable().
|
||||
* Data in the range [rangeStart, rangeEnd) will be decompressed.
|
||||
*
|
||||
* Call ZSTD_seekable_decompressStream() repetitively to consume input stream.
|
||||
* @return : There are a number of possible return codes for this function
|
||||
* - 0, the decompression operation has completed.
|
||||
* - An error code checkable with ZSTD_isError
|
||||
* + If this error code is ZSTD_error_needSeek, the user should seek
|
||||
* to the file position provided by ZSTD_seekable_getSeekOffset()
|
||||
* and indicate this to the stream with
|
||||
* ZSTD_seekable_updateOffset(), before resuming decompression
|
||||
* + Otherwise, this is a regular decompression error and the input
|
||||
* file is likely corrupted or the API was incorrectly used.
|
||||
* - A size hint, the preferred nb of bytes to provide as input to the
|
||||
* next function call to improve latency.
|
||||
*
|
||||
* ZSTD_seekable_getSeekOffset() and ZSTD_seekable_updateOffset() are helper
|
||||
* functions to indicate where the user should seek their file stream to, when
|
||||
* a different position is required to continue decompression.
|
||||
* Note that ZSTD_seekable_updateOffset will error if given an offset other
|
||||
* than the one requested from ZSTD_seekable_getSeekOffset().
|
||||
******************************************************************************/
|
||||
|
||||
/*===== Seekable decompressor management =====*/
|
||||
ZSTDLIB_API ZSTD_seekable_DStream* ZSTD_seekable_createDStream(void);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_freeDStream(ZSTD_seekable_DStream* zds);
|
||||
|
||||
/*===== Seekable decompression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable_DStream* zds, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_initDStream(ZSTD_seekable_DStream* zds, unsigned long long rangeStart, unsigned long long rangeEnd);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_decompressStream(ZSTD_seekable_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
ZSTDLIB_API unsigned long long ZSTD_seekable_getSeekOffset(ZSTD_seekable_DStream* zds);
|
||||
ZSTDLIB_API size_t ZSTD_seekable_updateOffset(ZSTD_seekable_DStream* zds, unsigned long long offset);
|
||||
|
||||
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user