2015-01-24 00:58:16 +00:00
/* ******************************************************************
2016-03-04 18:26:59 +00:00
FSE : Finite State Entropy codec
Public Prototypes declaration
Copyright ( C ) 2013 - 2016 , Yann Collet .
2015-01-24 00:58:16 +00:00
BSD 2 - Clause License ( http : //www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms , with or without
modification , are permitted provided that the following conditions are
met :
* Redistributions of source code must retain the above copyright
notice , this list of conditions and the following disclaimer .
* Redistributions in binary form must reproduce the above
copyright notice , this list of conditions and the following disclaimer
in the documentation and / or other materials provided with the
distribution .
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
" AS IS " AND ANY EXPRESS OR IMPLIED WARRANTIES , INCLUDING , BUT NOT
LIMITED TO , THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED . IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT , INDIRECT , INCIDENTAL ,
SPECIAL , EXEMPLARY , OR CONSEQUENTIAL DAMAGES ( INCLUDING , BUT NOT
LIMITED TO , PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES ; LOSS OF USE ,
DATA , OR PROFITS ; OR BUSINESS INTERRUPTION ) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY , WHETHER IN CONTRACT , STRICT LIABILITY , OR TORT
( INCLUDING NEGLIGENCE OR OTHERWISE ) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE , EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE .
You can contact the author at :
- Source repository : https : //github.com/Cyan4973/FiniteStateEntropy
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# if defined (__cplusplus)
extern " C " {
# endif
2017-07-13 02:08:24 +00:00
# ifndef FSE_H
# define FSE_H
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*-*****************************************
* Dependencies
2015-01-24 00:58:16 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-07-04 23:56:41 +00:00
# include <stddef.h> /* size_t, ptrdiff_t */
2015-01-24 00:58:16 +00:00
2017-02-15 20:00:03 +00:00
/*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
# elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
# define FSE_PUBLIC_API __declspec(dllexport)
# elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
# else
# define FSE_PUBLIC_API
# endif
/*------ Version ------*/
# define FSE_VERSION_MAJOR 0
# define FSE_VERSION_MINOR 9
# define FSE_VERSION_RELEASE 0
# define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
# define FSE_QUOTE(str) #str
# define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
# define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
# define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
FSE_PUBLIC_API unsigned FSE_versionNumber ( void ) ; /**< library version number; to be used when checking dll version */
2016-01-27 23:18:06 +00:00
/*-****************************************
2015-01-24 00:58:16 +00:00
* FSE simple functions
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-03-04 18:26:59 +00:00
/*! FSE_compress() :
2015-01-24 00:58:16 +00:00
Compress content of buffer ' src ' , of size ' srcSize ' , into destination buffer ' dst ' .
2016-03-04 18:26:59 +00:00
' dst ' buffer must be already allocated . Compression runs faster is dstCapacity > = FSE_compressBound ( srcSize ) .
@ return : size of compressed data ( < = dstCapacity ) .
2015-08-07 14:21:00 +00:00
Special values : if return = = 0 , srcData is not compressible = > Nothing is stored within dst ! ! !
if return = = 1 , srcData is a single byte symbol * srcSize times . Use RLE compression instead .
if FSE_isError ( return ) , compression failed ( more details using FSE_getErrorName ( ) )
2016-03-04 18:26:59 +00:00
*/
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_compress ( void * dst , size_t dstCapacity ,
const void * src , size_t srcSize ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_decompress():
2015-01-24 00:58:16 +00:00
Decompress FSE data from buffer ' cSrc ' , of size ' cSrcSize ' ,
2016-03-04 18:26:59 +00:00
into already allocated destination buffer ' dst ' , of size ' dstCapacity ' .
@ return : size of regenerated data ( < = maxDstSize ) ,
or an error code , which can be tested using FSE_isError ( ) .
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
* * Important * * : FSE_decompress ( ) does not decompress non - compressible nor RLE data ! ! !
2015-07-04 23:56:41 +00:00
Why ? : making this distinction requires a header .
2015-07-25 23:23:57 +00:00
Header management is intentionally delegated to the user layer , which can better manage special cases .
*/
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_decompress ( void * dst , size_t dstCapacity ,
const void * cSrc , size_t cSrcSize ) ;
2015-07-25 23:23:57 +00:00
2016-03-04 18:26:59 +00:00
/*-*****************************************
2015-01-24 00:58:16 +00:00
* Tool functions
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_compressBound ( size_t size ) ; /* maximum compressed size */
2015-01-24 00:58:16 +00:00
/* Error Management */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API unsigned FSE_isError ( size_t code ) ; /* tells if a return value is an error code */
FSE_PUBLIC_API const char * FSE_getErrorName ( size_t code ) ; /* provides error code string (useful for debugging) */
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*-*****************************************
2015-01-24 00:58:16 +00:00
* FSE advanced functions
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-03-04 18:26:59 +00:00
/*! FSE_compress2() :
2015-01-24 00:58:16 +00:00
Same as FSE_compress ( ) , but allows the selection of ' maxSymbolValue ' and ' tableLog '
Both parameters can be defined as ' 0 ' to mean : use default value
2016-03-04 18:26:59 +00:00
@ return : size of compressed data
2015-07-04 23:56:41 +00:00
Special values : if return = = 0 , srcData is not compressible = > Nothing is stored within cSrc ! ! !
if return = = 1 , srcData is a single byte symbol * srcSize times . Use RLE compression .
if FSE_isError ( return ) , it ' s an error code .
2015-01-24 00:58:16 +00:00
*/
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_compress2 ( void * dst , size_t dstSize , const void * src , size_t srcSize , unsigned maxSymbolValue , unsigned tableLog ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*-*****************************************
2015-07-04 23:56:41 +00:00
* FSE detailed API
2015-01-24 00:58:16 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-01-06 00:58:37 +00:00
/*!
2015-07-04 23:56:41 +00:00
FSE_compress ( ) does the following :
2015-01-24 00:58:16 +00:00
1. count symbol occurrence from source [ ] into table count [ ]
2. normalize counters so that sum ( count [ ] ) = = Power_of_2 ( 2 ^ tableLog )
2015-08-07 14:21:00 +00:00
3. save normalized counters to memory buffer using writeNCount ( )
2015-01-24 00:58:16 +00:00
4. build encoding table ' CTable ' from normalized counters
2015-07-04 23:56:41 +00:00
5. encode the data stream using encoding table ' CTable '
2015-01-24 00:58:16 +00:00
2015-07-04 23:56:41 +00:00
FSE_decompress ( ) does the following :
2015-08-07 14:21:00 +00:00
1. read normalized counters with readNCount ( )
2015-01-24 00:58:16 +00:00
2. build decoding table ' DTable ' from normalized counters
2015-07-04 23:56:41 +00:00
3. decode the data stream using decoding table ' DTable '
2015-01-24 00:58:16 +00:00
2015-08-07 14:21:00 +00:00
The following API allows targeting specific sub - functions for advanced tasks .
2015-07-04 23:56:41 +00:00
For example , it ' s possible to compress several blocks using the same ' CTable ' ,
2015-08-07 14:21:00 +00:00
or to save and provide normalized distribution using external method .
2015-01-24 00:58:16 +00:00
*/
/* *** COMPRESSION *** */
2016-03-04 18:26:59 +00:00
/*! FSE_count():
Provides the precise count of each byte within a table ' count ' .
' count ' is a table of unsigned int , of minimum size ( * maxSymbolValuePtr + 1 ) .
* maxSymbolValuePtr will be updated if detected smaller than initial value .
@ return : the count of the most frequent symbol ( which is not identified ) .
if return = = srcSize , there is only one symbol .
Can also return an error code , which can be tested with FSE_isError ( ) . */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_count ( unsigned * count , unsigned * maxSymbolValuePtr , const void * src , size_t srcSize ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_optimalTableLog():
dynamically downsize ' tableLog ' when conditions are met .
It saves CPU time , by using smaller tables , while preserving or even improving compression ratio .
2016-06-16 14:39:55 +00:00
@ return : recommended tableLog ( necessarily < = ' maxTableLog ' ) */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API unsigned FSE_optimalTableLog ( unsigned maxTableLog , size_t srcSize , unsigned maxSymbolValue ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_normalizeCount():
normalize counts so that sum ( count [ ] ) = = Power_of_2 ( 2 ^ tableLog )
' normalizedCounter ' is a table of short , of minimum size ( maxSymbolValue + 1 ) .
@ return : tableLog ,
or an errorCode , which can be tested using FSE_isError ( ) */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_normalizeCount ( short * normalizedCounter , unsigned tableLog , const unsigned * count , size_t srcSize , unsigned maxSymbolValue ) ;
2015-07-04 23:56:41 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table , given ' maxSymbolValue ' and ' tableLog ' .
Typically useful for allocation purpose . */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_NCountWriteBound ( unsigned maxSymbolValue , unsigned tableLog ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_writeNCount():
Compactly save ' normalizedCounter ' into ' buffer ' .
@ return : size of the compressed table ,
or an errorCode , which can be tested using FSE_isError ( ) . */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_writeNCount ( void * buffer , size_t bufferSize , const short * normalizedCounter , unsigned maxSymbolValue , unsigned tableLog ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! Constructor and Destructor of FSE_CTable.
Note that FSE_CTable size depends on ' tableLog ' and ' maxSymbolValue ' */
2016-01-06 00:58:37 +00:00
typedef unsigned FSE_CTable ; /* don't allocate that. It's only meant to be more restrictive than void* */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API FSE_CTable * FSE_createCTable ( unsigned tableLog , unsigned maxSymbolValue ) ;
FSE_PUBLIC_API void FSE_freeCTable ( FSE_CTable * ct ) ;
2015-07-04 23:56:41 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_buildCTable():
Builds ` ct ` , which must be already allocated , using FSE_createCTable ( ) .
@ return : 0 , or an errorCode , which can be tested using FSE_isError ( ) */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_buildCTable ( FSE_CTable * ct , const short * normalizedCounter , unsigned maxSymbolValue , unsigned tableLog ) ;
2015-07-04 23:56:41 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_compress_usingCTable():
Compress ` src ` using ` ct ` into ` dst ` which must be already allocated .
@ return : size of compressed data ( < = ` dstCapacity ` ) ,
or 0 if compressed data could not fit into ` dst ` ,
or an errorCode , which can be tested using FSE_isError ( ) */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_compress_usingCTable ( void * dst , size_t dstCapacity , const void * src , size_t srcSize , const FSE_CTable * ct ) ;
2015-07-04 23:56:41 +00:00
2016-01-06 00:58:37 +00:00
/*!
2015-07-04 23:56:41 +00:00
Tutorial :
- - - - - - - - - -
2015-08-07 14:21:00 +00:00
The first step is to count all symbols . FSE_count ( ) does this job very fast .
2015-07-04 23:56:41 +00:00
Result will be saved into ' count ' , a table of unsigned int , which must be already allocated , and have ' maxSymbolValuePtr [ 0 ] + 1 ' cells .
' src ' is a table of bytes of size ' srcSize ' . All values within ' src ' MUST be < = maxSymbolValuePtr [ 0 ]
maxSymbolValuePtr [ 0 ] will be updated , with its real value ( necessarily < = original value )
2015-01-24 00:58:16 +00:00
FSE_count ( ) will return the number of occurrence of the most frequent symbol .
2015-08-07 14:21:00 +00:00
This can be used to know if there is a single symbol within ' src ' , and to quickly evaluate its compressibility .
2015-01-24 00:58:16 +00:00
If there is an error , the function will return an ErrorCode ( which can be tested using FSE_isError ( ) ) .
The next step is to normalize the frequencies .
FSE_normalizeCount ( ) will ensure that sum of frequencies is = = 2 ^ ' tableLog ' .
2015-08-07 14:21:00 +00:00
It also guarantees a minimum of 1 to any Symbol with frequency > = 1.
You can use ' tableLog ' = = 0 to mean " use default tableLog value " .
If you are unsure of which tableLog value to use , you can ask FSE_optimalTableLog ( ) ,
2015-01-24 00:58:16 +00:00
which will provide the optimal valid tableLog given sourceSize , maxSymbolValue , and a user - defined maximum ( 0 means " default " ) .
The result of FSE_normalizeCount ( ) will be saved into a table ,
called ' normalizedCounter ' , which is a table of signed short .
' normalizedCounter ' must be already allocated , and have at least ' maxSymbolValue + 1 ' cells .
The return value is tableLog if everything proceeded as expected .
It is 0 if there is a single symbol within distribution .
2015-08-07 14:21:00 +00:00
If there is an error ( ex : invalid tableLog value ) , the function will return an ErrorCode ( which can be tested using FSE_isError ( ) ) .
2015-01-24 00:58:16 +00:00
2015-08-07 14:21:00 +00:00
' normalizedCounter ' can be saved in a compact manner to a memory area using FSE_writeNCount ( ) .
' buffer ' must be already allocated .
2015-01-24 00:58:16 +00:00
For guaranteed success , buffer size must be at least FSE_headerBound ( ) .
2015-08-07 14:21:00 +00:00
The result of the function is the number of bytes written into ' buffer ' .
If there is an error , the function will return an ErrorCode ( which can be tested using FSE_isError ( ) ; ex : buffer size too small ) .
2015-01-24 00:58:16 +00:00
2015-07-04 23:56:41 +00:00
' normalizedCounter ' can then be used to create the compression table ' CTable ' .
2015-08-07 14:21:00 +00:00
The space required by ' CTable ' must be already allocated , using FSE_createCTable ( ) .
2015-01-24 00:58:16 +00:00
You can then use FSE_buildCTable ( ) to fill ' CTable ' .
2015-08-07 14:21:00 +00:00
If there is an error , both functions will return an ErrorCode ( which can be tested using FSE_isError ( ) ) .
2015-01-24 00:58:16 +00:00
2015-07-04 23:56:41 +00:00
' CTable ' can then be used to compress ' src ' , with FSE_compress_usingCTable ( ) .
Similar to FSE_count ( ) , the convention is that ' src ' is assumed to be a table of char of size ' srcSize '
2016-03-04 18:26:59 +00:00
The function returns the size of compressed data ( without header ) , necessarily < = ` dstCapacity ` .
2015-08-07 14:21:00 +00:00
If it returns ' 0 ' , compressed data could not fit into ' dst ' .
2015-07-04 23:56:41 +00:00
If there is an error , the function will return an ErrorCode ( which can be tested using FSE_isError ( ) ) .
2015-01-24 00:58:16 +00:00
*/
/* *** DECOMPRESSION *** */
2016-03-04 18:26:59 +00:00
/*! FSE_readNCount():
Read compactly saved ' normalizedCounter ' from ' rBuffer ' .
@ return : size read from ' rBuffer ' ,
or an errorCode , which can be tested using FSE_isError ( ) .
maxSymbolValuePtr [ 0 ] and tableLogPtr [ 0 ] will also be updated with their respective values */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_readNCount ( short * normalizedCounter , unsigned * maxSymbolValuePtr , unsigned * tableLogPtr , const void * rBuffer , size_t rBuffSize ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! Constructor and Destructor of FSE_DTable.
2015-08-07 14:21:00 +00:00
Note that its size depends on ' tableLog ' */
typedef unsigned FSE_DTable ; /* don't allocate that. It's just a way to be more restrictive than void* */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API FSE_DTable * FSE_createDTable ( unsigned tableLog ) ;
FSE_PUBLIC_API void FSE_freeDTable ( FSE_DTable * dt ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_buildDTable():
Builds ' dt ' , which must be already allocated , using FSE_createDTable ( ) .
return : 0 , or an errorCode , which can be tested using FSE_isError ( ) */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_buildDTable ( FSE_DTable * dt , const short * normalizedCounter , unsigned maxSymbolValue , unsigned tableLog ) ;
2015-01-24 00:58:16 +00:00
2016-03-04 18:26:59 +00:00
/*! FSE_decompress_usingDTable():
Decompress compressed source ` cSrc ` of size ` cSrcSize ` using ` dt `
into ` dst ` which must be already allocated .
@ return : size of regenerated data ( necessarily < = ` dstCapacity ` ) ,
or an errorCode , which can be tested using FSE_isError ( ) */
2017-02-15 20:00:03 +00:00
FSE_PUBLIC_API size_t FSE_decompress_usingDTable ( void * dst , size_t dstCapacity , const void * cSrc , size_t cSrcSize , const FSE_DTable * dt ) ;
2015-07-04 23:56:41 +00:00
2016-01-06 00:58:37 +00:00
/*!
2015-07-04 23:56:41 +00:00
Tutorial :
- - - - - - - - - -
( Note : these functions only decompress FSE - compressed blocks .
If block is uncompressed , use memcpy ( ) instead
If block is a single repeated byte , use memset ( ) instead )
2015-01-24 00:58:16 +00:00
The first step is to obtain the normalized frequencies of symbols .
2015-08-07 14:21:00 +00:00
This can be performed by FSE_readNCount ( ) if it was saved using FSE_writeNCount ( ) .
' normalizedCounter ' must be already allocated , and have at least ' maxSymbolValuePtr [ 0 ] + 1 ' cells of signed short .
2015-01-24 00:58:16 +00:00
In practice , that means it ' s necessary to know ' maxSymbolValue ' beforehand ,
or size the table to handle worst case situations ( typically 256 ) .
2015-08-07 14:21:00 +00:00
FSE_readNCount ( ) will provide ' tableLog ' and ' maxSymbolValue ' .
The result of FSE_readNCount ( ) is the number of bytes read from ' rBuffer ' .
Note that ' rBufferSize ' must be at least 4 bytes , even if useful information is less than that .
2015-01-24 00:58:16 +00:00
If there is an error , the function will return an error code , which can be tested using FSE_isError ( ) .
2015-08-07 14:21:00 +00:00
The next step is to build the decompression tables ' FSE_DTable ' from ' normalizedCounter ' .
2015-01-24 00:58:16 +00:00
This is performed by the function FSE_buildDTable ( ) .
2015-07-04 23:56:41 +00:00
The space required by ' FSE_DTable ' must be already allocated using FSE_createDTable ( ) .
2015-01-24 00:58:16 +00:00
If there is an error , the function will return an error code , which can be tested using FSE_isError ( ) .
2016-03-04 18:26:59 +00:00
` FSE_DTable ` can then be used to decompress ` cSrc ` , with FSE_decompress_usingDTable ( ) .
` cSrcSize ` must be strictly correct , otherwise decompression will fail .
FSE_decompress_usingDTable ( ) result will tell how many bytes were regenerated ( < = ` dstCapacity ` ) .
2015-08-07 14:21:00 +00:00
If there is an error , the function will return an error code , which can be tested using FSE_isError ( ) . ( ex : dst buffer too small )
2015-01-24 00:58:16 +00:00
*/
2017-07-13 02:08:24 +00:00
# endif /* FSE_H */
2015-01-24 00:58:16 +00:00
2017-07-13 02:08:24 +00:00
# if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
# define FSE_H_FSE_STATIC_LINKING_ONLY
2016-06-04 22:58:01 +00:00
/* *** Dependency *** */
# include "bitstream.h"
/* *****************************************
* Static allocation
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* FSE buffer bounds */
# define FSE_NCOUNTBOUND 512
# define FSE_BLOCKBOUND(size) (size + (size>>7))
# define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
2016-11-30 20:36:45 +00:00
/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
2016-06-04 22:58:01 +00:00
# define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
# define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
2017-04-26 18:39:35 +00:00
/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
# define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
# define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
2016-06-04 22:58:01 +00:00
/* *****************************************
* FSE advanced API
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-12-02 00:13:35 +00:00
/* FSE_count_wksp() :
* Same as FSE_count ( ) , but using an externally provided scratch buffer .
* ` workSpace ` size must be table of > = ` 1024 ` unsigned
*/
size_t FSE_count_wksp ( unsigned * count , unsigned * maxSymbolValuePtr ,
const void * source , size_t sourceSize , unsigned * workSpace ) ;
/** FSE_countFast() :
* same as FSE_count ( ) , but blindly trusts that all byte values within src are < = * maxSymbolValuePtr
*/
2016-06-04 22:58:01 +00:00
size_t FSE_countFast ( unsigned * count , unsigned * maxSymbolValuePtr , const void * src , size_t srcSize ) ;
2016-12-02 00:13:35 +00:00
/* FSE_countFast_wksp() :
* Same as FSE_countFast ( ) , but using an externally provided scratch buffer .
* ` workSpace ` must be a table of minimum ` 1024 ` unsigned
*/
size_t FSE_countFast_wksp ( unsigned * count , unsigned * maxSymbolValuePtr , const void * src , size_t srcSize , unsigned * workSpace ) ;
/*! FSE_count_simple
* Same as FSE_countFast ( ) , but does not use any additional memory ( not even on stack ) .
* This function is unsafe , and will segfault if any value within ` src ` is ` > * maxSymbolValuePtr ` ( presuming it ' s also the size of ` count ` ) .
*/
size_t FSE_count_simple ( unsigned * count , unsigned * maxSymbolValuePtr , const void * src , size_t srcSize ) ;
2016-06-04 22:58:01 +00:00
unsigned FSE_optimalTableLog_internal ( unsigned maxTableLog , size_t srcSize , unsigned maxSymbolValue , unsigned minus ) ;
/**< same as FSE_optimalTableLog(), which used `minus==2` */
2016-11-30 23:52:20 +00:00
/* FSE_compress_wksp() :
* Same as FSE_compress2 ( ) , but using an externally allocated scratch buffer ( ` workSpace ` ) .
2016-12-01 00:46:13 +00:00
* FSE_WKSP_SIZE_U32 ( ) provides the minimum size required for ` workSpace ` as a table of FSE_CTable .
2016-11-30 23:52:20 +00:00
*/
2017-04-04 23:17:21 +00:00
# define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
2016-12-01 00:46:13 +00:00
size_t FSE_compress_wksp ( void * dst , size_t dstSize , const void * src , size_t srcSize , unsigned maxSymbolValue , unsigned tableLog , void * workSpace , size_t wkspSize ) ;
2016-11-30 23:52:20 +00:00
2016-06-04 22:58:01 +00:00
size_t FSE_buildCTable_raw ( FSE_CTable * ct , unsigned nbBits ) ;
2016-12-01 00:46:13 +00:00
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
2016-06-04 22:58:01 +00:00
size_t FSE_buildCTable_rle ( FSE_CTable * ct , unsigned char symbolValue ) ;
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
2016-11-30 23:52:20 +00:00
/* FSE_buildCTable_wksp() :
* Same as FSE_buildCTable ( ) , but using an externally allocated scratch buffer ( ` workSpace ` ) .
2016-12-01 00:46:13 +00:00
* ` wkspSize ` must be > = ` ( 1 < < tableLog ) ` .
2016-11-30 23:52:20 +00:00
*/
size_t FSE_buildCTable_wksp ( FSE_CTable * ct , const short * normalizedCounter , unsigned maxSymbolValue , unsigned tableLog , void * workSpace , size_t wkspSize ) ;
2016-06-04 22:58:01 +00:00
size_t FSE_buildDTable_raw ( FSE_DTable * dt , unsigned nbBits ) ;
2016-12-01 00:46:13 +00:00
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
2016-06-04 22:58:01 +00:00
size_t FSE_buildDTable_rle ( FSE_DTable * dt , unsigned char symbolValue ) ;
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
2016-11-30 20:36:45 +00:00
size_t FSE_decompress_wksp ( void * dst , size_t dstCapacity , const void * cSrc , size_t cSrcSize , FSE_DTable * workSpace , unsigned maxLog ) ;
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
2017-07-13 19:45:39 +00:00
typedef enum {
FSE_repeat_none , /**< Cannot use the previous table */
FSE_repeat_check , /**< Can use the previous table but it must be checked */
FSE_repeat_valid /**< Can use the previous table and it is asumed to be valid */
} FSE_repeat ;
2016-06-04 22:58:01 +00:00
/* *****************************************
* FSE symbol compression API
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*!
This API consists of small unitary functions , which highly benefit from being inlined .
2016-12-02 00:13:35 +00:00
Hence their body are included in next section .
2016-06-04 22:58:01 +00:00
*/
2016-12-02 00:13:35 +00:00
typedef struct {
2016-06-04 22:58:01 +00:00
ptrdiff_t value ;
const void * stateTable ;
const void * symbolTT ;
unsigned stateLog ;
} FSE_CState_t ;
static void FSE_initCState ( FSE_CState_t * CStatePtr , const FSE_CTable * ct ) ;
static void FSE_encodeSymbol ( BIT_CStream_t * bitC , FSE_CState_t * CStatePtr , unsigned symbol ) ;
static void FSE_flushCState ( BIT_CStream_t * bitC , const FSE_CState_t * CStatePtr ) ;
/**<
These functions are inner components of FSE_compress_usingCTable ( ) .
They allow the creation of custom streams , mixing multiple tables and bit sources .
A key property to keep in mind is that encoding and decoding are done * * in reverse direction * * .
So the first symbol you will encode is the last you will decode , like a LIFO stack .
You will need a few variables to track your CStream . They are :
FSE_CTable ct ; // Provided by FSE_buildCTable()
BIT_CStream_t bitStream ; // bitStream tracking structure
FSE_CState_t state ; // State tracking structure (can have several)
The first thing to do is to init bitStream and state .
size_t errorCode = BIT_initCStream ( & bitStream , dstBuffer , maxDstSize ) ;
FSE_initCState ( & state , ct ) ;
Note that BIT_initCStream ( ) can produce an error code , so its result should be tested , using FSE_isError ( ) ;
You can then encode your input data , byte after byte .
FSE_encodeSymbol ( ) outputs a maximum of ' tableLog ' bits at a time .
Remember decoding will be done in reverse direction .
FSE_encodeByte ( & bitStream , & state , symbol ) ;
At any time , you can also add any bit sequence .
Note : maximum allowed nbBits is 25 , for compatibility with 32 - bits decoders
BIT_addBits ( & bitStream , bitField , nbBits ) ;
The above methods don ' t commit data to memory , they just store it into local register , for speed .
Local register size is 64 - bits on 64 - bits systems , 32 - bits on 32 - bits systems ( size_t ) .
Writing data to memory is a manual operation , performed by the flushBits function .
BIT_flushBits ( & bitStream ) ;
Your last FSE encoding operation shall be to flush your last state value ( s ) .
FSE_flushState ( & bitStream , & state ) ;
Finally , you must close the bitStream .
The function returns the size of CStream in bytes .
If data couldn ' t fit into dstBuffer , it will return a 0 ( = = not compressible )
If there is an error , it returns an errorCode ( which can be tested using FSE_isError ( ) ) .
size_t size = BIT_closeCStream ( & bitStream ) ;
*/
/* *****************************************
* FSE symbol decompression API
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-12-02 00:13:35 +00:00
typedef struct {
2016-06-04 22:58:01 +00:00
size_t state ;
const void * table ; /* precise table may vary, depending on U16 */
} FSE_DState_t ;
static void FSE_initDState ( FSE_DState_t * DStatePtr , BIT_DStream_t * bitD , const FSE_DTable * dt ) ;
static unsigned char FSE_decodeSymbol ( FSE_DState_t * DStatePtr , BIT_DStream_t * bitD ) ;
static unsigned FSE_endOfDState ( const FSE_DState_t * DStatePtr ) ;
/**<
Let ' s now decompose FSE_decompress_usingDTable ( ) into its unitary components .
You will decode FSE - encoded symbols from the bitStream ,
and also any other bitFields you put in , * * in reverse order * * .
You will need a few variables to track your bitStream . They are :
BIT_DStream_t DStream ; // Stream context
FSE_DState_t DState ; // State context. Multiple ones are possible
FSE_DTable * DTablePtr ; // Decoding table, provided by FSE_buildDTable()
The first thing to do is to init the bitStream .
errorCode = BIT_initDStream ( & DStream , srcBuffer , srcSize ) ;
You should then retrieve your initial state ( s )
( in reverse flushing order if you have several ones ) :
errorCode = FSE_initDState ( & DState , & DStream , DTablePtr ) ;
You can then decode your data , symbol after symbol .
For information the maximum number of bits read by FSE_decodeSymbol ( ) is ' tableLog ' .
Keep in mind that symbols are decoded in reverse order , like a LIFO stack ( last in , first out ) .
unsigned char symbol = FSE_decodeSymbol ( & DState , & DStream ) ;
You can retrieve any bitfield you eventually stored into the bitStream ( in reverse order )
Note : maximum allowed nbBits is 25 , for 32 - bits compatibility
size_t bitField = BIT_readBits ( & DStream , nbBits ) ;
All above operations only read from local register ( which size depends on size_t ) .
Refueling the register from memory is manually performed by the reload method .
endSignal = FSE_reloadDStream ( & DStream ) ;
BIT_reloadDStream ( ) result tells if there is still some more data to read from DStream .
BIT_DStream_unfinished : there is still some data left into the DStream .
BIT_DStream_endOfBuffer : Dstream reached end of buffer . Its container may no longer be completely filled .
BIT_DStream_completed : Dstream reached its exact end , corresponding in general to decompression completed .
BIT_DStream_tooFar : Dstream went too far . Decompression result is corrupted .
When reaching end of buffer ( BIT_DStream_endOfBuffer ) , progress slowly , notably if you decode multiple symbols per loop ,
to properly detect the exact end of stream .
After each decoded symbol , check if DStream is fully consumed using this simple test :
BIT_reloadDStream ( & DStream ) > = BIT_DStream_completed
When it ' s done , verify decompression is fully completed , by checking both DStream and the relevant states .
Checking if DStream has reached its end is performed by :
BIT_endOfDStream ( & DStream ) ;
Check also the states . There might be some symbols left there , if some high probability ones ( > 50 % ) are possible .
FSE_endOfDState ( & DState ) ;
*/
/* *****************************************
* FSE unsafe API
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
static unsigned char FSE_decodeSymbolFast ( FSE_DState_t * DStatePtr , BIT_DStream_t * bitD ) ;
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/* *****************************************
* Implementation of inlined functions
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
typedef struct {
int deltaFindState ;
U32 deltaNbBits ;
} FSE_symbolCompressionTransform ; /* total 8 bytes */
MEM_STATIC void FSE_initCState ( FSE_CState_t * statePtr , const FSE_CTable * ct )
{
const void * ptr = ct ;
const U16 * u16ptr = ( const U16 * ) ptr ;
const U32 tableLog = MEM_read16 ( ptr ) ;
statePtr - > value = ( ptrdiff_t ) 1 < < tableLog ;
statePtr - > stateTable = u16ptr + 2 ;
statePtr - > symbolTT = ( ( const U32 * ) ct + 1 + ( tableLog ? ( 1 < < ( tableLog - 1 ) ) : 1 ) ) ;
statePtr - > stateLog = tableLog ;
}
/*! FSE_initCState2() :
* Same as FSE_initCState ( ) , but the first symbol to include ( which will be the last to be read )
* uses the smallest state value possible , saving the cost of this symbol */
MEM_STATIC void FSE_initCState2 ( FSE_CState_t * statePtr , const FSE_CTable * ct , U32 symbol )
{
FSE_initCState ( statePtr , ct ) ;
{ const FSE_symbolCompressionTransform symbolTT = ( ( const FSE_symbolCompressionTransform * ) ( statePtr - > symbolTT ) ) [ symbol ] ;
const U16 * stateTable = ( const U16 * ) ( statePtr - > stateTable ) ;
U32 nbBitsOut = ( U32 ) ( ( symbolTT . deltaNbBits + ( 1 < < 15 ) ) > > 16 ) ;
statePtr - > value = ( nbBitsOut < < 16 ) - symbolTT . deltaNbBits ;
statePtr - > value = stateTable [ ( statePtr - > value > > nbBitsOut ) + symbolTT . deltaFindState ] ;
}
}
MEM_STATIC void FSE_encodeSymbol ( BIT_CStream_t * bitC , FSE_CState_t * statePtr , U32 symbol )
{
2017-04-21 00:28:31 +00:00
FSE_symbolCompressionTransform const symbolTT = ( ( const FSE_symbolCompressionTransform * ) ( statePtr - > symbolTT ) ) [ symbol ] ;
2016-06-04 22:58:01 +00:00
const U16 * const stateTable = ( const U16 * ) ( statePtr - > stateTable ) ;
2017-04-21 00:28:31 +00:00
U32 const nbBitsOut = ( U32 ) ( ( statePtr - > value + symbolTT . deltaNbBits ) > > 16 ) ;
2016-06-04 22:58:01 +00:00
BIT_addBits ( bitC , statePtr - > value , nbBitsOut ) ;
statePtr - > value = stateTable [ ( statePtr - > value > > nbBitsOut ) + symbolTT . deltaFindState ] ;
}
MEM_STATIC void FSE_flushCState ( BIT_CStream_t * bitC , const FSE_CState_t * statePtr )
{
BIT_addBits ( bitC , statePtr - > value , statePtr - > stateLog ) ;
BIT_flushBits ( bitC ) ;
}
2016-10-11 15:21:09 +00:00
2016-08-25 23:43:47 +00:00
/* ====== Decompression ====== */
2016-06-04 22:58:01 +00:00
typedef struct {
U16 tableLog ;
U16 fastMode ;
} FSE_DTableHeader ; /* sizeof U32 */
typedef struct
{
unsigned short newState ;
unsigned char symbol ;
unsigned char nbBits ;
} FSE_decode_t ; /* size == U32 */
MEM_STATIC void FSE_initDState ( FSE_DState_t * DStatePtr , BIT_DStream_t * bitD , const FSE_DTable * dt )
{
const void * ptr = dt ;
const FSE_DTableHeader * const DTableH = ( const FSE_DTableHeader * ) ptr ;
DStatePtr - > state = BIT_readBits ( bitD , DTableH - > tableLog ) ;
BIT_reloadDStream ( bitD ) ;
DStatePtr - > table = dt + 1 ;
}
MEM_STATIC BYTE FSE_peekSymbol ( const FSE_DState_t * DStatePtr )
{
FSE_decode_t const DInfo = ( ( const FSE_decode_t * ) ( DStatePtr - > table ) ) [ DStatePtr - > state ] ;
return DInfo . symbol ;
}
MEM_STATIC void FSE_updateState ( FSE_DState_t * DStatePtr , BIT_DStream_t * bitD )
{
FSE_decode_t const DInfo = ( ( const FSE_decode_t * ) ( DStatePtr - > table ) ) [ DStatePtr - > state ] ;
U32 const nbBits = DInfo . nbBits ;
size_t const lowBits = BIT_readBits ( bitD , nbBits ) ;
DStatePtr - > state = DInfo . newState + lowBits ;
}
MEM_STATIC BYTE FSE_decodeSymbol ( FSE_DState_t * DStatePtr , BIT_DStream_t * bitD )
{
FSE_decode_t const DInfo = ( ( const FSE_decode_t * ) ( DStatePtr - > table ) ) [ DStatePtr - > state ] ;
U32 const nbBits = DInfo . nbBits ;
BYTE const symbol = DInfo . symbol ;
size_t const lowBits = BIT_readBits ( bitD , nbBits ) ;
DStatePtr - > state = DInfo . newState + lowBits ;
return symbol ;
}
/*! FSE_decodeSymbolFast() :
unsafe , only works if no symbol has a probability > 50 % */
MEM_STATIC BYTE FSE_decodeSymbolFast ( FSE_DState_t * DStatePtr , BIT_DStream_t * bitD )
{
FSE_decode_t const DInfo = ( ( const FSE_decode_t * ) ( DStatePtr - > table ) ) [ DStatePtr - > state ] ;
U32 const nbBits = DInfo . nbBits ;
BYTE const symbol = DInfo . symbol ;
size_t const lowBits = BIT_readBitsFast ( bitD , nbBits ) ;
DStatePtr - > state = DInfo . newState + lowBits ;
return symbol ;
}
MEM_STATIC unsigned FSE_endOfDState ( const FSE_DState_t * DStatePtr )
{
return DStatePtr - > state = = 0 ;
}
# ifndef FSE_COMMONDEFS_ONLY
/* **************************************************************
* Tuning parameters
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*!MEMORY_USAGE :
* Memory usage formula : N - > 2 ^ N Bytes ( examples : 10 - > 1 KB ; 12 - > 4 KB ; 16 - > 64 KB ; 20 - > 1 MB ; etc . )
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed , due to cache effect
* Recommended max value is 14 , for 16 KB , which nicely fits into Intel x86 L1 cache */
2016-10-11 15:21:09 +00:00
# ifndef FSE_MAX_MEMORY_USAGE
# define FSE_MAX_MEMORY_USAGE 14
# endif
# ifndef FSE_DEFAULT_MEMORY_USAGE
# define FSE_DEFAULT_MEMORY_USAGE 13
# endif
2016-06-04 22:58:01 +00:00
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized .
* Required for proper stack allocation */
2016-10-11 15:21:09 +00:00
# ifndef FSE_MAX_SYMBOL_VALUE
# define FSE_MAX_SYMBOL_VALUE 255
# endif
2016-06-04 22:58:01 +00:00
/* **************************************************************
* template functions type & suffix
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# define FSE_FUNCTION_TYPE BYTE
# define FSE_FUNCTION_EXTENSION
# define FSE_DECODE_TYPE FSE_decode_t
# endif /* !FSE_COMMONDEFS_ONLY */
/* ***************************************************************
* Constants
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
# define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
# define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
# define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
# define FSE_MIN_TABLELOG 5
# define FSE_TABLELOG_ABSOLUTE_MAX 15
# if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
# endif
# define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
# endif /* FSE_STATIC_LINKING_ONLY */
2015-01-24 00:58:16 +00:00
# if defined (__cplusplus)
}
# endif