2017-08-18 23:52:05 +00:00
/*
2020-03-26 22:19:05 +00:00
* Copyright ( c ) 2016 - 2020 , Yann Collet , Facebook , Inc .
2016-08-30 17:04:33 +00:00
* All rights reserved .
*
2017-08-18 23:52:05 +00:00
* This source code is licensed under both the BSD - style license ( found in the
* LICENSE file in the root directory of this source tree ) and the GPLv2 ( found
* in the COPYING file in the root directory of this source tree ) .
2017-09-08 07:09:23 +00:00
* You may select , at your option , one of the above - listed licenses .
2016-08-30 17:04:33 +00:00
*/
2016-02-11 23:07:30 +00:00
# ifndef DICTBUILDER_H_001
# define DICTBUILDER_H_001
# if defined (__cplusplus)
extern " C " {
# endif
2016-08-18 11:12:49 +00:00
2016-08-30 17:04:33 +00:00
/*====== Dependencies ======*/
# include <stddef.h> /* size_t */
2016-12-16 21:27:30 +00:00
/* ===== ZDICTLIB_API : control library symbols visibility ===== */
2017-05-20 01:01:59 +00:00
# ifndef ZDICTLIB_VISIBILITY
# if defined(__GNUC__) && (__GNUC__ >= 4)
# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
# else
# define ZDICTLIB_VISIBILITY
# endif
2016-12-16 21:27:30 +00:00
# endif
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
2016-08-18 11:12:49 +00:00
# else
2016-12-16 21:27:30 +00:00
# define ZDICTLIB_API ZDICTLIB_VISIBILITY
2016-08-18 11:12:49 +00:00
# endif
2017-06-27 04:07:14 +00:00
/*! ZDICT_trainFromBuffer():
2018-01-11 17:42:38 +00:00
* Train a dictionary from an array of samples .
2018-08-23 19:06:20 +00:00
* Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover ( ) single - threaded , with d = 8 , steps = 4 ,
2018-09-05 00:12:35 +00:00
* f = 20 , and accel = 1.
2018-01-11 17:42:38 +00:00
* Samples must be stored concatenated in a single flat buffer ` samplesBuffer ` ,
* supplied with an array of sizes ` samplesSizes ` , providing the size of each sample , in order .
* The resulting dictionary will be saved into ` dictBuffer ` .
2017-06-27 04:07:14 +00:00
* @ return : size of dictionary stored into ` dictBuffer ` ( < = ` dictBufferCapacity ` )
2018-01-11 17:42:38 +00:00
* or an error code , which can be tested with ZDICT_isError ( ) .
2019-02-01 23:19:32 +00:00
* Note : Dictionary training will fail if there are not enough samples to construct a
* dictionary , or if most of the samples are too small ( < 8 bytes being the lower limit ) .
* If dictionary training fails , you should use zstd without a dictionary , as the dictionary
* would ' ve been ineffective anyways . If you believe your samples would benefit from a dictionary
* please open an issue with details , and we can look into it .
* Note : ZDICT_trainFromBuffer ( ) ' s memory usage is about 6 MB .
2018-01-11 17:42:38 +00:00
* Tips : In general , a reasonable dictionary has a size of ~ 100 KB .
* It ' s possible to select smaller or larger size , just by specifying ` dictBufferCapacity ` .
* In general , it ' s recommended to provide a few thousands samples , though this can vary a lot .
2017-06-27 04:07:14 +00:00
* It ' s recommended that total size of all samples be about ~ x100 times the target size of dictionary .
*/
2016-08-18 11:47:06 +00:00
ZDICTLIB_API size_t ZDICT_trainFromBuffer ( void * dictBuffer , size_t dictBufferCapacity ,
2018-08-27 02:29:12 +00:00
const void * samplesBuffer ,
const size_t * samplesSizes , unsigned nbSamples ) ;
2016-05-31 01:49:58 +00:00
2020-05-07 01:51:14 +00:00
typedef struct {
int compressionLevel ; /*< optimize for a specific zstd compression level; 0 means default */
unsigned notificationLevel ; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
unsigned dictID ; /*< force dictID value; 0 means auto mode (32-bits random value) */
} ZDICT_params_t ;
/*! ZDICT_finalizeDictionary():
* Given a custom content as a basis for dictionary , and a set of samples ,
* finalize dictionary by adding headers and statistics according to the zstd
* dictionary format .
*
* Samples must be stored concatenated in a flat buffer ` samplesBuffer ` ,
* supplied with an array of sizes ` samplesSizes ` , providing the size of each
* sample in order . The samples are used to construct the statistics , so they
* should be representative of what you will compress with this dictionary .
*
* The compression level can be set in ` parameters ` . You should pass the
* compression level you expect to use in production . The statistics for each
* compression level differ , so tuning the dictionary for the compression level
* can help quite a bit .
*
* You can set an explicit dictionary ID in ` parameters ` , or allow us to pick
* a random dictionary ID for you , but we can ' t guarantee no collisions .
*
* The dstDictBuffer and the dictContent may overlap , and the content will be
* appended to the end of the header . If the header + the content doesn ' t fit in
* maxDictSize the beginning of the content is truncated to make room , since it
* is presumed that the most profitable content is at the end of the dictionary ,
* since that is the cheapest to reference .
*
* ` dictContentSize ` must be > = ZDICT_CONTENTSIZE_MIN bytes .
* ` maxDictSize ` must be > = max ( dictContentSize , ZSTD_DICTSIZE_MIN ) .
*
* @ return : size of dictionary stored into ` dstDictBuffer ` ( < = ` maxDictSize ` ) ,
* or an error code , which can be tested by ZDICT_isError ( ) .
* Note : ZDICT_finalizeDictionary ( ) will push notifications into stderr if
* instructed to , using notificationLevel > 0.
* NOTE : This function currently may fail in several edge cases including :
* * Not enough samples
* * Samples are uncompressible
* * Samples are all exactly the same
*/
ZDICTLIB_API size_t ZDICT_finalizeDictionary ( void * dstDictBuffer , size_t maxDictSize ,
const void * dictContent , size_t dictContentSize ,
const void * samplesBuffer , const size_t * samplesSizes , unsigned nbSamples ,
ZDICT_params_t parameters ) ;
2016-05-31 01:49:58 +00:00
2016-07-28 01:47:45 +00:00
/*====== Helper functions ======*/
2016-08-19 12:23:58 +00:00
ZDICTLIB_API unsigned ZDICT_getDictID ( const void * dictBuffer , size_t dictSize ) ; /**< extracts dictID; @return zero if error (not a valid dictionary) */
2019-11-07 16:46:25 +00:00
ZDICTLIB_API size_t ZDICT_getDictHeaderSize ( const void * dictBuffer , size_t dictSize ) ; /* returns dict header size; returns a ZSTD error code on failure */
2016-08-18 11:47:06 +00:00
ZDICTLIB_API unsigned ZDICT_isError ( size_t errorCode ) ;
ZDICTLIB_API const char * ZDICT_getErrorName ( size_t errorCode ) ;
2016-02-11 23:07:30 +00:00
2016-07-28 01:47:45 +00:00
2016-06-04 16:56:23 +00:00
# ifdef ZDICT_STATIC_LINKING_ONLY
/* ====================================================================================
* The definitions in this section are considered experimental .
2016-06-15 11:53:34 +00:00
* They should never be used with a dynamic library , as they may change in the future .
2016-06-04 16:56:23 +00:00
* They are provided for advanced usages .
* Use them only in association with static linking .
* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */
2020-05-07 01:51:14 +00:00
# define ZDICT_CONTENTSIZE_MIN 128
# define ZDICT_DICTSIZE_MIN 256
2016-06-04 16:56:23 +00:00
2017-06-27 04:07:14 +00:00
/*! ZDICT_cover_params_t:
* k and d are the only required parameters .
2018-01-11 17:42:38 +00:00
* For others , value 0 means default .
2017-06-27 04:07:14 +00:00
*/
2017-01-01 05:07:44 +00:00
typedef struct {
2017-01-02 20:40:43 +00:00
unsigned k ; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
unsigned d ; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
2018-08-30 22:37:29 +00:00
unsigned steps ; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
2017-01-27 19:56:02 +00:00
unsigned nbThreads ; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
2018-08-30 21:53:59 +00:00
double splitPoint ; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
2019-06-27 23:26:57 +00:00
unsigned shrinkDict ; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
unsigned shrinkDictMaxRegression ; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
2017-06-27 04:07:14 +00:00
ZDICT_params_t zParams ;
} ZDICT_cover_params_t ;
2018-08-23 19:06:20 +00:00
typedef struct {
unsigned k ; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
unsigned d ; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
2018-09-05 00:12:35 +00:00
unsigned f ; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/
2018-08-30 22:37:29 +00:00
unsigned steps ; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
2018-08-23 19:06:20 +00:00
unsigned nbThreads ; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
2018-08-30 21:53:59 +00:00
double splitPoint ; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
2018-08-23 19:06:20 +00:00
unsigned accel ; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
2019-06-27 23:26:57 +00:00
unsigned shrinkDict ; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
unsigned shrinkDictMaxRegression ; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
2018-08-23 19:06:20 +00:00
ZDICT_params_t zParams ;
} ZDICT_fastCover_params_t ;
2017-06-27 04:07:14 +00:00
/*! ZDICT_trainFromBuffer_cover():
2018-01-11 17:42:38 +00:00
* Train a dictionary from an array of samples using the COVER algorithm .
* Samples must be stored concatenated in a single flat buffer ` samplesBuffer ` ,
* supplied with an array of sizes ` samplesSizes ` , providing the size of each sample , in order .
* The resulting dictionary will be saved into ` dictBuffer ` .
2017-06-27 04:07:14 +00:00
* @ return : size of dictionary stored into ` dictBuffer ` ( < = ` dictBufferCapacity ` )
2018-01-11 17:42:38 +00:00
* or an error code , which can be tested with ZDICT_isError ( ) .
2019-02-01 23:19:32 +00:00
* See ZDICT_trainFromBuffer ( ) for details on failure modes .
2018-01-11 17:42:38 +00:00
* Note : ZDICT_trainFromBuffer_cover ( ) requires about 9 bytes of memory for each input byte .
* Tips : In general , a reasonable dictionary has a size of ~ 100 KB .
* It ' s possible to select smaller or larger size , just by specifying ` dictBufferCapacity ` .
* In general , it ' s recommended to provide a few thousands samples , though this can vary a lot .
2017-06-27 04:07:14 +00:00
* It ' s recommended that total size of all samples be about ~ x100 times the target size of dictionary .
*/
ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover (
2018-01-28 06:23:26 +00:00
void * dictBuffer , size_t dictBufferCapacity ,
2018-01-11 17:42:38 +00:00
const void * samplesBuffer , const size_t * samplesSizes , unsigned nbSamples ,
2018-01-28 06:23:26 +00:00
ZDICT_cover_params_t parameters ) ;
2017-06-27 04:07:14 +00:00
/*! ZDICT_optimizeTrainFromBuffer_cover():
* The same requirements as above hold for all the parameters except ` parameters ` .
* This function tries many parameter combinations and picks the best parameters .
2018-01-11 17:42:38 +00:00
* ` * parameters ` is filled with the best parameters found ,
* dictionary constructed with those parameters is stored in ` dictBuffer ` .
2017-06-27 04:07:14 +00:00
*
* All of the parameters d , k , steps are optional .
2018-08-23 19:06:20 +00:00
* If d is non - zero then we don ' t check multiple values of d , otherwise we check d = { 6 , 8 } .
2017-06-27 04:07:14 +00:00
* if steps is zero it defaults to its default value .
2018-08-23 19:06:20 +00:00
* If k is non - zero then we don ' t check multiple values of k , otherwise we check steps values in [ 50 , 2000 ] .
2017-06-27 04:07:14 +00:00
*
* @ return : size of dictionary stored into ` dictBuffer ` ( < = ` dictBufferCapacity ` )
2019-02-01 23:19:32 +00:00
* or an error code , which can be tested with ZDICT_isError ( ) .
* On success ` * parameters ` contains the parameters selected .
* See ZDICT_trainFromBuffer ( ) for details on failure modes .
2017-06-27 04:07:14 +00:00
* Note : ZDICT_optimizeTrainFromBuffer_cover ( ) requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread .
*/
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover (
2018-01-28 06:23:26 +00:00
void * dictBuffer , size_t dictBufferCapacity ,
2018-01-11 17:42:38 +00:00
const void * samplesBuffer , const size_t * samplesSizes , unsigned nbSamples ,
2018-01-28 06:23:26 +00:00
ZDICT_cover_params_t * parameters ) ;
2017-06-27 04:07:14 +00:00
2018-08-23 19:06:20 +00:00
/*! ZDICT_trainFromBuffer_fastCover():
* Train a dictionary from an array of samples using a modified version of COVER algorithm .
* Samples must be stored concatenated in a single flat buffer ` samplesBuffer ` ,
* supplied with an array of sizes ` samplesSizes ` , providing the size of each sample , in order .
* d and k are required .
* All other parameters are optional , will use default values if not provided
* The resulting dictionary will be saved into ` dictBuffer ` .
* @ return : size of dictionary stored into ` dictBuffer ` ( < = ` dictBufferCapacity ` )
* or an error code , which can be tested with ZDICT_isError ( ) .
2019-02-01 23:19:32 +00:00
* See ZDICT_trainFromBuffer ( ) for details on failure modes .
* Note : ZDICT_trainFromBuffer_fastCover ( ) requires 6 * 2 ^ f bytes of memory .
2018-08-23 19:06:20 +00:00
* Tips : In general , a reasonable dictionary has a size of ~ 100 KB .
* It ' s possible to select smaller or larger size , just by specifying ` dictBufferCapacity ` .
* In general , it ' s recommended to provide a few thousands samples , though this can vary a lot .
* It ' s recommended that total size of all samples be about ~ x100 times the target size of dictionary .
*/
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover ( void * dictBuffer ,
size_t dictBufferCapacity , const void * samplesBuffer ,
const size_t * samplesSizes , unsigned nbSamples ,
ZDICT_fastCover_params_t parameters ) ;
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
* The same requirements as above hold for all the parameters except ` parameters ` .
* This function tries many parameter combinations ( specifically , k and d combinations )
* and picks the best parameters . ` * parameters ` is filled with the best parameters found ,
* dictionary constructed with those parameters is stored in ` dictBuffer ` .
* All of the parameters d , k , steps , f , and accel are optional .
* If d is non - zero then we don ' t check multiple values of d , otherwise we check d = { 6 , 8 } .
* if steps is zero it defaults to its default value .
* If k is non - zero then we don ' t check multiple values of k , otherwise we check steps values in [ 50 , 2000 ] .
2018-09-05 00:12:35 +00:00
* If f is zero , default value of 20 is used .
2018-08-23 19:06:20 +00:00
* If accel is zero , default value of 1 is used .
*
* @ return : size of dictionary stored into ` dictBuffer ` ( < = ` dictBufferCapacity ` )
2019-02-01 23:19:32 +00:00
* or an error code , which can be tested with ZDICT_isError ( ) .
* On success ` * parameters ` contains the parameters selected .
* See ZDICT_trainFromBuffer ( ) for details on failure modes .
* Note : ZDICT_optimizeTrainFromBuffer_fastCover ( ) requires about 6 * 2 ^ f bytes of memory for each thread .
2018-08-23 19:06:20 +00:00
*/
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover ( void * dictBuffer ,
size_t dictBufferCapacity , const void * samplesBuffer ,
const size_t * samplesSizes , unsigned nbSamples ,
ZDICT_fastCover_params_t * parameters ) ;
2017-06-27 04:07:14 +00:00
typedef struct {
unsigned selectivityLevel ; /* 0 means default; larger => select more => larger dictionary */
ZDICT_params_t zParams ;
} ZDICT_legacy_params_t ;
/*! ZDICT_trainFromBuffer_legacy():
2018-01-11 17:42:38 +00:00
* Train a dictionary from an array of samples .
* Samples must be stored concatenated in a single flat buffer ` samplesBuffer ` ,
* supplied with an array of sizes ` samplesSizes ` , providing the size of each sample , in order .
* The resulting dictionary will be saved into ` dictBuffer ` .
2017-06-27 04:07:14 +00:00
* ` parameters ` is optional and can be provided with values set to 0 to mean " default " .
* @ return : size of dictionary stored into ` dictBuffer ` ( < = ` dictBufferCapacity ` )
2018-01-11 17:42:38 +00:00
* or an error code , which can be tested with ZDICT_isError ( ) .
2019-02-01 23:19:32 +00:00
* See ZDICT_trainFromBuffer ( ) for details on failure modes .
2018-01-11 17:42:38 +00:00
* Tips : In general , a reasonable dictionary has a size of ~ 100 KB .
* It ' s possible to select smaller or larger size , just by specifying ` dictBufferCapacity ` .
* In general , it ' s recommended to provide a few thousands samples , though this can vary a lot .
2017-06-27 04:07:14 +00:00
* It ' s recommended that total size of all samples be about ~ x100 times the target size of dictionary .
2018-01-11 17:42:38 +00:00
* Note : ZDICT_trainFromBuffer_legacy ( ) will send notifications into stderr if instructed to , using notificationLevel > 0.
2017-06-27 04:07:14 +00:00
*/
ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy (
2018-01-11 17:42:38 +00:00
void * dictBuffer , size_t dictBufferCapacity ,
const void * samplesBuffer , const size_t * samplesSizes , unsigned nbSamples ,
ZDICT_legacy_params_t parameters ) ;
2016-07-28 01:47:45 +00:00
2016-12-22 19:18:43 +00:00
/* Deprecation warnings */
/* It is generally possible to disable deprecation warnings from compiler,
for example with - Wno - deprecated - declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual .
Otherwise , it ' s also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
# ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
2016-12-23 01:28:49 +00:00
# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */
2016-12-22 19:18:43 +00:00
# else
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
2017-02-09 01:38:17 +00:00
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
2016-12-22 19:18:43 +00:00
# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
2016-12-23 01:28:49 +00:00
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
2016-12-22 19:18:43 +00:00
# elif (ZDICT_GCC_VERSION >= 301)
2016-12-23 01:28:49 +00:00
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
2016-12-22 19:18:43 +00:00
# elif defined(_MSC_VER)
2016-12-23 01:28:49 +00:00
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
2016-12-22 19:18:43 +00:00
# else
# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
2016-12-23 01:28:49 +00:00
# define ZDICT_DEPRECATED(message) ZDICTLIB_API
2016-12-22 19:18:43 +00:00
# endif
# endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
ZDICT_DEPRECATED ( " use ZDICT_finalizeDictionary() instead " )
size_t ZDICT_addEntropyTablesFromBuffer ( void * dictBuffer , size_t dictContentSize , size_t dictBufferCapacity ,
const void * samplesBuffer , const size_t * samplesSizes , unsigned nbSamples ) ;
2016-06-04 16:56:23 +00:00
# endif /* ZDICT_STATIC_LINKING_ONLY */
2016-02-11 23:07:30 +00:00
# if defined (__cplusplus)
}
# endif
2016-07-13 15:38:39 +00:00
# endif /* DICTBUILDER_H_001 */