AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

notificationLevel into ZDICT_param_t

Browse Source
This commit is contained in:
Yann Collet 2016-02-12 20:19:48 +01:00
parent 09ab681328
commit 6f3acbac0d
6 changed files with 28 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/zdict.c
View File

@ -102,7 +102,6 @@ static const size_t g_min_fast_dictContent = 192;
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
void ZDICT_setNotificationLevel(unsigned l) { g_displayLevel=l; }
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if (ZDICT_GetMilliSpan(g_time) > refreshRate) \
@ -111,7 +110,7 @@ void ZDICT_setNotificationLevel(unsigned l) { g_displayLevel=l; }
static const unsigned refreshRate = 300;
static clock_t g_time = 0;
void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
{
const BYTE* const b = (const BYTE*)ptr;
size_t u;
@ -813,6 +812,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; }
if (!dictList) return ERROR(memory_allocation);
ZDICT_initDictItem(dictList);
g_displayLevel = params.notificationLevel;
if (selectivity==0) selectivity = g_selectivity_default;
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;

28
lib/zdict_static.h
View File

@ -53,6 +53,8 @@ extern "C" {
typedef struct {
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
unsigned reserved[3]; /* space for future parameters */
} ZDICT_params_t;
@ -71,32 +73,6 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
ZDICT_params_t parameters);
/*-*************************************
* Helper functions
***************************************/
/*! ZDICT_setNotificationLevel() :
Set amount of notification to be displayed on the console.
default : 0 = no console notification.
1 = errors; 2 = notifications; 3 = details; 4 = debug;
Note : not thread-safe (uses a global constant)
*/
void ZDICT_setNotificationLevel(unsigned l);
/*-*************************************
* Private functions
***************************************/
/*! ZDICT_trainFromBuffer_unsafe() :
Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
note : `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
or an error code.
*/
size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t parameters);
#if defined (__cplusplus)
}
#endif

14
programs/dibio.c
View File

@ -85,7 +85,6 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
void DiB_setNotificationLevel(unsigned l) { g_displayLevel=l; ZDICT_setNotificationLevel(l); }
/*-*************************************
@ -216,6 +215,18 @@ static void DiB_saveDict(const char* dictFileName,
}
/*! ZDICT_trainFromBuffer_unsafe() :
Strictly Internal use only !!
Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
`samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
or an error code.
*/
size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t parameters);
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
const char** fileNamesTable, unsigned nbFiles,
ZDICT_params_t params)
@ -229,6 +240,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
int result = 0;
/* init */
g_displayLevel = params.notificationLevel;
benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)

11
programs/dibio.h
View File

@ -49,15 +49,4 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
ZDICT_params_t parameters);
/*-*************************************
* Helper functions
***************************************/
/*! DiB_setNotificationLevel
Set amount of notification to be displayed on the console.
default initial value : 0 = no console notification.
Note : not thread-safe (use a global constant)
*/
void DiB_setNotificationLevel(unsigned l);
#endif

18
programs/fileio.c
View File

@ -52,7 +52,7 @@
#define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
/* *************************************
/*-*************************************
* Includes
***************************************/
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
@ -68,12 +68,12 @@
#include "zbuff_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_legacy.h" /* legacy */
# include "fileio_legacy.h" /* legacy */
# include "zstd_legacy.h" /* ZSTD_isLegacy */
# include "fileio_legacy.h" /* FIO_decompressLegacyFrame */
#endif
/* *************************************
/*-*************************************
* OS-specific Includes
***************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
@ -92,7 +92,7 @@
#endif
/* *************************************
/*-*************************************
* Constants
***************************************/
#define KB *(1U<<10)
@ -112,12 +112,13 @@
#define BLOCKSIZE (128 KB)
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define FSE_CHECKSUM_SEED 0
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define FSE_CHECKSUM_SEED 0
#define CACHELINE 64
#define MAX_DICT_SIZE (512 KB)
#define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */
/* *************************************
* Macros
@ -138,7 +139,6 @@ static clock_t g_time = 0;
* Local Parameters
***************************************/
static U32 g_overwrite = 0;
void FIO_overwriteMode(void) { g_overwrite=1; }
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }

4
programs/zstdcli.c
View File

@ -133,7 +133,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -c : force write to standard output, even if it is the console\n");
#ifndef ZSTD_NODICT
DISPLAY( "Dictionary builder :\n");
DISPLAY( "--train : Create a dictionary from a set of files \n");
DISPLAY( "--train : create a dictionary from a training set of files \n");
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
@ -366,7 +366,7 @@ int main(int argCount, const char** argv)
ZDICT_params_t dictParams;
dictParams.compressionLevel = dictCLevel;
dictParams.selectivityLevel = dictSelect;
DiB_setNotificationLevel(displayLevel);
dictParams.notificationLevel = displayLevel;
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
#endif
goto _end;