AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reduced console display on loading lots of files with zstd --train. Reported by @KrzysFR, see #177

Browse Source
This commit is contained in:
Yann Collet 2016-05-09 04:44:45 +02:00
parent e1a69a88c0
commit f6ca09b5ff
5 changed files with 31 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
NEWS
View File

@ -1,3 +1,7 @@
v0.6.1
Fixed : Legacy codec v0.5 compatible with dictionary decompression
Fixed : Decoder corruption error (#173)
v0.6.0
Stronger high compression modes, thanks to Przemyslaw Skibinski
API : ZSTD_getFrameParams() provides size of decompressed content

5
lib/common/util.h
View File

@ -95,7 +95,6 @@ extern "C" {
#endif
/*-****************************************
* Utility functions
******************************************/
@ -116,8 +115,8 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
(void)ticksPerSecond;
UTIL_getTime(clockStart);
do {
UTIL_getTime(clockEnd);
do {
UTIL_getTime(clockEnd);
} while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
}

2
lib/legacy/zstd_v05.h
View File

@ -102,7 +102,7 @@ size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapac
*************************/
typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx;
ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void);
size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize);

28
programs/dibio.c
View File

@ -76,6 +76,19 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((DIB_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const unsigned refreshRate = 150;
static clock_t g_time = 0;
static unsigned DIB_GetMilliSpan(clock_t nPrevious)
{
clock_t const nCurrent = clock();
return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
}
/*-*************************************
* Exceptions
@ -118,7 +131,7 @@ static void DiB_loadFiles(void* buffer, size_t bufferSize,
unsigned long long fileSize = UTIL_getFileSize(fileNamesTable[n]);
FILE* f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */
readSize = fread(buff+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
@ -166,17 +179,14 @@ static void DiB_fillNoise(void* buffer, size_t length)
static void DiB_saveDict(const char* dictFileName,
const void* buff, size_t buffSize)
{
FILE* f;
size_t n;
f = fopen(dictFileName, "wb");
FILE* f = fopen(dictFileName, "wb");
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
n = fwrite(buff, 1, buffSize, f);
if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName)
{ size_t const n = fwrite(buff, 1, buffSize, f);
if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) }
n = (size_t)fclose(f);
if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName)
{ size_t const n = (size_t)fclose(f);
if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) }
}

13
programs/fileio.c
View File

@ -126,6 +126,11 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const unsigned refreshRate = 150;
static clock_t g_time = 0;
static unsigned FIO_GetMilliSpan(clock_t nPrevious)
{
clock_t const nCurrent = clock();
return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
}
/*-*************************************
@ -157,14 +162,6 @@ void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
/*-*************************************
* Functions
***************************************/
static unsigned FIO_GetMilliSpan(clock_t nPrevious)
{
clock_t nCurrent = clock();
unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
return nSpan;
}
static FILE* FIO_openSrcFile(const char* srcFileName)
{
FILE* f;