zstd/programs/datagen.c

204 lines
6.9 KiB
C
Raw Normal View History

2015-01-24 00:58:16 +00:00
/*
datagen.c - compressible data generator test tool
2016-04-10 23:12:32 +00:00
Copyright (C) Yann Collet 2012-2016
2015-01-24 00:58:16 +00:00
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
2016-04-10 23:12:32 +00:00
- zstd homepage : http://www.zstd.net/
- source repository : https://github.com/Cyan4973/zstd
2015-01-24 00:58:16 +00:00
*/
2016-02-02 13:36:49 +00:00
/*-************************************
2015-01-24 00:58:16 +00:00
* Includes
**************************************/
#include <stdlib.h> /* malloc */
#include <stdio.h> /* FILE, fwrite */
#include <string.h> /* memcpy */
2016-05-09 10:20:50 +00:00
#include "mem.h"
2015-01-24 00:58:16 +00:00
2016-02-02 13:36:49 +00:00
/*-************************************
2015-01-24 12:31:55 +00:00
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# define SET_BINARY_MODE(file) {int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
2015-01-24 12:31:55 +00:00
#else
# define SET_BINARY_MODE(file)
#endif
2016-02-02 13:36:49 +00:00
/*-************************************
2016-05-09 10:20:50 +00:00
* Macros
2015-01-24 00:58:16 +00:00
**************************************/
#define KB *(1 <<10)
2016-05-18 15:18:48 +00:00
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define RDG_DEBUG 0
#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ )
2015-01-24 00:58:16 +00:00
2016-02-02 13:36:49 +00:00
/*-************************************
2016-05-18 15:18:48 +00:00
* Local constants
2015-07-05 00:23:52 +00:00
**************************************/
#define LTLOG 13
#define LTSIZE (1<<LTLOG)
#define LTMASK (LTSIZE-1)
2016-02-02 13:36:49 +00:00
/*-*******************************************************
2015-01-24 00:58:16 +00:00
* Local Functions
*********************************************************/
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
2016-05-18 15:18:48 +00:00
static U32 RDG_rand(U32* src)
2015-01-24 00:58:16 +00:00
{
2016-04-10 23:12:32 +00:00
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
2015-01-24 00:58:16 +00:00
U32 rand32 = *src;
2016-04-10 23:12:32 +00:00
rand32 *= prime1;
rand32 ^= prime2;
rand32 = RDG_rotl32(rand32, 13);
2015-01-24 00:58:16 +00:00
*src = rand32;
2016-05-18 15:18:48 +00:00
return rand32 >> 5;
2015-01-24 00:58:16 +00:00
}
2016-05-18 15:18:48 +00:00
static void RDG_fillLiteralDistrib(BYTE* ldt, double ld)
2015-01-24 00:58:16 +00:00
{
2016-05-18 15:18:48 +00:00
BYTE const firstChar = (ld<=0.0) ? 0 : '(';
BYTE const lastChar = (ld<=0.0) ? 255 : '}';
2016-05-09 10:20:50 +00:00
BYTE character = (ld<=0.0) ? 0 : '0';
2016-05-18 15:18:48 +00:00
U32 u;
if (ld<=0.0) ld = 0.0;
//TRACE(" percent:%5.2f%% \n", ld*100.);
//TRACE(" start:(%c)[%02X] ", character, character);
for (u=0; u<LTSIZE; ) {
U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
U32 const end = MIN ( u + weight , LTSIZE);
while (u < end) ldt[u++] = character; // TRACE(" %u(%c)[%02X] ", u, character, character);
2015-01-24 00:58:16 +00:00
character++;
if (character > lastChar) character = firstChar;
}
}
2015-07-05 00:23:52 +00:00
2016-05-18 15:18:48 +00:00
static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
2015-01-24 00:58:16 +00:00
{
2016-04-10 23:12:32 +00:00
U32 const id = RDG_rand(seed) & LTMASK;
2016-05-18 15:18:48 +00:00
//TRACE(" %u : \n", id);
//TRACE(" %4u [%4u] ; val : %4u \n", id, id&255, ldt[id]);
return (ldt[id]); /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with 0.0. Checked : table is fully initialized */
2015-01-24 00:58:16 +00:00
}
2015-07-05 00:23:52 +00:00
2016-05-18 15:18:48 +00:00
#define RDG_RAND15BITS ( RDG_rand(seed) & 0x7FFF )
#define RDG_RANDLENGTH ( (RDG_rand(seed) & 7) ? (RDG_rand(seed) & 0xF) : (RDG_rand(seed) & 0x1FF) + 0xF)
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
2015-01-24 00:58:16 +00:00
{
BYTE* buffPtr = (BYTE*)buffer;
2015-02-10 17:15:20 +00:00
const U32 matchProba32 = (U32)(32768 * matchProba);
size_t pos = prefixSize;
U32* seed = seedPtr;
2015-11-20 11:00:25 +00:00
U32 prevOffset = 1;
2015-01-24 00:58:16 +00:00
2015-07-08 01:26:17 +00:00
/* special case : sparse content */
2016-02-02 13:36:49 +00:00
while (matchProba >= 1.0) {
size_t size0 = RDG_rand(seed) & 3;
2015-07-05 00:23:52 +00:00
size0 = (size_t)1 << (16 + size0 * 2);
size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
2016-02-02 13:36:49 +00:00
if (buffSize < pos + size0) {
memset(buffPtr+pos, 0, buffSize-pos);
return;
}
memset(buffPtr+pos, 0, size0);
pos += size0;
2016-05-18 15:18:48 +00:00
buffPtr[pos-1] = RDG_genChar(seed, ldt);
continue;
}
2015-01-24 12:31:55 +00:00
/* init */
2016-05-18 15:18:48 +00:00
if (pos==0) buffPtr[0] = RDG_genChar(seed, ldt), pos=1;
2015-02-10 17:15:20 +00:00
/* Generate compressible data */
2016-02-02 13:36:49 +00:00
while (pos < buffSize) {
2015-01-24 00:58:16 +00:00
/* Select : Literal (char) or Match (within 32K) */
2016-02-02 13:36:49 +00:00
if (RDG_RAND15BITS < matchProba32) {
2015-02-10 17:15:20 +00:00
/* Copy (within 32K) */
2016-05-18 15:18:48 +00:00
U32 const length = RDG_RANDLENGTH + 4;
2016-05-18 15:26:23 +00:00
U32 const d = (U32) MIN(pos + length , buffSize);
2016-05-18 15:18:48 +00:00
U32 const repeatOffset = (RDG_rand(seed) & 15) == 2;
U32 const randOffset = RDG_RAND15BITS + 1;
2016-05-18 15:26:23 +00:00
U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
2016-05-18 15:18:48 +00:00
size_t match = pos - offset;
//TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length);
2015-11-20 11:46:08 +00:00
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
2016-05-18 15:18:48 +00:00
prevOffset = offset;
2016-02-02 13:36:49 +00:00
} else {
2015-01-24 00:58:16 +00:00
/* Literal (noise) */
2016-05-18 15:18:48 +00:00
U32 const length = RDG_RANDLENGTH;
2016-05-18 15:26:23 +00:00
U32 const d = (U32) MIN(pos + length, buffSize);
2016-05-18 15:18:48 +00:00
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt);
2016-02-02 13:36:49 +00:00
} }
2015-02-10 17:15:20 +00:00
}
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{
2016-05-18 15:18:48 +00:00
BYTE ldt[LTSIZE];
memset(ldt, '0', sizeof(ldt));
if (litProba<=0.0) litProba = matchProba / 4.5;
//TRACE(" percent:%5.2f%% \n", litProba*100.);
RDG_fillLiteralDistrib(ldt, litProba);
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
2015-02-10 17:15:20 +00:00
}
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
2015-02-10 17:15:20 +00:00
{
2016-05-18 15:18:48 +00:00
size_t const stdBlockSize = 128 KB;
size_t const stdDictSize = 32 KB;
BYTE* buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
2015-02-10 17:15:20 +00:00
U64 total = 0;
2016-05-18 15:18:48 +00:00
BYTE ldt[LTSIZE];
2015-02-10 17:15:20 +00:00
/* init */
2015-07-08 01:26:17 +00:00
if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); }
2016-04-10 23:20:14 +00:00
if (litProba<=0.0) litProba = matchProba / 4.5;
2016-05-18 15:18:48 +00:00
memset(ldt, '0', sizeof(ldt));
2016-04-10 23:20:14 +00:00
RDG_fillLiteralDistrib(ldt, litProba);
2015-02-10 17:15:20 +00:00
SET_BINARY_MODE(stdout);
2015-07-08 01:26:17 +00:00
/* Generate initial dict */
2016-05-18 15:18:48 +00:00
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
2015-01-24 00:58:16 +00:00
/* Generate compressible data */
2016-02-02 13:36:49 +00:00
while (total < size) {
2016-05-18 15:18:48 +00:00
size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
2015-01-24 00:58:16 +00:00
total += genBlockSize;
2016-04-10 23:12:32 +00:00
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
2015-02-10 17:15:20 +00:00
/* update dict */
2016-05-18 15:18:48 +00:00
memcpy(buff, buff + stdBlockSize, stdDictSize);
2015-01-24 00:58:16 +00:00
}
2015-07-08 01:26:17 +00:00
2016-02-02 13:36:49 +00:00
/* cleanup */
2015-07-08 01:26:17 +00:00
free(buff);
2015-01-24 00:58:16 +00:00
}