zstd/programs/datagen.c
2016-09-15 17:02:06 +02:00

200 lines
6.5 KiB
C

/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/* *************************************
* Compiler Options
***************************************/
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
# define _CRT_SECURE_NO_DEPRECATE /* removes VS2005 warning on strerror() */
#endif
/*-************************************
* Dependencies
**************************************/
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE, fwrite, fprintf */
#include <string.h> /* memcpy */
#include "mem.h" /* U32 */
/*-************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# define SET_BINARY_MODE(file) {int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
#else
# define SET_BINARY_MODE(file)
#endif
/*-************************************
* Macros
**************************************/
#define KB *(1 <<10)
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define RDG_DEBUG 0
#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ )
/*-************************************
* Local constants
**************************************/
#define LTLOG 13
#define LTSIZE (1<<LTLOG)
#define LTMASK (LTSIZE-1)
/*-*******************************************************
* Local Functions
*********************************************************/
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static U32 RDG_rand(U32* src)
{
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
U32 rand32 = *src;
rand32 *= prime1;
rand32 ^= prime2;
rand32 = RDG_rotl32(rand32, 13);
*src = rand32;
return rand32 >> 5;
}
static void RDG_fillLiteralDistrib(BYTE* ldt, double ld)
{
BYTE const firstChar = (ld<=0.0) ? 0 : '(';
BYTE const lastChar = (ld<=0.0) ? 255 : '}';
BYTE character = (ld<=0.0) ? 0 : '0';
U32 u;
if (ld<=0.0) ld = 0.0;
for (u=0; u<LTSIZE; ) {
U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
U32 const end = MIN ( u + weight , LTSIZE);
while (u < end) ldt[u++] = character;
character++;
if (character > lastChar) character = firstChar;
}
}
static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
{
U32 const id = RDG_rand(seed) & LTMASK;
return ldt[id]; /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */
}
static U32 RDG_rand15Bits (unsigned* seedPtr)
{
return RDG_rand(seedPtr) & 0x7FFF;
}
static U32 RDG_randLength(unsigned* seedPtr)
{
if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
}
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
{
BYTE* const buffPtr = (BYTE*)buffer;
U32 const matchProba32 = (U32)(32768 * matchProba);
size_t pos = prefixSize;
U32 prevOffset = 1;
/* special case : sparse content */
while (matchProba >= 1.0) {
size_t size0 = RDG_rand(seedPtr) & 3;
size0 = (size_t)1 << (16 + size0 * 2);
size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/
if (buffSize < pos + size0) {
memset(buffPtr+pos, 0, buffSize-pos);
return;
}
memset(buffPtr+pos, 0, size0);
pos += size0;
buffPtr[pos-1] = RDG_genChar(seedPtr, ldt);
continue;
}
/* init */
if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1;
/* Generate compressible data */
while (pos < buffSize) {
/* Select : Literal (char) or Match (within 32K) */
if (RDG_rand15Bits(seedPtr) < matchProba32) {
/* Copy (within 32K) */
U32 const length = RDG_randLength(seedPtr) + 4;
U32 const d = (U32) MIN(pos + length , buffSize);
U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2;
U32 const randOffset = RDG_rand15Bits(seedPtr) + 1;
U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
size_t match = pos - offset;
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
prevOffset = offset;
} else {
/* Literal (noise) */
U32 const length = RDG_randLength(seedPtr);
U32 const d = (U32) MIN(pos + length, buffSize);
while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt);
} }
}
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{
BYTE ldt[LTSIZE];
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
if (litProba<=0.0) litProba = matchProba / 4.5;
RDG_fillLiteralDistrib(ldt, litProba);
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
}
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
{
size_t const stdBlockSize = 128 KB;
size_t const stdDictSize = 32 KB;
BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
U64 total = 0;
BYTE ldt[LTSIZE]; /* literals distribution table */
/* init */
if (buff==NULL) { perror("datagen"); exit(1); }
if (litProba<=0.0) litProba = matchProba / 4.5;
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
RDG_fillLiteralDistrib(ldt, litProba);
SET_BINARY_MODE(stdout);
/* Generate initial dict */
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
/* Generate compressible data */
while (total < size) {
size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
total += genBlockSize;
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
/* update dict */
memcpy(buff, buff + stdBlockSize, stdDictSize);
}
/* cleanup */
free(buff);
}