AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #1456 from yijinfb/example-ZSTD_compressCCtx

Browse Source 
Add multiple_simple_compression.c (demonstrates the use of ZSTD_compressCCtx(...)) to examples directory.
This commit is contained in:
Yann Collet 2018-12-18 09:24:18 -08:00 committed by GitHub
commit 345af8abc7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 214 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
examples/.gitignore vendored
View File

@ -1,6 +1,7 @@
#build
simple_compression
simple_decompression
multiple_simple_compression
dictionary_compression
dictionary_decompression
streaming_compression

36
examples/Makefile
View File

@ -17,6 +17,7 @@ LIB = ../lib/libzstd.a
default: all
all: simple_compression simple_decompression \
multiple_simple_compression\
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression \
multiple_streaming_compression streaming_memory_usage
@ -24,33 +25,37 @@ all: simple_compression simple_decompression \
$(LIB) :
$(MAKE) -C ../lib libzstd.a
simple_compression : simple_compression.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
simple_compression : simple_compression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
simple_decompression : simple_decompression.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
simple_decompression : simple_decompression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
dictionary_compression : dictionary_compression.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
multiple_simple_compression : multiple_simple_compression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
dictionary_decompression : dictionary_decompression.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
dictionary_compression : dictionary_compression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
streaming_compression : streaming_compression.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
dictionary_decompression : dictionary_decompression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
multiple_streaming_compression : multiple_streaming_compression.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
streaming_compression : streaming_compression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
streaming_decompression : streaming_decompression.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
multiple_streaming_compression : multiple_streaming_compression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
streaming_decompression : streaming_decompression.c utils.h $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
streaming_memory_usage : streaming_memory_usage.c $(LIB)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
clean:
@rm -f core *.o tmp* result* *.zst \
simple_compression simple_decompression \
multiple_simple_compression \
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression \
multiple_streaming_compression streaming_memory_usage
@ -62,6 +67,7 @@ test: all
@echo -- Simple compression tests
./simple_compression tmp
./simple_decompression tmp.zst
./multiple_simple_compression *.c
./streaming_decompression tmp.zst > /dev/null
@echo -- Streaming memory usage
./streaming_memory_usage

10
examples/README.md
View File

@ -11,8 +11,14 @@ Zstandard library : usage examples
Result remains in memory.
Introduces usage of : `ZSTD_decompress()`
- [Multiple simple compression](multiple_simple_compression.c) :
Compress multiple files (in simple mode) in a single command line.
Demonstrates memory preservation technique that
minimizes malloc()/free() calls by re-using existing resources.
Introduces usage of : `ZSTD_compressCCtx()`
- [Streaming memory usage](streaming_memory_usage.c) :
Provides amount of memory used by streaming context
Provides amount of memory used by streaming context.
Introduces usage of : `ZSTD_sizeof_CStream()`
- [Streaming compression](streaming_compression.c) :
@ -20,7 +26,7 @@ Zstandard library : usage examples
Introduces usage of : `ZSTD_compressStream()`
- [Multiple Streaming compression](multiple_streaming_compression.c) :
Compress multiple files in a single command line.
Compress multiple files (in streaming mode) in a single command line.
Introduces memory usage preservation technique,
reducing impact of malloc()/free() and memset() by re-using existing resources.

4
examples/dictionary_compression.c
View File

@ -21,7 +21,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
{
size_t dictSize;
printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
if (!cdict) {
fprintf(stderr, "ZSTD_createCDict error \n");
@ -35,7 +35,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
{
size_t fSize;
void* const fBuff = loadFile_orDie(fname, &fSize);
void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize);

4
examples/dictionary_decompression.c
View File

@ -25,7 +25,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
{
size_t dictSize;
printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
free(dictBuffer);
@ -35,7 +35,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
static void decompress(const char* fname, const ZSTD_DDict* ddict)
{
size_t cSize;
void* const cBuff = loadFile_orDie(fname, &cSize);
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==ZSTD_CONTENTSIZE_ERROR) {
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);

117
examples/multiple_simple_compression.c Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include <stdlib.h> // malloc, free, exit
#include <stdio.h> // fprintf, perror, fopen, etc.
#include <string.h> // strlen, strcat, memset, strerror
#include <errno.h> // errno
#include <sys/stat.h> // stat
#include <zstd.h> // presumes zstd library is installed
#include "utils.h"
typedef struct {
void* fBuffer;
void* cBuffer;
size_t fBufferSize;
size_t cBufferSize;
ZSTD_CCtx* cctx;
} resources;
/*
* allocate memory for buffers big enough to compress all files
* as well as memory for output file name (ofn)
*/
static resources createResources_orDie(int argc, const char** argv, char **ofn, int* ofnBufferLen)
{
size_t maxFilenameLength=0;
size_t maxFileSize = 0;
int argNb;
for (argNb = 1; argNb < argc; argNb++) {
const char* const filename = argv[argNb];
size_t const filenameLength = strlen(filename);
size_t const fileSize = fsize_orDie(filename);
if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength;
if (fileSize > maxFileSize) maxFileSize = fileSize;
}
resources ress;
ress.fBufferSize = maxFileSize;
ress.cBufferSize = ZSTD_compressBound(maxFileSize);
*ofnBufferLen = maxFilenameLength + 5;
*ofn = (char*)malloc_orDie(*ofnBufferLen);
ress.fBuffer = malloc_orDie(ress.fBufferSize);
ress.cBuffer = malloc_orDie(ress.cBufferSize);
ress.cctx = ZSTD_createCCtx();
if (ress.cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
return ress;
}
static void freeResources(resources ress, char *outFilename)
{
free(ress.fBuffer);
free(ress.cBuffer);
ZSTD_freeCCtx(ress.cctx); /* never fails */
free(outFilename);
}
/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
static void compressFile_orDie(resources ress, const char* fname, const char* oname)
{
size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
if (ZSTD_isError(cSize)) {
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(8);
}
saveFile_orDie(oname, ress.cBuffer, cSize);
/* success */
// printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc<2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE(s)\n", exeName);
return 1;
}
/* memory allocation for outFilename and resources */
char* outFilename;
int outFilenameBufferLen;
resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen);
/* compress files with shared context, input and output buffers */
int argNb;
for (argNb = 1; argNb < argc; argNb++) {
const char* const inFilename = argv[argNb];
int inFilenameLen = strlen(inFilename);
assert(inFilenameLen + 5 <= outFilenameBufferLen);
memcpy(outFilename, inFilename, inFilenameLen);
memcpy(outFilename+inFilenameLen, ".zst", 5);
compressFile_orDie(ress, inFilename, outFilename);
}
/* free momery */
freeResources(ress,outFilename);
printf("compressed %i files \n", argc-1);
return 0;
}

2
examples/simple_compression.c
View File

@ -19,7 +19,7 @@
static void compress_orDie(const char* fname, const char* oname)
{
size_t fSize;
void* const fBuff = loadFile_orDie(fname, &fSize);
void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize);

2
examples/simple_decompression.c
View File

@ -20,7 +20,7 @@
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = loadFile_orDie(fname, &cSize);
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==ZSTD_CONTENTSIZE_ERROR) {
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);

96
examples/utils.h
View File

@ -9,7 +9,7 @@
*/
/*
* This header file has common utility functions used in examples.
* This header file has common utility functions used in examples.
*/
#ifndef UTILS_H
#define UTILS_H
@ -18,10 +18,11 @@
#include <stdio.h> // fprintf, perror, fopen, etc.
#include <string.h> // strlen, strcat, memset, strerror
#include <errno.h> // errno
#include <assert.h> // assert
#include <sys/stat.h> // stat
/*
* Define the returned error code from utility functions.
* Define the returned error code from utility functions.
*/
typedef enum {
ERROR_fsize = 1,
@ -35,21 +36,34 @@ typedef enum {
ERROR_largeFile = 9,
} UTILS_ErrorCode;
/*! fsize_orDie() :
/*! fsize_orDie() :
* Get the size of a given file path.
*
*
* @return The size of a given file path.
*/
static off_t fsize_orDie(const char *filename)
static size_t fsize_orDie(const char *filename)
{
struct stat st;
if (stat(filename, &st) == 0) return st.st_size;
/* error */
perror(filename);
exit(ERROR_fsize);
if (stat(filename, &st) != 0) {
/* error */
perror(filename);
exit(ERROR_fsize);
}
off_t const fileSize = st.st_size;
size_t const size = (size_t)fileSize;
/* 1. fileSize should be non-negative,
* 2. if off_t -> size_t type conversion results in discrepancy,
* the file size is too large for type size_t.
*/
if ((fileSize < 0) || (fileSize != (off_t)size)) {
fprintf(stderr, "%s : filesize too large \n", filename);
exit(ERROR_largeFile);
}
return size;
}
/*! fopen_orDie() :
/*! fopen_orDie() :
* Open a file using given file path and open option.
*
* @return If successful this function will return a FILE pointer to an
@ -64,7 +78,7 @@ static FILE* fopen_orDie(const char *filename, const char *instruction)
exit(ERROR_fopen);
}
/*! fclose_orDie() :
/*! fclose_orDie() :
* Close an opened file using given FILE pointer.
*/
static void fclose_orDie(FILE* file)
@ -75,11 +89,11 @@ static void fclose_orDie(FILE* file)
exit(ERROR_fclose);
}
/*! fread_orDie() :
*
/*! fread_orDie() :
*
* Read sizeToRead bytes from a given file, storing them at the
* location given by buffer.
*
*
* @return The number of bytes read.
*/
static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
@ -93,12 +107,12 @@ static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
}
/*! fwrite_orDie() :
*
*
* Write sizeToWrite bytes to a file pointed to by file, obtaining
* them from a location given by buffer.
*
* Note: This function will send an error to stderr and exit if it
* cannot write data to the given file pointer.
* cannot write data to the given file pointer.
*
* @return The number of bytes written.
*/
@ -113,7 +127,7 @@ static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
/*! malloc_orDie() :
* Allocate memory.
*
*
* @return If successful this function returns a pointer to allo-
* cated memory. If there is an error, this function will send that
* error to stderr and exit.
@ -128,39 +142,51 @@ static void* malloc_orDie(size_t size)
}
/*! loadFile_orDie() :
* Read size bytes from a file.
*
* load file into buffer (memory).
*
* Note: This function will send an error to stderr and exit if it
* cannot read data from the given file path.
*
* @return If successful this function will return a pointer to read
* data otherwise it will printout an error to stderr and exit.
*
* @return If successful this function will load file into buffer and
* return file size, otherwise it will printout an error to stderr and exit.
*/
static void* loadFile_orDie(const char* fileName, size_t* size)
static size_t loadFile_orDie(const char* fileName, void* buffer, int bufferSize)
{
off_t const fileSize = fsize_orDie(fileName);
size_t const buffSize = (size_t)fileSize;
if ((off_t)buffSize < fileSize) { /* narrowcast overflow */
fprintf(stderr, "%s : filesize too large \n", fileName);
exit(ERROR_largeFile);
}
size_t const fileSize = fsize_orDie(fileName);
assert(fileSize <= bufferSize);
FILE* const inFile = fopen_orDie(fileName, "rb");
void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) {
size_t const readSize = fread(buffer, 1, fileSize, inFile);
if (readSize != (size_t)fileSize) {
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
exit(ERROR_fread);
}
fclose(inFile); /* can't fail, read only */
*size = buffSize;
return fileSize;
}
/*! mallocAndLoadFile_orDie() :
* allocate memory buffer and then load file into it.
*
* Note: This function will send an error to stderr and exit if memory allocation
* fails or it cannot read data from the given file path.
*
* @return If successful this function will return buffer and bufferSize(=fileSize),
* otherwise it will printout an error to stderr and exit.
*/
static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) {
size_t const fileSize = fsize_orDie(fileName);
*bufferSize = fileSize;
void* const buffer = malloc_orDie(*bufferSize);
loadFile_orDie(fileName, buffer, *bufferSize);
return buffer;
}
/*! saveFile_orDie() :
*
*
* Save buffSize bytes to a given file path, obtaining them from a location pointed
* to by buff.
*
*
* Note: This function will send an error to stderr and exit if it
* cannot write to a given file.
*/