2015-01-24 00:58:16 +00:00
/*
2016-02-11 03:17:50 +00:00
fileio . c - File i / o handler for zstd
Copyright ( C ) Yann Collet 2013 - 2016
2015-01-24 00:58:16 +00:00
GPL v2 License
This program is free software ; you can redistribute it and / or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation ; either version 2 of the License , or
( at your option ) any later version .
This program is distributed in the hope that it will be useful ,
but WITHOUT ANY WARRANTY ; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
GNU General Public License for more details .
You should have received a copy of the GNU General Public License along
with this program ; if not , write to the Free Software Foundation , Inc . ,
51 Franklin Street , Fifth Floor , Boston , MA 02110 - 1301 USA .
You can contact the author at :
2016-02-11 03:17:50 +00:00
- zstd homepage : http : //www.zstd.net
2015-01-24 00:58:16 +00:00
*/
/*
Note : this is stand - alone program .
It is not part of ZSTD compression library , it is a user program of ZSTD library .
The license of ZSTD library is BSD .
The license of this file is GPLv2 .
*/
2015-10-22 15:55:40 +00:00
/* *************************************
2015-10-18 21:18:32 +00:00
* Tuning options
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-10-18 21:18:32 +00:00
# ifndef ZSTD_LEGACY_SUPPORT
2016-02-11 03:17:50 +00:00
/* LEGACY_SUPPORT :
2015-10-18 21:18:32 +00:00
* decompressor can decode older formats ( starting from Zstd 0.1 + ) */
# define ZSTD_LEGACY_SUPPORT 1
2015-11-09 16:42:17 +00:00
# endif
2015-10-18 21:18:32 +00:00
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* Compiler Options
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-01-24 00:58:16 +00:00
/* Disable some Visual warning messages */
# ifdef _MSC_VER
# define _CRT_SECURE_NO_WARNINGS
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# endif
# define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */
# define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* Includes
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-11-09 16:42:17 +00:00
# include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
# include <stdlib.h> /* malloc, free */
# include <string.h> /* strcmp, strlen */
# include <time.h> /* clock */
# include <errno.h> /* errno */
# include <sys/types.h> /* stat64 */
# include <sys/stat.h> /* stat64 */
2015-10-18 21:18:32 +00:00
# include "mem.h"
2015-01-24 00:58:16 +00:00
# include "fileio.h"
2015-11-25 13:42:45 +00:00
# include "zstd_static.h" /* ZSTD_magicNumber */
# include "zstd_buffered_static.h"
2015-01-24 00:58:16 +00:00
2015-10-18 21:18:32 +00:00
# if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
2015-11-25 13:42:45 +00:00
# include "zstd_legacy.h" /* legacy */
2015-10-30 10:21:50 +00:00
# include "fileio_legacy.h" /* legacy */
2015-10-22 15:55:40 +00:00
# endif
2015-10-18 21:18:32 +00:00
2015-01-24 00:58:16 +00:00
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* OS - specific Includes
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-01-24 00:58:16 +00:00
# if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
2015-07-05 07:20:56 +00:00
# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
2015-01-24 00:58:16 +00:00
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
# else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
# endif
2015-11-09 17:20:39 +00:00
# if !defined(S_ISREG)
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
# endif
2015-01-24 00:58:16 +00:00
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* Constants
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-01-24 00:58:16 +00:00
# define KB *(1U<<10)
# define MB *(1U<<20)
# define GB *(1U<<30)
# define _1BIT 0x01
# define _2BITS 0x03
# define _3BITS 0x07
# define _4BITS 0x0F
# define _6BITS 0x3F
# define _8BITS 0xFF
# define BIT6 0x40
# define BIT7 0x80
2015-11-25 13:42:45 +00:00
# define BLOCKSIZE (128 KB)
# define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
2015-01-24 00:58:16 +00:00
# define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
# define FSE_CHECKSUM_SEED 0
# define CACHELINE 64
2015-12-13 12:35:21 +00:00
# define MAX_DICT_SIZE (512 KB)
2015-01-24 00:58:16 +00:00
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* Macros
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-01-24 00:58:16 +00:00
# define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
# define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2 ; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
# define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ( ( FIO_GetMilliSpan ( g_time ) > refreshRate ) | | ( g_displayLevel > = 4 ) ) \
{ g_time = clock ( ) ; DISPLAY ( __VA_ARGS__ ) ; \
if ( g_displayLevel > = 4 ) fflush ( stdout ) ; } }
static const unsigned refreshRate = 150 ;
static clock_t g_time = 0 ;
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* Local Parameters
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-01-24 00:58:16 +00:00
static U32 g_overwrite = 0 ;
void FIO_overwriteMode ( void ) { g_overwrite = 1 ; }
void FIO_setNotificationLevel ( unsigned level ) { g_displayLevel = level ; }
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* Exceptions
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# ifndef DEBUG
# define DEBUG 0
# endif
2015-01-24 00:58:16 +00:00
# define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
# define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT ( " Error defined at %s, line %i : \n " , __FILE__ , __LINE__ ) ; \
DISPLAYLEVEL ( 1 , " Error %i : " , error ) ; \
DISPLAYLEVEL ( 1 , __VA_ARGS__ ) ; \
DISPLAYLEVEL ( 1 , " \n " ) ; \
exit ( error ) ; \
}
2015-10-22 15:55:40 +00:00
/* *************************************
2015-01-24 00:58:16 +00:00
* Functions
2015-10-22 15:55:40 +00:00
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-01-24 00:58:16 +00:00
static unsigned FIO_GetMilliSpan ( clock_t nPrevious )
{
clock_t nCurrent = clock ( ) ;
unsigned nSpan = ( unsigned ) ( ( ( nCurrent - nPrevious ) * 1000 ) / CLOCKS_PER_SEC ) ;
return nSpan ;
}
2015-11-09 16:42:17 +00:00
static U64 FIO_getFileSize ( const char * infilename )
{
int r ;
# if defined(_MSC_VER)
struct _stat64 statbuf ;
r = _stat64 ( infilename , & statbuf ) ;
# else
struct stat statbuf ;
r = stat ( infilename , & statbuf ) ;
# endif
if ( r | | ! S_ISREG ( statbuf . st_mode ) ) return 0 ;
return ( U64 ) statbuf . st_size ;
}
2016-02-12 14:56:46 +00:00
static FILE * FIO_openSrcFile ( const char * srcFileName )
2015-12-17 01:23:58 +00:00
{
2016-02-12 14:56:46 +00:00
FILE * f ;
2016-02-02 13:36:49 +00:00
if ( ! strcmp ( srcFileName , stdinmark ) ) {
2015-12-17 01:23:58 +00:00
DISPLAYLEVEL ( 4 , " Using stdin for input \n " ) ;
2016-02-12 14:56:46 +00:00
f = stdin ;
2015-12-17 01:23:58 +00:00
SET_BINARY_MODE ( stdin ) ;
2016-02-02 13:36:49 +00:00
} else {
2016-02-12 14:56:46 +00:00
f = fopen ( srcFileName , " rb " ) ;
2015-12-17 01:23:58 +00:00
}
2016-02-12 14:56:46 +00:00
if ( f = = NULL ) DISPLAYLEVEL ( 1 , " zstd: %s: No such file \n " , srcFileName ) ;
return f ;
}
static FILE * FIO_openDstFile ( const char * dstFileName )
{
FILE * f ;
2015-12-17 01:23:58 +00:00
2016-02-02 13:36:49 +00:00
if ( ! strcmp ( dstFileName , stdoutmark ) ) {
2015-12-17 01:23:58 +00:00
DISPLAYLEVEL ( 4 , " Using stdout for output \n " ) ;
2016-02-12 14:56:46 +00:00
f = stdout ;
2015-12-17 01:23:58 +00:00
SET_BINARY_MODE ( stdout ) ;
2016-02-02 13:36:49 +00:00
} else {
if ( ! g_overwrite ) { /* Check if destination file already exists */
2016-02-12 14:56:46 +00:00
f = fopen ( dstFileName , " rb " ) ;
if ( f ! = 0 ) { /* dest file exists, prompt for overwrite authorization */
fclose ( f ) ;
if ( g_displayLevel < = 1 ) {
2015-12-17 19:30:14 +00:00
/* No interaction possible */
2016-02-12 14:56:46 +00:00
DISPLAY ( " zstd: %s already exists; not overwritten \n " , dstFileName ) ;
return 0 ;
2015-12-17 19:30:14 +00:00
}
2016-02-12 14:56:46 +00:00
DISPLAY ( " zstd: %s already exists; do you wish to overwrite (y/N) ? " , dstFileName ) ;
2015-12-17 19:30:14 +00:00
{
2015-12-29 10:57:15 +00:00
int ch = getchar ( ) ;
2016-02-02 13:36:49 +00:00
if ( ( ch ! = ' Y ' ) & & ( ch ! = ' y ' ) ) {
2016-02-12 14:56:46 +00:00
DISPLAY ( " not overwritten \n " ) ;
return 0 ;
2015-12-29 10:57:15 +00:00
}
while ( ( ch ! = EOF ) & & ( ch ! = ' \n ' ) ) ch = getchar ( ) ; /* flush rest of input line */
2016-02-02 13:36:49 +00:00
} } }
2016-02-12 14:56:46 +00:00
f = fopen ( dstFileName , " wb " ) ;
}
return f ;
}
2015-12-17 19:30:14 +00:00
/*!FIO_loadFile
* creates a buffer , pointed by * bufferPtr ,
* loads " filename " content into it
* up to MAX_DICT_SIZE bytes
*/
static size_t FIO_loadFile ( void * * bufferPtr , const char * fileName )
{
FILE * fileHandle ;
size_t readSize ;
U64 fileSize ;
* bufferPtr = NULL ;
2016-02-02 13:36:49 +00:00
if ( fileName = = NULL ) return 0 ;
2015-12-17 19:30:14 +00:00
DISPLAYLEVEL ( 4 , " Loading %s as dictionary \n " , fileName ) ;
fileHandle = fopen ( fileName , " rb " ) ;
if ( fileHandle = = 0 ) EXM_THROW ( 31 , " Error opening file %s " , fileName ) ;
fileSize = FIO_getFileSize ( fileName ) ;
2016-02-02 13:36:49 +00:00
if ( fileSize > MAX_DICT_SIZE ) {
2015-12-17 19:30:14 +00:00
int seekResult ;
if ( fileSize > 1 GB ) EXM_THROW ( 32 , " Dictionary file %s is too large " , fileName ) ; /* avoid extreme cases */
DISPLAYLEVEL ( 2 , " Dictionary %s is too large : using last %u bytes only \n " , fileName , MAX_DICT_SIZE ) ;
seekResult = fseek ( fileHandle , ( long int ) ( fileSize - MAX_DICT_SIZE ) , SEEK_SET ) ; /* use end of file */
if ( seekResult ! = 0 ) EXM_THROW ( 33 , " Error seeking into file %s " , fileName ) ;
fileSize = MAX_DICT_SIZE ;
}
* bufferPtr = ( BYTE * ) malloc ( ( size_t ) fileSize ) ;
if ( * bufferPtr = = NULL ) EXM_THROW ( 34 , " Allocation error : not enough memory for dictBuffer " ) ;
readSize = fread ( * bufferPtr , 1 , ( size_t ) fileSize , fileHandle ) ;
if ( readSize ! = fileSize ) EXM_THROW ( 35 , " Error reading dictionary file %s " , fileName ) ;
fclose ( fileHandle ) ;
return ( size_t ) fileSize ;
}
2015-12-17 01:23:58 +00:00
/* **********************************************************************
* Compression
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
typedef struct {
void * srcBuffer ;
size_t srcBufferSize ;
void * dstBuffer ;
size_t dstBufferSize ;
void * dictBuffer ;
size_t dictBufferSize ;
ZBUFF_CCtx * ctx ;
2016-02-12 14:56:46 +00:00
FILE * dstFile ;
2015-12-17 01:23:58 +00:00
} cRess_t ;
static cRess_t FIO_createCResources ( const char * dictFileName )
{
cRess_t ress ;
ress . ctx = ZBUFF_createCCtx ( ) ;
if ( ress . ctx = = NULL ) EXM_THROW ( 30 , " Allocation error : can't create ZBUFF context " ) ;
/* Allocate Memory */
ress . srcBufferSize = ZBUFF_recommendedCInSize ( ) ;
ress . srcBuffer = malloc ( ress . srcBufferSize ) ;
ress . dstBufferSize = ZBUFF_recommendedCOutSize ( ) ;
ress . dstBuffer = malloc ( ress . dstBufferSize ) ;
if ( ! ress . srcBuffer | | ! ress . dstBuffer ) EXM_THROW ( 31 , " Allocation error : not enough memory " ) ;
/* dictionary */
2015-12-17 19:30:14 +00:00
ress . dictBufferSize = FIO_loadFile ( & ( ress . dictBuffer ) , dictFileName ) ;
2015-12-17 01:23:58 +00:00
return ress ;
}
static void FIO_freeCResources ( cRess_t ress )
{
size_t errorCode ;
free ( ress . srcBuffer ) ;
free ( ress . dstBuffer ) ;
free ( ress . dictBuffer ) ;
errorCode = ZBUFF_freeCCtx ( ress . ctx ) ;
if ( ZBUFF_isError ( errorCode ) ) EXM_THROW ( 38 , " Error : can't release ZBUFF context resource : %s " , ZBUFF_getErrorName ( errorCode ) ) ;
}
2016-02-12 14:56:46 +00:00
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess ( ) , with ress . desFile already opened
* @ return : 0 : compression completed correctly ,
* 1 : missing or pb opening srcFileName
2015-12-17 01:23:58 +00:00
*/
2016-02-12 14:56:46 +00:00
static int FIO_compressFilename_internal ( cRess_t ress ,
const char * dstFileName , const char * srcFileName ,
int cLevel )
2015-12-17 01:23:58 +00:00
{
FILE * srcFile ;
2016-02-12 14:56:46 +00:00
FILE * dstFile = ress . dstFile ;
2015-12-17 01:23:58 +00:00
U64 filesize = 0 ;
U64 compressedfilesize = 0 ;
size_t dictSize = ress . dictBufferSize ;
size_t sizeCheck , errorCode ;
/* File check */
2016-02-12 14:56:46 +00:00
srcFile = FIO_openSrcFile ( srcFileName ) ;
if ( ! srcFile ) return 1 ; /* srcFile could not be opened */
2015-12-17 01:23:58 +00:00
/* init */
filesize = FIO_getFileSize ( srcFileName ) + dictSize ;
2016-01-26 15:31:22 +00:00
errorCode = ZBUFF_compressInit_advanced ( ress . ctx , ress . dictBuffer , ress . dictBufferSize , ZSTD_getParams ( cLevel , filesize ) ) ;
if ( ZBUFF_isError ( errorCode ) ) EXM_THROW ( 21 , " Error initializing compression : %s " , ZBUFF_getErrorName ( errorCode ) ) ;
2015-12-17 01:23:58 +00:00
/* Main compression loop */
filesize = 0 ;
2016-01-26 15:31:22 +00:00
while ( 1 ) {
2015-12-17 01:23:58 +00:00
/* Fill input Buffer */
2016-01-26 15:31:22 +00:00
size_t inSize = fread ( ress . srcBuffer , ( size_t ) 1 , ress . srcBufferSize , srcFile ) ;
2015-12-17 01:23:58 +00:00
if ( inSize = = 0 ) break ;
filesize + = inSize ;
DISPLAYUPDATE ( 2 , " \r Read : %u MB " , ( U32 ) ( filesize > > 20 ) ) ;
2016-02-12 14:56:46 +00:00
{ /* Compress using buffered streaming */
2015-12-17 01:23:58 +00:00
size_t usedInSize = inSize ;
size_t cSize = ress . dstBufferSize ;
size_t result = ZBUFF_compressContinue ( ress . ctx , ress . dstBuffer , & cSize , ress . srcBuffer , & usedInSize ) ;
if ( ZBUFF_isError ( result ) )
EXM_THROW ( 23 , " Compression error : %s " , ZBUFF_getErrorName ( result ) ) ;
if ( inSize ! = usedInSize )
/* inBuff should be entirely consumed since buffer sizes are recommended ones */
EXM_THROW ( 24 , " Compression error : input block not fully consumed " ) ;
/* Write cBlock */
sizeCheck = fwrite ( ress . dstBuffer , 1 , cSize , dstFile ) ;
if ( sizeCheck ! = cSize ) EXM_THROW ( 25 , " Write error : cannot write compressed block into %s " , dstFileName ) ;
compressedfilesize + = cSize ;
}
DISPLAYUPDATE ( 2 , " \r Read : %u MB ==> %.2f%% " , ( U32 ) ( filesize > > 20 ) , ( double ) compressedfilesize / filesize * 100 ) ;
}
/* End of Frame */
{
size_t cSize = ress . dstBufferSize ;
size_t result = ZBUFF_compressEnd ( ress . ctx , ress . dstBuffer , & cSize ) ;
if ( result ! = 0 ) EXM_THROW ( 26 , " Compression error : cannot create frame end " ) ;
sizeCheck = fwrite ( ress . dstBuffer , 1 , cSize , dstFile ) ;
if ( sizeCheck ! = cSize ) EXM_THROW ( 27 , " Write error : cannot write frame end into %s " , dstFileName ) ;
compressedfilesize + = cSize ;
}
/* Status */
DISPLAYLEVEL ( 2 , " \r %79s \r " , " " ) ;
DISPLAYLEVEL ( 2 , " Compressed %llu bytes into %llu bytes ==> %.2f%% \n " ,
( unsigned long long ) filesize , ( unsigned long long ) compressedfilesize , ( double ) compressedfilesize / filesize * 100 ) ;
/* clean */
fclose ( srcFile ) ;
return 0 ;
}
2016-02-12 14:56:46 +00:00
/*! FIO_compressFilename_extRess() :
* @ return : 0 : compression completed correctly ,
* 1 : missing or pb opening srcFileName
*/
static int FIO_compressFilename_extRess ( cRess_t ress ,
const char * dstFileName , const char * srcFileName ,
int cLevel )
{
int result ;
ress . dstFile = FIO_openDstFile ( dstFileName ) ;
if ( ress . dstFile = = 0 ) return 1 ;
result = FIO_compressFilename_internal ( ress , dstFileName , srcFileName , cLevel ) ;
if ( fclose ( ress . dstFile ) ) EXM_THROW ( 28 , " Write error : cannot properly close %s " , dstFileName ) ;
return result ;
}
2015-12-17 13:09:55 +00:00
int FIO_compressFilename ( const char * dstFileName , const char * srcFileName ,
const char * dictFileName , int compressionLevel )
2015-01-24 00:58:16 +00:00
{
2015-12-17 13:09:55 +00:00
clock_t start , end ;
cRess_t ress ;
int issueWithSrcFile = 0 ;
2015-01-24 00:58:16 +00:00
2015-12-17 13:09:55 +00:00
/* Init */
start = clock ( ) ;
ress = FIO_createCResources ( dictFileName ) ;
2015-11-25 13:42:45 +00:00
2015-12-17 13:09:55 +00:00
/* Compress File */
issueWithSrcFile + = FIO_compressFilename_extRess ( ress , dstFileName , srcFileName , compressionLevel ) ;
2015-01-24 00:58:16 +00:00
2015-12-17 13:09:55 +00:00
/* Free resources */
FIO_freeCResources ( ress ) ;
2015-01-24 00:58:16 +00:00
2015-12-17 13:09:55 +00:00
/* Final Status */
end = clock ( ) ;
2015-11-25 13:42:45 +00:00
{
2015-12-17 13:09:55 +00:00
double seconds = ( double ) ( end - start ) / CLOCKS_PER_SEC ;
DISPLAYLEVEL ( 4 , " Completed in %.2f sec \n " , seconds ) ;
2015-11-25 13:42:45 +00:00
}
2015-01-24 00:58:16 +00:00
2015-12-17 13:09:55 +00:00
return issueWithSrcFile ;
2015-01-24 00:58:16 +00:00
}
2015-12-17 01:23:58 +00:00
# define FNSPACE 30
2015-12-17 13:09:55 +00:00
int FIO_compressMultipleFilenames ( const char * * inFileNamesTable , unsigned nbFiles ,
2015-12-17 01:23:58 +00:00
const char * suffix ,
const char * dictFileName , int compressionLevel )
{
2015-12-17 13:09:55 +00:00
unsigned u ;
2015-12-17 01:23:58 +00:00
int missed_files = 0 ;
char * dstFileName = ( char * ) malloc ( FNSPACE ) ;
size_t dfnSize = FNSPACE ;
2016-02-12 14:56:46 +00:00
const size_t suffixSize = suffix ? strlen ( suffix ) : 0 ;
2015-12-17 01:23:58 +00:00
cRess_t ress ;
/* init */
ress = FIO_createCResources ( dictFileName ) ;
/* loop on each file */
2016-02-12 14:56:46 +00:00
if ( suffix ) {
for ( u = 0 ; u < nbFiles ; u + + ) {
size_t ifnSize = strlen ( inFileNamesTable [ u ] ) ;
if ( dfnSize < = ifnSize + suffixSize + 1 ) { free ( dstFileName ) ; dfnSize = ifnSize + 20 ; dstFileName = ( char * ) malloc ( dfnSize ) ; }
strcpy ( dstFileName , inFileNamesTable [ u ] ) ;
strcat ( dstFileName , suffix ) ;
missed_files + = FIO_compressFilename_extRess ( ress , dstFileName ,
inFileNamesTable [ u ] , compressionLevel ) ;
}
} else {
ress . dstFile = stdout ;
for ( u = 0 ; u < nbFiles ; u + + )
missed_files + = FIO_compressFilename_internal ( ress , stdoutmark ,
inFileNamesTable [ u ] , compressionLevel ) ;
if ( fclose ( ress . dstFile ) ) EXM_THROW ( 29 , " Write error : cannot properly close %s " , stdoutmark ) ;
2015-12-17 01:23:58 +00:00
}
/* Close & Free */
FIO_freeCResources ( ress ) ;
free ( dstFileName ) ;
return missed_files ;
}
/* **************************************************************************
* Decompression
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2015-12-17 19:30:14 +00:00
typedef struct {
void * srcBuffer ;
size_t srcBufferSize ;
void * dstBuffer ;
size_t dstBufferSize ;
void * dictBuffer ;
size_t dictBufferSize ;
ZBUFF_DCtx * dctx ;
2016-02-12 17:33:26 +00:00
FILE * dstFile ;
2015-12-17 19:30:14 +00:00
} dRess_t ;
static dRess_t FIO_createDResources ( const char * dictFileName )
{
dRess_t ress ;
/* init */
ress . dctx = ZBUFF_createDCtx ( ) ;
if ( ress . dctx = = NULL ) EXM_THROW ( 60 , " Can't create ZBUFF decompression context " ) ;
2015-12-17 01:23:58 +00:00
2015-12-17 19:30:14 +00:00
/* Allocate Memory */
ress . srcBufferSize = ZBUFF_recommendedDInSize ( ) ;
ress . srcBuffer = malloc ( ress . srcBufferSize ) ;
ress . dstBufferSize = ZBUFF_recommendedDOutSize ( ) ;
ress . dstBuffer = malloc ( ress . dstBufferSize ) ;
if ( ! ress . srcBuffer | | ! ress . dstBuffer ) EXM_THROW ( 61 , " Allocation error : not enough memory " ) ;
2015-12-17 01:23:58 +00:00
2015-12-17 19:30:14 +00:00
/* dictionary */
ress . dictBufferSize = FIO_loadFile ( & ( ress . dictBuffer ) , dictFileName ) ;
return ress ;
}
static void FIO_freeDResources ( dRess_t ress )
{
size_t errorCode = ZBUFF_freeDCtx ( ress . dctx ) ;
if ( ZBUFF_isError ( errorCode ) ) EXM_THROW ( 69 , " Error : can't free ZBUFF context resource : %s " , ZBUFF_getErrorName ( errorCode ) ) ;
free ( ress . srcBuffer ) ;
free ( ress . dstBuffer ) ;
free ( ress . dictBuffer ) ;
}
unsigned long long FIO_decompressFrame ( dRess_t ress ,
FILE * foutput , FILE * finput , size_t alreadyLoaded )
2015-01-24 00:58:16 +00:00
{
2015-11-25 13:42:45 +00:00
U64 frameSize = 0 ;
size_t readSize = alreadyLoaded ;
2015-01-24 00:58:16 +00:00
/* Main decompression Loop */
2016-01-26 14:58:49 +00:00
ZBUFF_decompressInitDictionary ( ress . dctx , ress . dictBuffer , ress . dictBufferSize ) ;
2016-02-02 13:36:49 +00:00
while ( 1 ) {
2015-11-25 13:42:45 +00:00
/* Decode */
size_t sizeCheck ;
2015-12-17 19:30:14 +00:00
size_t inSize = readSize , decodedSize = ress . dstBufferSize ;
size_t toRead = ZBUFF_decompressContinue ( ress . dctx , ress . dstBuffer , & decodedSize , ress . srcBuffer , & inSize ) ;
2015-11-25 13:42:45 +00:00
if ( ZBUFF_isError ( toRead ) ) EXM_THROW ( 36 , " Decoding error : %s " , ZBUFF_getErrorName ( toRead ) ) ;
readSize - = inSize ;
/* Write block */
2015-12-17 19:30:14 +00:00
sizeCheck = fwrite ( ress . dstBuffer , 1 , decodedSize , foutput ) ;
2015-11-25 13:42:45 +00:00
if ( sizeCheck ! = decodedSize ) EXM_THROW ( 37 , " Write error : unable to write data block to destination file " ) ;
frameSize + = decodedSize ;
DISPLAYUPDATE ( 2 , " \r Decoded : %u MB... " , ( U32 ) ( frameSize > > 20 ) ) ;
2015-11-27 13:07:36 +00:00
if ( toRead = = 0 ) break ;
2015-12-17 19:30:14 +00:00
if ( readSize ) EXM_THROW ( 38 , " Decoding error : should consume entire input " ) ;
2015-01-24 00:58:16 +00:00
/* Fill input buffer */
2015-12-17 19:30:14 +00:00
if ( toRead > ress . srcBufferSize ) EXM_THROW ( 34 , " too large block " ) ;
readSize = fread ( ress . srcBuffer , 1 , toRead , finput ) ;
2015-11-25 13:42:45 +00:00
if ( readSize ! = toRead ) EXM_THROW ( 35 , " Read error " ) ;
2015-01-24 00:58:16 +00:00
}
2015-11-25 13:42:45 +00:00
return frameSize ;
2015-09-10 22:26:09 +00:00
}
2016-02-12 17:33:26 +00:00
/** FIO_decompressSrcFile() :
Decompression ` srcFileName ` into ` ress . dstFile `
@ return : 0 : OK
1 : operation not started
*/
static int FIO_decompressSrcFile ( dRess_t ress , const char * srcFileName )
2015-10-18 21:18:32 +00:00
{
2015-12-17 19:30:14 +00:00
unsigned long long filesize = 0 ;
2016-02-12 17:33:26 +00:00
FILE * dstFile = ress . dstFile ;
FILE * srcFile = FIO_openSrcFile ( srcFileName ) ;
if ( srcFile = = 0 ) return 1 ;
2015-11-25 13:42:45 +00:00
2015-10-18 21:18:32 +00:00
/* for each frame */
2016-02-02 13:36:49 +00:00
for ( ; ; ) {
2015-11-30 10:53:11 +00:00
size_t sizeCheck ;
/* check magic number -> version */
2015-12-17 19:30:14 +00:00
size_t toRead = 4 ;
sizeCheck = fread ( ress . srcBuffer , ( size_t ) 1 , toRead , srcFile ) ;
2015-11-30 10:53:11 +00:00
if ( sizeCheck = = 0 ) break ; /* no more input */
2016-02-12 17:33:26 +00:00
if ( sizeCheck ! = toRead ) EXM_THROW ( 31 , " zstd: %s read error : cannot read header " , srcFileName ) ;
2015-10-18 21:18:32 +00:00
# if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
2016-02-02 13:36:49 +00:00
if ( ZSTD_isLegacy ( MEM_readLE32 ( ress . srcBuffer ) ) ) {
2015-12-17 19:30:14 +00:00
filesize + = FIO_decompressLegacyFrame ( dstFile , srcFile , MEM_readLE32 ( ress . srcBuffer ) ) ;
2015-11-30 10:53:11 +00:00
continue ;
2015-10-18 21:18:32 +00:00
}
2015-10-30 10:21:50 +00:00
# endif /* ZSTD_LEGACY_SUPPORT */
2015-10-18 21:18:32 +00:00
2015-12-17 19:30:14 +00:00
filesize + = FIO_decompressFrame ( ress , dstFile , srcFile , toRead ) ;
2015-10-18 21:18:32 +00:00
}
2015-12-17 19:30:14 +00:00
/* Final Status */
2015-10-18 21:18:32 +00:00
DISPLAYLEVEL ( 2 , " \r %79s \r " , " " ) ;
2015-12-17 19:30:14 +00:00
DISPLAYLEVEL ( 2 , " Successfully decoded %llu bytes \n " , filesize ) ;
2015-10-18 21:18:32 +00:00
2015-12-17 19:30:14 +00:00
/* Close */
fclose ( srcFile ) ;
2016-02-12 17:33:26 +00:00
return 0 ;
}
/** FIO_decompressFile_extRess() :
decompress ` srcFileName ` into ` dstFileName `
@ return : 0 : OK
1 : operation aborted ( src not available , dst already taken , etc . )
*/
static int FIO_decompressFile_extRess ( dRess_t ress ,
const char * dstFileName , const char * srcFileName )
{
ress . dstFile = FIO_openDstFile ( dstFileName ) ;
if ( ress . dstFile = = 0 ) return 1 ;
2015-12-17 19:30:14 +00:00
2016-02-12 17:33:26 +00:00
FIO_decompressSrcFile ( ress , srcFileName ) ;
if ( fclose ( ress . dstFile ) ) EXM_THROW ( 38 , " Write error : cannot properly close %s " , dstFileName ) ;
2015-12-17 19:30:14 +00:00
return 0 ;
2015-10-18 21:18:32 +00:00
}
2015-12-17 19:30:14 +00:00
int FIO_decompressFilename ( const char * dstFileName , const char * srcFileName ,
const char * dictFileName )
{
int missingFiles = 0 ;
dRess_t ress = FIO_createDResources ( dictFileName ) ;
missingFiles + = FIO_decompressFile_extRess ( ress , dstFileName , srcFileName ) ;
FIO_freeDResources ( ress ) ;
return missingFiles ;
}
# define MAXSUFFIXSIZE 8
int FIO_decompressMultipleFilenames ( const char * * srcNamesTable , unsigned nbFiles ,
const char * suffix ,
const char * dictFileName )
{
unsigned u ;
int skippedFiles = 0 ;
int missingFiles = 0 ;
char * dstFileName = ( char * ) malloc ( FNSPACE ) ;
size_t dfnSize = FNSPACE ;
2016-02-12 17:33:26 +00:00
const size_t suffixSize = suffix ? strlen ( suffix ) : 0 ;
2015-12-17 19:30:14 +00:00
dRess_t ress ;
if ( dstFileName = = NULL ) EXM_THROW ( 70 , " not enough memory for dstFileName " ) ;
ress = FIO_createDResources ( dictFileName ) ;
2016-02-12 17:33:26 +00:00
if ( suffix ) {
for ( u = 0 ; u < nbFiles ; u + + ) { /* create dstFileName */
const char * srcFileName = srcNamesTable [ u ] ;
size_t sfnSize = strlen ( srcFileName ) ;
const char * suffixPtr = srcFileName + sfnSize - suffixSize ;
if ( dfnSize < = sfnSize - suffixSize + 1 ) { free ( dstFileName ) ; dfnSize = sfnSize + 20 ; dstFileName = ( char * ) malloc ( dfnSize ) ; if ( dstFileName = = NULL ) EXM_THROW ( 71 , " not enough memory for dstFileName " ) ; }
if ( sfnSize < = suffixSize | | strcmp ( suffixPtr , suffix ) ! = 0 ) {
DISPLAYLEVEL ( 1 , " zstd: %s: unknown suffix (%4s expected) -- ignored \n " , srcFileName , suffix ) ;
skippedFiles + + ;
continue ;
}
memcpy ( dstFileName , srcFileName , sfnSize - suffixSize ) ;
dstFileName [ sfnSize - suffixSize ] = ' \0 ' ;
missingFiles + = FIO_decompressFile_extRess ( ress , dstFileName , srcFileName ) ;
2015-12-17 19:30:14 +00:00
}
2016-02-12 17:33:26 +00:00
} else {
ress . dstFile = stdout ;
for ( u = 0 ; u < nbFiles ; u + + )
missingFiles + = FIO_decompressSrcFile ( ress , srcNamesTable [ u ] ) ;
if ( fclose ( ress . dstFile ) ) EXM_THROW ( 39 , " Write error : cannot properly close %s " , stdoutmark ) ;
2015-12-17 19:30:14 +00:00
}
FIO_freeDResources ( ress ) ;
free ( dstFileName ) ;
return missingFiles + skippedFiles ;
}