Merge branch 'dev' of github.com:Cyan4973/lz4 into dev
This commit is contained in:
commit
468011c346
@ -118,7 +118,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return
|
||||
|
||||
/* The larger the block size, the (slightly) better the compression ratio,
|
||||
* though there are diminishing returns.
|
||||
* Larger blocks also increase memory usage on both compression and decompression sides. */
|
||||
* Larger blocks also increase memory usage on both compression and decompression sides.
|
||||
*/
|
||||
typedef enum {
|
||||
LZ4F_default=0,
|
||||
LZ4F_max64KB=4,
|
||||
|
261
programs/lz4io.c
261
programs/lz4io.c
@ -325,6 +325,7 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName)
|
||||
}
|
||||
|
||||
/** FIO_openDstFile() :
|
||||
* prefs is writable, because sparseFileSupport might be updated.
|
||||
* condition : `dstFileName` must be non-NULL.
|
||||
* @result : FILE* to `dstFileName`, or NULL if it fails */
|
||||
static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileName)
|
||||
@ -338,13 +339,14 @@ static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileNa
|
||||
SET_BINARY_MODE(stdout);
|
||||
if (prefs->sparseFileSupport==1) {
|
||||
prefs->sparseFileSupport = 0;
|
||||
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
|
||||
DISPLAYLEVEL(4, "Sparse File Support automatically disabled on stdout ;"
|
||||
" to force-enable it, add --sparse command \n");
|
||||
}
|
||||
} else {
|
||||
if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
|
||||
f = fopen( dstFileName, "rb" );
|
||||
if (f != NULL) { /* dest exists, prompt for overwrite authorization */
|
||||
fclose(f);
|
||||
FILE* const testf = fopen( dstFileName, "rb" );
|
||||
if (testf != NULL) { /* dest exists, prompt for overwrite authorization */
|
||||
fclose(testf);
|
||||
if (g_displayLevel <= 1) { /* No interaction possible */
|
||||
DISPLAY("%s already exists; not overwritten \n", dstFileName);
|
||||
return NULL;
|
||||
@ -392,7 +394,9 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi
|
||||
/* LZ4IO_compressFilename_Legacy :
|
||||
* This function is intentionally "hidden" (not published in .h)
|
||||
* It generates compressed streams using the old 'legacy' format */
|
||||
int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel)
|
||||
int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs,
|
||||
const char* input_filename, const char* output_filename,
|
||||
int compressionlevel)
|
||||
{
|
||||
typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
|
||||
compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC;
|
||||
@ -424,23 +428,22 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_
|
||||
|
||||
/* Write Archive Header */
|
||||
LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
|
||||
{ size_t const writeSize = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
|
||||
if (writeSize != MAGICNUMBER_SIZE)
|
||||
if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE)
|
||||
EXM_THROW(22, "Write error : cannot write header");
|
||||
}
|
||||
|
||||
/* Main Loop */
|
||||
while (1) {
|
||||
int outSize;
|
||||
/* Read Block */
|
||||
size_t const inSize = fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
|
||||
assert(inSize <= LEGACY_BLOCKSIZE);
|
||||
if (inSize == 0) break;
|
||||
assert(inSize <= LEGACY_BLOCKSIZE);
|
||||
filesize += inSize;
|
||||
|
||||
/* Compress Block */
|
||||
outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
|
||||
compressedfilesize += outSize+4;
|
||||
assert(outSize >= 0);
|
||||
compressedfilesize += (unsigned long long)outSize+4;
|
||||
DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ",
|
||||
(int)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
|
||||
@ -448,8 +451,7 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_
|
||||
assert(outSize > 0);
|
||||
assert(outSize < outBuffSize);
|
||||
LZ4IO_writeLE32(out_buff, (unsigned)outSize);
|
||||
{ size_t const writeSize = fwrite(out_buff, 1, outSize+4, foutput);
|
||||
if (writeSize != (size_t)(outSize+4))
|
||||
if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) {
|
||||
EXM_THROW(24, "Write error : cannot write compressed block");
|
||||
} }
|
||||
if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
|
||||
@ -537,22 +539,20 @@ typedef struct {
|
||||
LZ4F_CDict* cdict;
|
||||
} cRess_t;
|
||||
|
||||
static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) {
|
||||
static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
|
||||
{
|
||||
size_t readSize;
|
||||
size_t dictEnd = 0;
|
||||
size_t dictLen = 0;
|
||||
size_t dictStart;
|
||||
size_t circularBufSize = LZ4_MAX_DICT_SIZE;
|
||||
char* circularBuf;
|
||||
char* circularBuf = (char*)malloc(circularBufSize);
|
||||
char* dictBuf;
|
||||
const char* dictFilename = prefs->dictionaryFilename;
|
||||
FILE* dictFile;
|
||||
|
||||
if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer");
|
||||
if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
|
||||
|
||||
circularBuf = (char *) malloc(circularBufSize);
|
||||
if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory");
|
||||
|
||||
dictFile = LZ4IO_openSrcFile(dictFilename);
|
||||
if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
|
||||
|
||||
@ -595,14 +595,13 @@ static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) {
|
||||
return dictBuf;
|
||||
}
|
||||
|
||||
static LZ4F_CDict* LZ4IO_createCDict(LZ4IO_prefs_t* const prefs) {
|
||||
static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs)
|
||||
{
|
||||
size_t dictionarySize;
|
||||
void* dictionaryBuffer;
|
||||
LZ4F_CDict* cdict;
|
||||
if (!prefs->useDictionary) {
|
||||
return NULL;
|
||||
}
|
||||
dictionaryBuffer = LZ4IO_createDict(prefs, &dictionarySize);
|
||||
if (!prefs->useDictionary) return NULL;
|
||||
dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename);
|
||||
if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
|
||||
cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
|
||||
free(dictionaryBuffer);
|
||||
@ -643,6 +642,7 @@ static void LZ4IO_freeCResources(cRess_t ress)
|
||||
|
||||
/*
|
||||
* LZ4IO_compressFilename_extRess()
|
||||
* io_prefs is mutable, as it may update sparseFileSupport
|
||||
* result : 0 : compression completed correctly
|
||||
* 1 : missing or pb opening srcFileName
|
||||
*/
|
||||
@ -653,7 +653,6 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
unsigned long long compressedfilesize = 0;
|
||||
FILE* srcFile;
|
||||
FILE* dstFile;
|
||||
void* const srcBuffer = ress.srcBuffer;
|
||||
void* const dstBuffer = ress.dstBuffer;
|
||||
@ -664,13 +663,12 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
|
||||
LZ4F_preferences_t prefs;
|
||||
|
||||
/* Init */
|
||||
srcFile = LZ4IO_openSrcFile(srcFileName);
|
||||
FILE* const srcFile = LZ4IO_openSrcFile(srcFileName);
|
||||
if (srcFile == NULL) return 1;
|
||||
dstFile = LZ4IO_openDstFile(io_prefs, dstFileName);
|
||||
if (dstFile == NULL) { fclose(srcFile); return 1; }
|
||||
memset(&prefs, 0, sizeof(prefs));
|
||||
|
||||
|
||||
/* Set compression parameters */
|
||||
prefs.autoFlush = 1;
|
||||
prefs.compressionLevel = compressionLevel;
|
||||
@ -694,41 +692,41 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
|
||||
/* single-block file */
|
||||
if (readSize < blockSize) {
|
||||
/* Compress in single pass */
|
||||
size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
|
||||
if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
|
||||
size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
|
||||
if (LZ4F_isError(cSize))
|
||||
EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
|
||||
compressedfilesize = cSize;
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
|
||||
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
|
||||
|
||||
/* Write Block */
|
||||
{ size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile);
|
||||
if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block");
|
||||
if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) {
|
||||
EXM_THROW(32, "Write error : failed writing single-block compressed frame");
|
||||
} }
|
||||
|
||||
else
|
||||
|
||||
/* multiple-blocks file */
|
||||
{
|
||||
/* Write Archive Header */
|
||||
size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
|
||||
/* Write Frame Header */
|
||||
size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
|
||||
if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
|
||||
{ size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
|
||||
if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
|
||||
if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize)
|
||||
EXM_THROW(34, "Write error : cannot write header");
|
||||
compressedfilesize += headerSize;
|
||||
|
||||
/* Main Loop */
|
||||
/* Main Loop - one block at a time */
|
||||
while (readSize>0) {
|
||||
size_t outSize;
|
||||
|
||||
/* Compress Block */
|
||||
outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
|
||||
if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
|
||||
size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
|
||||
if (LZ4F_isError(outSize))
|
||||
EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
|
||||
compressedfilesize += outSize;
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
|
||||
(unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
|
||||
/* Write Block */
|
||||
{ size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile);
|
||||
if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
|
||||
if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize)
|
||||
EXM_THROW(36, "Write error : cannot write compressed block");
|
||||
|
||||
/* Read next block */
|
||||
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
|
||||
@ -736,14 +734,14 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
|
||||
}
|
||||
if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
|
||||
|
||||
/* End of Stream mark */
|
||||
headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
|
||||
if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
|
||||
|
||||
{ size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
|
||||
if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
|
||||
compressedfilesize += headerSize;
|
||||
}
|
||||
/* End of Frame mark */
|
||||
{ size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
|
||||
if (LZ4F_isError(endSize))
|
||||
EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize));
|
||||
if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
|
||||
EXM_THROW(39, "Write error : cannot write end of frame");
|
||||
compressedfilesize += endSize;
|
||||
} }
|
||||
|
||||
/* Release file handlers */
|
||||
fclose (srcFile);
|
||||
@ -861,7 +859,11 @@ static unsigned LZ4IO_readLE32 (const void* s)
|
||||
}
|
||||
|
||||
|
||||
static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
|
||||
static unsigned
|
||||
LZ4IO_fwriteSparse(FILE* file,
|
||||
const void* buffer, size_t bufferSize,
|
||||
int sparseFileSupport,
|
||||
unsigned storedSkips)
|
||||
{
|
||||
const size_t sizeT = sizeof(size_t);
|
||||
const size_t maskT = sizeT -1 ;
|
||||
@ -871,7 +873,7 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const
|
||||
const size_t* const bufferTEnd = bufferT + bufferSizeT;
|
||||
const size_t segmentSizeT = (32 KB) / sizeT;
|
||||
|
||||
if (!prefs->sparseFileSupport) { /* normal write */
|
||||
if (!sparseFileSupport) { /* normal write */
|
||||
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
|
||||
if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
|
||||
return 0;
|
||||
@ -919,7 +921,7 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const
|
||||
int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
|
||||
storedSkips = 0;
|
||||
{ size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
|
||||
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
|
||||
if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
|
||||
} }
|
||||
}
|
||||
@ -929,18 +931,18 @@ static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const
|
||||
|
||||
static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
|
||||
{
|
||||
if (storedSkips>0) { /* implies g_sparseFileSupport>0 */
|
||||
int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR);
|
||||
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
|
||||
{ const char lastZeroByte[1] = { 0 };
|
||||
size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
|
||||
if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
|
||||
} }
|
||||
if (storedSkips>0) { /* implies sparseFileSupport>0 */
|
||||
const char lastZeroByte[1] = { 0 };
|
||||
if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0)
|
||||
EXM_THROW(69, "Final skip error (sparse file)\n");
|
||||
if (fwrite(lastZeroByte, 1, 1, file) != 1)
|
||||
EXM_THROW(69, "Write error : cannot write last zero\n");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */
|
||||
static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput)
|
||||
static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
|
||||
{
|
||||
unsigned long long streamSize = 0;
|
||||
unsigned storedSkips = 0;
|
||||
@ -974,7 +976,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, F
|
||||
if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
|
||||
streamSize += (unsigned long long)decodeSize;
|
||||
/* Write Block */
|
||||
storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, (size_t)decodeSize, storedSkips); /* success or die */
|
||||
storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */
|
||||
} }
|
||||
if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
|
||||
|
||||
@ -1000,19 +1002,20 @@ typedef struct {
|
||||
size_t dictBufferSize;
|
||||
} dRess_t;
|
||||
|
||||
static void LZ4IO_loadDDict(LZ4IO_prefs_t* const prefs, dRess_t* ress) {
|
||||
static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs)
|
||||
{
|
||||
if (!prefs->useDictionary) {
|
||||
ress->dictBuffer = NULL;
|
||||
ress->dictBufferSize = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
ress->dictBuffer = LZ4IO_createDict(prefs, &ress->dictBufferSize);
|
||||
ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename);
|
||||
if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
|
||||
}
|
||||
|
||||
static const size_t LZ4IO_dBufferSize = 64 KB;
|
||||
static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs)
|
||||
static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
|
||||
{
|
||||
dRess_t ress;
|
||||
|
||||
@ -1027,7 +1030,7 @@ static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs)
|
||||
ress.dstBuffer = malloc(ress.dstBufferSize);
|
||||
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
|
||||
|
||||
LZ4IO_loadDDict(prefs, &ress);
|
||||
LZ4IO_loadDDict(&ress, prefs);
|
||||
|
||||
ress.dstFile = NULL;
|
||||
return ress;
|
||||
@ -1043,7 +1046,10 @@ static void LZ4IO_freeDResources(dRess_t ress)
|
||||
}
|
||||
|
||||
|
||||
static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, FILE* dstFile)
|
||||
static unsigned long long
|
||||
LZ4IO_decompressLZ4F(dRess_t ress,
|
||||
FILE* const srcFile, FILE* const dstFile,
|
||||
const LZ4IO_prefs_t* const prefs)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
LZ4F_errorCode_t nextToLoad;
|
||||
@ -1079,7 +1085,7 @@ static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess
|
||||
/* Write Block */
|
||||
if (decodedBytes) {
|
||||
if (!prefs->testMode)
|
||||
storedSkips = LZ4IO_fwriteSparse(prefs, dstFile, ress.dstBuffer, decodedBytes, storedSkips);
|
||||
storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips);
|
||||
filesize += decodedBytes;
|
||||
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
|
||||
}
|
||||
@ -1097,22 +1103,30 @@ static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess
|
||||
}
|
||||
|
||||
|
||||
/* LZ4IO_passThrough:
|
||||
* just output the same content as input, no decoding.
|
||||
* This is a capability of zcat, and by extension lz4cat
|
||||
* MNstore : contain the first MAGICNUMBER_SIZE bytes already read from finput
|
||||
*/
|
||||
#define PTSIZE (64 KB)
|
||||
#define PTSIZET (PTSIZE / sizeof(size_t))
|
||||
static unsigned long long LZ4IO_passThrough(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE])
|
||||
static unsigned long long
|
||||
LZ4IO_passThrough(FILE* finput, FILE* foutput,
|
||||
unsigned char MNstore[MAGICNUMBER_SIZE],
|
||||
int sparseFileSupport)
|
||||
{
|
||||
size_t buffer[PTSIZET];
|
||||
size_t readBytes = 1;
|
||||
unsigned long long total = MAGICNUMBER_SIZE;
|
||||
unsigned storedSkips = 0;
|
||||
|
||||
size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput);
|
||||
if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error");
|
||||
|
||||
if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) {
|
||||
EXM_THROW(50, "Pass-through write error");
|
||||
}
|
||||
while (readBytes) {
|
||||
readBytes = fread(buffer, 1, PTSIZE, finput);
|
||||
readBytes = fread(buffer, 1, sizeof(buffer), finput);
|
||||
total += readBytes;
|
||||
storedSkips = LZ4IO_fwriteSparse(prefs, foutput, buffer, readBytes, storedSkips);
|
||||
storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips);
|
||||
}
|
||||
if (ferror(finput)) EXM_THROW(51, "Read Error");
|
||||
|
||||
@ -1139,7 +1153,10 @@ static int fseek_u32(FILE *fp, unsigned offset, int where)
|
||||
}
|
||||
|
||||
#define ENDOFSTREAM ((unsigned long long)-1)
|
||||
static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* finput, FILE* foutput)
|
||||
static unsigned long long
|
||||
selectDecoder(dRess_t ress,
|
||||
FILE* finput, FILE* foutput,
|
||||
const LZ4IO_prefs_t* const prefs)
|
||||
{
|
||||
unsigned char MNstore[MAGICNUMBER_SIZE];
|
||||
unsigned magicNumber;
|
||||
@ -1165,10 +1182,10 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress
|
||||
switch(magicNumber)
|
||||
{
|
||||
case LZ4IO_MAGICNUMBER:
|
||||
return LZ4IO_decompressLZ4F(prefs, ress, finput, foutput);
|
||||
return LZ4IO_decompressLZ4F(ress, finput, foutput, prefs);
|
||||
case LEGACY_MAGICNUMBER:
|
||||
DISPLAYLEVEL(4, "Detected : Legacy format \n");
|
||||
return LZ4IO_decodeLegacyStream(prefs, finput, foutput);
|
||||
return LZ4IO_decodeLegacyStream(finput, foutput, prefs);
|
||||
case LZ4IO_SKIPPABLE0:
|
||||
DISPLAYLEVEL(4, "Skipping detected skippable area \n");
|
||||
{ size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
|
||||
@ -1187,7 +1204,7 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress
|
||||
/* Wrong magic number at the beginning of 1st stream */
|
||||
if (!prefs->testMode && prefs->overwrite && prefs->passThrough) {
|
||||
nbFrames = 0;
|
||||
return LZ4IO_passThrough(prefs, finput, foutput, MNstore);
|
||||
return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport);
|
||||
}
|
||||
EXM_THROW(44,"Unrecognized header : file cannot be decoded");
|
||||
}
|
||||
@ -1202,7 +1219,10 @@ static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress
|
||||
}
|
||||
|
||||
|
||||
static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename)
|
||||
static int
|
||||
LZ4IO_decompressSrcFile(dRess_t ress,
|
||||
const char* input_filename, const char* output_filename,
|
||||
const LZ4IO_prefs_t* const prefs)
|
||||
{
|
||||
FILE* const foutput = ress.dstFile;
|
||||
unsigned long long filesize = 0;
|
||||
@ -1210,11 +1230,12 @@ static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con
|
||||
/* Init */
|
||||
FILE* const finput = LZ4IO_openSrcFile(input_filename);
|
||||
if (finput==NULL) return 1;
|
||||
assert(foutput != NULL);
|
||||
|
||||
/* Loop over multiple streams */
|
||||
for ( ; ; ) { /* endless loop, see break condition */
|
||||
unsigned long long const decodedSize =
|
||||
selectDecoder(prefs, ress, finput, foutput);
|
||||
selectDecoder(ress, finput, foutput, prefs);
|
||||
if (decodedSize == ENDOFSTREAM) break;
|
||||
filesize += decodedSize;
|
||||
}
|
||||
@ -1235,7 +1256,10 @@ static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con
|
||||
}
|
||||
|
||||
|
||||
static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename)
|
||||
static int
|
||||
LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs,
|
||||
dRess_t ress,
|
||||
const char* input_filename, const char* output_filename)
|
||||
{
|
||||
stat_t statbuf;
|
||||
int stat_result = 0;
|
||||
@ -1247,7 +1271,7 @@ static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, con
|
||||
stat_result = 1;
|
||||
|
||||
ress.dstFile = foutput;
|
||||
LZ4IO_decompressSrcFile(prefs, ress, input_filename, output_filename);
|
||||
LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
|
||||
|
||||
fclose(foutput);
|
||||
|
||||
@ -1298,7 +1322,7 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs,
|
||||
size_t const ifnSize = strlen(inFileNamesTable[i]);
|
||||
const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
|
||||
if (!strcmp(suffix, stdoutmark)) {
|
||||
missingFiles += LZ4IO_decompressSrcFile(prefs, ress, inFileNamesTable[i], stdoutmark);
|
||||
missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs);
|
||||
continue;
|
||||
}
|
||||
if (ofnSize <= ifnSize-suffixSize+1) {
|
||||
@ -1363,9 +1387,11 @@ static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "Skippa
|
||||
returns 0 in case it can't succesfully skip block data.
|
||||
Assumes SEEK_CUR after frame header.
|
||||
*/
|
||||
static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
|
||||
static unsigned long long
|
||||
LZ4IO_skipBlocksData(FILE* finput,
|
||||
const LZ4F_blockChecksum_t blockChecksumFlag,
|
||||
const LZ4F_contentChecksum_t contentChecksumFlag) {
|
||||
const LZ4F_contentChecksum_t contentChecksumFlag)
|
||||
{
|
||||
unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE];
|
||||
unsigned long long totalBlocksSize = 0;
|
||||
for (;;) {
|
||||
@ -1374,8 +1400,7 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
|
||||
return 0;
|
||||
}
|
||||
totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE;
|
||||
{
|
||||
const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU;
|
||||
{ const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU;
|
||||
const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE);
|
||||
if (nextCBlockSize == 0) {
|
||||
/* Reached EndMark */
|
||||
@ -1390,11 +1415,9 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
|
||||
}
|
||||
totalBlocksSize += nextBlock;
|
||||
/* skip to the next block */
|
||||
if (UTIL_fseek(finput, nextBlock, SEEK_CUR) != 0) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(nextBlock < LONG_MAX);
|
||||
if (UTIL_fseek(finput, (long)nextBlock, SEEK_CUR) != 0) return 0;
|
||||
} }
|
||||
return totalBlocksSize;
|
||||
}
|
||||
|
||||
@ -1405,7 +1428,8 @@ static unsigned long long LZ4IO_skipBlocksData(FILE* finput,
|
||||
This works as long as legacy block header size = magic number size.
|
||||
Assumes SEEK_CUR after frame header.
|
||||
*/
|
||||
static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) {
|
||||
static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput)
|
||||
{
|
||||
unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE];
|
||||
unsigned long long totalBlocksSize = 0;
|
||||
LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
|
||||
@ -1428,14 +1452,15 @@ static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) {
|
||||
/* skip to the next block */
|
||||
if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
} } }
|
||||
return totalBlocksSize;
|
||||
}
|
||||
|
||||
/* buffer : must be a valid memory area of at least 4 bytes */
|
||||
const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) {
|
||||
/* LZ4IO_blockTypeID:
|
||||
* return human-readable block type, following command line convention
|
||||
* buffer : must be a valid memory area of at least 4 bytes */
|
||||
const char* LZ4IO_blockTypeID(LZ4F_blockSizeID_t sizeID, LZ4F_blockMode_t blockMode, char buffer[4])
|
||||
{
|
||||
buffer[0] = 'B';
|
||||
assert(sizeID >= 4); assert(sizeID <= 7);
|
||||
buffer[1] = (char)(sizeID + '0');
|
||||
@ -1445,7 +1470,8 @@ const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) {
|
||||
}
|
||||
|
||||
/* buffer : must be valid memory area of at least 10 bytes */
|
||||
static const char* LZ4IO_toHuman(long double size, char *buf) {
|
||||
static const char* LZ4IO_toHuman(long double size, char *buf)
|
||||
{
|
||||
const char units[] = {"\0KMGTPEZY"};
|
||||
size_t i = 0;
|
||||
for (; size >= 1024; i++) size /= 1024;
|
||||
@ -1454,14 +1480,15 @@ static const char* LZ4IO_toHuman(long double size, char *buf) {
|
||||
}
|
||||
|
||||
/* Get filename without path prefix */
|
||||
static const char* LZ4IO_baseName(const char* input_filename) {
|
||||
static const char* LZ4IO_baseName(const char* input_filename)
|
||||
{
|
||||
const char* b = strrchr(input_filename, '/');
|
||||
if (!b) b = strrchr(input_filename, '\\');
|
||||
if (!b) return input_filename;
|
||||
return b + 1;
|
||||
}
|
||||
|
||||
/* Report frame/s information in verbose mode.
|
||||
/* Report frame/s information (--list) in verbose mode (-v).
|
||||
* Will populate file info with fileName and frameSummary where applicable.
|
||||
* - TODO :
|
||||
* + report nb of blocks, hence max. possible decompressed size (when not reported in header)
|
||||
@ -1480,11 +1507,12 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
|
||||
LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO;
|
||||
unsigned magicNumber;
|
||||
/* Get MagicNumber */
|
||||
size_t nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput);
|
||||
{ size_t const nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput);
|
||||
if (nbReadBytes == 0) { break; } /* EOF */
|
||||
result = LZ4IO_format_not_known; /* default result (error) */
|
||||
if (nbReadBytes != MAGICNUMBER_SIZE)
|
||||
if (nbReadBytes != MAGICNUMBER_SIZE) {
|
||||
EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
|
||||
} }
|
||||
magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */
|
||||
if (LZ4IO_isSkippableMagicNumber(magicNumber))
|
||||
magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */
|
||||
@ -1497,7 +1525,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
|
||||
if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename);
|
||||
}
|
||||
{ size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN);
|
||||
if (!LZ4F_isError(hSize)) {
|
||||
if (LZ4F_isError(hSize)) break;
|
||||
if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
|
||||
/* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
|
||||
const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
|
||||
@ -1505,11 +1533,10 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
|
||||
}
|
||||
/* Create decompression context */
|
||||
{ LZ4F_dctx* dctx;
|
||||
unsigned isError = LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION));
|
||||
if (!isError) {
|
||||
isError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize));
|
||||
if ( LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) break;
|
||||
{ unsigned const frameInfoError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize));
|
||||
LZ4F_freeDecompressionContext(dctx);
|
||||
if (!isError) {
|
||||
if (frameInfoError) break;
|
||||
if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID ||
|
||||
cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode)
|
||||
&& cfinfo->frameCount != 0)
|
||||
@ -1540,13 +1567,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
|
||||
cfinfo->allContentSize = 0;
|
||||
}
|
||||
result = LZ4IO_LZ4F_OK;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} } } } }
|
||||
break;
|
||||
case LEGACY_MAGICNUMBER:
|
||||
frameInfo.frameType = legacyFrame;
|
||||
@ -1569,7 +1590,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
|
||||
if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0;
|
||||
cfinfo->eqBlockTypes = 0;
|
||||
cfinfo->allContentSize = 0;
|
||||
{ nbReadBytes = fread(buffer, 1, 4, finput);
|
||||
{ size_t const nbReadBytes = fread(buffer, 1, 4, finput);
|
||||
if (nbReadBytes != 4)
|
||||
EXM_THROW(42, "Stream error : skippable size unreadable");
|
||||
}
|
||||
@ -1594,12 +1615,10 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (result != LZ4IO_LZ4F_OK) {
|
||||
break;
|
||||
}
|
||||
if (result != LZ4IO_LZ4F_OK) break;
|
||||
cfinfo->frameSummary = frameInfo;
|
||||
cfinfo->frameCount++;
|
||||
}
|
||||
} /* while (!feof(finput)) */
|
||||
fclose(finput);
|
||||
return result;
|
||||
}
|
||||
|
@ -420,7 +420,7 @@ test-lz4-hugefile: lz4 datagen
|
||||
|
||||
test-lz4-testmode: lz4 datagen
|
||||
@echo "\n ---- bench mode ----"
|
||||
$(LZ4) -bi1
|
||||
$(LZ4) -bi0
|
||||
@echo "\n ---- test mode ----"
|
||||
! $(DATAGEN) | $(LZ4) -t
|
||||
! $(DATAGEN) | $(LZ4) -tf
|
||||
|
Loading…
Reference in New Issue
Block a user