AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #54 from t-mat/gc-issue/155

Browse Source 
Gc issue/155
This commit is contained in:
Yann Collet 2015-03-08 19:06:17 +01:00
commit e277511ab6
3 changed files with 167 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
programs/lz4cli.c
View File

@ -83,6 +83,11 @@
/*****************************
* Constants
******************************/
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
# undef LZ4_VERSION
# define LZ4_VERSION "EXPERIMENTAL_SPARSE_FILE"
#endif
#define COMPRESSOR_NAME "LZ4 command line interface"
#ifndef LZ4_VERSION
# define LZ4_VERSION "r128"
@ -187,6 +192,10 @@ static int usage_advanced(void)
DISPLAY( " -y : overwrite output without prompting \n");
DISPLAY( " -s : suppress warnings \n");
#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
DISPLAY( "Experimental : Sparse file\n");
DISPLAY( " -x : enable sparse file\n");
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
EXTENDED_HELP;
return 0;
}
@ -279,6 +288,9 @@ int main(int argc, char** argv)
/* Init */
programName = argv[0];
LZ4IO_setOverwrite(0);
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
LZ4IO_setSparseFile(0);
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
/* lz4cat predefined behavior */
@ -415,6 +427,11 @@ int main(int argc, char** argv)
/* Pause at the end (hidden option) */
case 'p': main_pause=1; BMK_SetPause(); break;
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
/* Experimental : Enable sparse file */
case 'x': LZ4IO_setSparseFile(1); break;
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
/* Specific commands for customized versions */
EXTENDED_ARGUMENTS;

145
programs/lz4io.c
View File

@ -62,6 +62,27 @@
#include "lz4frame.h"
/**************************************
Basic Types
**************************************/
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
/****************************
* OS-specific Includes
*****************************/
@ -76,10 +97,20 @@
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
# if defined(LZ4IO_ENABLE_SPARSE_FILE)
# include <windows.h>
# define SET_SPARSE_FILE_MODE(file) do { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } while(0)
# if defined(_MSC_VER) && (_MSC_VER >= 1400)
# define fseek _fseeki64
# endif
# endif /* LZ4IO_ENABLE_SPARSE_FILE */
#else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
# if defined(LZ4IO_ENABLE_SPARSE_FILE)
# define SET_SPARSE_FILE_MODE(file)
# endif /* LZ4IO_ENABLE_SPARSE_FILE */
#endif
@ -133,6 +164,9 @@ static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
static int blockChecksum = 0;
static int streamChecksum = 1;
static int blockIndependence = 1;
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
static int sparseFile = 0;
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
static const int minBlockSizeID = 4;
static const int maxBlockSizeID = 7;
@ -174,6 +208,75 @@ int LZ4IO_setOverwrite(int yes)
return overwrite;
}
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
/* Default setting : sparseFile = 0; (Disable)
return : sparse file mode (0:Disable / 1:Enable) */
int LZ4IO_setSparseFile(int yes)
{
sparseFile = yes;
return sparseFile;
}
static int isSparse(const void* p, size_t size)
{
#if 0
/* naive */
const char* p8 = p;
for(; size; --size)
{
if(*p8 != 0)
{
return 0;
}
++p8;
}
return 1;
#elif 0
/* xz method */
const U64* p64 = (const U64*) p;
const char* p8 = (const char*) p;
const size_t n = size / sizeof(*p64);
size_t i;
for (i = 0; i < n; ++i)
{
if (p64[i] != 0)
{
return 0;
}
}
for(i = n * sizeof(*p64); i < size; ++i)
{
if (p8[i] != 0)
{
return 0;
}
}
return 1;
#elif 0
/* Neil's */
const char* buf = (const char*) p;
return buf[0] == 0 && !memcmp(buf, buf + 1, size - 1);
#else
/* GNU Core Utilities : coreutils/src/system.h / is_nul() */
const U64* wp = (const U64*) p;
const char* cbuf = (const char*) p;
const char* cp;
// Find first nonzero *word*, or the word with the sentinel.
while(*wp++ == 0) ;
// Find the first nonzero *byte*, or the sentinel.
cp = (const char*) (wp - 1);
while(*cp++ == 0) ;
return cbuf + size < cp;
#endif
}
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
/* blockSizeID : valid values : 4-5-6-7 */
int LZ4IO_setBlockSizeID(int bsid)
{
@ -560,6 +663,9 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
LZ4F_decompressionContext_t ctx;
LZ4F_errorCode_t errorCode;
LZ4F_frameInfo_t frameInfo;
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
size_t sparsePending = 0;
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
/* init */
errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
@ -581,7 +687,12 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID);
inBuffSize = outBuffSize + 4;
inBuff = (char*)malloc(inBuffSize);
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
outBuff = (char*)malloc(outBuffSize+sizeof(U64));
*(U64*) &outBuff[outBuffSize] = (U64) -1; /* sentinel */
#else /* LZ4IO_ENABLE_SPARSE_FILE */
outBuff = (char*)malloc(outBuffSize);
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory");
/* Main Loop */
@ -601,9 +712,35 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
filesize += decodedBytes;
/* Write Block */
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
if(sparseFile)
{
if(isSparse(outBuff, decodedBytes))
{
sparsePending += decodedBytes;
continue;
}
if(sparsePending > 0)
{
fseek(foutput, sparsePending, SEEK_CUR);
sparsePending = 0;
}
}
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n");
}
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
if(sparseFile)
{
if(sparsePending > 0)
{
fseek(foutput, sparsePending-1, SEEK_CUR);
fputc(0, foutput);
sparsePending = 0;
}
}
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
/* Free */
free(inBuff);
@ -666,6 +803,14 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
start = clock();
get_fileHandle(input_filename, output_filename, &finput, &foutput);
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
if (sparseFile!=0 && foutput!=0)
{
DISPLAY("Experimental : Using sparse file\n");
SET_SPARSE_FILE_MODE(foutput);
}
#endif /* LZ4IO_ENABLE_SPARSE_FILE */
/* Loop over multiple streams */
do
{

5
programs/lz4io.h
View File

@ -77,3 +77,8 @@ int LZ4IO_setStreamChecksumMode(int xxhash);
/* Default setting : 0 (no notification) */
int LZ4IO_setNotificationLevel(int level);
#if defined(LZ4IO_ENABLE_SPARSE_FILE)
/* Default setting : 0 (sparseFile = 0; disable sparse file) */
int LZ4IO_setSparseFile(int yes);
#endif