From 67c25a5c1b03eaf61a89c634469fad3b0de76ae8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 1 Dec 2014 22:00:07 +0100 Subject: [PATCH 1/3] lz4frame.h : clarified a few comments --- lib/lz4frame.h | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/lz4frame.h b/lib/lz4frame.h index a4f5e36..7184cc3 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -1,7 +1,7 @@ /* LZ4 auto-framing library Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -29,6 +29,7 @@ You can contact the author at : - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source mirror : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -46,7 +47,7 @@ extern "C" { /**************************************** Note : experimental API. - Not yet integrated within lz4 library. + Not yet integrated within liblz4 ****************************************/ /************************************** @@ -56,8 +57,8 @@ extern "C" { /************************************** - Error management -**************************************/ + * Error management + * ************************************/ typedef size_t LZ4F_errorCode_t; #define LZ4F_LIST_ERRORS(ITEM) \ ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ @@ -72,15 +73,15 @@ typedef size_t LZ4F_errorCode_t; #define LZ4F_GENERATE_ENUM(ENUM) ENUM, typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */ -int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */ -const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return enum as string */ +int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */ +const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code string; useful for debugging */ /************************************** - Framing compression functions -**************************************/ + * Frame compression types + * ************************************/ -typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7} blockSizeID_t; +typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7 } blockSizeID_t; typedef enum { blockLinked=0, blockIndependent} blockMode_t; typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t; @@ -93,8 +94,8 @@ typedef struct { typedef struct { LZ4F_frameInfo_t frameInfo; - unsigned compressionLevel; /* Not yet supported : only fast compression for the time being */ - unsigned autoFlush; /* 1 == always flush; reduce need for tmp buffer */ + unsigned compressionLevel; /* 0 == default (fast mode); values above 16 count as 16 */ + unsigned autoFlush; /* 1 == always flush : reduce need for tmp buffer */ unsigned reserved[4]; } LZ4F_preferences_t; @@ -120,7 +121,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf /********************************** * Advanced compression functions - * *********************************/ + * ********************************/ typedef void* LZ4F_compressionContext_t; @@ -159,6 +160,7 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP /* LZ4F_compressBound() : * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. * preferencesPtr is optional : you can provide NULL as argument, all preferences will then be set to default. + * Note that different preferences will produce in different results. */ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr); @@ -207,19 +209,22 @@ typedef struct { /* Resource management */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned version); -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext); +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* ctxPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t ctx); /* LZ4F_createDecompressionContext() : * The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. * This is achieved using LZ4F_createDecompressionContext(). - * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. * Object can release its memory using LZ4F_freeDecompressionContext(); */ /* Decompression */ -size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); +size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); /* LZ4F_getFrameInfo() * This function decodes frame header information, such as blockSize. * It is optional : you could start by calling directly LZ4F_decompress() instead. @@ -231,7 +236,10 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_ * or an error code which can be tested using LZ4F_isError(). */ -size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr); +size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* optionsPtr); /* LZ4F_decompress() * Call this function repetitively to regenerate data compressed within srcBuffer. * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. @@ -248,8 +256,8 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* d * * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. * Basically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. - * Note that this is just a hint, you can always provide any srcSize you want. + * Respecting the hint provides some boost to performance, since it does not need intermediate buffers. + * This is just a hint, you can always provide any srcSize you want. * When a frame is fully decoded, the function result will be 0. * If decompression failed, function result is an error code which can be tested using LZ4F_isError(). */ From 118296aeb0f02b8978544793ae4cc897d7638765 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 Dec 2014 23:57:15 +0100 Subject: [PATCH 2/3] variable ref renamed --- lib/lz4.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index fb84955..4d3e67f 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -465,7 +465,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy { switch (tableType) { - case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } } @@ -550,7 +550,7 @@ static int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - const BYTE* ref; + const BYTE* match; BYTE* token; { const BYTE* forwardIp = ip; @@ -566,10 +566,10 @@ static int LZ4_compress_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - ref = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, ctx, tableType, base); if (dict==usingExtDict) { - if (ref<(const BYTE*)source) + if (match<(const BYTE*)source) { refDelta = dictDelta; lowLimit = dictionary; @@ -583,13 +583,13 @@ static int LZ4_compress_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, ctx, tableType, base); - } while ( ((dictIssue==dictSmall) ? (ref < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (ref + MAX_DISTANCE < ip)) - || (LZ4_read32(ref+refDelta) != LZ4_read32(ip)) ); + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); } /* Catch up */ - while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; } + while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } { /* Encode Literal length */ @@ -613,7 +613,7 @@ static int LZ4_compress_generic( _next_match: /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-ref)); op+=2; + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; /* Encode MatchLength */ { @@ -622,10 +622,10 @@ _next_match: if ((dict==usingExtDict) && (lowLimit==dictionary)) { const BYTE* limit; - ref += refDelta; - limit = ip + (dictEnd-ref); + match += refDelta; + limit = ip + (dictEnd-match); if (limit > matchlimit) limit = matchlimit; - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit); + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); ip += MINMATCH + matchLength; if (ip==limit) { @@ -636,7 +636,7 @@ _next_match: } else { - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit); + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); ip += MINMATCH + matchLength; } @@ -662,10 +662,10 @@ _next_match: LZ4_putPosition(ip-2, ctx, tableType, base); /* Test next position */ - ref = LZ4_getPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, ctx, tableType, base); if (dict==usingExtDict) { - if (ref<(const BYTE*)source) + if (match<(const BYTE*)source) { refDelta = dictDelta; lowLimit = dictionary; @@ -677,9 +677,9 @@ _next_match: } } LZ4_putPosition(ip, ctx, tableType, base); - if ( ((dictIssue==dictSmall) ? (ref>=lowRefLimit) : 1) - && (ref+MAX_DISTANCE>=ip) - && (LZ4_read32(ref+refDelta)==LZ4_read32(ip)) ) + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } /* Prepare next loop */ From 65ee6b09c4392814167e7a039d9312aabdb5119b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 3 Dec 2014 19:17:10 +0100 Subject: [PATCH 3/3] Fixed : deprecated function LZ4_slideInputBufferHC() --- lib/Makefile | 1 - lib/lz4.c | 4 +-- lib/lz4hc.c | 98 ++++++++++++++++++++++------------------------------ 3 files changed, 43 insertions(+), 60 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index f2d585f..3ae12fa 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -63,7 +63,6 @@ else endif default: liblz4 - @cd $(PRGDIR); $(MAKE) -e all: liblz4 diff --git a/lib/lz4.c b/lib/lz4.c index 4d3e67f..f8186b4 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -394,17 +394,17 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi pIn += LZ4_NbCommonBytes(diff); return (unsigned)(pIn - pStart); } + if (LZ4_64bits()) if ((pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } if ((pInhashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); hc4->nextToUpdate = 64 KB; - hc4->base = base - 64 KB; - hc4->inputBuffer = base; - hc4->end = base; - hc4->dictBase = base - 64 KB; + hc4->base = start - 64 KB; + hc4->inputBuffer = start; + hc4->end = start; + hc4->dictBase = start - 64 KB; hc4->dictLimit = 64 KB; hc4->lowLimit = 64 KB; } @@ -155,21 +155,6 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) } -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->end >= ctxPtr->base + 4) - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content - // Note : need to handle risk of index overflow - // Use only one memory segment for dict, so any previous External Dict is lost at this stage - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block -} - - FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos, @@ -571,7 +556,7 @@ int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, /***************************** - Using external allocation + Using external allocation *****************************/ int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } @@ -598,9 +583,10 @@ int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, c { return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); } + /************************************** - Experimental Streaming Functions -**************************************/ + * Streaming Functions + * ************************************/ /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }; @@ -616,55 +602,68 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; } - LZ4HC_init (streamPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (streamPtr, (const BYTE*)dictionary +(dictSize-3)); - streamPtr->end = (const BYTE*)dictionary + dictSize; + LZ4HC_init (ctxPtr, (const BYTE*)dictionary); + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); + ctxPtr->end = (const BYTE*)dictionary + dictSize; return dictSize; } /* compression */ -static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ +} + +static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, const char* source, char* dest, int inputSize, int maxOutputSize, limitedOutput_directive limit) { /* auto-init if forgotten */ - if (dsPtr->base == NULL) - LZ4HC_init (dsPtr, (const BYTE*) source); + if (ctxPtr->base == NULL) + LZ4HC_init (ctxPtr, (const BYTE*) source); /* Check overflow */ - if ((size_t)(dsPtr->end - dsPtr->base) > 2 GB) + if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { - size_t dictSize = (size_t)(dsPtr->end - dsPtr->base) - dsPtr->dictLimit; + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; if (dictSize > 64 KB) dictSize = 64 KB; - LZ4_loadDictHC((LZ4_streamHC_t*)dsPtr, (const char*)(dsPtr->end) - dictSize, (int)dictSize); + LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } /* Check if blocks follow each other */ - if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source); + if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; - const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit; + const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; + const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd)) { if (sourceEnd > dictEnd) sourceEnd = dictEnd; - dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase); - if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit; + ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; } } - return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit); + return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); } int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) @@ -688,7 +687,6 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; memcpy(safeBuffer, streamPtr->end - dictSize, dictSize); - //LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); { U32 endIndex = (U32)(streamPtr->end - streamPtr->base); streamPtr->end = (const BYTE*)safeBuffer + dictSize; @@ -749,20 +747,6 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer; - - if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */ - - distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ - LZ4HC_Insert(hc4, hc4->end - MINMATCH); - memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); - hc4->base -= distance; - if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ - { - int i; - hc4->base += 1 GB; - for (i=0; ihashTable[i] -= 1 GB; - } - hc4->end -= distance; - return (char*)(hc4->end); + int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); + return (char*)(hc4->inputBuffer + dictSize); }