AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Strict end-of-buffer checks

Browse Source 
License changed to BSD

git-svn-id: https://lz4.googlecode.com/svn/trunk@4 650e7d94-2a16-8b24-b05c-7c0b3f6821cd
This commit is contained in:
yann.collet.73@gmail.com 2011-04-26 23:49:24 +00:00
parent 1920856128
commit cfcae8a50f
3 changed files with 103 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
LZ4.c
View File

@ -1,20 +1,29 @@
/* /*
LZ4 - Fast LZ compression algorithm LZ4 - Fast LZ compression algorithm
Copyright (C) Yann Collet 2011, Copyright (C) 2011, Yann Collet.
This program is free software; you can redistribute it and/or modify Redistribution and use in source and binary forms, with or without
it under the terms of the GNU General Public License as published by modification, are permitted provided that the following conditions are
the Free Software Foundation; either version 2 of the License, or met:
(at your option) any later version.
* Redistributions of source code must retain the above copyright
This program is distributed in the hope that it will be useful, notice, this list of conditions and the following disclaimer.
but WITHOUT ANY WARRANTY; without even the implied warranty of * Redistributions in binary form must reproduce the above
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the copyright notice, this list of conditions and the following disclaimer
GNU General Public License for more details. in the documentation and/or other materials provided with the
distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
//************************************** //**************************************
@ -46,7 +55,7 @@
#define MAXD_LOG 16 #define MAXD_LOG 16
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) #define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
#define HASH_LOG 17 #define HASH_LOG 17 // <--- Lower this value to lower memory usage. N->2^(N+2) Bytes (ex : 17 -> 512KB)
#define HASHTABLESIZE (1 << HASH_LOG) #define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1) #define HASH_MASK (HASHTABLESIZE - 1)
@ -91,6 +100,17 @@ int LZ4_compress(char* source,
int LZ4_singleThread_compress(
char* source,
char* dest,
int isize)
{
static void* ctx = NULL;
return LZ4_compressCtx(&ctx, source, dest, isize);
}
int LZ4_compressCtx(void** ctx, int LZ4_compressCtx(void** ctx,
char* source, char* source,
char* dest, char* dest,
@ -133,11 +153,7 @@ int LZ4_compressCtx(void** ctx,
// Min Match // Min Match
if (( ((ip-ref) >> MAXD_LOG) != 0) || (*(U32*)ref != sequence)) if (( ((ip-ref) >> MAXD_LOG) != 0) || (*(U32*)ref != sequence))
{ {
if (ip-anchor>limit) if (ip-anchor>limit) { limit<<=1; step += 1 + (step>>2); }
{
limit<<=1;
step += 1 + (step>>2);
}
ip+=step; ip+=step;
continue; continue;
} }
@ -169,8 +185,7 @@ int LZ4_compressCtx(void** ctx,
// Start Counting // Start Counting
ip+=MINMATCH; ref+=MINMATCH; // MinMatch verified ip+=MINMATCH; ref+=MINMATCH; // MinMatch verified
anchor = ip; anchor = ip;
while (*ref == *ip) { ip++; ref++; } // Ends at *ip!=*ref while ((ip<iend) && (*ref == *ip)) { ip++; ref++; } // Ends at *ip!=*ref
if (ip>iend) ip=iend;
len = (ip - anchor); len = (ip - anchor);
// Encode MatchLength // Encode MatchLength
@ -184,15 +199,11 @@ int LZ4_compressCtx(void** ctx,
// Encode Last Literals // Encode Last Literals
len = length = iend - anchor; len = length = iend - anchor;
// if (length > 0) orun=op++;
{ if (len>(RUN_MASK-1)) { *orun=(RUN_MASK<<ML_BITS); len-=RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE) len; }
orun=op++; else *orun = (len<<ML_BITS);
if (len>(RUN_MASK-1)) for(;length>0;length-=4) { *(U32*)op = *(U32*)anchor; op+=4; anchor+=4; }
{ *orun=(RUN_MASK<<ML_BITS); len-=RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE) len; } op += length; // correction
else *orun = (len<<ML_BITS);
for(;length>0;length-=4) { *(U32*)op = *(U32*)anchor; op+=4; anchor+=4; }
op += length; // correction
}
// End // End
@ -229,6 +240,10 @@ int LZ4_decode ( char* source,
// copy literals // copy literals
ref = op+length; ref = op+length;
#ifdef SAFEWRITEBUFFER
if (ref>iend-4) { while(op<iend-3) { *(U32*)op=*(U32*)ip; op+=4; ip+=4; } while(op<ref) *op++=*ip++; }
else
#endif
while (op<ref) { *(U32*)op = *(U32*)ip; op+=4; ip+=4; } while (op<ref) { *(U32*)op = *(U32*)ip; op+=4; ip+=4; }
ip-=(op-ref); op=ref; // correction ip-=(op-ref); op=ref; // correction
if (ip>=iend) break; // Check EOF if (ip>=iend) break; // Check EOF
@ -250,6 +265,10 @@ int LZ4_decode ( char* source,
*op++ = *ref++; *op++ = *ref++;
ref -= dec[op-ref]; ref -= dec[op-ref];
} }
#ifdef SAFEWRITEBUFFER
if (cpy>iend-4) { while(op<iend-3) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; } while(op<cpy) *op++=*ref++; }
else
#endif
while(op<cpy) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; } while(op<cpy) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }
op=cpy; // correction op=cpy; // correction
} }

65
LZ4.h
View File

@ -1,21 +1,30 @@
/* /*
LZ4 - Fast LZ compression algorithm LZ4 - Fast LZ compression algorithm
Header File Header File
Copyright (C) Yann Collet 2011, Copyright (C) 2011, Yann Collet.
This program is free software; you can redistribute it and/or modify Redistribution and use in source and binary forms, with or without
it under the terms of the GNU General Public License as published by modification, are permitted provided that the following conditions are
the Free Software Foundation; either version 2 of the License, or met:
(at your option) any later version.
* Redistributions of source code must retain the above copyright
This program is distributed in the hope that it will be useful, notice, this list of conditions and the following disclaimer.
but WITHOUT ANY WARRANTY; without even the implied warranty of * Redistributions in binary form must reproduce the above
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the copyright notice, this list of conditions and the following disclaimer
GNU General Public License for more details. in the documentation and/or other materials provided with the
distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
#if defined (__cplusplus) #if defined (__cplusplus)
@ -23,6 +32,16 @@ extern "C" {
#endif #endif
//****************************
// Instructions
//****************************
// Uncomment next line to ensure that LZ4_Decode will never write beyond dest + originalSize
// If commented, the decoder may write up to 3 bytes more than originalSize, so provide extra room in dest buffer for that
// Recommendation : commented, for improved performance; ensure that destination buffer is at least originalSize + 3 Bytes
// #define SAFEWRITEBUFFER
//**************************** //****************************
// Simple Functions // Simple Functions
//**************************** //****************************
@ -33,28 +52,34 @@ int LZ4_decode (char* source, char* dest, int isize);
/* /*
LZ4_compress : LZ4_compress :
return : the number of bytes in compressed buffer dest return : the number of bytes in compressed buffer dest
note 1 : this function may sometimes read beyond source+isize, so provide some extra buffer room for that. note : this simple function explicitly allocate/deallocate memory **at each call**
note 2 : this simple function explicitly allocate/deallocate memory **at each call**
LZ4_decode : LZ4_decode :
return : the number of bytes in decoded buffer dest return : the number of bytes in decoded buffer dest
note : this function may write up to 3 bytes more than decoded data length, so provide some extra buffer room for that.
*/ */
//**************************** //****************************
// More Complex Functions // Advanced Functions
//**************************** //****************************
int LZ4_singleThread_compress (char* source, char* dest, int isize);
int LZ4_compressCtx(void** ctx, char* source, char* dest, int isize); int LZ4_compressCtx(void** ctx, char* source, char* dest, int isize);
/* /*
LZ4_singleThread_compress :
Same as LZ4_compress, but avoids re-allocation effect between each call.
Better performance, but only use it in single-threading scenario !
note : performance difference is only noticeable when repetitively calling the compression algorithm on many small data packets.
LZ4_compressCtx : LZ4_compressCtx :
This function allows to handle explicitly the CTX memory structure. This function explicitly handle the CTX memory structure.
It avoids allocating/deallocating this memory at each call, for better performance. It avoids allocating/deallocating memory between each call, improving performance.
On first call : provide a *ctx=NULL; It will be automatically allocated. On first call : provide a *ctx=NULL; It will be automatically allocated.
On next calls : reuse the same ctx pointer. On next calls : reuse the same ctx pointer.
Use different pointers for different threads when doing multi-threading. Use different pointers for different threads when doing multi-threading.
note : performance difference is only noticeable when repetitively calling the compression algorithm on many small data packets.
*/ */

17
main.c
View File

@ -84,9 +84,8 @@ compress_file(char* input_filename, char* output_filename)
if ( foutput==0) { printf("Pb opening %s\n", output_filename); return 3; } if ( foutput==0) { printf("Pb opening %s\n", output_filename); return 3; }
// Allocate Memory // Allocate Memory
in_buff = malloc(CHUNKSIZE + CACHELINE); in_buff = malloc(CHUNKSIZE);
out_buff = malloc(OUT_CHUNKSIZE); out_buff = malloc(OUT_CHUNKSIZE);
{ int i; int prime = PRIME; for (i=CHUNKSIZE; i<CHUNKSIZE+CACHELINE; i++) { prime *= prime; in_buff[i] = prime; } }
// Write Archive Header // Write Archive Header
* (unsigned long*) out_buff = ARCHIVE_MAGICNUMBER; * (unsigned long*) out_buff = ARCHIVE_MAGICNUMBER;
@ -97,9 +96,9 @@ compress_file(char* input_filename, char* output_filename)
{ {
int outSize; int outSize;
// Read Block // Read Block
int inSize = fread( in_buff, 1, CHUNKSIZE, finput ); int inSize = fread( in_buff, 1, CHUNKSIZE, finput );
if( inSize<=0 ) break; if( inSize<=0 ) break;
filesize += inSize; filesize += inSize;
// Compress Block // Compress Block
outSize = LZ4_compress(in_buff, out_buff+4, inSize); outSize = LZ4_compress(in_buff, out_buff+4, inSize);
@ -144,10 +143,10 @@ decode_file(char* input_filename, char* output_filename)
{ {
int outSize; int outSize;
// Read Block // Read Block
int inSize = fread( in_buff, 1, 4, finput ); int inSize = fread( in_buff, 1, 4, finput );
if( inSize<=0 ) break; if( inSize<=0 ) break;
inSize = (int) * (unsigned long*) in_buff; inSize = (int) * (unsigned long*) in_buff;
fread( in_buff, 1, inSize, finput ); fread( in_buff, 1, inSize, finput );
// Decode Block // Decode Block
outSize = LZ4_decode(in_buff, out_buff, inSize); outSize = LZ4_decode(in_buff, out_buff, inSize);