From ea53527d10d57a468727b3f797175a2a671338c8 Mon Sep 17 00:00:00 2001 From: "yann.collet.73@gmail.com" Date: Tue, 27 Sep 2011 11:59:58 +0000 Subject: [PATCH] New : ARM Validated code, thanks to Vlad Grachov. LZ4Demo : compress/decompress output time git-svn-id: https://lz4.googlecode.com/svn/trunk@33 650e7d94-2a16-8b24-b05c-7c0b3f6821cd --- lz4.c | 47 +++++++++++----- lz4demo.c | 163 +++++++++++++++++++++++++++++------------------------- 2 files changed, 122 insertions(+), 88 deletions(-) diff --git a/lz4.c b/lz4.c index c8e4072..c72609d 100644 --- a/lz4.c +++ b/lz4.c @@ -95,13 +95,32 @@ struct refTables const BYTE* hashTable[HASHTABLESIZE]; }; +#ifdef __GNUC__ +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +typedef struct _U32_S +{ + U32 v; +} _PACKED U32_S; + +typedef struct _U16_S +{ + U16 v; +} _PACKED U16_S; + +#define A32(x) (((U32_S *)(x))->v) +#define A16(x) (((U16_S *)(x))->v) + //************************************** // Macros //************************************** #define LZ4_HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) -#define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(*(U32*)(p)) -#define LZ4_COPYPACKET(s,d) *(U32*)d = *(U32*)s; d+=4; s+=4; *(U32*)d = *(U32*)s; d+=4; s+=4; +#define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p)) +#define LZ4_COPYPACKET(s,d) A32(d) = A32(s); d+=4; s+=4; A32(d) = A32(s); d+=4; s+=4; #define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (danchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; } @@ -193,19 +212,19 @@ int LZ4_compressCtx(void** ctx, _next_match: // Encode Offset - *(U16*)op = (ip-ref); op+=2; + A16(op) = (ip-ref); op+=2; // Start Counting ip+=MINMATCH; ref+=MINMATCH; // MinMatch verified anchor = ip; while (ip ip - (MAX_DISTANCE + 1)) && (*(U32*)ref == *(U32*)ip)) { token = op++; *token=0; goto _next_match; } + if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; } // Prepare next loop anchor = ip++; @@ -308,7 +327,7 @@ int LZ4_uncompress(char* source, // get offset - ref -= *(U16*)ip; ip+=2; + ref -= A16(ip); ip+=2; // get matchlength if ((length=(token&ML_MASK)) == ML_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; } @@ -321,8 +340,8 @@ int LZ4_uncompress(char* source, *op++ = *ref++; *op++ = *ref++; ref -= dec[op-ref]; - *(U32*)op=*(U32*)ref; - } else { *(U32*)op=*(U32*)ref; op+=4; ref+=4; } + A32(op)=A32(ref); + } else { A32(op)=A32(ref); op+=4; ref+=4; } cpy = op + length; if (cpy > oend-COPYLENGTH) { @@ -388,7 +407,7 @@ int LZ4_uncompress_unknownOutputSize( // get offset - ref -= *(U16*)ip; ip+=2; + ref -= A16(ip); ip+=2; // get matchlength if ((length=(token&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; } @@ -401,8 +420,8 @@ int LZ4_uncompress_unknownOutputSize( *op++ = *ref++; *op++ = *ref++; ref -= dec[op-ref]; - *(U32*)op=*(U32*)ref; - } else { *(U32*)op=*(U32*)ref; op+=4; ref+=4; } + A32(op)=A32(ref); + } else { A32(op)=A32(ref); op+=4; ref+=4; } cpy = op + length; if (cpy>oend-COPYLENGTH) { diff --git a/lz4demo.c b/lz4demo.c index 2652e4e..ed0782d 100644 --- a/lz4demo.c +++ b/lz4demo.c @@ -33,6 +33,7 @@ #include // fprintf, fopen, fread #include // malloc #include // strcmp +#include // clock #ifdef _WIN32 #include // _setmode #include // _O_BINARY @@ -43,20 +44,7 @@ //************************************** // Basic Types //************************************** -#if defined(_MSC_VER) || defined(_WIN32) || defined(__WIN32__) -#define BYTE unsigned __int8 -#define U16 unsigned __int16 -#define U32 unsigned __int32 -#define S32 __int32 -#define U64 unsigned __int64 -#else -#include -#define BYTE uint8_t -#define U16 uint16_t -#define U32 uint32_t -#define S32 int32_t -#define U64 uint64_t -#endif + //**************************** @@ -66,7 +54,7 @@ #define COMPRESSOR_VERSION "" #define COMPILED __DATE__ #define AUTHOR "Yann Collet" -#define BINARY_NAME "LZ4.exe" +#define BINARY_NAME "lz4demo.exe" #define EXTENSION ".lz4" #define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED @@ -77,73 +65,97 @@ #define ARCHIVE_MAGICNUMBER_SIZE 4 +//************************************** +// MACRO +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + + //**************************** // Functions //**************************** int usage() { - fprintf(stderr, "Usage :\n"); - fprintf(stderr, " %s [arg] input output\n",BINARY_NAME); - fprintf(stderr, "Arguments :\n"); - fprintf(stderr, " -c : compression (default)\n"); - fprintf(stderr, " -d : decompression \n"); - fprintf(stderr, " -t : test compressed file \n"); - fprintf(stderr, " -h : help (this text)\n"); - fprintf(stderr, "input : can be 'stdin' (pipe) or a filename\n"); - fprintf(stderr, "output : can be 'stdout'(pipe) or a filename or 'null'\n"); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] input output\n",BINARY_NAME); + DISPLAY( "Arguments :\n"); + DISPLAY( " -c : compression (default)\n"); + DISPLAY( " -d : decompression \n"); + DISPLAY( " -t : test compressed file \n"); + DISPLAY( " -h : help (this text)\n"); + DISPLAY( "input : can be 'stdin' (pipe) or a filename\n"); + DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n"); return 0; } int badusage() { - fprintf(stderr, "Wrong parameters\n"); + DISPLAY("Wrong parameters\n"); usage(); return 0; } -int compress_file(char* input_filename, char* output_filename) + +int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) { - U64 filesize = 0; - U64 compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE; - char* in_buff; - char* out_buff; - FILE* finput; - FILE* foutput; char stdinmark[] = "stdin"; char stdoutmark[] = "stdout"; if (!strcmp (input_filename, stdinmark)) { - fprintf(stderr, "Using stdin for input\n"); - finput = stdin; + DISPLAY( "Using stdin for input\n"); + *pfinput = stdin; #ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows _setmode( _fileno( stdin ), _O_BINARY ); #endif } else { - finput = fopen( input_filename, "rb" ); + *pfinput = fopen( input_filename, "rb" ); } if (!strcmp (output_filename, stdoutmark)) { - fprintf(stderr, "Using stdout for output\n"); - foutput = stdout; + DISPLAY( "Using stdout for output\n"); + *pfoutput = stdout; #ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows _setmode( _fileno( stdout ), _O_BINARY ); #endif } else { - foutput = fopen( output_filename, "wb" ); + *pfoutput = fopen( output_filename, "wb" ); } - if ( finput==0 ) { fprintf(stderr, "Pb opening %s\n", input_filename); return 2; } - if ( foutput==0) { fprintf(stderr, "Pb opening %s\n", output_filename); return 3; } + if ( *pfinput==0 ) { DISPLAY( "Pb opening %s\n", input_filename); return 2; } + if ( *pfoutput==0) { DISPLAY( "Pb opening %s\n", output_filename); return 3; } + return 0; +} + + + +int compress_file(char* input_filename, char* output_filename) +{ + unsigned long long filesize = 0; + unsigned long long compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE; + char* in_buff; + char* out_buff; + FILE* finput; + FILE* foutput; + int r; + clock_t start, end; + + + // Init + start = clock(); + r = get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (r) return r; + // Allocate Memory - in_buff = malloc(CHUNKSIZE); - out_buff = malloc(OUT_CHUNKSIZE); + in_buff = (char*)malloc(CHUNKSIZE); + out_buff = (char*)malloc(OUT_CHUNKSIZE); + if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 8; } // Write Archive Header - *(U32*)out_buff = ARCHIVE_MAGICNUMBER; + *(unsigned long*)out_buff = ARCHIVE_MAGICNUMBER; fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); // Main Loop @@ -157,7 +169,7 @@ int compress_file(char* input_filename, char* output_filename) // Compress Block outSize = LZ4_compress(in_buff, out_buff+4, inSize); - * (U32*) out_buff = outSize; + * (unsigned long*) out_buff = outSize; compressedfilesize += outSize+4; // Write Block @@ -165,9 +177,17 @@ int compress_file(char* input_filename, char* output_filename) } // Status - fprintf(stderr, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + end = clock(); + DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + // Close & Free + free(in_buff); + free(out_buff); fclose(finput); fclose(foutput); @@ -177,46 +197,33 @@ int compress_file(char* input_filename, char* output_filename) int decode_file(char* input_filename, char* output_filename) { - U64 filesize = 0; + unsigned long long filesize = 0; char* in_buff; char* out_buff; size_t uselessRet; int sinkint; - U32 nextSize; + unsigned long nextSize; FILE* finput; FILE* foutput; - char stdinmark[] = "stdin"; - char stdoutmark[] = "stdout"; + clock_t start, end; + int r; - if (!strcmp (input_filename, stdinmark)) { - fprintf(stderr, "Using stdin for input\n"); - finput = stdin; -#ifdef _WIN32 // need to set stdin/stdout to binary mode - _setmode( _fileno( stdin ), _O_BINARY ); -#endif - } else { - finput = fopen( input_filename, "rb" ); - } - if (!strcmp (output_filename, stdoutmark)) { - fprintf(stderr, "Using stdout for output\n"); - foutput = stdout; -#ifdef _WIN32 // need to set stdin/stdout to binary mode - _setmode( _fileno( stdout ), _O_BINARY ); -#endif - } else { - foutput = fopen( output_filename, "wb" ); - } + // Init + start = clock(); + r = get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (r) return r; // Allocate Memory - in_buff = malloc(OUT_CHUNKSIZE); - out_buff = malloc(CHUNKSIZE); + in_buff = (char*)malloc(OUT_CHUNKSIZE); + out_buff = (char*)malloc(CHUNKSIZE); + if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 7; } // Check Archive Header uselessRet = fread(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, finput); - if (*(U32*)out_buff != ARCHIVE_MAGICNUMBER) { fprintf(stderr,"Unrecognized header : file cannot be decoded\n"); return 6; } + if (*(unsigned long*)out_buff != ARCHIVE_MAGICNUMBER) { DISPLAY("Unrecognized header : file cannot be decoded\n"); return 6; } uselessRet = fread(in_buff, 1, 4, finput); - nextSize = *(U32*)in_buff; + nextSize = *(unsigned long*)in_buff; // Main Loop while (1) @@ -225,7 +232,7 @@ int decode_file(char* input_filename, char* output_filename) uselessRet = fread(in_buff, 1, nextSize, finput); // Check Next Block - uselessRet = (U32) fread(&nextSize, 1, 4, finput); + uselessRet = (unsigned long) fread(&nextSize, 1, 4, finput); if( uselessRet==0 ) break; // Decode Block @@ -243,8 +250,16 @@ int decode_file(char* input_filename, char* output_filename) fwrite(out_buff, 1, sinkint, foutput); // Status - fprintf(stderr, "Successfully decoded %llu bytes \n", (unsigned long long)filesize); + end = clock(); + DISPLAY( "Successfully decoded %llu bytes \n", (unsigned long long)filesize); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + // Close & Free + free(in_buff); + free(out_buff); fclose(finput); fclose(foutput); @@ -267,7 +282,7 @@ int main(int argc, char** argv) char nullinput[] = "null"; // Welcome message - fprintf(stderr, WELCOME_MESSAGE); + DISPLAY( WELCOME_MESSAGE); if (argc<2) { badusage(); return 1; }