AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'dev' into fasterCount

Browse Source
This commit is contained in:
Yann Collet 2020-08-25 22:18:54 -07:00
commit a7ba363514
3 changed files with 146 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
programs/lz4io.c
View File

@ -680,7 +680,7 @@ LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress,
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum; prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum;
prefs.favorDecSpeed = io_prefs->favorDecSpeed; prefs.favorDecSpeed = io_prefs->favorDecSpeed;
if (io_prefs->contentSizeFlag) { if (io_prefs->contentSizeFlag) {
U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const fileSize = UTIL_getOpenFileSize(srcFile);
prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */
if (fileSize==0) if (fileSize==0)
DISPLAYLEVEL(3, "Warning : cannot determine input content size \n"); DISPLAYLEVEL(3, "Warning : cannot determine input content size \n");
@ -1472,7 +1472,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */
unsigned char buffer[LZ4F_HEADER_SIZE_MAX]; unsigned char buffer[LZ4F_HEADER_SIZE_MAX];
FILE* const finput = LZ4IO_openSrcFile(input_filename); FILE* const finput = LZ4IO_openSrcFile(input_filename);
cfinfo->fileSize = UTIL_getFileSize(input_filename); cfinfo->fileSize = (finput == NULL) ? 0 : UTIL_getOpenFileSize(finput);
while (!feof(finput)) { while (!feof(finput)) {
LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO;

66
programs/util.h
View File

@ -33,7 +33,7 @@ extern "C" {
#include <stddef.h> /* size_t, ptrdiff_t */ #include <stddef.h> /* size_t, ptrdiff_t */
#include <stdlib.h> /* malloc */ #include <stdlib.h> /* malloc */
#include <string.h> /* strlen, strncpy */ #include <string.h> /* strlen, strncpy */
#include <stdio.h> /* fprintf */ #include <stdio.h> /* fprintf, fileno */
#include <assert.h> #include <assert.h>
#include <sys/types.h> /* stat, utime */ #include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */ #include <sys/stat.h> /* stat */
@ -122,6 +122,36 @@ extern "C" {
#endif #endif
/*-****************************************
* stat() functions
******************************************/
#if defined(_MSC_VER)
# define UTIL_TYPE_stat __stat64
# define UTIL_stat _stat64
# define UTIL_fstat _fstat64
# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG)
#elif defined(__MINGW32__) && defined (__MSVCRT__)
# define UTIL_TYPE_stat _stati64
# define UTIL_stat _stati64
# define UTIL_fstat _fstati64
# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG)
#else
# define UTIL_TYPE_stat stat
# define UTIL_stat stat
# define UTIL_fstat fstat
# define UTIL_STAT_MODE_ISREG(st_mode) (S_ISREG(st_mode))
#endif
/*-****************************************
* fileno() function
******************************************/
#if defined(_MSC_VER)
# define UTIL_fileno _fileno
#else
# define UTIL_fileno fileno
#endif
/* ************************************* /* *************************************
* Constants * Constants
***************************************/ ***************************************/
@ -357,22 +387,30 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
} }
UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file)
{
int r;
int fd;
struct UTIL_TYPE_stat statbuf;
fd = UTIL_fileno(file);
if (fd < 0) {
perror("fileno");
exit(1);
}
r = UTIL_fstat(fd, &statbuf);
if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
{ {
int r; int r;
#if defined(_MSC_VER) struct UTIL_TYPE_stat statbuf;
struct __stat64 statbuf;
r = _stat64(infilename, &statbuf); r = UTIL_stat(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */
#elif defined(__MINGW32__) && defined (__MSVCRT__)
struct _stati64 statbuf;
r = _stati64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
#endif
return (U64)statbuf.st_size; return (U64)statbuf.st_size;
} }

173
tests/Makefile
View File

@ -146,7 +146,9 @@ ifneq (,$(filter $(shell uname),SunOS))
DIFF:=gdiff DIFF:=gdiff
endif endif
CAT:=cat
DD:=dd DD:=dd
DATAGEN:=./datagen
.PHONY: list .PHONY: list
list: list:
@ -168,14 +170,14 @@ lz4_all.o: lz4_all.c
$(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@ $(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@
lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c
cat $^ > $@ $(CAT) $^ > $@
test-install: lz4 lib liblz4.pc test-install: lz4 lib liblz4.pc
lz4_root=.. ./test_install.sh lz4_root=.. ./test_install.sh
test-lz4-sparse: lz4 datagen test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----" @echo "\n ---- test sparse file support ----"
./datagen -g5M -P100 > tmplsdg5M $(DATAGEN) -g5M -P100 > tmplsdg5M
$(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4 $(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4
$(DIFF) -s tmplsdg5M tmplscB4 $(DIFF) -s tmplsdg5M tmplscB4
$(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5 $(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5
@ -187,17 +189,17 @@ test-lz4-sparse: lz4 datagen
$(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse $(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse
$(DIFF) -s tmplsdg5M tmplsnosparse $(DIFF) -s tmplsdg5M tmplsnosparse
ls -ls tmpls* ls -ls tmpls*
./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block) $(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block)
./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd $(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
ls -ls tmplsodd ls -ls tmplsodd
@$(RM) tmpls* @$(RM) tmpls*
@echo "\n Compatibility with Console :" @echo "\n Compatibility with Console :"
echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | $(CAT)
echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
@echo "\n Compatibility with Append :" @echo "\n Compatibility with Append :"
./datagen -P100 -g1M > tmplsdg1M $(DATAGEN) -P100 -g1M > tmplsdg1M
cat tmplsdg1M tmplsdg1M > tmpls2M $(CAT) tmplsdg1M tmplsdg1M > tmpls2M
$(LZ4) -B5 -v tmplsdg1M tmplsc $(LZ4) -B5 -v tmplsdg1M tmplsc
$(LZ4) -d -v tmplsc tmplsr $(LZ4) -d -v tmplsc tmplsr
$(LZ4) -d -v tmplsc -c >> tmplsr $(LZ4) -d -v tmplsc -c >> tmplsr
@ -207,20 +209,29 @@ test-lz4-sparse: lz4 datagen
test-lz4-contentSize: lz4 datagen test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----" @echo "\n ---- test original size support ----"
./datagen -g15M > tmplc1 $(DATAGEN) -g15M > tmplc1
$(LZ4) -v tmplc1 -c | $(LZ4) -t $(LZ4) -v tmplc1 -c | $(LZ4) -t
$(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2 $(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2
$(DIFF) -s tmplc1 tmplc2 $(DIFF) tmplc1 tmplc2
$(LZ4) -f tmplc1 -c > tmplc1.lz4
$(LZ4) --content-size tmplc1 -c > tmplc2.lz4
! $(DIFF) tmplc1.lz4 tmplc2.lz4 # must differ, due to content size
$(LZ4) --content-size < tmplc1 > tmplc3.lz4
$(DIFF) tmplc2.lz4 tmplc3.lz4 # both must contain content size
$(CAT) tmplc1 | $(LZ4) > tmplc4.lz4
$(DIFF) tmplc1.lz4 tmplc4.lz4 # both don't have content size
$(CAT) tmplc1 | $(LZ4) --content-size > tmplc5.lz4 # can't determine content size
$(DIFF) tmplc1.lz4 tmplc5.lz4 # both don't have content size
@$(RM) tmplc* @$(RM) tmplc*
test-lz4-frame-concatenation: lz4 datagen test-lz4-frame-concatenation: lz4 datagen
@echo "\n ---- test frame concatenation ----" @echo "\n ---- test frame concatenation ----"
@echo -n > tmp-lfc-empty @echo -n > tmp-lfc-empty
@echo hi > tmp-lfc-nonempty @echo hi > tmp-lfc-nonempty
cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src $(CAT) tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
$(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4 $(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4
$(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4 $(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4
cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4 $(CAT) tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
$(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result $(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result
$(CMP) tmp-lfc-src tmp-lfc-result $(CMP) tmp-lfc-src tmp-lfc-result
@$(RM) tmp-lfc-* @$(RM) tmp-lfc-*
@ -228,9 +239,9 @@ test-lz4-frame-concatenation: lz4 datagen
test-lz4-multiple: lz4 datagen test-lz4-multiple: lz4 datagen
@echo "\n ---- test multiple files ----" @echo "\n ---- test multiple files ----"
@./datagen -s1 > tmp-tlm1 2> $(VOID) @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
@./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
@./datagen -s3 -g200K > tmp-tlm3 2> $(VOID) @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
# compress multiple files : one .lz4 per source file # compress multiple files : one .lz4 per source file
$(LZ4) -f -m tmp-tlm* $(LZ4) -f -m tmp-tlm*
test -f tmp-tlm1.lz4 test -f tmp-tlm1.lz4
@ -245,7 +256,7 @@ test-lz4-multiple: lz4 datagen
$(CMP) tmp-tlm2 tmp-tlm2-orig $(CMP) tmp-tlm2 tmp-tlm2-orig
$(CMP) tmp-tlm3 tmp-tlm3-orig $(CMP) tmp-tlm3 tmp-tlm3-orig
# compress multiple files into stdout # compress multiple files into stdout
cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1 $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
$(RM) *.lz4 $(RM) *.lz4
$(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2 $(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
@ -253,7 +264,7 @@ test-lz4-multiple: lz4 datagen
# decompress multiple files into stdout # decompress multiple files into stdout
$(RM) tmp-tlm-concat1 tmp-tlm-concat2 $(RM) tmp-tlm-concat1 tmp-tlm-concat2
$(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress $(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
cat tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
$(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
test ! -f tmp-tlm1 # must not create file artefact test ! -f tmp-tlm1 # must not create file artefact
@ -264,9 +275,9 @@ test-lz4-multiple: lz4 datagen
test-lz4-multiple-legacy: lz4 datagen test-lz4-multiple-legacy: lz4 datagen
@echo "\n ---- test multiple files (Legacy format) ----" @echo "\n ---- test multiple files (Legacy format) ----"
@./datagen -s1 > tmp-tlm1 2> $(VOID) @$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
@./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) @$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
@./datagen -s3 -g200K > tmp-tlm3 2> $(VOID) @$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
# compress multiple files using legacy format: one .lz4 per source file # compress multiple files using legacy format: one .lz4 per source file
$(LZ4) -f -l -m tmp-tlm* $(LZ4) -f -l -m tmp-tlm*
test -f tmp-tlm1.lz4 test -f tmp-tlm1.lz4
@ -282,7 +293,7 @@ test-lz4-multiple-legacy: lz4 datagen
$(CMP) tmp-tlm2 tmp-tlm2-orig $(CMP) tmp-tlm2 tmp-tlm2-orig
$(CMP) tmp-tlm3 tmp-tlm3-orig $(CMP) tmp-tlm3 tmp-tlm3-orig
# compress multiple files into stdout using legacy format # compress multiple files into stdout using legacy format
cat tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1 $(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
$(RM) *.lz4 $(RM) *.lz4
$(LZ4) -l -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2 $(LZ4) -l -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
@ -290,7 +301,7 @@ test-lz4-multiple-legacy: lz4 datagen
# # # decompress multiple files into stdout using legacy format # # # decompress multiple files into stdout using legacy format
$(RM) tmp-tlm-concat1 tmp-tlm-concat2 $(RM) tmp-tlm-concat1 tmp-tlm-concat2
$(LZ4) -l -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress $(LZ4) -l -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
cat tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference $(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 $(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
$(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 $(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
$(LZ4) -d -l -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 # -l mustn't impact option -d $(LZ4) -d -l -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 # -l mustn't impact option -d
@ -302,18 +313,18 @@ test-lz4-multiple-legacy: lz4 datagen
test-lz4-basic: lz4 datagen unlz4 lz4cat test-lz4-basic: lz4 datagen unlz4 lz4cat
@echo "\n ---- test lz4 basic compression/decompression ----" @echo "\n ---- test lz4 basic compression/decompression ----"
./datagen -g0 | $(LZ4) -v | $(LZ4) -t $(DATAGEN) -g0 | $(LZ4) -v | $(LZ4) -t
./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t $(DATAGEN) -g16KB | $(LZ4) -9 | $(LZ4) -t
./datagen -g20KB > tmp-tlb-dg20k $(DATAGEN) -g20KB > tmp-tlb-dg20k
$(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec $(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
$(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
./datagen | $(LZ4) -BI | $(LZ4) -t $(DATAGEN) | $(LZ4) -BI | $(LZ4) -t
./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t $(DATAGEN) -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt $(DATAGEN) -g17M | $(LZ4) -9v | $(LZ4) -qt
./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t $(DATAGEN) -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
@echo "hello world" > tmp-tlb-hw @echo "hello world" > tmp-tlb-hw
$(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4 $(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
test ! -f tmp-tlb-hw # must fail (--rm) test ! -f tmp-tlb-hw # must fail (--rm)
@ -342,18 +353,18 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
$(DIFF) -q tmp-tlb-hw tmp-tlb4 $(DIFF) -q tmp-tlb-hw tmp-tlb4
$(LZ4) -f tmp-tlb-hw $(LZ4) -f tmp-tlb-hw
$(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file $(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file
cat tmp-tlb-hw >> tmp-tlb-hw.lz4 $(CAT) tmp-tlb-hw >> tmp-tlb-hw.lz4
$(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data
$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum
# ./datagen -g20KB generates the same file every single time # $(DATAGEN) -g20KB generates the same file every single time
# cannot save output of ./datagen -g20KB as input file to lz4 because the following shell commands are run before ./datagen -g20KB # cannot save output of $(DATAGEN) -g20KB as input file to lz4 because the following shell commands are run before $(DATAGEN) -g20KB
test "$(shell ./datagen -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell ./datagen -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9 test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9
test "$(shell ./datagen -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell ./datagen -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1 test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1
test "$(shell ./datagen -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell ./datagen -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1 test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1
! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0 ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0
! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1 ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1
# High --fast values can result in out-of-bound dereferences #876 # High --fast values can result in out-of-bound dereferences #876
./datagen -g1M | $(LZ4) -c --fast=999999999 > /dev/null $(DATAGEN) -g1M | $(LZ4) -c --fast=999999999 > /dev/null
# Test for #596 # Test for #596
@echo "TEST" > tmp-tlb-test @echo "TEST" > tmp-tlb-test
$(LZ4) -m tmp-tlb-test $(LZ4) -m tmp-tlb-test
@ -365,10 +376,10 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
test-lz4-dict: lz4 datagen test-lz4-dict: lz4 datagen
@echo "\n ---- test lz4 compression/decompression with dictionary ----" @echo "\n ---- test lz4 compression/decompression with dictionary ----"
./datagen -g16KB > tmp-dict $(DATAGEN) -g16KB > tmp-dict
./datagen -g32KB > tmp-dict-sample-32k $(DATAGEN) -g32KB > tmp-dict-sample-32k
< tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k < tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k
./datagen -g128MB > tmp-dict-sample-128m $(DATAGEN) -g128MB > tmp-dict-sample-128m
< tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m < tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m
touch tmp-dict-sample-0 touch tmp-dict-sample-0
< tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0 < tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0
@ -377,10 +388,10 @@ test-lz4-dict: lz4 datagen
< tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0 < tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0
@echo "\n ---- test lz4 dictionary loading ----" @echo "\n ---- test lz4 dictionary loading ----"
./datagen -g128KB > tmp-dict-data-128KB $(DATAGEN) -g128KB > tmp-dict-data-128KB
set -e; \ set -e; \
for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \ for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
./datagen -g$$l > tmp-dict-$$l; \ $(DATAGEN) -g$$l > tmp-dict-$$l; \
$(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \ $(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
< tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \ < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
< tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \ < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
@ -390,12 +401,12 @@ test-lz4-dict: lz4 datagen
test-lz4-hugefile: lz4 datagen test-lz4-hugefile: lz4 datagen
@echo "\n ---- test huge files compression/decompression ----" @echo "\n ---- test huge files compression/decompression ----"
./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt $(DATAGEN) -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
./datagen -g5GB | $(LZ4) -v4BD | $(LZ4) -qt $(DATAGEN) -g5GB | $(LZ4) -v4BD | $(LZ4) -qt
# test large file size [2-4] GB # test large file size [2-4] GB
@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1 @$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
@ls -ls tmphf1 @ls -ls tmphf1
@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2 @$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
@ls -ls tmphf2 @ls -ls tmphf2
$(DIFF) -s tmphf1 tmphf2 $(DIFF) -s tmphf1 tmphf2
@$(RM) tmphf* @$(RM) tmphf*
@ -404,14 +415,14 @@ test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----" @echo "\n ---- bench mode ----"
$(LZ4) -bi1 $(LZ4) -bi1
@echo "\n ---- test mode ----" @echo "\n ---- test mode ----"
! ./datagen | $(LZ4) -t ! $(DATAGEN) | $(LZ4) -t
! ./datagen | $(LZ4) -tf ! $(DATAGEN) | $(LZ4) -tf
@echo "\n ---- pass-through mode ----" @echo "\n ---- pass-through mode ----"
@echo "Why hello there " > tmp-tlt2.lz4 @echo "Why hello there " > tmp-tlt2.lz4
! $(LZ4) -f tmp-tlt2.lz4 > $(VOID) ! $(LZ4) -f tmp-tlt2.lz4 > $(VOID)
! ./datagen | $(LZ4) -dc > $(VOID) ! $(DATAGEN) | $(LZ4) -dc > $(VOID)
! ./datagen | $(LZ4) -df > $(VOID) ! $(DATAGEN) | $(LZ4) -df > $(VOID)
./datagen | $(LZ4) -dcf > $(VOID) $(DATAGEN) | $(LZ4) -dcf > $(VOID)
@echo "Hello World !" > tmp-tlt1 @echo "Hello World !" > tmp-tlt1
$(LZ4) -dcf tmp-tlt1 $(LZ4) -dcf tmp-tlt1
@echo "from underground..." > tmp-tlt2 @echo "from underground..." > tmp-tlt2
@ -425,16 +436,16 @@ test-lz4-testmode: lz4 datagen
test-lz4-opt-parser: lz4 datagen test-lz4-opt-parser: lz4 datagen
@echo "\n ---- test opt-parser ----" @echo "\n ---- test opt-parser ----"
./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t $(DATAGEN) -g16KB | $(LZ4) -12 | $(LZ4) -t
./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t $(DATAGEN) -P10 | $(LZ4) -12B4 | $(LZ4) -t
./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t $(DATAGEN) -g256K | $(LZ4) -12B4D | $(LZ4) -t
./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t $(DATAGEN) -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t
./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t $(DATAGEN) -g1M | $(LZ4) -12B5 | $(LZ4) -t
./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t $(DATAGEN) -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t
./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt $(DATAGEN) -g4M | $(LZ4) -11vq | $(LZ4) -qt
./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t $(DATAGEN) -g8M | $(LZ4) -11B4 | $(LZ4) -t
./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t $(DATAGEN) -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t
./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t $(DATAGEN) -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t
test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-multiple-legacy \ test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-multiple-legacy \
test-lz4-frame-concatenation test-lz4-testmode \ test-lz4-frame-concatenation test-lz4-testmode \
@ -447,35 +458,35 @@ test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
test-lz4c: lz4c datagen test-lz4c: lz4c datagen
@echo "\n ---- test lz4c variant ----" @echo "\n ---- test lz4c variant ----"
./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t $(DATAGEN) -g256MB | $(LZ4)c -l -v | $(LZ4)c -t
test-lz4c32: CFLAGS+=-m32 test-lz4c32: CFLAGS+=-m32
test-lz4c32: test-lz4 test-lz4c32: test-lz4
test-interop-32-64: lz4 lz4c32 datagen test-interop-32-64: lz4 lz4c32 datagen
@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----" @echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4) -t $(DATAGEN) -g16KB | $(LZ4)c32 -9 | $(LZ4) -t
./datagen -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t $(DATAGEN) -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t
./datagen | $(LZ4)c32 | $(LZ4) -t $(DATAGEN) | $(LZ4)c32 | $(LZ4) -t
./datagen -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t $(DATAGEN) -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t
./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt $(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt
./datagen -g1G -P90 | $(LZ4) | $(LZ4)c32 -t $(DATAGEN) -g1G -P90 | $(LZ4) | $(LZ4)c32 -t
./datagen -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt $(DATAGEN) -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt
test-lz4c32-basic: lz4c32 datagen test-lz4c32-basic: lz4c32 datagen
@echo "\n ---- test lz4c32 32-bits version ----" @echo "\n ---- test lz4c32 32-bits version ----"
./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t $(DATAGEN) -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t
./datagen | $(LZ4)c32 | $(LZ4)c32 -t $(DATAGEN) | $(LZ4)c32 | $(LZ4)c32 -t
./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt $(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt
./datagen -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt $(DATAGEN) -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt
test-platform: test-platform:
@echo "\n ---- test lz4 $(QEMU_SYS) platform ----" @echo "\n ---- test lz4 $(QEMU_SYS) platform ----"
$(QEMU_SYS) ./datagen -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t $(QEMU_SYS) $(DATAGEN) -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t
$(QEMU_SYS) ./datagen | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t $(QEMU_SYS) $(DATAGEN) | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t
$(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt $(QEMU_SYS) $(DATAGEN) -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt
ifneq ($(QEMU_SYS),qemu-arm-static) ifneq ($(QEMU_SYS),qemu-arm-static)
$(QEMU_SYS) ./datagen -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt $(QEMU_SYS) $(DATAGEN) -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt
endif endif
test-fullbench: fullbench test-fullbench: fullbench
@ -498,13 +509,13 @@ test-frametest32: test-frametest
test-mem: lz4 datagen fuzzer frametest fullbench test-mem: lz4 datagen fuzzer frametest fullbench
@echo "\n ---- valgrind tests : memory analyzer ----" @echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) valgrind --leak-check=yes --error-exitcode=1 $(DATAGEN) -g50M > $(VOID)
./datagen -g16KB > ftmdg16K $(DATAGEN) -g16KB > ftmdg16K
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID) valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID)
./datagen -g16KB -s2 > ftmdg16K2 $(DATAGEN) -g16KB -s2 > ftmdg16K2
./datagen -g16KB -s3 > ftmdg16K3 $(DATAGEN) -g16KB -s3 > ftmdg16K3
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
./datagen -g7MB > ftmdg7M $(DATAGEN) -g7MB > ftmdg7M
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M