diff --git a/Makefile b/Makefile index 679e0562..9afdf658 100644 --- a/Makefile +++ b/Makefile @@ -224,10 +224,10 @@ asan-%: clean LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $* msan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason msan-%: clean - LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) $* + LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $* asan32: clean $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" diff --git a/programs/Makefile b/programs/Makefile index ab2db728..8b080d44 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -139,16 +139,13 @@ all: zstd $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) -zstd xzstd zstd4 xzstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) -zstd xzstd zstd4 xzstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD) -xzstd xzstd4 : CPPFLAGS += $(LZMACPP) -xzstd xzstd4 : LDFLAGS += $(LZMALD) -zstd4 xzstd4 : CPPFLAGS += $(LZ4CPP) -zstd4 xzstd4 : LDFLAGS += $(LZ4LD) -zstd zstd4 : LZMA_MSG := - xz/lzma support is disabled -zstd xzstd : LZ4_MSG := - lz4 support is disabled -zstd xzstd zstd4 xzstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -zstd xzstd zstd4 xzstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o +zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) +zstd zstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) +zstd4 : CPPFLAGS += $(LZ4CPP) +zstd4 : LDFLAGS += $(LZ4LD) +zstd : LZ4_MSG := - lz4 support is disabled +zstd zstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +zstd zstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o @echo "$(THREAD_MSG)" @echo "$(ZLIB_MSG)" @echo "$(LZMA_MSG)" @@ -181,6 +178,11 @@ zstd-nogz : ZLIBLD := zstd-nogz : ZLIB_MSG := - gzip support is disabled zstd-nogz : zstd +zstd-noxz : LZMACPP := +zstd-noxz : LZMALD := +zstd-noxz : LZMA_MSG := - xz/lzma support is disabled +zstd-noxz : zstd + zstd-pgo : MOREFLAGS = -fprofile-generate zstd-pgo : clean zstd diff --git a/programs/README.md b/programs/README.md index d7922a09..bd8fba06 100644 --- a/programs/README.md +++ b/programs/README.md @@ -24,13 +24,23 @@ There are however other Makefile targets that create different variations of CLI - __HAVE_ZLIB__ : `zstd` can compress and decompress files in `.gz` format. This is done through command `--format=gzip`. Alternatively, symlinks named `gzip` or `gunzip` will mimic intended behavior. - .gz support is automatically enabled when `zlib` library is detected at build time. - It's possible to disable .gz support, by either compiling `zstd-nogz` target or using HAVE_ZLIB=0 variable. + `.gz` support is automatically enabled when `zlib` library is detected at build time. + It's possible to disable `.gz` support, by either compiling `zstd-nogz` target or using HAVE_ZLIB=0 variable. Example : make zstd HAVE_ZLIB=0 It's also possible to force compilation with zlib support, using HAVE_ZLIB=1. In which case, linking stage will fail if `zlib` library cannot be found. This might be useful to prevent silent feature disabling. +- __HAVE_LZMA__ : `zstd` can compress and decompress files in `.xz` and `.lzma` formats. + This is done through commands `--format=xz` and `--format=lzma` respectively. + Alternatively, symlinks named `xz`, `unxz`, `lzma`, or `unlzma` will mimic intended behavior. + `.xz` and `.lzma` support is automatically enabled when `lzma` library is detected at build time. + It's possible to disable `.xz` and `.lzma` support, by either compiling `zstd-noxz` target or using HAVE_LZMA=0 variable. + Example : make zstd HAVE_LZMA=0 + It's also possible to force compilation with lzma support, using HAVE_LZMA=1. + In which case, linking stage will fail if `lzma` library cannot be found. + This might be useful to prevent silent feature disabling. + #### Aggregation of parameters CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. diff --git a/programs/fileio.c b/programs/fileio.c index 78ac3ba6..77943717 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -559,7 +559,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, strm.next_in = 0; strm.avail_in = 0; - strm.next_out = ress->dstBuffer; + strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; while (1) { @@ -567,7 +567,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); if (inSize == 0) action = LZMA_FINISH; inFileSize += inSize; - strm.next_in = ress->srcBuffer; + strm.next_in = (BYTE const*)ress->srcBuffer; strm.avail_in = inSize; } @@ -580,7 +580,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) EXM_THROW(73, "Write error : cannot write to output file"); outFileSize += compBytes; - strm.next_out = ress->dstBuffer; + strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; } } if (!srcFileSize) @@ -1490,16 +1490,16 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d", srcFileName, ret); - strm.next_out = ress->dstBuffer; + strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; + strm.next_in = (BYTE const*)ress->srcBuffer; strm.avail_in = ress->srcBufferLoaded; - strm.next_in = ress->srcBuffer; for ( ; ; ) { if (strm.avail_in == 0) { ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; - strm.next_in = ress->srcBuffer; + strm.next_in = (BYTE const*)ress->srcBuffer; strm.avail_in = ress->srcBufferLoaded; } ret = lzma_code(&strm, action); @@ -1515,7 +1515,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); outFileSize += decompBytes; - strm.next_out = ress->dstBuffer; + strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; } } if (ret == LZMA_STREAM_END) break; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 57943863..f59f9785 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -56,7 +56,9 @@ #define ZSTD_GUNZIP "gunzip" #define ZSTD_GZCAT "gzcat" #define ZSTD_LZMA "lzma" +#define ZSTD_UNLZMA "unlzma" #define ZSTD_XZ "xz" +#define ZSTD_UNXZ "unxz" #define KB *(1 <<10) #define MB *(1 <<20) @@ -379,7 +381,9 @@ int main(int argCount, const char* argv[]) if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat */ if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ + if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma */ if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ + if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz */ memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ diff --git a/tests/playTests.sh b/tests/playTests.sh index fa82ae9e..2e1cc682 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -442,6 +442,7 @@ if [ $LZMAMODE -eq 1 ]; then XZEXE=1 xz -V && lzma -V || XZEXE=0 if [ $XZEXE -eq 1 ]; then + $ECHO "Testing zstd xz and lzma support" ./datagen > tmp $ZSTD --format=lzma -f tmp $ZSTD --format=xz -f tmp @@ -452,6 +453,24 @@ if [ $LZMAMODE -eq 1 ]; then $ZSTD -d -f -v tmp.xz $ZSTD -d -f -v tmp.lzma rm tmp* + $ECHO "Creating symlinks" + ln -s $ZSTD ./xz + ln -s $ZSTD ./unxz + ln -s $ZSTD ./lzma + ln -s $ZSTD ./unlzma + $ECHO "Testing xz and lzma symlinks" + ./datagen > tmp + ./xz tmp + xz -d tmp.xz + ./lzma tmp + lzma -d tmp.lzma + $ECHO "Testing unxz and unlzma symlinks" + xz tmp + ./xz -d tmp.xz + lzma tmp + ./lzma -d tmp.lzma + rm xz unxz lzma unlzma + rm tmp* else $ECHO "xz binary not detected" fi