AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #735 from terrelln/xz-default

Browse Source 
[zstdcli] Support xz by default when liblzma is available
This commit is contained in:
Yann Collet 2017-06-27 09:42:02 -07:00 committed by GitHub
commit 09a5bbe22e
6 changed files with 56 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -224,10 +224,10 @@ asan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $*
msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
msan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) $*
LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
asan32: clean
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"

22
programs/Makefile
View File

@ -139,16 +139,13 @@ all: zstd
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd xzstd zstd4 xzstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP)
zstd xzstd zstd4 xzstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD)
xzstd xzstd4 : CPPFLAGS += $(LZMACPP)
xzstd xzstd4 : LDFLAGS += $(LZMALD)
zstd4 xzstd4 : CPPFLAGS += $(LZ4CPP)
zstd4 xzstd4 : LDFLAGS += $(LZ4LD)
zstd zstd4 : LZMA_MSG := - xz/lzma support is disabled
zstd xzstd : LZ4_MSG := - lz4 support is disabled
zstd xzstd zstd4 xzstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd xzstd zstd4 xzstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP)
zstd zstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD)
zstd4 : CPPFLAGS += $(LZ4CPP)
zstd4 : LDFLAGS += $(LZ4LD)
zstd : LZ4_MSG := - lz4 support is disabled
zstd zstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd zstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@ -181,6 +178,11 @@ zstd-nogz : ZLIBLD :=
zstd-nogz : ZLIB_MSG := - gzip support is disabled
zstd-nogz : zstd
zstd-noxz : LZMACPP :=
zstd-noxz : LZMALD :=
zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
zstd-noxz : zstd
zstd-pgo : MOREFLAGS = -fprofile-generate
zstd-pgo : clean zstd

14
programs/README.md
View File

@ -24,13 +24,23 @@ There are however other Makefile targets that create different variations of CLI
- __HAVE_ZLIB__ : `zstd` can compress and decompress files in `.gz` format.
This is done through command `--format=gzip`.
Alternatively, symlinks named `gzip` or `gunzip` will mimic intended behavior.
.gz support is automatically enabled when `zlib` library is detected at build time.
It's possible to disable .gz support, by either compiling `zstd-nogz` target or using HAVE_ZLIB=0 variable.
`.gz` support is automatically enabled when `zlib` library is detected at build time.
It's possible to disable `.gz` support, by either compiling `zstd-nogz` target or using HAVE_ZLIB=0 variable.
Example : make zstd HAVE_ZLIB=0
It's also possible to force compilation with zlib support, using HAVE_ZLIB=1.
In which case, linking stage will fail if `zlib` library cannot be found.
This might be useful to prevent silent feature disabling.
- __HAVE_LZMA__ : `zstd` can compress and decompress files in `.xz` and `.lzma` formats.
This is done through commands `--format=xz` and `--format=lzma` respectively.
Alternatively, symlinks named `xz`, `unxz`, `lzma`, or `unlzma` will mimic intended behavior.
`.xz` and `.lzma` support is automatically enabled when `lzma` library is detected at build time.
It's possible to disable `.xz` and `.lzma` support, by either compiling `zstd-noxz` target or using HAVE_LZMA=0 variable.
Example : make zstd HAVE_LZMA=0
It's also possible to force compilation with lzma support, using HAVE_LZMA=1.
In which case, linking stage will fail if `lzma` library cannot be found.
This might be useful to prevent silent feature disabling.
#### Aggregation of parameters
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`.

14
programs/fileio.c
View File

@ -559,7 +559,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
strm.next_in = 0;
strm.avail_in = 0;
strm.next_out = ress->dstBuffer;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
while (1) {
@ -567,7 +567,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
if (inSize == 0) action = LZMA_FINISH;
inFileSize += inSize;
strm.next_in = ress->srcBuffer;
strm.next_in = (BYTE const*)ress->srcBuffer;
strm.avail_in = inSize;
}
@ -580,7 +580,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
EXM_THROW(73, "Write error : cannot write to output file");
outFileSize += compBytes;
strm.next_out = ress->dstBuffer;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
} }
if (!srcFileSize)
@ -1490,16 +1490,16 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d",
srcFileName, ret);
strm.next_out = ress->dstBuffer;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
strm.next_in = (BYTE const*)ress->srcBuffer;
strm.avail_in = ress->srcBufferLoaded;
strm.next_in = ress->srcBuffer;
for ( ; ; ) {
if (strm.avail_in == 0) {
ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
if (ress->srcBufferLoaded == 0) action = LZMA_FINISH;
strm.next_in = ress->srcBuffer;
strm.next_in = (BYTE const*)ress->srcBuffer;
strm.avail_in = ress->srcBufferLoaded;
}
ret = lzma_code(&strm, action);
@ -1515,7 +1515,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
EXM_THROW(31, "Write error : cannot write to output file");
outFileSize += decompBytes;
strm.next_out = ress->dstBuffer;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
} }
if (ret == LZMA_STREAM_END) break;

4
programs/zstdcli.c
View File

@ -56,7 +56,9 @@
#define ZSTD_GUNZIP "gunzip"
#define ZSTD_GZCAT "gzcat"
#define ZSTD_LZMA "lzma"
#define ZSTD_UNLZMA "unlzma"
#define ZSTD_XZ "xz"
#define ZSTD_UNXZ "unxz"
#define KB *(1 <<10)
#define MB *(1 <<20)
@ -379,7 +381,9 @@ int main(int argCount, const char* argv[])
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */
if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz */
memset(&compressionParams, 0, sizeof(compressionParams));
/* command switches */

19
tests/playTests.sh
View File

@ -442,6 +442,7 @@ if [ $LZMAMODE -eq 1 ]; then
XZEXE=1
xz -V && lzma -V || XZEXE=0
if [ $XZEXE -eq 1 ]; then
$ECHO "Testing zstd xz and lzma support"
./datagen > tmp
$ZSTD --format=lzma -f tmp
$ZSTD --format=xz -f tmp
@ -452,6 +453,24 @@ if [ $LZMAMODE -eq 1 ]; then
$ZSTD -d -f -v tmp.xz
$ZSTD -d -f -v tmp.lzma
rm tmp*
$ECHO "Creating symlinks"
ln -s $ZSTD ./xz
ln -s $ZSTD ./unxz
ln -s $ZSTD ./lzma
ln -s $ZSTD ./unlzma
$ECHO "Testing xz and lzma symlinks"
./datagen > tmp
./xz tmp
xz -d tmp.xz
./lzma tmp
lzma -d tmp.lzma
$ECHO "Testing unxz and unlzma symlinks"
xz tmp
./xz -d tmp.xz
lzma tmp
./lzma -d tmp.lzma
rm xz unxz lzma unlzma
rm tmp*
else
$ECHO "xz binary not detected"
fi