AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fixed zstd-compress

Browse Source 
file-information is dependent on decompression functions.
it should only be enabled when ZSTD_NODECOMPRESS is not set.

also : added zstd-compress compilation test into `make shortest`
This commit is contained in:
Yann Collet 2017-08-18 18:30:41 -07:00
parent f207b39f55
commit 166645e7b3
4 changed files with 261 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -76,6 +76,7 @@ zlibwrapper:
.PHONY: test shortest
test shortest:
$(MAKE) -C $(PRGDIR) allVariants
$(MAKE) -C $(TESTDIR) $@
.PHONY: examples

24
programs/Makefile
View File

@ -132,12 +132,15 @@ else
LZ4_MSG := $(NO_LZ4_MSG)
endif
.PHONY: default all clean clean_decomp_o install uninstall generate_res
.PHONY: default
default: zstd-release
.PHONY: all
all: zstd
.PHONY: allVariants
allVariants: zstd zstd-compress
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP)
@ -156,6 +159,7 @@ ifneq (,$(filter Windows%,$(OS)))
endif
$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
.PHONY: zstd-release
zstd-release: DEBUGFLAGS :=
zstd-release: zstd
@ -166,7 +170,7 @@ ifneq (,$(filter Windows%,$(OS)))
endif
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
zstd-nolegacy : clean_decomp_o
zstd-nolegacy :
$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
zstd-nomt : THREAD_CPP :=
@ -211,9 +215,11 @@ zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
# zstd is now built with Multi-threading by default
zstdmt: zstd
.PHONY: generate_res
generate_res:
windres/generate_res.bat
.PHONY: clean
clean:
$(MAKE) -C $(ZSTDDIR) clean
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
@ -222,20 +228,20 @@ clean:
*.gcda default.profraw have_zlib$(EXT)
@echo Cleaning completed
clean_decomp_o:
@$(RM) $(ZSTDDECOMP_O)
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
zstd.1: zstd.1.md
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
.PHONY: man
man: zstd.1
.PHONY: clean-man
clean-man:
rm zstd.1
.PHONY: preview-man
preview-man: clean-man man
man ./zstd.1
@ -244,6 +250,10 @@ preview-man: clean-man man
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
.PHONY: list
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
else
@ -264,6 +274,7 @@ INSTALL_PROGRAM ?= $(INSTALL) -m 755
INSTALL_SCRIPT ?= $(INSTALL) -m 755
INSTALL_MAN ?= $(INSTALL) -m 644
.PHONY: install
install: zstd
@echo Installing binaries
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
@ -279,6 +290,7 @@ install: zstd
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
@echo zstd installation completed
.PHONY: uninstall
uninstall:
@$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
@$(RM) $(DESTDIR)$(BINDIR)/zstdless

450
programs/fileio.c
View File

@ -925,223 +925,6 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
return result;
}
typedef struct {
int numActualFrames;
int numSkippableFrames;
unsigned long long decompressedSize;
int decompUnavailable;
unsigned long long compressedSize;
int usesCheck;
} fileInfo_t;
/*
* Reads information from file, stores in *info
* if successful, returns 0, returns 1 for frame analysis error, returns 2 for file not compressed with zstd
* returns 3 for cases in which file could not be opened.
*/
static int getFileInfo(fileInfo_t* info, const char* inFileName){
int detectError = 0;
FILE* const srcFile = FIO_openSrcFile(inFileName);
if (srcFile == NULL) {
DISPLAY("Error: could not open source file %s\n", inFileName);
return 3;
}
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
/* begin analyzing frame */
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
break;
}
else if (feof(srcFile)) {
DISPLAY("Error: reached end of file with incomplete frame\n");
detectError = 2;
break;
}
else {
DISPLAY("Error: did not reach end of file but ran out of frames\n");
detectError = 1;
break;
}
}
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
/* Zstandard frame */
if (magicNumber == ZSTD_MAGICNUMBER) {
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
info->decompUnavailable = 1;
} else {
info->decompressedSize += frameContentSize;
}
/* move to the end of the frame header */
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
if (ZSTD_isError(headerSize)) {
DISPLAY("Error: could not determine frame header size\n");
detectError = 1;
break;
}
{ int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not move to end of frame header\n");
detectError = 1;
break;
} } }
/* skip the rest of the blocks in the frame */
{ int lastBlock = 0;
do {
BYTE blockHeaderBuffer[3];
size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
if (readBytes != 3) {
DISPLAY("There was a problem reading the block header\n");
detectError = 1;
break;
}
{ U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
U32 const blockTypeID = (blockHeader >> 1) & 3;
U32 const isRLE = (blockTypeID == 1);
U32 const isWrongBlock = (blockTypeID == 3);
long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
if (isWrongBlock) {
DISPLAY("Error: unsupported block type \n");
detectError = 1;
break;
}
lastBlock = blockHeader & 1;
{ int const ret = fseek(srcFile, blockSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not skip to end of block\n");
detectError = 1;
break;
} } }
} while (lastBlock != 1);
if (detectError) break;
}
/* check if checksum is used */
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
int const ret = fseek(srcFile, 4, SEEK_CUR);
info->usesCheck = 1;
if (ret != 0) {
DISPLAY("Error: could not skip past checksum\n");
detectError = 1;
break;
} } }
info->numActualFrames++;
}
/* Skippable frame */
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead);
int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1;
break;
}
info->numSkippableFrames++;
}
/* unknown content */
else {
detectError = 2;
break;
}
}
} /* end analyzing frame */
fclose(srcFile);
return detectError;
}
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)info->compressedSize / unit;
double const decompressedSizeUnit = (double)info->decompressedSize / unit;
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) {
if (!info->decompUnavailable) {
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, inFileName);
} else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, checkString, inFileName);
}
} else {
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
compressedSizeUnit, unitStr, info->compressedSize);
if (!info->decompUnavailable) {
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
decompressedSizeUnit, unitStr, info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio);
}
DISPLAYOUT("Check: %s\n", checkString);
DISPLAYOUT("\n");
}
}
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
/* initialize info to avoid warnings */
fileInfo_t info;
memset(&info, 0, sizeof(info));
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
{
int const error = getFileInfo(&info, inFileName);
if (error == 1) {
/* display error, but provide output */
DISPLAY("An error occurred with getting file info\n");
}
else if (error == 2) {
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
else if (error == 3) {
/* error occurred with opening the file */
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
displayInfo(inFileName, &info, displayLevel);
return error;
}
}
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
if (numFiles == 0) {
DISPLAYOUT("No files given\n");
return 0;
}
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Printing information about compressed files\n");
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Number of files listed: %u\n", numFiles);
{
int error = 0;
unsigned u;
for (u=0; u<numFiles;u++) {
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
}
return error;
}
}
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
const char* suffix,
@ -1187,10 +970,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
} }
/* Close & Free */
FIO_freeCResources(ress);
free(dstFileName);
return missed_files;
}
@ -1201,8 +982,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
#ifndef ZSTD_NODECOMPRESS
/* **************************************************************************
* Decompression
****************************************************************************/
* Decompression
***************************************************************************/
typedef struct {
void* srcBuffer;
size_t srcBufferLoaded;
@ -1871,4 +1652,231 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
return missingFiles + skippedFiles;
}
/* **************************************************************************
* .zst file info (--list command)
***************************************************************************/
typedef struct {
int numActualFrames;
int numSkippableFrames;
unsigned long long decompressedSize;
int decompUnavailable;
unsigned long long compressedSize;
int usesCheck;
} fileInfo_t;
/** getFileInfo() :
* Reads information from file, stores in *info
* @return : 0 if successful
* 1 for frame analysis error
* 2 for file not compressed with zstd
* 3 for cases in which file could not be opened.
*/
static int getFileInfo(fileInfo_t* info, const char* inFileName){
int detectError = 0;
FILE* const srcFile = FIO_openSrcFile(inFileName);
if (srcFile == NULL) {
DISPLAY("Error: could not open source file %s\n", inFileName);
return 3;
}
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
/* begin analyzing frame */
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
break;
}
else if (feof(srcFile)) {
DISPLAY("Error: reached end of file with incomplete frame\n");
detectError = 2;
break;
}
else {
DISPLAY("Error: did not reach end of file but ran out of frames\n");
detectError = 1;
break;
}
}
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
/* Zstandard frame */
if (magicNumber == ZSTD_MAGICNUMBER) {
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
info->decompUnavailable = 1;
} else {
info->decompressedSize += frameContentSize;
}
/* move to the end of the frame header */
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
if (ZSTD_isError(headerSize)) {
DISPLAY("Error: could not determine frame header size\n");
detectError = 1;
break;
}
{ int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not move to end of frame header\n");
detectError = 1;
break;
} } }
/* skip the rest of the blocks in the frame */
{ int lastBlock = 0;
do {
BYTE blockHeaderBuffer[3];
size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
if (readBytes != 3) {
DISPLAY("There was a problem reading the block header\n");
detectError = 1;
break;
}
{ U32 const blockHeader = MEM_readLE24(blockHeaderBuffer);
U32 const blockTypeID = (blockHeader >> 1) & 3;
U32 const isRLE = (blockTypeID == 1);
U32 const isWrongBlock = (blockTypeID == 3);
long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3);
if (isWrongBlock) {
DISPLAY("Error: unsupported block type \n");
detectError = 1;
break;
}
lastBlock = blockHeader & 1;
{ int const ret = fseek(srcFile, blockSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not skip to end of block\n");
detectError = 1;
break;
} } }
} while (lastBlock != 1);
if (detectError) break;
}
/* check if checksum is used */
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
int const ret = fseek(srcFile, 4, SEEK_CUR);
info->usesCheck = 1;
if (ret != 0) {
DISPLAY("Error: could not skip past checksum\n");
detectError = 1;
break;
} } }
info->numActualFrames++;
}
/* Skippable frame */
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead);
int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1;
break;
}
info->numSkippableFrames++;
}
/* unknown content */
else {
detectError = 2;
break;
}
}
} /* end analyzing frame */
fclose(srcFile);
return detectError;
}
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)info->compressedSize / unit;
double const decompressedSizeUnit = (double)info->decompressedSize / unit;
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) {
if (!info->decompUnavailable) {
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, inFileName);
} else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, checkString, inFileName);
}
} else {
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
compressedSizeUnit, unitStr, info->compressedSize);
if (!info->decompUnavailable) {
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
decompressedSizeUnit, unitStr, info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio);
}
DISPLAYOUT("Check: %s\n", checkString);
DISPLAYOUT("\n");
}
}
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
/* initialize info to avoid warnings */
fileInfo_t info;
memset(&info, 0, sizeof(info));
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
{
int const error = getFileInfo(&info, inFileName);
if (error == 1) {
/* display error, but provide output */
DISPLAY("An error occurred with getting file info\n");
}
else if (error == 2) {
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
else if (error == 3) {
/* error occurred with opening the file */
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
return 1;
}
displayInfo(inFileName, &info, displayLevel);
return error;
}
}
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){
if (numFiles == 0) {
DISPLAYOUT("No files given\n");
return 0;
}
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Printing information about compressed files\n");
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Number of files listed: %u\n", numFiles);
{
int error = 0;
unsigned u;
for (u=0; u<numFiles;u++) {
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
}
return error;
}
}
#endif /* #ifndef ZSTD_NODECOMPRESS */

16
programs/zstdcli.c
View File

@ -634,7 +634,10 @@ int main(int argCount, const char* argv[])
filenameTable[filenameIdx++] = argument;
}
if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); } /* forgotten argument */
if (lastCommand) { /* forgotten argument */
DISPLAY("error : command must be followed by argument \n");
CLEAN_RETURN(1);
}
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
@ -642,7 +645,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
#endif
#ifdef _POSIX_VERSION
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
#endif
#ifdef PLATFORM_POSIX_VERSION
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
@ -651,7 +654,7 @@ int main(int argCount, const char* argv[])
if (nbThreads == 0) {
/* try to guess */
nbThreads = UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected\n", nbThreads);
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbThreads);
}
g_utilDisplayLevel = g_displayLevel;
@ -678,10 +681,17 @@ int main(int argCount, const char* argv[])
}
}
#endif
if (operation == zom_list) {
#ifndef ZSTD_NODECOMPRESS
int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
CLEAN_RETURN(ret);
#else
DISPLAY("file information is not supported \n");
CLEAN_RETURN(1);
#endif
}
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH