AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improved and fixed --list command, original patches by @ib (#772)

Browse Source 
accepts all skippable frame identifiers.
display in MB or KB, depending on frame size.
fixed combination of skippable and zstd frames.
This commit is contained in:
Yann Collet 2017-08-18 15:57:53 -07:00
parent 3f54d788e9
commit c523c93b26
2 changed files with 42 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,6 +1,7 @@
v1.3.1 v1.3.1
perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt
perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760) perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
cli : improved and fixed --list command, by @ib (#772)
build: fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (#718) build: fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (#718)
API exp : breaking change : ZSTD_getframeHeader() provides more information API exp : breaking change : ZSTD_getframeHeader() provides more information
API exp : breaking change : pinned down values of error codes API exp : breaking change : pinned down values of error codes

95
programs/fileio.c
View File

@ -947,6 +947,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
return 3; return 3;
} }
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
/* begin analyzing frame */ /* begin analyzing frame */
for ( ; ; ) { for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
@ -966,37 +967,31 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
break; break;
} }
} }
{ { U32 const magicNumber = MEM_readLE32(headerBuffer);
U32 const magicNumber = MEM_readLE32(headerBuffer); /* Zstandard frame */
if (magicNumber == ZSTD_MAGICNUMBER) { if (magicNumber == ZSTD_MAGICNUMBER) {
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
info->decompUnavailable = 1; info->decompUnavailable = 1;
} } else {
else {
info->decompressedSize += frameContentSize; info->decompressedSize += frameContentSize;
} }
{ /* move to the end of the frame header */
/* move to the end of the frame header */ { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
if (ZSTD_isError(headerSize)) { if (ZSTD_isError(headerSize)) {
DISPLAY("Error: could not determine frame header size\n"); DISPLAY("Error: could not determine frame header size\n");
detectError = 1; detectError = 1;
break; break;
} }
{ { int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
if (ret != 0) { if (ret != 0) {
DISPLAY("Error: could not move to end of frame header\n"); DISPLAY("Error: could not move to end of frame header\n");
detectError = 1; detectError = 1;
break; break;
} } } }
}
}
/* skip the rest of the blocks in the frame */ /* skip the rest of the blocks in the frame */
{ { int lastBlock = 0;
int lastBlock = 0;
do { do {
BYTE blockHeaderBuffer[3]; BYTE blockHeaderBuffer[3];
U32 blockHeader; U32 blockHeader;
@ -1009,24 +1004,20 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
} }
blockHeader = MEM_readLE24(blockHeaderBuffer); blockHeader = MEM_readLE24(blockHeaderBuffer);
lastBlock = blockHeader & 1; lastBlock = blockHeader & 1;
blockSize = blockHeader >> 3; blockSize = blockHeader >> 3; /* Warning : does not work when block is RLE type */
{ { int const ret = fseek(srcFile, blockSize, SEEK_CUR);
int const ret = fseek(srcFile, blockSize, SEEK_CUR);
if (ret != 0) { if (ret != 0) {
DISPLAY("Error: could not skip to end of block\n"); DISPLAY("Error: could not skip to end of block\n");
detectError = 1; detectError = 1;
break; break;
} } }
}
} while (lastBlock != 1); } while (lastBlock != 1);
if (detectError) { if (detectError) break;
break;
}
} }
{
/* check if checksum is used */ /* check if checksum is used */
BYTE const frameHeaderDescriptor = headerBuffer[4]; { BYTE const frameHeaderDescriptor = headerBuffer[4];
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) { if (contentChecksumFlag) {
int const ret = fseek(srcFile, 4, SEEK_CUR); int const ret = fseek(srcFile, 4, SEEK_CUR);
@ -1035,64 +1026,60 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
DISPLAY("Error: could not skip past checksum\n"); DISPLAY("Error: could not skip past checksum\n");
detectError = 1; detectError = 1;
break; break;
} } } }
}
}
info->numActualFrames++; info->numActualFrames++;
} }
else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) { /* Skippable frame */
BYTE frameSizeBuffer[4]; else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t const readBytes = fread(frameSizeBuffer, 1, 4, srcFile); U32 const frameSize = MEM_readLE32(headerBuffer + 4);
if (readBytes != 4) { long const seek = -numBytesRead + 8 + frameSize;
DISPLAY("There was an error reading skippable frame size"); int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1; detectError = 1;
break; break;
} }
{
U32 const frameSize = MEM_readLE32(frameSizeBuffer);
int const ret = LONG_SEEK(srcFile, frameSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1;
break;
}
}
info->numSkippableFrames++; info->numSkippableFrames++;
} }
/* unknown content */
else { else {
detectError = 2; detectError = 2;
break; break;
} }
} }
} } /* end analyzing frame */
fclose(srcFile); fclose(srcFile);
return detectError; return detectError;
} }
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
double const compressedSizeMB = (double)info->compressedSize/(1 MB); unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
double const decompressedSizeMB = (double)info->decompressedSize/(1 MB); const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)info->compressedSize / unit;
double const decompressedSizeUnit = (double)info->decompressedSize / unit;
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize; double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
const char* const checkString = (info->usesCheck ? "XXH64" : "None"); const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) { if (displayLevel <= 2) {
if (!info->decompUnavailable) { if (!info->decompUnavailable) {
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %5s %s\n", DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, inFileName); ratio, checkString, inFileName);
} } else {
else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n", DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName); info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, checkString, inFileName);
} }
} } else {
else{
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
compressedSizeUnit, unitStr, info->compressedSize);
if (!info->decompUnavailable) { if (!info->decompUnavailable) {
DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
decompressedSizeUnit, unitStr, info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio); DISPLAYOUT("Ratio: %.4f\n", ratio);
} }
DISPLAYOUT("Check: %s\n", checkString); DISPLAYOUT("Check: %s\n", checkString);