AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'list' into dev

Browse Source
This commit is contained in:
Paul Cruz 2017-06-15 16:46:54 -07:00
commit dc80b3c798
4 changed files with 146 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/decompress/zstd_decompress.c
View File

@ -204,7 +204,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
/** ZSTD_frameHeaderSize() : /** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_prefix. * srcSize must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */ * @return : size of the Frame Header */
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
{ {
if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
{ BYTE const fhd = ((const BYTE*)src)[4]; { BYTE const fhd = ((const BYTE*)src)[4];

5
lib/zstd.h
View File

@ -461,6 +461,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
* however it does mean that all frame data must be present and valid. */ * however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_frameHeaderSize() :
* `src` should point to the start of a ZSTD frame
* `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
/*************************************** /***************************************
* Context memory usage * Context memory usage

273
programs/fileio.c
View File

@ -91,6 +91,7 @@
* Macros * Macros
***************************************/ ***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
@ -871,182 +872,182 @@ typedef struct {
int usesCheck; int usesCheck;
} fileInfo_t; } fileInfo_t;
int calcFrameHeaderSize(BYTE frameHeaderDescriptor){
int frameContentSizeBytes = 0;
int windowDescriptorBytes;
int dictionaryIDBytes;
int frameContentSizeFlag = frameHeaderDescriptor >> 6;
int singleSegmentFlag = (frameHeaderDescriptor & (1 << 5)) >> 5;
int dictionaryIDFlag = frameHeaderDescriptor & 3;
if(frameContentSizeFlag!=0){
frameContentSizeBytes = 1 << frameContentSizeFlag;
}
else if(singleSegmentFlag){
frameContentSizeBytes = 1;
}
windowDescriptorBytes = singleSegmentFlag ? 0 : 1;
dictionaryIDBytes = dictionaryIDFlag ? 1 << (dictionaryIDFlag - 1): 0;
return 4 + 1 + windowDescriptorBytes + frameContentSizeBytes + dictionaryIDBytes;
}
/* /*
* Reads information from file, stores in *info * Reads information from file, stores in *info
* if successful, returns 0, otherwise returns 1 * if successful, returns 0, otherwise returns 1
*/ */
int getFileInfo(fileInfo_t* info, const char* inFileName){ static int getFileInfo(fileInfo_t* info, const char* inFileName){
FILE* srcFile = FIO_openSrcFile(inFileName); int detectError = 0;
if(srcFile==NULL){ FILE* const srcFile = FIO_openSrcFile(inFileName);
if (srcFile == NULL) {
DISPLAY("Error: could not open source file %s\n", inFileName);
return 1; return 1;
} }
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
info->decompressedSize = 0; info->decompressedSize = 0;
info->numActualFrames = 0; info->numActualFrames = 0;
info-> numSkippableFrames = 0; info->numSkippableFrames = 0;
info->canComputeDecompSize = 1; info->canComputeDecompSize = 1;
/* begin analyzing frame */ /* begin analyzing frame */
while(1){ for( ; ; ){
BYTE magicNumberBuffer[4]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t numBytesRead = fread(magicNumberBuffer, 1, 4, srcFile); size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
U32 magicNumber; if (numBytesRead < ZSTD_frameHeaderSize_min) {
if(numBytesRead != 4) break; if(feof(srcFile)){
magicNumber = MEM_readLE32(magicNumberBuffer); break;
if(magicNumber==ZSTD_MAGICNUMBER){
BYTE frameHeaderDescriptor;
int totalFrameHeaderBytes;
BYTE* frameHeader;
int lastBlock = 0;
size_t readBytes = fread(&frameHeaderDescriptor, 1, 1, srcFile);
info->numActualFrames++;
if(readBytes != 1){
DISPLAY("There was an error with reading frame header descriptor\n");
exit(1);
} }
/* calculate actual frame header size */ else{
totalFrameHeaderBytes = calcFrameHeaderSize(frameHeaderDescriptor); DISPLAY("Error: did not reach end of file but ran out of frames\n");
detectError = 1;
/* reset to beginning of from and read entire header */ break;
fseek(srcFile, -5, SEEK_CUR);
frameHeader = (BYTE*)malloc(totalFrameHeaderBytes);
readBytes = fread(frameHeader, 1, totalFrameHeaderBytes, srcFile);
if(readBytes != (size_t)totalFrameHeaderBytes){
DISPLAY("There was an error reading the frame header\n");
exit(1);
} }
}
/* get decompressed file size */ {
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
U64 additional = ZSTD_getFrameContentSize(frameHeader, totalFrameHeaderBytes); if (magicNumber == ZSTD_MAGICNUMBER) {
if(additional!=ZSTD_CONTENTSIZE_UNKNOWN && additional!=ZSTD_CONTENTSIZE_ERROR){ U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
info->decompressedSize += additional; if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
}
else{
info->canComputeDecompSize = 0; info->canComputeDecompSize = 0;
} }
} else {
info->decompressedSize += frameContentSize;
}
{
/* move to the end of the frame header */
size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
if (ZSTD_isError(headerSize)) {
DISPLAY("Error: could not determine frame header size\n");
detectError = 1;
break;
}
{
int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not move to end of frame header\n");
detectError = 1;
break;
}
}
}
/* skip the rest of the blocks in the frame */ /* skip the rest of the blocks in the frame */
do{ {
BYTE blockHeaderBuffer[3]; int lastBlock = 0;
U32 blockHeader; size_t readBytes = 0;
int blockSize; do{
readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); BYTE blockHeaderBuffer[3];
if(readBytes != 3){ U32 blockHeader;
DISPLAY("There was a problem reading the block header\n"); int blockSize;
exit(1); readBytes = fread(blockHeaderBuffer, 1, 3, srcFile);
if (readBytes != 3) {
DISPLAY("There was a problem reading the block header\n");
detectError = 1;
break;
}
blockHeader = MEM_readLE24(blockHeaderBuffer);
lastBlock = blockHeader & 1;
blockSize = (blockHeader - (blockHeader & 7)) >> 3;
{
int const ret = fseek(srcFile, blockSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not skip to end of block\n");
detectError = 1;
break;
}
}
} while (lastBlock != 1);
if (detectError) {
break;
}
} }
blockHeader = MEM_readLE24(blockHeaderBuffer); {
lastBlock = blockHeader & 1; /* check if checksum is used */
blockSize = (blockHeader - (blockHeader & 7)) >> 3; BYTE const frameHeaderDescriptor = headerBuffer[4];
fseek(srcFile, blockSize, SEEK_CUR); int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
}while(lastBlock != 1); if (contentChecksumFlag) {
{ info->usesCheck = 1;
/* check if checksum is used */ }
int contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; if (contentChecksumFlag) {
if(contentChecksumFlag){ int const ret = fseek(srcFile, 4, SEEK_CUR);
info->usesCheck = 1; if (ret != 0) {
DISPLAY("Error: could not skip past checksum\n");
detectError = 1;
break;
}
}
} }
if(contentChecksumFlag){ info->numActualFrames++;
fseek(srcFile, 4, SEEK_CUR); }
else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) {
BYTE frameSizeBuffer[4];
size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
if (readBytes != 4) {
DISPLAY("There was an error reading skippable frame size");
detectError = 1;
break;
} }
{
long const frameSize = MEM_readLE32(frameSizeBuffer);
int const ret = fseek(srcFile, frameSize, SEEK_CUR);
if (ret != 0) {
DISPLAY("Error: could not find end of skippable frame\n");
detectError = 1;
break;
}
}
info->numSkippableFrames++;
} }
} }
else if(magicNumber==ZSTD_MAGIC_SKIPPABLE_START){
BYTE frameSizeBuffer[4];
long frameSize;
size_t readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
info->numSkippableFrames++;
if(readBytes != 4){
DISPLAY("There was an error reading skippable frame size");
exit(1);
}
frameSize = MEM_readLE32(frameSizeBuffer);
fseek(srcFile, frameSize, SEEK_CUR);
}
} }
return 0; fclose(srcFile);
return detectError;
} }
void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
double compressedSizeMB = (double)info->compressedSize/(1 MB);
double decompressedSizeMB = (double)info->decompressedSize/(1 MB);
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
double const compressedSizeMB = (double)info->compressedSize/(1 MB);
double const decompressedSizeMB = (double)info->decompressedSize/(1 MB);
const char* checkString = (info->usesCheck ? "XXH64" : "None");
if(displayLevel<=2){ if(displayLevel<=2){
if(info->usesCheck && info->canComputeDecompSize){ if (info->canComputeDecompSize) {
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f XXH64 %s\n", DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
compressedSizeMB/decompressedSizeMB, inFileName); compressedSizeMB/decompressedSizeMB, checkString, inFileName);
} }
else if(!info->usesCheck){ else {
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAY("%9d %13d %7.2f MB %7.2f MB %5.3f %s\n", DISPLAYOUT("%9d %13d %7.2f MB %s %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB, info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName);
compressedSizeMB/decompressedSizeMB, inFileName);
}
else if(!info->canComputeDecompSize){
DISPLAY("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAY("%9d %13d %7.2f MB XXH64 %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName);
}
else{
DISPLAY("Skippable Non-Skippable Filename\n");
DISPLAY("%9d %13d %7.2f MB %s\n",
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, inFileName);
} }
} }
else{ else{
DISPLAY("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAY("# Skippable Frames: %d\n", info->numSkippableFrames); DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAY("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize); DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
if(info->canComputeDecompSize){ if (info->canComputeDecompSize) {
DISPLAY("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize); DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
DISPLAY("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB); DISPLAYOUT("Ratio: %.4f\n", compressedSizeMB/decompressedSizeMB);
} }
if(info->usesCheck){ if (info->usesCheck) {
DISPLAY("Check: XXH64\n"); DISPLAYOUT("Check: XXH64\n");
} }
} }
DISPLAYOUT("\n");
} }
int FIO_listFile(const char* inFileName, int displayLevel){ int FIO_listFile(const char* inFileName, int displayLevel){
const char* const suffixPtr = strrchr(inFileName, '.'); fileInfo_t info;
DISPLAY("File: %s\n", inFileName); DISPLAYOUT("File: %s\n", inFileName);
if(!suffixPtr || strcmp(suffixPtr, ZSTD_EXTENSION)){ {
DISPLAYLEVEL(1, "file %s was not compressed with zstd -- ignoring\n\n", inFileName); int const error = getFileInfo(&info, inFileName);
return 1; if (error == 1) {
}
else{
fileInfo_t* info = (fileInfo_t*)malloc(sizeof(fileInfo_t));
int error = getFileInfo(info, inFileName);
if(error==1){
DISPLAY("An error occurred with getting file info\n"); DISPLAY("An error occurred with getting file info\n");
exit(1); return 1;
} }
displayInfo(inFileName, info, displayLevel);
} }
DISPLAY("\n"); displayInfo(inFileName, &info, displayLevel);
return 0; return 0;
} }

3
programs/zstdcli.c
View File

@ -149,6 +149,7 @@ static int usage_advanced(const char* programName)
#endif #endif
#endif #endif
DISPLAY( " -M# : Set a memory usage limit for decompression \n"); DISPLAY( " -M# : Set a memory usage limit for decompression \n");
DISPLAY( "--list : list information about a zstd compressed file \n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT #ifndef ZSTD_NODICT
DISPLAY( "\n"); DISPLAY( "\n");
@ -402,6 +403,7 @@ int main(int argCount, const char* argv[])
if (argument[1]=='-') { if (argument[1]=='-') {
/* long commands (--long-word) */ /* long commands (--long-word) */
if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */ if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
@ -674,6 +676,7 @@ int main(int argCount, const char* argv[])
} }
#endif #endif
if(operation==zom_list){ if(operation==zom_list){
g_displayOut = stdout;
if(filenameIdx==0){ if(filenameIdx==0){
DISPLAY("No files given\n"); DISPLAY("No files given\n");
CLEAN_RETURN(0); CLEAN_RETURN(0);