AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cli : -d and -t do not stop after a failed decompression

Browse Source 
The problematic srcfile will be named on console/log,
but decompression/test will continue onto next file in the list.
This commit is contained in:
Yann Collet 2017-05-05 19:15:24 -07:00
parent 11bff3fbd3
commit 01a1abfdb5
2 changed files with 67 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
programs/fileio.c
View File

@ -206,8 +206,8 @@ void FIO_setOverlapLog(unsigned overlapLog){
static int FIO_remove(const char* path)
{
#if defined(_WIN32) || defined(WIN32)
/* windows doesn't allow remove read-only files, so try to make it
* writable first */
/* windows doesn't allow remove read-only files,
* so try to make it writable first */
chmod(path, _S_IWRITE);
#endif
return remove(path);
@ -983,16 +983,19 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
/** FIO_decompressFrame() :
@return : size of decoded frame
@return : size of decoded frame, or an error code
*/
#define FIO_ERROR_ZSTD_DECODING ((unsigned long long)(-2))
unsigned long long FIO_decompressFrame(dRess_t* ress,
FILE* finput,
const char* srcFileName,
U64 alreadyDecoded)
{
U64 frameSize = 0;
U32 storedSkips = 0;
ZSTD_resetDStream(ress->dctx);
if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
/* Header loading (optional, saves one loop) */
{ size_t const toRead = 9;
@ -1005,12 +1008,17 @@ unsigned long long FIO_decompressFrame(dRess_t* ress,
ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint));
return FIO_ERROR_ZSTD_DECODING;
}
/* Write block */
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
srcFileName, (U32)((alreadyDecoded+frameSize)>>20) );
if (inBuff.pos > 0) {
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
@ -1018,14 +1026,22 @@ unsigned long long FIO_decompressFrame(dRess_t* ress,
}
if (readSizeHint == 0) break; /* end of frame */
if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
if (inBuff.size != inBuff.pos) {
DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n",
srcFileName);
return FIO_ERROR_ZSTD_DECODING;
}
/* Fill input buffer */
{ size_t const toRead = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */
if (ress->srcBufferLoaded < toRead)
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end");
} }
ress->srcBufferLoaded += fread((char*)ress->srcBuffer + ress->srcBufferLoaded,
1, toRead - ress->srcBufferLoaded, finput);
if (ress->srcBufferLoaded < toRead) {
DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
srcFileName);
return FIO_ERROR_ZSTD_DECODING;
} } }
FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
@ -1221,9 +1237,14 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
size_t const toRead = 4;
const BYTE* buf = (const BYTE*)ress.srcBuffer;
if (ress.srcBufferLoaded < toRead)
ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded,
(size_t)1, toRead - ress.srcBufferLoaded, srcFile);
if (ress.srcBufferLoaded==0) {
if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
if (readSomething==0) {
DISPLAY("zstd: %s: unexpected end of file \n", srcFileName);
fclose(srcFile);
return 1;
} /* srcFileName is empty */
break; /* no more input */
}
readSomething = 1; /* there is at least >= 4 bytes in srcFile */
@ -1259,7 +1280,8 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
} else {
if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
unsigned const result = FIO_passThrough(ress.dstFile, srcFile,
ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
return result;
} else {
@ -1267,9 +1289,15 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
fclose(srcFile);
return 1;
} }
filesize += FIO_decompressFrame(&ress, srcFile, filesize);
{ unsigned long long const frameSize = FIO_decompressFrame(&ress, srcFile, srcFileName, filesize);
if (frameSize == FIO_ERROR_ZSTD_DECODING) {
fclose(srcFile);
return 1;
}
filesize += frameSize;
}
}
}
} /* for each frame */
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
@ -1364,15 +1392,20 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
dstFileName = (char*)malloc(dfnSize);
if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName");
}
if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION) && strcmp(suffixPtr, LZ4_EXTENSION))) {
DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s expected) -- ignored \n", srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION);
if (sfnSize <= suffixSize
|| (strcmp(suffixPtr, GZ_EXTENSION)
&& strcmp(suffixPtr, XZ_EXTENSION)
&& strcmp(suffixPtr, ZSTD_EXTENSION)
&& strcmp(suffixPtr, LZMA_EXTENSION)
&& strcmp(suffixPtr, LZ4_EXTENSION)) ) {
DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s/%s expected) -- ignored \n",
srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION, LZ4_EXTENSION);
skippedFiles++;
continue;
} else {
memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
dstFileName[sfnSize-suffixSize] = '\0';
}
missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
}
free(dstFileName);

31
tests/playTests.sh
View File

@ -92,7 +92,7 @@ $ZSTD tmp --stdout > tmpCompressed # long command format
$ECHO "test : compress to named file"
rm tmpCompressed
$ZSTD tmp -o tmpCompressed
ls tmpCompressed # must work
test -f tmpCompressed # file must be created
$ECHO "test : -o must be followed by filename (must fail)"
$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename "
$ECHO "test : force write, correct order"
@ -142,21 +142,21 @@ $ZSTD -q -f tmpro
rm -f tmpro tmpro.zst
$ECHO "test : file removal"
$ZSTD -f --rm tmp
ls tmp && die "tmp should no longer be present"
test ! -f tmp # tmp should no longer be present
$ZSTD -f -d --rm tmp.zst
ls tmp.zst && die "tmp.zst should no longer be present"
test ! -f tmp.zst # tmp.zst should no longer be present
$ECHO "test : --rm on stdin"
$ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent
rm tmp
$ZSTD -f tmp && die "tmp not present : should have failed"
ls tmp.zst && die "tmp.zst should not be created"
test ! -f tmp.zst # tmp.zst should not be created
$ECHO "\n**** Advanced compression parameters **** "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
ls tmp.zst && die "tmp.zst should not be created"
test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
@ -201,16 +201,17 @@ $ECHO foo | $ZSTD > /dev/full && die "write error not detected!"
$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
$ECHO "\n**** symbolic link test **** "
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
$ECHO "hello world" > hello.tmp
ln -s hello.tmp world.tmp
$ZSTD world.tmp hello.tmp
ls hello.tmp.zst || die "regular file should have been compressed!"
ls world.tmp.zst && die "symbolic link should not have been compressed!"
test -f hello.tmp.zst # regular file should have been compressed!
test ! -f world.tmp.zst # symbolic link should not have been compressed!
$ZSTD world.tmp hello.tmp -f
ls world.tmp.zst || die "symbol link should have been compressed with --force"
test -f world.tmp.zst # symbolic link should have been compressed with --force
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
fi
@ -225,10 +226,10 @@ $ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse
$DIFF -s tmpSparse tmpOutSparse
$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
$DIFF -s tmpSparse tmpOutNoSparse
ls -ls tmpSparse*
ls -ls tmpSparse* # look at file size and block size on disk
./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
ls -ls tmpSparseOdd
ls -ls tmpSparseOdd # look at file size and block size on disk
$ECHO "\n Sparse Compatibility with Console :"
$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c
$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat
@ -238,7 +239,7 @@ cat tmpSparse1M tmpSparse1M > tmpSparse2M
$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
ls -ls tmpSparse*
ls -ls tmpSparse* # look at file size and block size on disk
$DIFF tmpSparse2M tmpSparseRegenerated
rm tmpSparse*
@ -257,11 +258,11 @@ $ZSTD -df *.zst
ls -ls tmp*
$ECHO "compress tmp* into stdout > tmpall : "
$ZSTD -c tmp1 tmp2 tmp3 > tmpall
ls -ls tmp*
ls -ls tmp* # check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst)
$ECHO "decompress tmpall* into stdout > tmpdec : "
cp tmpall tmpall2
$ZSTD -dc tmpall* > tmpdec
ls -ls tmp*
ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
$ECHO "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
@ -379,7 +380,9 @@ $ZSTD -t tmp2.zst && die "bad file not detected !"
$ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad
$ECHO "test --rm and --test combined "
$ZSTD -t --rm tmp1.zst
ls -ls tmp1.zst # check file is still present
test -f tmp1.zst # check file is still present
split -b16384 tmp1.zst tmpSplit.
$ZSTD -t tmpSplit.* && die "bad file not detected !"
$ECHO "\n**** benchmark mode tests **** "