AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

support custom block sizes: remove unneeded fudging, fix leaked filehandles, other nits

Browse Source
This commit is contained in:
Blaise Sanouillet 2018-10-01 05:22:50 -07:00
parent 7f18842244
commit d4703852b1
2 changed files with 88 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
tests/checkFrame.c
View File

@ -1,84 +1,84 @@
/*
checkFrame - verify frame headers
Copyright (C) Yann Collet 2014-2016
/*
checkFrame - verify frame headers
Copyright (C) Yann Collet 2014-2016
GPL v2 License
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
You can contact the author at :
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
/*-************************************
* Compiler specific
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
#endif
/*-************************************
* Compiler specific
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
#endif
/*-************************************
* Includes
**************************************/
#include "util.h" /* U32 */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* fprintf */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
#include <assert.h>
#include "lz4frame.h" /* include multiple times to test correctness/safety */
#include "lz4frame.h"
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
#include "lz4frame.h"
#include "lz4.h" /* LZ4_VERSION_STRING */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
/*-************************************
* Includes
**************************************/
#include "util.h" /* U32 */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* fprintf */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
#include <assert.h>
#include "lz4frame.h" /* include multiple times to test correctness/safety */
#include "lz4frame.h"
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
#include "lz4frame.h"
#include "lz4.h" /* LZ4_VERSION_STRING */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
/*-************************************
* Constants
**************************************/
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
/*-************************************
* Constants
**************************************/
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
/*-************************************
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/*-************************************
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/**************************************
* Exceptions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
/**************************************
* Exceptions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
return(error); \
}
@ -97,24 +97,21 @@ static U32 use_pause = 0;
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
void frameCheck(const char *srcFileName, unsigned bsid, size_t blockSize)
int frameCheck(FILE* const srcFile, unsigned bsid, size_t blockSize)
{
FILE *srcFile;
LZ4F_decompressionContext_t dctx = NULL;
const size_t srcBufferSize = 4 MB;
void *srcBuffer = malloc(srcBufferSize);
void* const srcBuffer = malloc(srcBufferSize);
const size_t dstBufferSize = 4 MB;
void *dstBuffer = malloc(dstBufferSize);
void* const dstBuffer = malloc(dstBufferSize);
LZ4F_errorCode_t nextToLoad = 0;
size_t curblocksize = 0;
int partialBlock = 0;
if (!srcBuffer || !dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
srcFile = fopen(srcFileName, "rb");
if ( srcFile==NULL ) EXM_THROW(1, "%s: %s \n", srcFileName, strerror(errno));
if (!srcBuffer || !dstBuffer) EXM_THROW(20, "Allocation error : not enough memory");
if (LZ4F_isError( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) )) {
EXM_THROW(32, "Unable to create decompression context");
EXM_THROW(21, "Unable to create decompression context");
}
/* Main Loop */
for (;;) {
@ -135,14 +132,14 @@ void frameCheck(const char *srcFileName, unsigned bsid, size_t blockSize)
curblocksize = 0;
remaining = readSize - pos;
nextToLoad = LZ4F_getFrameInfo(dctx, &frameInfo, (char*)(srcBuffer)+pos, &remaining);
if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Error getting frame info: %s", LZ4F_getErrorName(nextToLoad)); /* XXX */
if (frameInfo.blockSizeID != bsid) EXM_THROW(66, "Block size ID %u != expected %u", frameInfo.blockSizeID, bsid);
if (LZ4F_isError(nextToLoad)) EXM_THROW(22, "Error getting frame info: %s", LZ4F_getErrorName(nextToLoad)); /* XXX */
if (frameInfo.blockSizeID != bsid) EXM_THROW(23, "Block size ID %u != expected %u", frameInfo.blockSizeID, bsid);
pos += remaining;
/* nextToLoad should be block header size */
remaining = nextToLoad;
decodedBytes = dstBufferSize;
nextToLoad = LZ4F_decompress(dctx, dstBuffer, &decodedBytes, (char*)(srcBuffer)+pos, &remaining, NULL);
if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
pos += remaining;
}
decodedBytes = dstBufferSize;
@ -158,25 +155,25 @@ void frameCheck(const char *srcFileName, unsigned bsid, size_t blockSize)
remaining = nextToLoad;
}
nextToLoad = LZ4F_decompress(dctx, dstBuffer, &decodedBytes, (char*)(srcBuffer)+pos, &remaining, NULL);
if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
curblocksize += decodedBytes;
pos += remaining;
if (!partialBlock) {
/* detect small block due to end of frame; the final 4-byte frame checksum could be left in the buffer */
if ((curblocksize != 0) && (nextToLoad > 4)) {
/* allow 4-byte fudge */
if ((MAX(curblocksize, blockSize) - MIN(curblocksize, blockSize)) > 4)
EXM_THROW(66, "Block size %zu != expected %zu", curblocksize, blockSize);
if (curblocksize != blockSize)
EXM_THROW(25, "Block size %zu != expected %zu, pos %zu\n", curblocksize, blockSize, pos);
}
curblocksize = 0;
}
}
}
/* can be out because readSize == 0, which could be an fread() error */
if (ferror(srcFile)) EXM_THROW(67, "Read error");
if (ferror(srcFile)) EXM_THROW(26, "Read error");
if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream");
if (nextToLoad!=0) EXM_THROW(27, "Unfinished stream");
return 0;
}
int FUZ_usage(const char* programName)
@ -186,7 +183,7 @@ int FUZ_usage(const char* programName)
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -b# : expected blocksizeID [4-7] (required)\n");
DISPLAY( " -B# : expected blocksize [4-7] (required)\n");
DISPLAY( " -B# : expected blocksize [32-4194304] (required)\n");
DISPLAY( " -v : verbose\n");
DISPLAY( " -h : display help and exit\n");
return 0;
@ -196,6 +193,7 @@ int FUZ_usage(const char* programName)
int main(int argc, const char** argv)
{
int argNb;
int err;
int bsid=0;
size_t blockSize=0;
const char* const programName = argv[0];
@ -259,10 +257,15 @@ int main(int argc, const char** argv)
}
}
} else {
FILE *srcFile;
if (bsid == 0 || blockSize == 0)
return FUZ_usage(programName);
DISPLAY("Starting frame checker (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING);
frameCheck(argument, bsid, blockSize);
srcFile = fopen(argument, "rb");
if ( srcFile==NULL ) EXM_THROW(1, "%s: %s \n", argument, strerror(errno));
err = frameCheck(srcFile, bsid, blockSize);
fclose(srcFile);
return (err);
}
}
return 0;

6
tests/test_custom_block_sizes.sh
View File

@ -13,10 +13,10 @@ TMPFILE2=/tmp/test_custom_block_sizes2.$$
$DATAGEN -g12345678 > $TMPFILE1
$DATAGEN -g12345678 > $TMPFILE2
echo Testing -B32
$LZ4 -f -B32 $TMPFILE && failures="31 (should fail) "
echo Testing -B31
$LZ4 -f -B31 $TMPFILE1 && failures="31 (should fail) "
for blocksize in 512 65535 65536
for blocksize in 32 65535 65536
do
echo Testing -B$blocksize
$LZ4 -f -B$blocksize $TMPFILE1