AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add warning flags to makefile and clean up code to remove warnings

Browse Source
This commit is contained in:
Stella Lau 2017-07-10 13:08:19 -07:00
parent 10a71d9f1c
commit e4155b11d7
6 changed files with 171 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
contrib/long_distance_matching/Makefile
View File

@ -9,6 +9,14 @@
# This Makefile presumes libzstd is installed, using `sudo make install`
CFLAGS ?= -O3
DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS)
LDFLAGS += -lzstd
@ -22,7 +30,7 @@ all: main-ldm
#main : ldm.c main.c
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
main-ldm : ldm.c main-ldm.c
main-ldm : util.c ldm.c main-ldm.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
clean:

57
contrib/long_distance_matching/ldm.c
View File

@ -4,6 +4,7 @@
#include <stdio.h>
#include "ldm.h"
#include "util.h"
#define HASH_EVERY 7
@ -36,6 +37,7 @@ typedef uint32_t hash_t;
// typedef uint64_t tag;
/*
static unsigned LDM_isLittleEndian(void) {
const union { U32 u; BYTE c[4]; } one = { 1 };
return one.c[0];
@ -85,6 +87,8 @@ static U64 LDM_read64(const void *ptr) {
static void LDM_copy8(void *dst, const void *src) {
memcpy(dst, src, 8);
}
*/
typedef struct LDM_hashEntry {
offset_t offset;
} LDM_hashEntry;
@ -144,6 +148,7 @@ static hash_t LDM_hash(U32 sequence) {
return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
}
/*
static hash_t LDM_hash5(U64 sequence) {
static const U64 prime5bytes = 889523592379ULL;
static const U64 prime8bytes = 11400714785074694791ULL;
@ -153,35 +158,40 @@ static hash_t LDM_hash5(U64 sequence) {
else
return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
}
*/
static hash_t LDM_hash_position(const void * const p) {
return LDM_hash(LDM_read32(p));
}
static void LDM_put_position_on_hash(const BYTE *p, hash_t h,
void *tableBase, const BYTE *srcBase) {
static void LDM_putHashOfPosition(const BYTE *p, hash_t h,
void *tableBase, const BYTE *srcBase) {
LDM_hashEntry *hashTable;
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
return;
}
LDM_hashEntry *hashTable = (LDM_hashEntry *) tableBase;
hashTable[h] = (LDM_hashEntry) { (hash_t )(p - srcBase) };
hashTable = (LDM_hashEntry *) tableBase;
hashTable[h] = (LDM_hashEntry) { (hash_t)(p - srcBase) };
}
static void LDM_putPosition(const BYTE *p, void *tableBase,
const BYTE *srcBase) {
const BYTE *srcBase) {
hash_t hash;
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
return;
}
hash_t const h = LDM_hash_position(p);
LDM_put_position_on_hash(p, h, tableBase, srcBase);
hash = LDM_hash_position(p);
LDM_putHashOfPosition(p, hash, tableBase, srcBase);
}
static void LDM_putHashOfCurrentPosition(LDM_CCtx *const cctx) {
LDM_putPosition(cctx->ip, cctx->hashTable, cctx->ibase);
hash_t hash = LDM_hash_position(cctx->ip);
LDM_putHashOfPosition(cctx->ip, hash, cctx->hashTable, cctx->ibase);
cctx->lastPosHashed = cctx->ip;
cctx->lastHash = hash;
}
static const BYTE *LDM_get_position_on_hash(
hash_t h, void *tableBase, const BYTE *srcBase) {
const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
@ -209,8 +219,8 @@ static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
return (unsigned)(pIn - pStart);
}
void LDM_read_header(const void *src, size_t *compressSize,
size_t *decompressSize) {
void LDM_readHeader(const void *src, size_t *compressSize,
size_t *decompressSize) {
const U32 *ip = (const U32 *)src;
*compressSize = *ip++;
*decompressSize = *ip;
@ -230,7 +240,7 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
cctx->imatchLimit = cctx->iend - MINMATCH;
cctx->obase = (BYTE *)dst;
cctx->op = (BYTE *)cctx->obase;
cctx->op = (BYTE *)dst;
cctx->anchor = cctx->ibase;
@ -244,13 +254,12 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
size_t LDM_compress(const void *src, size_t srcSize,
void *dst, size_t maxDstSize) {
LDM_CCtx cctx;
U32 forwardH;
LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
U32 forwardH;
/* Hash the first position and put it into the hash table. */
LDM_putHashOfCurrentPosition(&cctx);
const BYTE *lastHash = cctx.ip;
cctx.ip++;
forwardH = LDM_hash_position(cctx.ip);
@ -276,8 +285,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
match = LDM_get_position_on_hash(h, cctx.hashTable, cctx.ibase);
forwardH = LDM_hash_position(forwardIp);
LDM_put_position_on_hash(cctx.ip, h, cctx.hashTable, cctx.ibase);
lastHash = cctx.ip;
LDM_putHashOfPosition(cctx.ip, h, cctx.hashTable, cctx.ibase);
} while (cctx.ip - match > WINDOW_SIZE ||
LDM_read64(match) != LDM_read64(cctx.ip));
}
@ -319,7 +327,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
memcpy(cctx.op, cctx.anchor, litLength);
cctx.op += litLength;
}
_next_match:
/* Encode offset */
{
/*
@ -334,6 +342,7 @@ _next_match:
/* Encode Match Length */
{
unsigned matchCode;
unsigned ctr = 1;
matchCode = LDM_count(cctx.ip + MINMATCH, match + MINMATCH,
cctx.ihashLimit);
#ifdef LDM_DEBUG
@ -342,7 +351,6 @@ _next_match:
printf("\n");
#endif
cctx.stats.total_match_length += matchCode + MINMATCH;
unsigned ctr = 1;
cctx.ip++;
for (; ctr < MINMATCH + matchCode; cctx.ip++, ctr++) {
LDM_putHashOfCurrentPosition(&cctx);
@ -372,7 +380,6 @@ _next_match:
LDM_putPosition(cctx.ip, cctx.hashTable, cctx.ibase);
forwardH = LDM_hash_position(++cctx.ip);
lastHash = cctx.ip;
}
_last_literals:
/* Encode last literals */
@ -392,7 +399,7 @@ _last_literals:
cctx.op += lastRun;
}
LDM_printCompressStats(&cctx.stats);
return (cctx.op - (BYTE *)cctx.obase);
return (cctx.op - (const BYTE *)cctx.obase);
}
typedef struct LDM_DCtx {
@ -427,12 +434,12 @@ size_t LDM_decompress(const void *src, size_t compressSize,
LDM_DCtx dctx;
LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
BYTE *cpy;
size_t length;
const BYTE *match;
size_t offset;
while (dctx.ip < dctx.iend) {
BYTE *cpy;
size_t length;
const BYTE *match;
size_t offset;
/* get literal length */
unsigned const token = *(dctx.ip)++;
if ((length = (token >> ML_BITS)) == RUN_MASK) {

4
contrib/long_distance_matching/ldm.h
View File

@ -13,7 +13,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
size_t LDM_decompress(const void *src, size_t srcSize,
void *dst, size_t maxDstSize);
void LDM_read_header(const void *src, size_t *compressSize,
size_t *decompressSize);
void LDM_readHeader(const void *src, size_t *compressSize,
size_t *decompressSize);
#endif /* LDM_H */

77
contrib/long_distance_matching/main-ldm.c
View File

@ -25,6 +25,7 @@ static int compress(const char *fname, const char *oname) {
int fdin, fdout;
struct stat statbuf;
char *src, *dst;
size_t maxCompressSize, compressSize;
/* Open the input file. */
if ((fdin = open(fname, O_RDONLY)) < 0) {
@ -44,10 +45,10 @@ static int compress(const char *fname, const char *oname) {
return 1;
}
size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
/* Go to the location corresponding to the last byte. */
/* TODO: fallocate? */
/* TODO: fallocate? */
if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
perror("lseek error");
return 1;
@ -74,14 +75,14 @@ static int compress(const char *fname, const char *oname) {
}
#ifdef ZSTD
size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
compressSize = ZSTD_compress(dst, statbuf.st_size,
src, statbuf.st_size, 1);
#else
size_t compressSize = LDM_HEADER_SIZE +
compressSize = LDM_HEADER_SIZE +
LDM_compress(src, statbuf.st_size,
dst + LDM_HEADER_SIZE, statbuf.st_size);
// Write compress and decompress size to header
// Write compress and decompress size to header
// TODO: should depend on LDM_DECOMPRESS_SIZE write32
memcpy(dst, &compressSize, 4);
memcpy(dst + 4, &(statbuf.st_size), 4);
@ -107,12 +108,13 @@ static int compress(const char *fname, const char *oname) {
/* Decompress file compressed using LDM_compress.
* The input file should have the LDM_HEADER followed by payload.
* Returns 0 if succesful, and an error code otherwise.
* Returns 0 if succesful, and an error code otherwise.
*/
static int decompress(const char *fname, const char *oname) {
int fdin, fdout;
struct stat statbuf;
char *src, *dst;
size_t compressSize, decompressSize, outSize;
/* Open the input file. */
if ((fdin = open(fname, O_RDONLY)) < 0) {
@ -140,8 +142,7 @@ static int decompress(const char *fname, const char *oname) {
}
/* Read the header. */
size_t compressSize, decompressSize;
LDM_read_header(src, &compressSize, &decompressSize);
LDM_readHeader(src, &compressSize, &decompressSize);
#ifdef DEBUG
printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
@ -168,11 +169,11 @@ static int decompress(const char *fname, const char *oname) {
}
#ifdef ZSTD
size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
outSize = ZSTD_decompress(dst, decomrpessed_size,
src + LDM_HEADER_SIZE,
statbuf.st_size - LDM_HEADER_SIZE);
#else
size_t outSize = LDM_decompress(
outSize = LDM_decompress(
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
dst, decompressSize);
@ -211,12 +212,14 @@ static void verify(const char *inpFilename, const char *decFilename) {
FILE *decFp = fopen(decFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("verify : OK\n");
} else {
printf("verify : NG\n");
}
{
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("verify : OK\n");
} else {
printf("verify : NG\n");
}
}
fclose(decFp);
fclose(inpFp);
@ -243,32 +246,34 @@ int main(int argc, const char *argv[]) {
printf("ldm = [%s]\n", ldmFilename);
printf("dec = [%s]\n", decFilename);
struct timeval tv1, tv2;
/* Compress */
gettimeofday(&tv1, NULL);
if (compress(inpFilename, ldmFilename)) {
printf("Compress error");
return 1;
{
struct timeval tv1, tv2;
gettimeofday(&tv1, NULL);
if (compress(inpFilename, ldmFilename)) {
printf("Compress error");
return 1;
}
gettimeofday(&tv2, NULL);
printf("Total time = %f seconds\n",
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
(double) (tv2.tv_sec - tv1.tv_sec));
}
gettimeofday(&tv2, NULL);
printf("Total time = %f seconds\n",
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
(double) (tv2.tv_sec - tv1.tv_sec));
/* Decompress */
gettimeofday(&tv1, NULL);
if (decompress(ldmFilename, decFilename)) {
printf("Decompress error");
return 1;
{
struct timeval tv1, tv2;
gettimeofday(&tv1, NULL);
if (decompress(ldmFilename, decFilename)) {
printf("Decompress error");
return 1;
}
gettimeofday(&tv2, NULL);
printf("Total time = %f seconds\n",
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
(double) (tv2.tv_sec - tv1.tv_sec));
}
gettimeofday(&tv2, NULL);
printf("Total time = %f seconds\n",
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
(double) (tv2.tv_sec - tv1.tv_sec));
/* verify */
verify(inpFilename, decFilename);
return 0;

64
contrib/long_distance_matching/util.c Normal file
View File

@ -0,0 +1,64 @@
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stdio.h>
#include "util.h"
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
unsigned LDM_isLittleEndian(void) {
const union { U32 u; BYTE c[4]; } one = { 1 };
return one.c[0];
}
U16 LDM_read16(const void *memPtr) {
U16 val;
memcpy(&val, memPtr, sizeof(val));
return val;
}
U16 LDM_readLE16(const void *memPtr) {
if (LDM_isLittleEndian()) {
return LDM_read16(memPtr);
} else {
const BYTE *p = (const BYTE *)memPtr;
return (U16)((U16)p[0] + (p[1] << 8));
}
}
void LDM_write16(void *memPtr, U16 value){
memcpy(memPtr, &value, sizeof(value));
}
void LDM_write32(void *memPtr, U32 value) {
memcpy(memPtr, &value, sizeof(value));
}
void LDM_writeLE16(void *memPtr, U16 value) {
if (LDM_isLittleEndian()) {
LDM_write16(memPtr, value);
} else {
BYTE* p = (BYTE *)memPtr;
p[0] = (BYTE) value;
p[1] = (BYTE)(value>>8);
}
}
U32 LDM_read32(const void *ptr) {
return *(const U32 *)ptr;
}
U64 LDM_read64(const void *ptr) {
return *(const U64 *)ptr;
}
void LDM_copy8(void *dst, const void *src) {
memcpy(dst, src, 8);
}

23
contrib/long_distance_matching/util.h Normal file
View File

@ -0,0 +1,23 @@
#ifndef LDM_UTIL_H
#define LDM_UTIL_H
unsigned LDM_isLittleEndian(void);
uint16_t LDM_read16(const void *memPtr);
uint16_t LDM_readLE16(const void *memPtr);
void LDM_write16(void *memPtr, uint16_t value);
void LDM_write32(void *memPtr, uint32_t value);
void LDM_writeLE16(void *memPtr, uint16_t value);
uint32_t LDM_read32(const void *ptr);
uint64_t LDM_read64(const void *ptr);
void LDM_copy8(void *dst, const void *src);
#endif /* LDM_UTIL_H */