[ldm] Clean up versions
This commit is contained in:
Stella Lau
parent
8ff8cdb15b
commit
8de82b6eb0
@ -19,8 +19,8 @@
|
||||
#define WINDOW_SIZE (1 << 20)
|
||||
|
||||
//These should be multiples of four.
|
||||
#define LDM_HASH_LENGTH 100
|
||||
#define MINMATCH 100
|
||||
#define LDM_HASH_LENGTH 4
|
||||
#define MINMATCH 4
|
||||
|
||||
#define ML_BITS 4
|
||||
#define ML_MASK ((1U<<ML_BITS)-1)
|
||||
@ -585,7 +585,6 @@ static void LDM_initializeDCtx(LDM_DCtx *dctx,
|
||||
dctx->iend = dctx->ip + dctx->compressSize;
|
||||
dctx->op = dst;
|
||||
dctx->oend = dctx->op + dctx->maxDecompressSize;
|
||||
|
||||
}
|
||||
|
||||
size_t LDM_decompress(const void *src, size_t compressSize,
|
||||
|
||||
@ -66,4 +66,3 @@ BYTE LDM_readByte(const void *memPtr) {
|
||||
memcpy(&val, memPtr, 1);
|
||||
return val;
|
||||
}
|
||||
|
||||
|
||||
@ -1,394 +0,0 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ldm.h"
|
||||
|
||||
#define LDM_MEMORY_USAGE 14
|
||||
#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
|
||||
#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
|
||||
#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
|
||||
#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
|
||||
|
||||
#define WINDOW_SIZE (1 << 20)
|
||||
#define MAX_WINDOW_SIZE 31
|
||||
#define HASH_SIZE 4
|
||||
#define MINMATCH 4
|
||||
|
||||
#define ML_BITS 4
|
||||
#define ML_MASK ((1U<<ML_BITS)-1)
|
||||
#define RUN_BITS (8-ML_BITS)
|
||||
#define RUN_MASK ((1U<<RUN_BITS)-1)
|
||||
|
||||
//#define LDM_DEBUG
|
||||
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
|
||||
typedef uint64_t tag;
|
||||
|
||||
static unsigned LDM_isLittleEndian(void)
|
||||
{
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 };
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
static U16 LDM_read16(const void *memPtr) {
|
||||
U16 val;
|
||||
memcpy(&val, memPtr, sizeof(val));
|
||||
return val;
|
||||
}
|
||||
|
||||
static U16 LDM_readLE16(const void *memPtr) {
|
||||
if (LDM_isLittleEndian()) {
|
||||
return LDM_read16(memPtr);
|
||||
} else {
|
||||
const BYTE *p = (const BYTE *)memPtr;
|
||||
return (U16)((U16)p[0] + (p[1] << 8));
|
||||
}
|
||||
}
|
||||
|
||||
static void LDM_write32(void *memPtr, U32 value) {
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
static void LDM_write16(void *memPtr, U16 value) {
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
static void LDM_writeLE16(void *memPtr, U16 value) {
|
||||
if (LDM_isLittleEndian()) {
|
||||
LDM_write16(memPtr, value);
|
||||
} else {
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE) value;
|
||||
p[1] = (BYTE)(value>>8);
|
||||
}
|
||||
}
|
||||
|
||||
static U32 LDM_read32(const void *ptr) {
|
||||
return *(const U32 *)ptr;
|
||||
}
|
||||
|
||||
static U64 LDM_read64(const void *ptr) {
|
||||
return *(const U64 *)ptr;
|
||||
}
|
||||
|
||||
|
||||
static void LDM_copy8(void *dst, const void *src) {
|
||||
memcpy(dst, src, 8);
|
||||
}
|
||||
|
||||
static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
|
||||
BYTE *d = (BYTE *)dstPtr;
|
||||
const BYTE *s = (const BYTE *)srcPtr;
|
||||
BYTE * const e = (BYTE *)dstEnd;
|
||||
|
||||
do {
|
||||
LDM_copy8(d, s);
|
||||
d += 8;
|
||||
s += 8;
|
||||
} while (d < e);
|
||||
|
||||
}
|
||||
|
||||
struct hash_entry {
|
||||
U64 offset;
|
||||
tag t;
|
||||
};
|
||||
|
||||
static U32 LDM_hash(U32 sequence) {
|
||||
return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
|
||||
}
|
||||
|
||||
static U32 LDM_hash5(U64 sequence) {
|
||||
static const U64 prime5bytes = 889523592379ULL;
|
||||
static const U64 prime8bytes = 11400714785074694791ULL;
|
||||
const U32 hashLog = LDM_HASHLOG;
|
||||
if (LDM_isLittleEndian())
|
||||
return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
|
||||
else
|
||||
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
|
||||
}
|
||||
|
||||
static U32 LDM_hash_position(const void * const p) {
|
||||
return LDM_hash(LDM_read32(p));
|
||||
}
|
||||
|
||||
static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
|
||||
const BYTE *srcBase) {
|
||||
U32 *hashTable = (U32 *) tableBase;
|
||||
hashTable[h] = (U32)(p - srcBase);
|
||||
}
|
||||
|
||||
static void LDM_put_position(const BYTE *p, void *tableBase,
|
||||
const BYTE *srcBase) {
|
||||
U32 const h = LDM_hash_position(p);
|
||||
LDM_put_position_on_hash(p, h, tableBase, srcBase);
|
||||
}
|
||||
|
||||
static const BYTE *LDM_get_position_on_hash(
|
||||
U32 h, void *tableBase, const BYTE *srcBase) {
|
||||
const U32 * const hashTable = (U32*)tableBase;
|
||||
return hashTable[h] + srcBase;
|
||||
}
|
||||
|
||||
static BYTE LDM_read_byte(const void *memPtr) {
|
||||
BYTE val;
|
||||
memcpy(&val, memPtr, 1);
|
||||
return val;
|
||||
}
|
||||
|
||||
static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
|
||||
const BYTE *pInLimit) {
|
||||
const BYTE * const pStart = pIn;
|
||||
while (pIn < pInLimit - 1) {
|
||||
BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
|
||||
if (!diff) {
|
||||
pIn++;
|
||||
pMatch++;
|
||||
continue;
|
||||
}
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
|
||||
void LDM_read_header(void const *source, size_t *compressed_size,
|
||||
size_t *decompressed_size) {
|
||||
const U32 *ip = (const U32 *)source;
|
||||
*compressed_size = *ip++;
|
||||
*decompressed_size = *ip;
|
||||
}
|
||||
|
||||
size_t LDM_compress(void const *source, void *dest, size_t source_size,
|
||||
size_t max_dest_size) {
|
||||
const BYTE * const istart = (const BYTE*)source;
|
||||
const BYTE *ip = istart;
|
||||
const BYTE * const iend = istart + source_size;
|
||||
const BYTE *ilimit = iend - HASH_SIZE;
|
||||
const BYTE * const matchlimit = iend - HASH_SIZE;
|
||||
const BYTE * const mflimit = iend - MINMATCH;
|
||||
BYTE *op = (BYTE*) dest;
|
||||
U32 hashTable[LDM_HASHTABLESIZE_U32];
|
||||
memset(hashTable, 0, sizeof(hashTable));
|
||||
|
||||
const BYTE *anchor = (const BYTE *)source;
|
||||
// struct LDM_cctx cctx;
|
||||
size_t output_size = 0;
|
||||
|
||||
U32 forwardH;
|
||||
|
||||
/* Hash first byte: put into hash table */
|
||||
|
||||
LDM_put_position(ip, hashTable, istart);
|
||||
ip++;
|
||||
forwardH = LDM_hash_position(ip);
|
||||
|
||||
//TODO Loop terminates before ip>=ilimit.
|
||||
while (ip < ilimit) {
|
||||
const BYTE *match;
|
||||
BYTE *token;
|
||||
|
||||
/* Find a match */
|
||||
{
|
||||
const BYTE *forwardIp = ip;
|
||||
unsigned step = 1;
|
||||
|
||||
do {
|
||||
U32 const h = forwardH;
|
||||
ip = forwardIp;
|
||||
forwardIp += step;
|
||||
|
||||
if (forwardIp > mflimit) {
|
||||
goto _last_literals;
|
||||
}
|
||||
|
||||
match = LDM_get_position_on_hash(h, hashTable, istart);
|
||||
|
||||
forwardH = LDM_hash_position(forwardIp);
|
||||
LDM_put_position_on_hash(ip, h, hashTable, istart);
|
||||
} while (ip - match > WINDOW_SIZE ||
|
||||
LDM_read64(match) != LDM_read64(ip));
|
||||
}
|
||||
|
||||
// TODO catchup
|
||||
while (ip > anchor && match > istart && ip[-1] == match[-1]) {
|
||||
ip--;
|
||||
match--;
|
||||
}
|
||||
|
||||
/* Encode literals */
|
||||
{
|
||||
unsigned const litLength = (unsigned)(ip - anchor);
|
||||
token = op++;
|
||||
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Cur position: %zu\n", anchor - istart);
|
||||
printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
|
||||
#endif
|
||||
/*
|
||||
fwrite(match, 4, 1, stdout);
|
||||
printf("\n");
|
||||
*/
|
||||
|
||||
if (litLength >= RUN_MASK) {
|
||||
int len = (int)litLength - RUN_MASK;
|
||||
*token = (RUN_MASK << ML_BITS);
|
||||
for (; len >= 255; len -= 255) {
|
||||
*op++ = 255;
|
||||
}
|
||||
*op++ = (BYTE)len;
|
||||
} else {
|
||||
*token = (BYTE)(litLength << ML_BITS);
|
||||
}
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Literals ");
|
||||
fwrite(anchor, litLength, 1, stdout);
|
||||
printf("\n");
|
||||
#endif
|
||||
memcpy(op, anchor, litLength);
|
||||
//LDM_wild_copy(op, anchor, op + litLength);
|
||||
op += litLength;
|
||||
}
|
||||
_next_match:
|
||||
/* Encode offset */
|
||||
{
|
||||
LDM_write32(op, ip - match);
|
||||
op += 4;
|
||||
}
|
||||
|
||||
/* Encode Match Length */
|
||||
{
|
||||
unsigned matchCode;
|
||||
matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
|
||||
matchlimit);
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Match length %zu\n", matchCode + MINMATCH);
|
||||
fwrite(ip, MINMATCH + matchCode, 1, stdout);
|
||||
printf("\n");
|
||||
#endif
|
||||
ip += MINMATCH + matchCode;
|
||||
if (matchCode >= ML_MASK) {
|
||||
*token += ML_MASK;
|
||||
matchCode -= ML_MASK;
|
||||
LDM_write32(op, 0xFFFFFFFF);
|
||||
while (matchCode >= 4*0xFF) {
|
||||
op += 4;
|
||||
LDM_write32(op, 0xffffffff);
|
||||
matchCode -= 4*0xFF;
|
||||
}
|
||||
op += matchCode / 255;
|
||||
*op++ = (BYTE)(matchCode % 255);
|
||||
} else {
|
||||
*token += (BYTE)(matchCode);
|
||||
}
|
||||
#ifdef LDM_DEBUG
|
||||
printf("\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
anchor = ip;
|
||||
|
||||
LDM_put_position(ip, hashTable, istart);
|
||||
forwardH = LDM_hash_position(++ip);
|
||||
}
|
||||
_last_literals:
|
||||
/* Encode last literals */
|
||||
{
|
||||
size_t const lastRun = (size_t)(iend - anchor);
|
||||
if (lastRun >= RUN_MASK) {
|
||||
size_t accumulator = lastRun - RUN_MASK;
|
||||
*op++ = RUN_MASK << ML_BITS;
|
||||
for(; accumulator >= 255; accumulator -= 255) {
|
||||
*op++ = 255;
|
||||
}
|
||||
*op++ = (BYTE)accumulator;
|
||||
} else {
|
||||
*op++ = (BYTE)(lastRun << ML_BITS);
|
||||
}
|
||||
memcpy(op, anchor, lastRun);
|
||||
op += lastRun;
|
||||
}
|
||||
return (op - (BYTE *)dest);
|
||||
}
|
||||
|
||||
size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
|
||||
size_t max_decompressed_size) {
|
||||
const BYTE *ip = (const BYTE *)source;
|
||||
const BYTE * const iend = ip + compressed_size;
|
||||
BYTE *op = (BYTE *)dest;
|
||||
BYTE * const oend = op + max_decompressed_size;
|
||||
BYTE *cpy;
|
||||
|
||||
while (ip < iend) {
|
||||
size_t length;
|
||||
const BYTE *match;
|
||||
size_t offset;
|
||||
|
||||
/* get literal length */
|
||||
unsigned const token = *ip++;
|
||||
if ((length=(token >> ML_BITS)) == RUN_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *ip++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Literal length: %zu\n", length);
|
||||
#endif
|
||||
|
||||
/* copy literals */
|
||||
cpy = op + length;
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Literals ");
|
||||
fwrite(ip, length, 1, stdout);
|
||||
printf("\n");
|
||||
#endif
|
||||
memcpy(op, ip, length);
|
||||
// LDM_wild_copy(op, ip, cpy);
|
||||
ip += length;
|
||||
op = cpy;
|
||||
|
||||
/* get offset */
|
||||
offset = LDM_read32(ip);
|
||||
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Offset: %zu\n", offset);
|
||||
#endif
|
||||
ip += 4;
|
||||
match = op - offset;
|
||||
// LDM_write32(op, (U32)offset);
|
||||
|
||||
/* get matchlength */
|
||||
length = token & ML_MASK;
|
||||
if (length == ML_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *ip++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
length += MINMATCH;
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Match length: %zu\n", length);
|
||||
#endif
|
||||
/* copy match */
|
||||
cpy = op + length;
|
||||
|
||||
// Inefficient for now
|
||||
|
||||
while (match < cpy - offset && op < oend) {
|
||||
*op++ = *match++;
|
||||
}
|
||||
}
|
||||
// memcpy(dest, source, compressed_size);
|
||||
return op - (BYTE *)dest;
|
||||
}
|
||||
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
#ifndef LDM_H
|
||||
#define LDM_H
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
#define LDM_COMPRESS_SIZE 4
|
||||
#define LDM_DECOMPRESS_SIZE 4
|
||||
#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
|
||||
|
||||
size_t LDM_compress(void const *source, void *dest, size_t source_size,
|
||||
size_t max_dest_size);
|
||||
|
||||
size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
|
||||
size_t max_decompressed_size);
|
||||
|
||||
void LDM_read_header(void const *source, size_t *compressed_size,
|
||||
size_t *decompressed_size);
|
||||
|
||||
#endif /* LDM_H */
|
||||
@ -1,459 +0,0 @@
|
||||
// TODO: file size must fit into a U32
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <zstd.h>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include "ldm.h"
|
||||
|
||||
// #define BUF_SIZE 16*1024 // Block size
|
||||
#define DEBUG
|
||||
|
||||
//#define ZSTD
|
||||
|
||||
#if 0
|
||||
static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
|
||||
size_t *size_out) {
|
||||
char *src, *buf = NULL;
|
||||
size_t r = 1;
|
||||
size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
|
||||
|
||||
src = malloc(BUF_SIZE);
|
||||
if (!src) {
|
||||
printf("Not enough memory\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
size = BUF_SIZE + LDM_HEADER_SIZE;
|
||||
buf = malloc(size);
|
||||
if (!buf) {
|
||||
printf("Not enough memory\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
|
||||
for (;;) {
|
||||
k = fread(src, 1, BUF_SIZE, in);
|
||||
if (k == 0)
|
||||
break;
|
||||
count_in += k;
|
||||
|
||||
n = LDM_compress(src, buf, k, BUF_SIZE);
|
||||
|
||||
// n = k;
|
||||
// offset += n;
|
||||
offset = k;
|
||||
count_out += k;
|
||||
|
||||
// k = fwrite(src, 1, offset, out);
|
||||
|
||||
k = fwrite(buf, 1, offset, out);
|
||||
if (k < offset) {
|
||||
if (ferror(out))
|
||||
printf("Write failed\n");
|
||||
else
|
||||
printf("Short write\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
}
|
||||
*size_in = count_in;
|
||||
*size_out = count_out;
|
||||
r = 0;
|
||||
cleanup:
|
||||
free(src);
|
||||
free(buf);
|
||||
return r;
|
||||
}
|
||||
|
||||
static size_t decompress_file(FILE *in, FILE *out) {
|
||||
void *src = malloc(BUF_SIZE);
|
||||
void *dst = NULL;
|
||||
size_t dst_capacity = BUF_SIZE;
|
||||
size_t ret = 1;
|
||||
size_t bytes_written = 0;
|
||||
|
||||
if (!src) {
|
||||
perror("decompress_file(src)");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
while (ret != 0) {
|
||||
/* Load more input */
|
||||
size_t src_size = fread(src, 1, BUF_SIZE, in);
|
||||
void *src_ptr = src;
|
||||
void *src_end = src_ptr + src_size;
|
||||
if (src_size == 0 || ferror(in)) {
|
||||
printf("(TODO): Decompress: not enough input or error reading file\n");
|
||||
//TODO
|
||||
ret = 0;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Allocate destination buffer if it hasn't been allocated already */
|
||||
if (!dst) {
|
||||
dst = malloc(dst_capacity);
|
||||
if (!dst) {
|
||||
perror("decompress_file(dst)");
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO
|
||||
|
||||
/* Decompress:
|
||||
* Continue while there is more input to read.
|
||||
*/
|
||||
while (src_ptr != src_end && ret != 0) {
|
||||
// size_t dst_size = src_size;
|
||||
size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
|
||||
size_t written = fwrite(dst, 1, dst_size, out);
|
||||
// printf("Writing %zu bytes\n", dst_size);
|
||||
bytes_written += dst_size;
|
||||
if (written != dst_size) {
|
||||
printf("Decompress: Failed to write to file\n");
|
||||
goto cleanup;
|
||||
}
|
||||
src_ptr += src_size;
|
||||
src_size = src_end - src_ptr;
|
||||
}
|
||||
|
||||
/* Update input */
|
||||
|
||||
}
|
||||
|
||||
printf("Wrote %zu bytes\n", bytes_written);
|
||||
|
||||
cleanup:
|
||||
free(src);
|
||||
free(dst);
|
||||
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
static size_t compress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
|
||||
/* open the input file */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* open the output file */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* find size of input file */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
size_t size_in = statbuf.st_size;
|
||||
|
||||
/* go to the location corresponding to the last byte */
|
||||
if (lseek(fdout, size_in + LDM_HEADER_SIZE - 1, SEEK_SET) == -1) {
|
||||
perror("lseek error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* write a dummy byte at the last location */
|
||||
if (write(fdout, "", 1) != 1) {
|
||||
perror("write error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the input file */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
size_t out_size = statbuf.st_size + LDM_HEADER_SIZE;
|
||||
|
||||
/* mmap the output file */
|
||||
if ((dst = mmap(0, out_size, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef ZSTD
|
||||
size_t size_out = ZSTD_compress(dst, statbuf.st_size,
|
||||
src, statbuf.st_size, 1);
|
||||
#else
|
||||
size_t size_out = LDM_compress(src, dst + LDM_HEADER_SIZE, statbuf.st_size,
|
||||
statbuf.st_size);
|
||||
size_out += LDM_HEADER_SIZE;
|
||||
|
||||
// TODO: should depend on LDM_DECOMPRESS_SIZE write32
|
||||
memcpy(dst, &size_out, 4);
|
||||
memcpy(dst + 4, &(statbuf.st_size), 4);
|
||||
printf("Compressed size: %zu\n", size_out);
|
||||
printf("Decompressed size: %zu\n", statbuf.st_size);
|
||||
#endif
|
||||
ftruncate(fdout, size_out);
|
||||
|
||||
printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
|
||||
(unsigned)statbuf.st_size, (unsigned)size_out, oname,
|
||||
(double)size_out / (statbuf.st_size) * 100);
|
||||
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t decompress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
|
||||
/* open the input file */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* open the output file */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* find size of input file */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the input file */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* read header */
|
||||
size_t compressed_size, decompressed_size;
|
||||
LDM_read_header(src, &compressed_size, &decompressed_size);
|
||||
|
||||
printf("Size, compressed_size, decompressed_size: %zu %zu %zu\n",
|
||||
statbuf.st_size, compressed_size, decompressed_size);
|
||||
|
||||
/* go to the location corresponding to the last byte */
|
||||
if (lseek(fdout, decompressed_size - 1, SEEK_SET) == -1) {
|
||||
perror("lseek error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* write a dummy byte at the last location */
|
||||
if (write(fdout, "", 1) != 1) {
|
||||
perror("write error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the output file */
|
||||
if ((dst = mmap(0, decompressed_size, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Copy input file to output file */
|
||||
// memcpy(dst, src, statbuf.st_size);
|
||||
|
||||
#ifdef ZSTD
|
||||
size_t size_out = ZSTD_decompress(dst, decomrpessed_size,
|
||||
src + LDM_HEADER_SIZE,
|
||||
statbuf.st_size - LDM_HEADER_SIZE);
|
||||
#else
|
||||
size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
|
||||
statbuf.st_size - LDM_HEADER_SIZE,
|
||||
decompressed_size);
|
||||
printf("Ret size out: %zu\n", size_out);
|
||||
#endif
|
||||
ftruncate(fdout, size_out);
|
||||
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int compare(FILE *fp0, FILE *fp1) {
|
||||
int result = 0;
|
||||
while (result == 0) {
|
||||
char b0[1024];
|
||||
char b1[1024];
|
||||
const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
|
||||
const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
|
||||
|
||||
result = (int)r0 - (int)r1;
|
||||
|
||||
if (0 == r0 || 0 == r1) {
|
||||
break;
|
||||
}
|
||||
if (0 == result) {
|
||||
result = memcmp(b0, b1, r0);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static void verify(const char *inpFilename, const char *decFilename) {
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *decFp = fopen(decFilename, "rb");
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
const int cmp = compare(inpFp, decFp);
|
||||
if(0 == cmp) {
|
||||
printf("verify : OK\n");
|
||||
} else {
|
||||
printf("verify : NG\n");
|
||||
}
|
||||
|
||||
fclose(decFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
const char * const exeName = argv[0];
|
||||
char inpFilename[256] = { 0 };
|
||||
char ldmFilename[256] = { 0 };
|
||||
char decFilename[256] = { 0 };
|
||||
|
||||
if (argc < 2) {
|
||||
printf("Wrong arguments\n");
|
||||
printf("Usage:\n");
|
||||
printf("%s FILE\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
snprintf(inpFilename, 256, "%s", argv[1]);
|
||||
snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
|
||||
snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
|
||||
|
||||
printf("inp = [%s]\n", inpFilename);
|
||||
printf("ldm = [%s]\n", ldmFilename);
|
||||
printf("dec = [%s]\n", decFilename);
|
||||
|
||||
struct timeval tv1, tv2;
|
||||
/* compress */
|
||||
{
|
||||
gettimeofday(&tv1, NULL);
|
||||
if (compress(inpFilename, ldmFilename)) {
|
||||
printf("Compress error");
|
||||
return 1;
|
||||
}
|
||||
gettimeofday(&tv2, NULL);
|
||||
printf("Total time = %f seconds\n",
|
||||
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec));
|
||||
}
|
||||
|
||||
/* decompress */
|
||||
|
||||
gettimeofday(&tv1, NULL);
|
||||
if (decompress(ldmFilename, decFilename)) {
|
||||
printf("Decompress error");
|
||||
return 1;
|
||||
}
|
||||
gettimeofday(&tv2, NULL);
|
||||
printf("Total time = %f seconds\n",
|
||||
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec));
|
||||
|
||||
/* verify */
|
||||
verify(inpFilename, decFilename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if 0
|
||||
int main2(int argc, char *argv[]) {
|
||||
char inpFilename[256] = { 0 };
|
||||
char ldmFilename[256] = { 0 };
|
||||
char decFilename[256] = { 0 };
|
||||
|
||||
if (argc < 2) {
|
||||
printf("Please specify input filename\n");
|
||||
return 0;
|
||||
}
|
||||
snprintf(inpFilename, 256, "%s", argv[1]);
|
||||
snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
|
||||
snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
|
||||
|
||||
printf("inp = [%s]\n", inpFilename);
|
||||
printf("ldm = [%s]\n", ldmFilename);
|
||||
printf("dec = [%s]\n", decFilename);
|
||||
|
||||
/* compress */
|
||||
{
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *outFp = fopen(ldmFilename, "wb");
|
||||
size_t sizeIn = 0;
|
||||
size_t sizeOut = 0;
|
||||
size_t ret;
|
||||
printf("compress : %s -> %s\n", inpFilename, ldmFilename);
|
||||
ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
|
||||
if (ret) {
|
||||
printf("compress : failed with code %zu\n", ret);
|
||||
return ret;
|
||||
}
|
||||
printf("%s: %zu → %zu bytes, %.1f%%\n",
|
||||
inpFilename, sizeIn, sizeOut,
|
||||
(double)sizeOut / sizeIn * 100);
|
||||
printf("compress : done\n");
|
||||
|
||||
fclose(outFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
/* decompress */
|
||||
{
|
||||
FILE *inpFp = fopen(ldmFilename, "rb");
|
||||
FILE *outFp = fopen(decFilename, "wb");
|
||||
size_t ret;
|
||||
|
||||
printf("decompress : %s -> %s\n", ldmFilename, decFilename);
|
||||
ret = decompress_file(inpFp, outFp);
|
||||
if (ret) {
|
||||
printf("decompress : failed with code %zu\n", ret);
|
||||
return ret;
|
||||
}
|
||||
printf("decompress : done\n");
|
||||
|
||||
fclose(outFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
/* verify */
|
||||
{
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *decFp = fopen(decFilename, "rb");
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
const int cmp = compare(inpFp, decFp);
|
||||
if(0 == cmp) {
|
||||
printf("verify : OK\n");
|
||||
} else {
|
||||
printf("verify : NG\n");
|
||||
}
|
||||
|
||||
fclose(decFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ################################################################
|
||||
|
||||
# This Makefile presumes libzstd is installed, using `sudo make install`
|
||||
|
||||
|
||||
LDFLAGS += -lzstd
|
||||
|
||||
.PHONY: default all clean
|
||||
|
||||
default: all
|
||||
|
||||
all: main-ldm
|
||||
|
||||
|
||||
#main : ldm.c main.c
|
||||
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
main-ldm : ldm.c main-ldm.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
|
||||
main main-ldm
|
||||
@echo Cleaning completed
|
||||
|
||||
@ -1,436 +0,0 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ldm.h"
|
||||
|
||||
#define HASH_EVERY 7
|
||||
|
||||
#define LDM_MEMORY_USAGE 14
|
||||
#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
|
||||
#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
|
||||
#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
|
||||
#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
|
||||
|
||||
#define WINDOW_SIZE (1 << 20)
|
||||
#define MAX_WINDOW_SIZE 31
|
||||
#define HASH_SIZE 8
|
||||
#define MINMATCH 8
|
||||
|
||||
#define ML_BITS 4
|
||||
#define ML_MASK ((1U<<ML_BITS)-1)
|
||||
#define RUN_BITS (8-ML_BITS)
|
||||
#define RUN_MASK ((1U<<RUN_BITS)-1)
|
||||
|
||||
//#define LDM_DEBUG
|
||||
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
|
||||
typedef uint64_t tag;
|
||||
|
||||
static unsigned LDM_isLittleEndian(void) {
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 };
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
static U16 LDM_read16(const void *memPtr) {
|
||||
U16 val;
|
||||
memcpy(&val, memPtr, sizeof(val));
|
||||
return val;
|
||||
}
|
||||
|
||||
static U16 LDM_readLE16(const void *memPtr) {
|
||||
if (LDM_isLittleEndian()) {
|
||||
return LDM_read16(memPtr);
|
||||
} else {
|
||||
const BYTE *p = (const BYTE *)memPtr;
|
||||
return (U16)((U16)p[0] + (p[1] << 8));
|
||||
}
|
||||
}
|
||||
|
||||
static void LDM_write16(void *memPtr, U16 value){
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
static void LDM_write32(void *memPtr, U32 value) {
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
static void LDM_writeLE16(void *memPtr, U16 value) {
|
||||
if (LDM_isLittleEndian()) {
|
||||
LDM_write16(memPtr, value);
|
||||
} else {
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE) value;
|
||||
p[1] = (BYTE)(value>>8);
|
||||
}
|
||||
}
|
||||
|
||||
static U32 LDM_read32(const void *ptr) {
|
||||
return *(const U32 *)ptr;
|
||||
}
|
||||
|
||||
static U64 LDM_read64(const void *ptr) {
|
||||
return *(const U64 *)ptr;
|
||||
}
|
||||
|
||||
static void LDM_copy8(void *dst, const void *src) {
|
||||
memcpy(dst, src, 8);
|
||||
}
|
||||
|
||||
typedef struct compress_stats {
|
||||
U32 num_matches;
|
||||
U32 total_match_length;
|
||||
U32 total_literal_length;
|
||||
U64 total_offset;
|
||||
} compress_stats;
|
||||
|
||||
static void LDM_printCompressStats(const compress_stats *stats) {
|
||||
printf("=====================\n");
|
||||
printf("Compression statistics\n");
|
||||
printf("Total number of matches: %u\n", stats->num_matches);
|
||||
printf("Average match length: %.1f\n", ((double)stats->total_match_length) /
|
||||
(double)stats->num_matches);
|
||||
printf("Average literal length: %.1f\n",
|
||||
((double)stats->total_literal_length) / (double)stats->num_matches);
|
||||
printf("Average offset length: %.1f\n",
|
||||
((double)stats->total_offset) / (double)stats->num_matches);
|
||||
printf("=====================\n");
|
||||
}
|
||||
|
||||
// TODO: unused.
|
||||
struct hash_entry {
|
||||
U64 offset;
|
||||
tag t;
|
||||
};
|
||||
|
||||
static U32 LDM_hash(U32 sequence) {
|
||||
return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
|
||||
}
|
||||
|
||||
static U32 LDM_hash5(U64 sequence) {
|
||||
static const U64 prime5bytes = 889523592379ULL;
|
||||
static const U64 prime8bytes = 11400714785074694791ULL;
|
||||
const U32 hashLog = LDM_HASHLOG;
|
||||
if (LDM_isLittleEndian())
|
||||
return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
|
||||
else
|
||||
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
|
||||
}
|
||||
|
||||
static U32 LDM_hash_position(const void * const p) {
|
||||
return LDM_hash(LDM_read32(p));
|
||||
}
|
||||
|
||||
static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
|
||||
const BYTE *srcBase) {
|
||||
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
|
||||
return;
|
||||
}
|
||||
|
||||
U32 *hashTable = (U32 *) tableBase;
|
||||
hashTable[h] = (U32)(p - srcBase);
|
||||
}
|
||||
|
||||
static void LDM_put_position(const BYTE *p, void *tableBase,
|
||||
const BYTE *srcBase) {
|
||||
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
|
||||
return;
|
||||
}
|
||||
U32 const h = LDM_hash_position(p);
|
||||
LDM_put_position_on_hash(p, h, tableBase, srcBase);
|
||||
}
|
||||
|
||||
static const BYTE *LDM_get_position_on_hash(
|
||||
U32 h, void *tableBase, const BYTE *srcBase) {
|
||||
const U32 * const hashTable = (U32*)tableBase;
|
||||
return hashTable[h] + srcBase;
|
||||
}
|
||||
|
||||
static BYTE LDM_read_byte(const void *memPtr) {
|
||||
BYTE val;
|
||||
memcpy(&val, memPtr, 1);
|
||||
return val;
|
||||
}
|
||||
|
||||
static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
|
||||
const BYTE *pInLimit) {
|
||||
const BYTE * const pStart = pIn;
|
||||
while (pIn < pInLimit - 1) {
|
||||
BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
|
||||
if (!diff) {
|
||||
pIn++;
|
||||
pMatch++;
|
||||
continue;
|
||||
}
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
|
||||
void LDM_read_header(const void *src, size_t *compressSize,
|
||||
size_t *decompressSize) {
|
||||
const U32 *ip = (const U32 *)src;
|
||||
*compressSize = *ip++;
|
||||
*decompressSize = *ip;
|
||||
}
|
||||
|
||||
// TODO: maxDstSize is unused
|
||||
size_t LDM_compress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize) {
|
||||
const BYTE * const istart = (const BYTE*)src;
|
||||
const BYTE *ip = istart;
|
||||
const BYTE * const iend = istart + srcSize;
|
||||
const BYTE *ilimit = iend - HASH_SIZE;
|
||||
const BYTE * const matchlimit = iend - HASH_SIZE;
|
||||
const BYTE * const mflimit = iend - MINMATCH;
|
||||
BYTE *op = (BYTE*) dst;
|
||||
|
||||
compress_stats compressStats = { 0 };
|
||||
|
||||
U32 hashTable[LDM_HASHTABLESIZE_U32];
|
||||
memset(hashTable, 0, sizeof(hashTable));
|
||||
|
||||
const BYTE *anchor = (const BYTE *)src;
|
||||
// struct LDM_cctx cctx;
|
||||
size_t output_size = 0;
|
||||
|
||||
U32 forwardH;
|
||||
|
||||
/* Hash first byte: put into hash table */
|
||||
|
||||
LDM_put_position(ip, hashTable, istart);
|
||||
const BYTE *lastHash = ip;
|
||||
ip++;
|
||||
forwardH = LDM_hash_position(ip);
|
||||
|
||||
//TODO Loop terminates before ip>=ilimit.
|
||||
while (ip < ilimit) {
|
||||
const BYTE *match;
|
||||
BYTE *token;
|
||||
|
||||
/* Find a match */
|
||||
{
|
||||
const BYTE *forwardIp = ip;
|
||||
unsigned step = 1;
|
||||
|
||||
do {
|
||||
U32 const h = forwardH;
|
||||
ip = forwardIp;
|
||||
forwardIp += step;
|
||||
|
||||
if (forwardIp > mflimit) {
|
||||
goto _last_literals;
|
||||
}
|
||||
|
||||
match = LDM_get_position_on_hash(h, hashTable, istart);
|
||||
|
||||
forwardH = LDM_hash_position(forwardIp);
|
||||
LDM_put_position_on_hash(ip, h, hashTable, istart);
|
||||
lastHash = ip;
|
||||
} while (ip - match > WINDOW_SIZE ||
|
||||
LDM_read64(match) != LDM_read64(ip));
|
||||
}
|
||||
compressStats.num_matches++;
|
||||
|
||||
/* Catchup: look back to extend match from found match */
|
||||
while (ip > anchor && match > istart && ip[-1] == match[-1]) {
|
||||
ip--;
|
||||
match--;
|
||||
}
|
||||
|
||||
/* Encode literals */
|
||||
{
|
||||
unsigned const litLength = (unsigned)(ip - anchor);
|
||||
token = op++;
|
||||
|
||||
compressStats.total_literal_length += litLength;
|
||||
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Cur position: %zu\n", anchor - istart);
|
||||
printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
|
||||
#endif
|
||||
|
||||
if (litLength >= RUN_MASK) {
|
||||
int len = (int)litLength - RUN_MASK;
|
||||
*token = (RUN_MASK << ML_BITS);
|
||||
for (; len >= 255; len -= 255) {
|
||||
*op++ = 255;
|
||||
}
|
||||
*op++ = (BYTE)len;
|
||||
} else {
|
||||
*token = (BYTE)(litLength << ML_BITS);
|
||||
}
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Literals ");
|
||||
fwrite(anchor, litLength, 1, stdout);
|
||||
printf("\n");
|
||||
#endif
|
||||
memcpy(op, anchor, litLength);
|
||||
op += litLength;
|
||||
}
|
||||
_next_match:
|
||||
/* Encode offset */
|
||||
{
|
||||
/*
|
||||
LDM_writeLE16(op, ip-match);
|
||||
op += 2;
|
||||
*/
|
||||
LDM_write32(op, ip - match);
|
||||
op += 4;
|
||||
compressStats.total_offset += (ip - match);
|
||||
}
|
||||
|
||||
/* Encode Match Length */
|
||||
{
|
||||
unsigned matchCode;
|
||||
matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
|
||||
matchlimit);
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Match length %zu\n", matchCode + MINMATCH);
|
||||
fwrite(ip, MINMATCH + matchCode, 1, stdout);
|
||||
printf("\n");
|
||||
#endif
|
||||
compressStats.total_match_length += matchCode + MINMATCH;
|
||||
unsigned ctr = 1;
|
||||
ip++;
|
||||
for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
|
||||
LDM_put_position(ip, hashTable, istart);
|
||||
}
|
||||
// ip += MINMATCH + matchCode;
|
||||
if (matchCode >= ML_MASK) {
|
||||
*token += ML_MASK;
|
||||
matchCode -= ML_MASK;
|
||||
LDM_write32(op, 0xFFFFFFFF);
|
||||
while (matchCode >= 4*0xFF) {
|
||||
op += 4;
|
||||
LDM_write32(op, 0xffffffff);
|
||||
matchCode -= 4*0xFF;
|
||||
}
|
||||
op += matchCode / 255;
|
||||
*op++ = (BYTE)(matchCode % 255);
|
||||
} else {
|
||||
*token += (BYTE)(matchCode);
|
||||
}
|
||||
#ifdef LDM_DEBUG
|
||||
printf("\n");
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
anchor = ip;
|
||||
|
||||
LDM_put_position(ip, hashTable, istart);
|
||||
forwardH = LDM_hash_position(++ip);
|
||||
lastHash = ip;
|
||||
}
|
||||
_last_literals:
|
||||
/* Encode last literals */
|
||||
{
|
||||
size_t const lastRun = (size_t)(iend - anchor);
|
||||
if (lastRun >= RUN_MASK) {
|
||||
size_t accumulator = lastRun - RUN_MASK;
|
||||
*op++ = RUN_MASK << ML_BITS;
|
||||
for(; accumulator >= 255; accumulator -= 255) {
|
||||
*op++ = 255;
|
||||
}
|
||||
*op++ = (BYTE)accumulator;
|
||||
} else {
|
||||
*op++ = (BYTE)(lastRun << ML_BITS);
|
||||
}
|
||||
memcpy(op, anchor, lastRun);
|
||||
op += lastRun;
|
||||
}
|
||||
LDM_printCompressStats(&compressStats);
|
||||
return (op - (BYTE *)dst);
|
||||
}
|
||||
|
||||
typedef struct LDM_DCtx {
|
||||
const BYTE * const ibase; /* Pointer to base of input */
|
||||
const BYTE *ip; /* Pointer to current input position */
|
||||
const BYTE *iend; /* End of source */
|
||||
BYTE *op; /* Pointer to output */
|
||||
const BYTE * const oend; /* Pointer to end of output */
|
||||
|
||||
} LDM_DCtx;
|
||||
|
||||
size_t LDM_decompress(const void *src, size_t compressed_size,
|
||||
void *dst, size_t max_decompressed_size) {
|
||||
const BYTE *ip = (const BYTE *)src;
|
||||
const BYTE * const iend = ip + compressed_size;
|
||||
BYTE *op = (BYTE *)dst;
|
||||
BYTE * const oend = op + max_decompressed_size;
|
||||
BYTE *cpy;
|
||||
|
||||
while (ip < iend) {
|
||||
size_t length;
|
||||
const BYTE *match;
|
||||
size_t offset;
|
||||
|
||||
/* get literal length */
|
||||
unsigned const token = *ip++;
|
||||
if ((length=(token >> ML_BITS)) == RUN_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *ip++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Literal length: %zu\n", length);
|
||||
#endif
|
||||
|
||||
/* copy literals */
|
||||
cpy = op + length;
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Literals ");
|
||||
fwrite(ip, length, 1, stdout);
|
||||
printf("\n");
|
||||
#endif
|
||||
memcpy(op, ip, length);
|
||||
ip += length;
|
||||
op = cpy;
|
||||
|
||||
/* get offset */
|
||||
/*
|
||||
offset = LDM_readLE16(ip);
|
||||
ip += 2;
|
||||
*/
|
||||
offset = LDM_read32(ip);
|
||||
ip += 4;
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Offset: %zu\n", offset);
|
||||
#endif
|
||||
match = op - offset;
|
||||
// LDM_write32(op, (U32)offset);
|
||||
|
||||
/* get matchlength */
|
||||
length = token & ML_MASK;
|
||||
if (length == ML_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *ip++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
length += MINMATCH;
|
||||
#ifdef LDM_DEBUG
|
||||
printf("Match length: %zu\n", length);
|
||||
#endif
|
||||
/* copy match */
|
||||
cpy = op + length;
|
||||
|
||||
// Inefficient for now
|
||||
while (match < cpy - offset && op < oend) {
|
||||
*op++ = *match++;
|
||||
}
|
||||
}
|
||||
return op - (BYTE *)dst;
|
||||
}
|
||||
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
#ifndef LDM_H
|
||||
#define LDM_H
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
#define LDM_COMPRESS_SIZE 4
|
||||
#define LDM_DECOMPRESS_SIZE 4
|
||||
#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
|
||||
|
||||
size_t LDM_compress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
size_t LDM_decompress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
void LDM_read_header(const void *src, size_t *compressSize,
|
||||
size_t *decompressSize);
|
||||
|
||||
#endif /* LDM_H */
|
||||
@ -1,474 +0,0 @@
|
||||
// TODO: file size must fit into a U32
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <zstd.h>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include "ldm.h"
|
||||
|
||||
// #define BUF_SIZE 16*1024 // Block size
|
||||
#define DEBUG
|
||||
|
||||
//#define ZSTD
|
||||
|
||||
/* Compress file given by fname and output to oname.
|
||||
* Returns 0 if successful, error code otherwise.
|
||||
*/
|
||||
static int compress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
|
||||
/* Open the input file. */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Open the output file. */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Find the size of the input file. */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
|
||||
|
||||
/* Go to the location corresponding to the last byte. */
|
||||
/* TODO: fallocate? */
|
||||
if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
|
||||
perror("lseek error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Write a dummy byte at the last location. */
|
||||
if (write(fdout, "", 1) != 1) {
|
||||
perror("write error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the input file. */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the output file */
|
||||
if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef ZSTD
|
||||
size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
|
||||
src, statbuf.st_size, 1);
|
||||
#else
|
||||
size_t compressSize = LDM_HEADER_SIZE +
|
||||
LDM_compress(src, statbuf.st_size,
|
||||
dst + LDM_HEADER_SIZE, statbuf.st_size);
|
||||
|
||||
// Write compress and decompress size to header
|
||||
// TODO: should depend on LDM_DECOMPRESS_SIZE write32
|
||||
memcpy(dst, &compressSize, 4);
|
||||
memcpy(dst + 4, &(statbuf.st_size), 4);
|
||||
|
||||
#ifdef DEBUG
|
||||
printf("Compressed size: %zu\n", compressSize);
|
||||
printf("Decompressed size: %zu\n", statbuf.st_size);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Truncate file to compressSize.
|
||||
ftruncate(fdout, compressSize);
|
||||
|
||||
printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
|
||||
(unsigned)statbuf.st_size, (unsigned)compressSize, oname,
|
||||
(double)compressSize / (statbuf.st_size) * 100);
|
||||
|
||||
// Close files.
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Decompress file compressed using LDM_compress.
|
||||
* The input file should have the LDM_HEADER followed by payload.
|
||||
* Returns 0 if succesful, and an error code otherwise.
|
||||
*/
|
||||
static int decompress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
|
||||
/* Open the input file. */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Open the output file. */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Find the size of the input file. */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the input file. */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Read the header. */
|
||||
size_t compressSize, decompressSize;
|
||||
LDM_read_header(src, &compressSize, &decompressSize);
|
||||
|
||||
#ifdef DEBUG
|
||||
printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
|
||||
statbuf.st_size, compressSize, decompressSize);
|
||||
#endif
|
||||
|
||||
/* Go to the location corresponding to the last byte. */
|
||||
if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
|
||||
perror("lseek error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* write a dummy byte at the last location */
|
||||
if (write(fdout, "", 1) != 1) {
|
||||
perror("write error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the output file */
|
||||
if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef ZSTD
|
||||
size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
|
||||
src + LDM_HEADER_SIZE,
|
||||
statbuf.st_size - LDM_HEADER_SIZE);
|
||||
#else
|
||||
size_t outSize = LDM_decompress(
|
||||
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
|
||||
dst, decompressSize);
|
||||
|
||||
printf("Ret size out: %zu\n", outSize);
|
||||
#endif
|
||||
ftruncate(fdout, outSize);
|
||||
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Compare two files.
|
||||
* Returns 0 iff they are the same.
|
||||
*/
|
||||
static int compare(FILE *fp0, FILE *fp1) {
|
||||
int result = 0;
|
||||
while (result == 0) {
|
||||
char b0[1024];
|
||||
char b1[1024];
|
||||
const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
|
||||
const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
|
||||
|
||||
result = (int)r0 - (int)r1;
|
||||
|
||||
if (0 == r0 || 0 == r1) break;
|
||||
|
||||
if (0 == result) result = memcmp(b0, b1, r0);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Verify the input file is the same as the decompressed file. */
|
||||
static void verify(const char *inpFilename, const char *decFilename) {
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *decFp = fopen(decFilename, "rb");
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
const int cmp = compare(inpFp, decFp);
|
||||
if(0 == cmp) {
|
||||
printf("verify : OK\n");
|
||||
} else {
|
||||
printf("verify : NG\n");
|
||||
}
|
||||
|
||||
fclose(decFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
const char * const exeName = argv[0];
|
||||
char inpFilename[256] = { 0 };
|
||||
char ldmFilename[256] = { 0 };
|
||||
char decFilename[256] = { 0 };
|
||||
|
||||
if (argc < 2) {
|
||||
printf("Wrong arguments\n");
|
||||
printf("Usage:\n");
|
||||
printf("%s FILE\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
snprintf(inpFilename, 256, "%s", argv[1]);
|
||||
snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
|
||||
snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
|
||||
|
||||
printf("inp = [%s]\n", inpFilename);
|
||||
printf("ldm = [%s]\n", ldmFilename);
|
||||
printf("dec = [%s]\n", decFilename);
|
||||
|
||||
struct timeval tv1, tv2;
|
||||
|
||||
/* Compress */
|
||||
|
||||
gettimeofday(&tv1, NULL);
|
||||
if (compress(inpFilename, ldmFilename)) {
|
||||
printf("Compress error");
|
||||
return 1;
|
||||
}
|
||||
gettimeofday(&tv2, NULL);
|
||||
printf("Total time = %f seconds\n",
|
||||
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec));
|
||||
|
||||
/* Decompress */
|
||||
|
||||
gettimeofday(&tv1, NULL);
|
||||
if (decompress(ldmFilename, decFilename)) {
|
||||
printf("Decompress error");
|
||||
return 1;
|
||||
}
|
||||
gettimeofday(&tv2, NULL);
|
||||
printf("Total time = %f seconds\n",
|
||||
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec));
|
||||
|
||||
/* verify */
|
||||
verify(inpFilename, decFilename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
#if 0
|
||||
static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
|
||||
size_t *size_out) {
|
||||
char *src, *buf = NULL;
|
||||
size_t r = 1;
|
||||
size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
|
||||
|
||||
src = malloc(BUF_SIZE);
|
||||
if (!src) {
|
||||
printf("Not enough memory\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
size = BUF_SIZE + LDM_HEADER_SIZE;
|
||||
buf = malloc(size);
|
||||
if (!buf) {
|
||||
printf("Not enough memory\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
|
||||
for (;;) {
|
||||
k = fread(src, 1, BUF_SIZE, in);
|
||||
if (k == 0)
|
||||
break;
|
||||
count_in += k;
|
||||
|
||||
n = LDM_compress(src, buf, k, BUF_SIZE);
|
||||
|
||||
// n = k;
|
||||
// offset += n;
|
||||
offset = k;
|
||||
count_out += k;
|
||||
|
||||
// k = fwrite(src, 1, offset, out);
|
||||
|
||||
k = fwrite(buf, 1, offset, out);
|
||||
if (k < offset) {
|
||||
if (ferror(out))
|
||||
printf("Write failed\n");
|
||||
else
|
||||
printf("Short write\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
}
|
||||
*size_in = count_in;
|
||||
*size_out = count_out;
|
||||
r = 0;
|
||||
cleanup:
|
||||
free(src);
|
||||
free(buf);
|
||||
return r;
|
||||
}
|
||||
|
||||
static size_t decompress_file(FILE *in, FILE *out) {
|
||||
void *src = malloc(BUF_SIZE);
|
||||
void *dst = NULL;
|
||||
size_t dst_capacity = BUF_SIZE;
|
||||
size_t ret = 1;
|
||||
size_t bytes_written = 0;
|
||||
|
||||
if (!src) {
|
||||
perror("decompress_file(src)");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
while (ret != 0) {
|
||||
/* Load more input */
|
||||
size_t src_size = fread(src, 1, BUF_SIZE, in);
|
||||
void *src_ptr = src;
|
||||
void *src_end = src_ptr + src_size;
|
||||
if (src_size == 0 || ferror(in)) {
|
||||
printf("(TODO): Decompress: not enough input or error reading file\n");
|
||||
//TODO
|
||||
ret = 0;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Allocate destination buffer if it hasn't been allocated already */
|
||||
if (!dst) {
|
||||
dst = malloc(dst_capacity);
|
||||
if (!dst) {
|
||||
perror("decompress_file(dst)");
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO
|
||||
|
||||
/* Decompress:
|
||||
* Continue while there is more input to read.
|
||||
*/
|
||||
while (src_ptr != src_end && ret != 0) {
|
||||
// size_t dst_size = src_size;
|
||||
size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
|
||||
size_t written = fwrite(dst, 1, dst_size, out);
|
||||
// printf("Writing %zu bytes\n", dst_size);
|
||||
bytes_written += dst_size;
|
||||
if (written != dst_size) {
|
||||
printf("Decompress: Failed to write to file\n");
|
||||
goto cleanup;
|
||||
}
|
||||
src_ptr += src_size;
|
||||
src_size = src_end - src_ptr;
|
||||
}
|
||||
|
||||
/* Update input */
|
||||
|
||||
}
|
||||
|
||||
printf("Wrote %zu bytes\n", bytes_written);
|
||||
|
||||
cleanup:
|
||||
free(src);
|
||||
free(dst);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int main2(int argc, char *argv[]) {
|
||||
char inpFilename[256] = { 0 };
|
||||
char ldmFilename[256] = { 0 };
|
||||
char decFilename[256] = { 0 };
|
||||
|
||||
if (argc < 2) {
|
||||
printf("Please specify input filename\n");
|
||||
return 0;
|
||||
}
|
||||
snprintf(inpFilename, 256, "%s", argv[1]);
|
||||
snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
|
||||
snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
|
||||
|
||||
printf("inp = [%s]\n", inpFilename);
|
||||
printf("ldm = [%s]\n", ldmFilename);
|
||||
printf("dec = [%s]\n", decFilename);
|
||||
|
||||
/* compress */
|
||||
{
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *outFp = fopen(ldmFilename, "wb");
|
||||
size_t sizeIn = 0;
|
||||
size_t sizeOut = 0;
|
||||
size_t ret;
|
||||
printf("compress : %s -> %s\n", inpFilename, ldmFilename);
|
||||
ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
|
||||
if (ret) {
|
||||
printf("compress : failed with code %zu\n", ret);
|
||||
return ret;
|
||||
}
|
||||
printf("%s: %zu → %zu bytes, %.1f%%\n",
|
||||
inpFilename, sizeIn, sizeOut,
|
||||
(double)sizeOut / sizeIn * 100);
|
||||
printf("compress : done\n");
|
||||
|
||||
fclose(outFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
/* decompress */
|
||||
{
|
||||
FILE *inpFp = fopen(ldmFilename, "rb");
|
||||
FILE *outFp = fopen(decFilename, "wb");
|
||||
size_t ret;
|
||||
|
||||
printf("decompress : %s -> %s\n", ldmFilename, decFilename);
|
||||
ret = decompress_file(inpFp, outFp);
|
||||
if (ret) {
|
||||
printf("decompress : failed with code %zu\n", ret);
|
||||
return ret;
|
||||
}
|
||||
printf("decompress : done\n");
|
||||
|
||||
fclose(outFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
/* verify */
|
||||
{
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *decFp = fopen(decFilename, "rb");
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
const int cmp = compare(inpFp, decFp);
|
||||
if(0 == cmp) {
|
||||
printf("verify : OK\n");
|
||||
} else {
|
||||
printf("verify : NG\n");
|
||||
}
|
||||
|
||||
fclose(decFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -1,12 +1,3 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ################################################################
|
||||
|
||||
# This Makefile presumes libzstd is installed, using `sudo make install`
|
||||
|
||||
CFLAGS ?= -O3
|
||||
@ -26,7 +17,6 @@ default: all
|
||||
|
||||
all: main-ldm
|
||||
|
||||
|
||||
#main : ldm.c main.c
|
||||
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
|
||||
3
contrib/long_distance_matching/versions/v0.3/README
Normal file
3
contrib/long_distance_matching/versions/v0.3/README
Normal file
@ -0,0 +1,3 @@
|
||||
This version uses simple lz4-style compression:
|
||||
- A 4-byte hash is inserted into the hash table for every position.
|
||||
- Hash table replacement policy: direct overwrite.
|
||||
@ -1,40 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ################################################################
|
||||
|
||||
# This Makefile presumes libzstd is installed, using `sudo make install`
|
||||
|
||||
CFLAGS ?= -O3
|
||||
DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS)
|
||||
|
||||
LDFLAGS += -lzstd
|
||||
|
||||
.PHONY: default all clean
|
||||
|
||||
default: all
|
||||
|
||||
all: main-ldm
|
||||
|
||||
|
||||
#main : ldm.c main.c
|
||||
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
main-ldm : util.c ldm.c main-ldm.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
|
||||
main main-ldm
|
||||
@echo Cleaning completed
|
||||
|
||||
@ -1,729 +0,0 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
|
||||
#include "ldm.h"
|
||||
#include "util.h"
|
||||
|
||||
#define HASH_EVERY 1
|
||||
|
||||
#define LDM_MEMORY_USAGE 22
|
||||
#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
|
||||
#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
|
||||
#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
|
||||
#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
|
||||
|
||||
#define LDM_OFFSET_SIZE 4
|
||||
|
||||
#define WINDOW_SIZE (1 << 23)
|
||||
#define MAX_WINDOW_SIZE 31
|
||||
#define HASH_SIZE 4
|
||||
#define LDM_HASH_LENGTH 4
|
||||
|
||||
// Should be multiple of four
|
||||
#define MINMATCH 4
|
||||
|
||||
#define ML_BITS 4
|
||||
#define ML_MASK ((1U<<ML_BITS)-1)
|
||||
#define RUN_BITS (8-ML_BITS)
|
||||
#define RUN_MASK ((1U<<RUN_BITS)-1)
|
||||
|
||||
#define LDM_ROLLING_HASH
|
||||
#define COMPUTE_STATS
|
||||
//#define RUN_CHECKS
|
||||
//#define LDM_DEBUG
|
||||
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
|
||||
typedef uint32_t offset_t;
|
||||
typedef uint32_t hash_t;
|
||||
typedef signed char schar;
|
||||
|
||||
|
||||
typedef struct LDM_hashEntry {
|
||||
offset_t offset;
|
||||
} LDM_hashEntry;
|
||||
|
||||
typedef struct LDM_compressStats {
|
||||
U32 numMatches;
|
||||
U32 totalMatchLength;
|
||||
U32 totalLiteralLength;
|
||||
U64 totalOffset;
|
||||
|
||||
U32 numCollisions;
|
||||
U32 numHashInserts;
|
||||
} LDM_compressStats;
|
||||
|
||||
static void LDM_printCompressStats(const LDM_compressStats *stats) {
|
||||
printf("=====================\n");
|
||||
printf("Compression statistics\n");
|
||||
printf("Total number of matches: %u\n", stats->numMatches);
|
||||
printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
|
||||
(double)stats->numMatches);
|
||||
printf("Average literal length: %.1f\n",
|
||||
((double)stats->totalLiteralLength) / (double)stats->numMatches);
|
||||
printf("Average offset length: %.1f\n",
|
||||
((double)stats->totalOffset) / (double)stats->numMatches);
|
||||
printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
|
||||
stats->numCollisions, stats->numHashInserts,
|
||||
stats->numHashInserts == 0 ?
|
||||
1.0 : (100.0 * (double)stats->numCollisions) /
|
||||
(double)stats->numHashInserts);
|
||||
printf("=====================\n");
|
||||
}
|
||||
|
||||
typedef struct LDM_CCtx {
|
||||
size_t isize; /* Input size */
|
||||
size_t maxOSize; /* Maximum output size */
|
||||
|
||||
const BYTE *ibase; /* Base of input */
|
||||
const BYTE *ip; /* Current input position */
|
||||
const BYTE *iend; /* End of input */
|
||||
|
||||
// Maximum input position such that hashing at the position does not exceed
|
||||
// end of input.
|
||||
const BYTE *ihashLimit;
|
||||
|
||||
// Maximum input position such that finding a match of at least the minimum
|
||||
// match length does not exceed end of input.
|
||||
const BYTE *imatchLimit;
|
||||
|
||||
const BYTE *obase; /* Base of output */
|
||||
BYTE *op; /* Output */
|
||||
|
||||
const BYTE *anchor; /* Anchor to start of current (match) block */
|
||||
|
||||
LDM_compressStats stats; /* Compression statistics */
|
||||
|
||||
LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
|
||||
|
||||
const BYTE *lastPosHashed; /* Last position hashed */
|
||||
hash_t lastHash; /* Hash corresponding to lastPosHashed */
|
||||
const BYTE *nextIp;
|
||||
const BYTE *nextPosHashed;
|
||||
hash_t nextHash; /* Hash corresponding to nextPosHashed */
|
||||
|
||||
// Members for rolling hash.
|
||||
U32 lastSum;
|
||||
U32 nextSum;
|
||||
|
||||
unsigned step;
|
||||
|
||||
// DEBUG
|
||||
const BYTE *DEBUG_setNextHash;
|
||||
} LDM_CCtx;
|
||||
|
||||
static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
|
||||
U16 lengthLeft = MINMATCH;
|
||||
const BYTE *curP = p;
|
||||
const BYTE *curMatch = match;
|
||||
|
||||
for (; lengthLeft >= 8; lengthLeft -= 8) {
|
||||
if (LDM_read64(curP) != LDM_read64(curMatch)) {
|
||||
return 0;
|
||||
}
|
||||
curP += 8;
|
||||
curMatch += 8;
|
||||
}
|
||||
if (lengthLeft > 0) {
|
||||
return LDM_read32(curP) == LDM_read32(curMatch);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifdef LDM_ROLLING_HASH
|
||||
/**
|
||||
* Convert a sum computed from LDM_getRollingHash to a hash value in the range
|
||||
* of the hash table.
|
||||
*/
|
||||
static hash_t LDM_sumToHash(U32 sum) {
|
||||
return sum & (LDM_HASH_SIZE_U32 - 1);
|
||||
}
|
||||
|
||||
static U32 LDM_getRollingHash(const char *data, U32 len) {
|
||||
U32 i;
|
||||
U32 s1, s2;
|
||||
const schar *buf = (const schar *)data;
|
||||
|
||||
s1 = s2 = 0;
|
||||
for (i = 0; i < (len - 4); i += 4) {
|
||||
s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
|
||||
(2 * buf[i + 2]) + (buf[i + 3]);
|
||||
s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
|
||||
}
|
||||
for(; i < len; i++) {
|
||||
s1 += buf[i];
|
||||
s2 += s1;
|
||||
}
|
||||
return (s1 & 0xffff) + (s2 << 16);
|
||||
}
|
||||
|
||||
typedef struct LDM_sumStruct {
|
||||
U16 s1, s2;
|
||||
} LDM_sumStruct;
|
||||
|
||||
static U32 LDM_updateRollingHash(U32 sum, U32 len,
|
||||
schar toRemove, schar toAdd) {
|
||||
U32 s1 = (sum & 0xffff) - toRemove + toAdd;
|
||||
U32 s2 = (sum >> 16) - (toRemove * len) + s1;
|
||||
|
||||
return (s1 & 0xffff) + (s2 << 16);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
static hash_t LDM_hashPosition(const void * const p) {
|
||||
return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
|
||||
}
|
||||
*/
|
||||
|
||||
/*
|
||||
static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
|
||||
sumStruct->s1 = sum & 0xffff;
|
||||
sumStruct->s2 = sum >> 16;
|
||||
}
|
||||
*/
|
||||
|
||||
static void LDM_setNextHash(LDM_CCtx *cctx) {
|
||||
|
||||
#ifdef RUN_CHECKS
|
||||
U32 check;
|
||||
if ((cctx->nextIp - cctx->ibase != 1) &&
|
||||
(cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
|
||||
printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
|
||||
cctx->DEBUG_setNextHash - cctx->ibase);
|
||||
}
|
||||
|
||||
cctx->DEBUG_setNextHash = cctx->nextIp;
|
||||
#endif
|
||||
|
||||
// cctx->nextSum = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
|
||||
cctx->nextSum = LDM_updateRollingHash(
|
||||
cctx->lastSum, LDM_HASH_LENGTH,
|
||||
(schar)((cctx->lastPosHashed)[0]),
|
||||
(schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
|
||||
|
||||
#ifdef RUN_CHECKS
|
||||
check = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
|
||||
|
||||
if (check != cctx->nextSum) {
|
||||
printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
|
||||
// printf("INFO: %u %u %u\n", LDM_read32(cctx->nextIp),
|
||||
}
|
||||
#endif
|
||||
cctx->nextPosHashed = cctx->nextIp;
|
||||
cctx->nextHash = LDM_sumToHash(cctx->nextSum);
|
||||
|
||||
#ifdef RUN_CHECKS
|
||||
if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
|
||||
printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
|
||||
cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
|
||||
cctx->ip - cctx->ibase);
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
static void LDM_putHashOfCurrentPositionFromHash(
|
||||
LDM_CCtx *cctx, hash_t hash, U32 sum) {
|
||||
/*
|
||||
if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
|
||||
return;
|
||||
}
|
||||
*/
|
||||
#ifdef COMPUTE_STATS
|
||||
if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
|
||||
offset_t offset = (cctx->hashTable)[hash].offset;
|
||||
cctx->stats.numHashInserts++;
|
||||
if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
|
||||
cctx->stats.numCollisions++;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
(cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
|
||||
cctx->lastPosHashed = cctx->ip;
|
||||
cctx->lastHash = hash;
|
||||
cctx->lastSum = sum;
|
||||
}
|
||||
|
||||
static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
|
||||
#ifdef RUN_CHECKS
|
||||
if (cctx->ip != cctx->nextPosHashed) {
|
||||
printf("CHECK failed: updateLastHashFromNextHash %zu\n", cctx->ip - cctx->ibase);
|
||||
}
|
||||
#endif
|
||||
LDM_putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
|
||||
}
|
||||
|
||||
static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
|
||||
U32 sum = LDM_getRollingHash((const char *)cctx->ip, LDM_HASH_LENGTH);
|
||||
hash_t hash = LDM_sumToHash(sum);
|
||||
#ifdef RUN_CHECKS
|
||||
if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
|
||||
printf("CHECK failed: putHashOfCurrentPosition %zu\n", cctx->ip - cctx->ibase);
|
||||
}
|
||||
#endif
|
||||
// hash_t hash = LDM_hashPosition(cctx->ip);
|
||||
LDM_putHashOfCurrentPositionFromHash(cctx, hash, sum);
|
||||
// printf("Offset %zu\n", cctx->ip - cctx->ibase);
|
||||
}
|
||||
|
||||
#else
|
||||
static hash_t LDM_hash(U32 sequence) {
|
||||
return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
|
||||
}
|
||||
|
||||
static hash_t LDM_hashPosition(const void * const p) {
|
||||
return LDM_hash(LDM_read32(p));
|
||||
}
|
||||
|
||||
static void LDM_putHashOfCurrentPositionFromHash(
|
||||
LDM_CCtx *cctx, hash_t hash) {
|
||||
/*
|
||||
if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
|
||||
return;
|
||||
}
|
||||
*/
|
||||
#ifdef COMPUTE_STATS
|
||||
if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
|
||||
offset_t offset = (cctx->hashTable)[hash].offset;
|
||||
cctx->stats.numHashInserts++;
|
||||
if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
|
||||
cctx->stats.numCollisions++;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
(cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
|
||||
#ifdef RUN_CHECKS
|
||||
if (cctx->ip - cctx->lastPosHashed != 1) {
|
||||
printf("putHashError\n");
|
||||
}
|
||||
#endif
|
||||
cctx->lastPosHashed = cctx->ip;
|
||||
cctx->lastHash = hash;
|
||||
}
|
||||
|
||||
static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
|
||||
hash_t hash = LDM_hashPosition(cctx->ip);
|
||||
LDM_putHashOfCurrentPositionFromHash(cctx, hash);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
static hash_t LDM_hash5(U64 sequence) {
|
||||
static const U64 prime5bytes = 889523592379ULL;
|
||||
static const U64 prime8bytes = 11400714785074694791ULL;
|
||||
const U32 hashLog = LDM_HASHLOG;
|
||||
if (LDM_isLittleEndian())
|
||||
return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
|
||||
else
|
||||
return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
static const BYTE *LDM_getPositionOnHash(
|
||||
hash_t h, void *tableBase, const BYTE *srcBase) {
|
||||
const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
|
||||
return hashTable[h].offset + srcBase;
|
||||
}
|
||||
|
||||
|
||||
static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
|
||||
const BYTE *pInLimit) {
|
||||
const BYTE * const pStart = pIn;
|
||||
while (pIn < pInLimit - 1) {
|
||||
BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
|
||||
if (!diff) {
|
||||
pIn++;
|
||||
pMatch++;
|
||||
continue;
|
||||
}
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
|
||||
void LDM_readHeader(const void *src, size_t *compressSize,
|
||||
size_t *decompressSize) {
|
||||
const U32 *ip = (const U32 *)src;
|
||||
*compressSize = *ip++;
|
||||
*decompressSize = *ip;
|
||||
}
|
||||
|
||||
static void LDM_initializeCCtx(LDM_CCtx *cctx,
|
||||
const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize) {
|
||||
cctx->isize = srcSize;
|
||||
cctx->maxOSize = maxDstSize;
|
||||
|
||||
cctx->ibase = (const BYTE *)src;
|
||||
cctx->ip = cctx->ibase;
|
||||
cctx->iend = cctx->ibase + srcSize;
|
||||
|
||||
#ifdef LDM_ROLLING_HASH
|
||||
cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
|
||||
#else
|
||||
cctx->ihashLimit = cctx->iend - HASH_SIZE;
|
||||
#endif
|
||||
cctx->imatchLimit = cctx->iend - MINMATCH;
|
||||
|
||||
cctx->obase = (BYTE *)dst;
|
||||
cctx->op = (BYTE *)dst;
|
||||
|
||||
cctx->anchor = cctx->ibase;
|
||||
|
||||
memset(&(cctx->stats), 0, sizeof(cctx->stats));
|
||||
memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
|
||||
|
||||
cctx->lastPosHashed = NULL;
|
||||
|
||||
cctx->step = 1;
|
||||
cctx->nextIp = cctx->ip + cctx->step;
|
||||
cctx->nextPosHashed = 0;
|
||||
|
||||
cctx->DEBUG_setNextHash = 0;
|
||||
}
|
||||
|
||||
#ifdef LDM_ROLLING_HASH
|
||||
static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
|
||||
cctx->nextIp = cctx->ip + cctx->step;
|
||||
|
||||
do {
|
||||
hash_t h;
|
||||
U32 sum;
|
||||
// printf("Call A\n");
|
||||
LDM_setNextHash(cctx);
|
||||
// printf("End call a\n");
|
||||
h = cctx->nextHash;
|
||||
sum = cctx->nextSum;
|
||||
cctx->ip = cctx->nextIp;
|
||||
cctx->nextIp += cctx->step;
|
||||
|
||||
if (cctx->ip > cctx->imatchLimit) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
*match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
|
||||
|
||||
// // Compute cctx->nextSum and cctx->nextHash from cctx->nextIp.
|
||||
// LDM_setNextHash(cctx);
|
||||
LDM_putHashOfCurrentPositionFromHash(cctx, h, sum);
|
||||
|
||||
// printf("%u %u\n", cctx->lastHash, cctx->nextHash);
|
||||
} while (cctx->ip - *match > WINDOW_SIZE ||
|
||||
!LDM_isValidMatch(cctx->ip, *match));
|
||||
// LDM_read64(*match) != LDM_read64(cctx->ip));
|
||||
LDM_setNextHash(cctx);
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
|
||||
cctx->nextIp = cctx->ip;
|
||||
|
||||
do {
|
||||
hash_t const h = cctx->nextHash;
|
||||
cctx->ip = cctx->nextIp;
|
||||
cctx->nextIp += cctx->step;
|
||||
|
||||
if (cctx->ip > cctx->imatchLimit) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
*match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
|
||||
|
||||
cctx->nextHash = LDM_hashPosition(cctx->nextIp);
|
||||
LDM_putHashOfCurrentPositionFromHash(cctx, h);
|
||||
|
||||
} while (cctx->ip - *match > WINDOW_SIZE ||
|
||||
!LDM_isValidMatch(cctx->ip, *match));
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Write current block (literals, literal length, match offset,
|
||||
* match length).
|
||||
*
|
||||
* Update input pointer, inserting hashes into hash table along the
|
||||
* way.
|
||||
*/
|
||||
static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
|
||||
unsigned const literalLength = (unsigned)(cctx->ip - cctx->anchor);
|
||||
unsigned const offset = cctx->ip - match;
|
||||
unsigned const matchLength = LDM_count(
|
||||
cctx->ip + MINMATCH, match + MINMATCH, cctx->ihashLimit);
|
||||
BYTE *token = cctx->op++;
|
||||
|
||||
cctx->stats.totalLiteralLength += literalLength;
|
||||
cctx->stats.totalOffset += offset;
|
||||
cctx->stats.totalMatchLength += matchLength + MINMATCH;
|
||||
|
||||
/* Encode the literal length. */
|
||||
if (literalLength >= RUN_MASK) {
|
||||
int len = (int)literalLength - RUN_MASK;
|
||||
*token = (RUN_MASK << ML_BITS);
|
||||
for (; len >= 255; len -= 255) {
|
||||
*(cctx->op)++ = 255;
|
||||
}
|
||||
*(cctx->op)++ = (BYTE)len;
|
||||
} else {
|
||||
*token = (BYTE)(literalLength << ML_BITS);
|
||||
}
|
||||
|
||||
/* Encode the literals. */
|
||||
memcpy(cctx->op, cctx->anchor, literalLength);
|
||||
cctx->op += literalLength;
|
||||
|
||||
/* Encode the offset. */
|
||||
LDM_write32(cctx->op, offset);
|
||||
cctx->op += LDM_OFFSET_SIZE;
|
||||
|
||||
/* Encode match length */
|
||||
if (matchLength >= ML_MASK) {
|
||||
unsigned matchLengthRemaining = matchLength;
|
||||
*token += ML_MASK;
|
||||
matchLengthRemaining -= ML_MASK;
|
||||
LDM_write32(cctx->op, 0xFFFFFFFF);
|
||||
while (matchLengthRemaining >= 4*0xFF) {
|
||||
cctx->op += 4;
|
||||
LDM_write32(cctx->op, 0xffffffff);
|
||||
matchLengthRemaining -= 4*0xFF;
|
||||
}
|
||||
cctx->op += matchLengthRemaining / 255;
|
||||
*(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
|
||||
} else {
|
||||
*token += (BYTE)(matchLength);
|
||||
}
|
||||
|
||||
// LDM_setNextHash(cctx);
|
||||
// cctx->ip = cctx->lastPosHashed + 1;
|
||||
// cctx->nextIp = cctx->ip + cctx->step;
|
||||
// printf("HERE: %zu %zu %zu\n", cctx->ip - cctx->ibase,
|
||||
// cctx->lastPosHashed - cctx->ibase, cctx->nextIp - cctx->ibase);
|
||||
|
||||
cctx->nextIp = cctx->ip + cctx->step;
|
||||
|
||||
while (cctx->ip < cctx->anchor + MINMATCH + matchLength + literalLength) {
|
||||
// printf("Loop\n");
|
||||
if (cctx->ip > cctx->lastPosHashed) {
|
||||
LDM_updateLastHashFromNextHash(cctx);
|
||||
// LDM_putHashOfCurrentPosition(cctx);
|
||||
#ifdef LDM_ROLLING_HASH
|
||||
LDM_setNextHash(cctx);
|
||||
#endif
|
||||
}
|
||||
/*
|
||||
printf("Call b %zu %zu %zu\n",
|
||||
cctx->lastPosHashed - cctx->ibase,
|
||||
cctx->nextIp - cctx->ibase,
|
||||
cctx->ip - cctx->ibase);
|
||||
*/
|
||||
// printf("end call b\n");
|
||||
cctx->ip++;
|
||||
cctx->nextIp++;
|
||||
}
|
||||
|
||||
// printf("There: %zu %zu\n", cctx->ip - cctx->ibase, cctx->lastPosHashed - cctx->ibase);
|
||||
}
|
||||
|
||||
// TODO: srcSize and maxDstSize is unused
|
||||
size_t LDM_compress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize) {
|
||||
LDM_CCtx cctx;
|
||||
LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
|
||||
|
||||
/* Hash the first position and put it into the hash table. */
|
||||
LDM_putHashOfCurrentPosition(&cctx);
|
||||
#ifdef LDM_ROLLING_HASH
|
||||
// LDM_setNextHash(&cctx);
|
||||
// tmp_hash = LDM_updateRollingHash(cctx.lastSum, LDM_HASH_LENGTH,
|
||||
// cctx.ip[0], cctx.ip[LDM_HASH_LENGTH]);
|
||||
// printf("Update test: %u %u\n", tmp_hash, cctx.nextSum);
|
||||
// cctx.ip++;
|
||||
#else
|
||||
cctx.ip++;
|
||||
cctx.nextHash = LDM_hashPosition(cctx.ip);
|
||||
#endif
|
||||
|
||||
// TODO: loop condition is not accurate.
|
||||
while (1) {
|
||||
const BYTE *match;
|
||||
// printf("Start of loop\n");
|
||||
|
||||
/**
|
||||
* Find a match.
|
||||
* If no more matches can be found (i.e. the length of the remaining input
|
||||
* is less than the minimum match length), then stop searching for matches
|
||||
* and encode the final literals.
|
||||
*/
|
||||
if (LDM_findBestMatch(&cctx, &match) != 0) {
|
||||
goto _last_literals;
|
||||
}
|
||||
// printf("End of match finding\n");
|
||||
|
||||
cctx.stats.numMatches++;
|
||||
|
||||
/**
|
||||
* Catch up: look back to extend the match backwards from the found match.
|
||||
*/
|
||||
while (cctx.ip > cctx.anchor && match > cctx.ibase &&
|
||||
cctx.ip[-1] == match[-1]) {
|
||||
// printf("Catch up\n");
|
||||
cctx.ip--;
|
||||
match--;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write current block (literals, literal length, match offset, match
|
||||
* length) and update pointers and hashes.
|
||||
*/
|
||||
LDM_outputBlock(&cctx, match);
|
||||
// printf("End of loop\n");
|
||||
|
||||
// Set start of next block to current input pointer.
|
||||
cctx.anchor = cctx.ip;
|
||||
LDM_updateLastHashFromNextHash(&cctx);
|
||||
// LDM_putHashOfCurrentPosition(&cctx);
|
||||
#ifndef LDM_ROLLING_HASH
|
||||
cctx.ip++;
|
||||
#endif
|
||||
|
||||
/*
|
||||
LDM_putHashOfCurrentPosition(&cctx);
|
||||
printf("Call c\n");
|
||||
LDM_setNextHash(&cctx);
|
||||
printf("End call c\n");
|
||||
cctx.ip++;
|
||||
cctx.nextIp++;
|
||||
*/
|
||||
}
|
||||
_last_literals:
|
||||
/* Encode the last literals (no more matches). */
|
||||
{
|
||||
size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
|
||||
if (lastRun >= RUN_MASK) {
|
||||
size_t accumulator = lastRun - RUN_MASK;
|
||||
*(cctx.op)++ = RUN_MASK << ML_BITS;
|
||||
for(; accumulator >= 255; accumulator -= 255) {
|
||||
*(cctx.op)++ = 255;
|
||||
}
|
||||
*(cctx.op)++ = (BYTE)accumulator;
|
||||
} else {
|
||||
*(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
|
||||
}
|
||||
memcpy(cctx.op, cctx.anchor, lastRun);
|
||||
cctx.op += lastRun;
|
||||
}
|
||||
LDM_printCompressStats(&cctx.stats);
|
||||
return (cctx.op - (const BYTE *)cctx.obase);
|
||||
}
|
||||
|
||||
typedef struct LDM_DCtx {
|
||||
size_t compressSize;
|
||||
size_t maxDecompressSize;
|
||||
|
||||
const BYTE *ibase; /* Base of input */
|
||||
const BYTE *ip; /* Current input position */
|
||||
const BYTE *iend; /* End of source */
|
||||
|
||||
const BYTE *obase; /* Base of output */
|
||||
BYTE *op; /* Current output position */
|
||||
const BYTE *oend; /* End of output */
|
||||
} LDM_DCtx;
|
||||
|
||||
static void LDM_initializeDCtx(LDM_DCtx *dctx,
|
||||
const void *src, size_t compressSize,
|
||||
void *dst, size_t maxDecompressSize) {
|
||||
dctx->compressSize = compressSize;
|
||||
dctx->maxDecompressSize = maxDecompressSize;
|
||||
|
||||
dctx->ibase = src;
|
||||
dctx->ip = (const BYTE *)src;
|
||||
dctx->iend = dctx->ip + dctx->compressSize;
|
||||
dctx->op = dst;
|
||||
dctx->oend = dctx->op + dctx->maxDecompressSize;
|
||||
|
||||
}
|
||||
|
||||
size_t LDM_decompress(const void *src, size_t compressSize,
|
||||
void *dst, size_t maxDecompressSize) {
|
||||
LDM_DCtx dctx;
|
||||
LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
|
||||
|
||||
while (dctx.ip < dctx.iend) {
|
||||
BYTE *cpy;
|
||||
const BYTE *match;
|
||||
size_t length, offset;
|
||||
|
||||
/* Get the literal length. */
|
||||
unsigned const token = *(dctx.ip)++;
|
||||
if ((length = (token >> ML_BITS)) == RUN_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *(dctx.ip)++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
|
||||
/* Copy literals. */
|
||||
cpy = dctx.op + length;
|
||||
memcpy(dctx.op, dctx.ip, length);
|
||||
dctx.ip += length;
|
||||
dctx.op = cpy;
|
||||
|
||||
//TODO : dynamic offset size
|
||||
offset = LDM_read32(dctx.ip);
|
||||
dctx.ip += LDM_OFFSET_SIZE;
|
||||
match = dctx.op - offset;
|
||||
|
||||
/* Get the match length. */
|
||||
length = token & ML_MASK;
|
||||
if (length == ML_MASK) {
|
||||
unsigned s;
|
||||
do {
|
||||
s = *(dctx.ip)++;
|
||||
length += s;
|
||||
} while (s == 255);
|
||||
}
|
||||
length += MINMATCH;
|
||||
|
||||
/* Copy match. */
|
||||
cpy = dctx.op + length;
|
||||
|
||||
// Inefficient for now.
|
||||
while (match < cpy - offset && dctx.op < dctx.oend) {
|
||||
*(dctx.op)++ = *match++;
|
||||
}
|
||||
}
|
||||
return dctx.op - (BYTE *)dst;
|
||||
}
|
||||
|
||||
void LDM_test(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize) {
|
||||
#ifdef LDM_ROLLING_HASH
|
||||
const BYTE *ip = (const BYTE *)src + 1125;
|
||||
U32 sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
|
||||
U32 sum2;
|
||||
++ip;
|
||||
for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
|
||||
sum2 = LDM_updateRollingHash(sum, LDM_HASH_LENGTH,
|
||||
ip[-1], ip[LDM_HASH_LENGTH - 1]);
|
||||
sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
|
||||
printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
#ifndef LDM_H
|
||||
#define LDM_H
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
#define LDM_COMPRESS_SIZE 4
|
||||
#define LDM_DECOMPRESS_SIZE 4
|
||||
#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
|
||||
|
||||
size_t LDM_compress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
size_t LDM_decompress(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
void LDM_readHeader(const void *src, size_t *compressSize,
|
||||
size_t *decompressSize);
|
||||
|
||||
void LDM_test(const void *src, size_t srcSize,
|
||||
void *dst, size_t maxDstSize);
|
||||
|
||||
#endif /* LDM_H */
|
||||
@ -1,480 +0,0 @@
|
||||
// TODO: file size must fit into a U32
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <zstd.h>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include "ldm.h"
|
||||
|
||||
// #define BUF_SIZE 16*1024 // Block size
|
||||
#define DEBUG
|
||||
//#define TEST
|
||||
|
||||
//#define ZSTD
|
||||
|
||||
/* Compress file given by fname and output to oname.
|
||||
* Returns 0 if successful, error code otherwise.
|
||||
*/
|
||||
static int compress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
size_t maxCompressSize, compressSize;
|
||||
|
||||
/* Open the input file. */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Open the output file. */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Find the size of the input file. */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
|
||||
|
||||
/* Go to the location corresponding to the last byte. */
|
||||
/* TODO: fallocate? */
|
||||
if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
|
||||
perror("lseek error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Write a dummy byte at the last location. */
|
||||
if (write(fdout, "", 1) != 1) {
|
||||
perror("write error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the input file. */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the output file */
|
||||
if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef TEST
|
||||
LDM_test(src, statbuf.st_size,
|
||||
dst + LDM_HEADER_SIZE, statbuf.st_size);
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD
|
||||
compressSize = ZSTD_compress(dst, statbuf.st_size,
|
||||
src, statbuf.st_size, 1);
|
||||
#else
|
||||
compressSize = LDM_HEADER_SIZE +
|
||||
LDM_compress(src, statbuf.st_size,
|
||||
dst + LDM_HEADER_SIZE, statbuf.st_size);
|
||||
|
||||
// Write compress and decompress size to header
|
||||
// TODO: should depend on LDM_DECOMPRESS_SIZE write32
|
||||
memcpy(dst, &compressSize, 4);
|
||||
memcpy(dst + 4, &(statbuf.st_size), 4);
|
||||
|
||||
#ifdef DEBUG
|
||||
printf("Compressed size: %zu\n", compressSize);
|
||||
printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Truncate file to compressSize.
|
||||
ftruncate(fdout, compressSize);
|
||||
|
||||
printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
|
||||
(unsigned)statbuf.st_size, (unsigned)compressSize, oname,
|
||||
(double)compressSize / (statbuf.st_size) * 100);
|
||||
|
||||
// Close files.
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Decompress file compressed using LDM_compress.
|
||||
* The input file should have the LDM_HEADER followed by payload.
|
||||
* Returns 0 if succesful, and an error code otherwise.
|
||||
*/
|
||||
static int decompress(const char *fname, const char *oname) {
|
||||
int fdin, fdout;
|
||||
struct stat statbuf;
|
||||
char *src, *dst;
|
||||
size_t compressSize, decompressSize, outSize;
|
||||
|
||||
/* Open the input file. */
|
||||
if ((fdin = open(fname, O_RDONLY)) < 0) {
|
||||
perror("Error in file opening");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Open the output file. */
|
||||
if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
|
||||
perror("Can't create output file");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Find the size of the input file. */
|
||||
if (fstat (fdin, &statbuf) < 0) {
|
||||
perror("Fstat error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the input file. */
|
||||
if ((src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0))
|
||||
== (caddr_t) - 1) {
|
||||
perror("mmap error for input");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Read the header. */
|
||||
LDM_readHeader(src, &compressSize, &decompressSize);
|
||||
|
||||
/* Go to the location corresponding to the last byte. */
|
||||
if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
|
||||
perror("lseek error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* write a dummy byte at the last location */
|
||||
if (write(fdout, "", 1) != 1) {
|
||||
perror("write error");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* mmap the output file */
|
||||
if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
|
||||
MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
|
||||
perror("mmap error for output");
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef ZSTD
|
||||
outSize = ZSTD_decompress(dst, decomrpessed_size,
|
||||
src + LDM_HEADER_SIZE,
|
||||
statbuf.st_size - LDM_HEADER_SIZE);
|
||||
#else
|
||||
outSize = LDM_decompress(
|
||||
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
|
||||
dst, decompressSize);
|
||||
|
||||
printf("Ret size out: %zu\n", outSize);
|
||||
#endif
|
||||
ftruncate(fdout, outSize);
|
||||
|
||||
close(fdin);
|
||||
close(fdout);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Compare two files.
|
||||
* Returns 0 iff they are the same.
|
||||
*/
|
||||
static int compare(FILE *fp0, FILE *fp1) {
|
||||
int result = 0;
|
||||
while (result == 0) {
|
||||
char b0[1024];
|
||||
char b1[1024];
|
||||
const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
|
||||
const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
|
||||
|
||||
result = (int)r0 - (int)r1;
|
||||
|
||||
if (0 == r0 || 0 == r1) break;
|
||||
|
||||
if (0 == result) result = memcmp(b0, b1, r0);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Verify the input file is the same as the decompressed file. */
|
||||
static void verify(const char *inpFilename, const char *decFilename) {
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *decFp = fopen(decFilename, "rb");
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
{
|
||||
const int cmp = compare(inpFp, decFp);
|
||||
if(0 == cmp) {
|
||||
printf("verify : OK\n");
|
||||
} else {
|
||||
printf("verify : NG\n");
|
||||
}
|
||||
}
|
||||
|
||||
fclose(decFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
const char * const exeName = argv[0];
|
||||
char inpFilename[256] = { 0 };
|
||||
char ldmFilename[256] = { 0 };
|
||||
char decFilename[256] = { 0 };
|
||||
|
||||
if (argc < 2) {
|
||||
printf("Wrong arguments\n");
|
||||
printf("Usage:\n");
|
||||
printf("%s FILE\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
snprintf(inpFilename, 256, "%s", argv[1]);
|
||||
snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
|
||||
snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
|
||||
|
||||
printf("inp = [%s]\n", inpFilename);
|
||||
printf("ldm = [%s]\n", ldmFilename);
|
||||
printf("dec = [%s]\n", decFilename);
|
||||
|
||||
|
||||
/* Compress */
|
||||
{
|
||||
struct timeval tv1, tv2;
|
||||
gettimeofday(&tv1, NULL);
|
||||
if (compress(inpFilename, ldmFilename)) {
|
||||
printf("Compress error");
|
||||
return 1;
|
||||
}
|
||||
gettimeofday(&tv2, NULL);
|
||||
printf("Total compress time = %f seconds\n",
|
||||
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec));
|
||||
}
|
||||
|
||||
/* Decompress */
|
||||
{
|
||||
struct timeval tv1, tv2;
|
||||
gettimeofday(&tv1, NULL);
|
||||
if (decompress(ldmFilename, decFilename)) {
|
||||
printf("Decompress error");
|
||||
return 1;
|
||||
}
|
||||
gettimeofday(&tv2, NULL);
|
||||
printf("Total decompress time = %f seconds\n",
|
||||
(double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
|
||||
(double) (tv2.tv_sec - tv1.tv_sec));
|
||||
}
|
||||
/* verify */
|
||||
verify(inpFilename, decFilename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
#if 0
|
||||
static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
|
||||
size_t *size_out) {
|
||||
char *src, *buf = NULL;
|
||||
size_t r = 1;
|
||||
size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
|
||||
|
||||
src = malloc(BUF_SIZE);
|
||||
if (!src) {
|
||||
printf("Not enough memory\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
size = BUF_SIZE + LDM_HEADER_SIZE;
|
||||
buf = malloc(size);
|
||||
if (!buf) {
|
||||
printf("Not enough memory\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
|
||||
for (;;) {
|
||||
k = fread(src, 1, BUF_SIZE, in);
|
||||
if (k == 0)
|
||||
break;
|
||||
count_in += k;
|
||||
|
||||
n = LDM_compress(src, buf, k, BUF_SIZE);
|
||||
|
||||
// n = k;
|
||||
// offset += n;
|
||||
offset = k;
|
||||
count_out += k;
|
||||
|
||||
// k = fwrite(src, 1, offset, out);
|
||||
|
||||
k = fwrite(buf, 1, offset, out);
|
||||
if (k < offset) {
|
||||
if (ferror(out))
|
||||
printf("Write failed\n");
|
||||
else
|
||||
printf("Short write\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
}
|
||||
*size_in = count_in;
|
||||
*size_out = count_out;
|
||||
r = 0;
|
||||
cleanup:
|
||||
free(src);
|
||||
free(buf);
|
||||
return r;
|
||||
}
|
||||
|
||||
static size_t decompress_file(FILE *in, FILE *out) {
|
||||
void *src = malloc(BUF_SIZE);
|
||||
void *dst = NULL;
|
||||
size_t dst_capacity = BUF_SIZE;
|
||||
size_t ret = 1;
|
||||
size_t bytes_written = 0;
|
||||
|
||||
if (!src) {
|
||||
perror("decompress_file(src)");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
while (ret != 0) {
|
||||
/* Load more input */
|
||||
size_t src_size = fread(src, 1, BUF_SIZE, in);
|
||||
void *src_ptr = src;
|
||||
void *src_end = src_ptr + src_size;
|
||||
if (src_size == 0 || ferror(in)) {
|
||||
printf("(TODO): Decompress: not enough input or error reading file\n");
|
||||
//TODO
|
||||
ret = 0;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Allocate destination buffer if it hasn't been allocated already */
|
||||
if (!dst) {
|
||||
dst = malloc(dst_capacity);
|
||||
if (!dst) {
|
||||
perror("decompress_file(dst)");
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO
|
||||
|
||||
/* Decompress:
|
||||
* Continue while there is more input to read.
|
||||
*/
|
||||
while (src_ptr != src_end && ret != 0) {
|
||||
// size_t dst_size = src_size;
|
||||
size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
|
||||
size_t written = fwrite(dst, 1, dst_size, out);
|
||||
// printf("Writing %zu bytes\n", dst_size);
|
||||
bytes_written += dst_size;
|
||||
if (written != dst_size) {
|
||||
printf("Decompress: Failed to write to file\n");
|
||||
goto cleanup;
|
||||
}
|
||||
src_ptr += src_size;
|
||||
src_size = src_end - src_ptr;
|
||||
}
|
||||
|
||||
/* Update input */
|
||||
|
||||
}
|
||||
|
||||
printf("Wrote %zu bytes\n", bytes_written);
|
||||
|
||||
cleanup:
|
||||
free(src);
|
||||
free(dst);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int main2(int argc, char *argv[]) {
|
||||
char inpFilename[256] = { 0 };
|
||||
char ldmFilename[256] = { 0 };
|
||||
char decFilename[256] = { 0 };
|
||||
|
||||
if (argc < 2) {
|
||||
printf("Please specify input filename\n");
|
||||
return 0;
|
||||
}
|
||||
snprintf(inpFilename, 256, "%s", argv[1]);
|
||||
snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
|
||||
snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
|
||||
|
||||
printf("inp = [%s]\n", inpFilename);
|
||||
printf("ldm = [%s]\n", ldmFilename);
|
||||
printf("dec = [%s]\n", decFilename);
|
||||
|
||||
/* compress */
|
||||
{
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *outFp = fopen(ldmFilename, "wb");
|
||||
size_t sizeIn = 0;
|
||||
size_t sizeOut = 0;
|
||||
size_t ret;
|
||||
printf("compress : %s -> %s\n", inpFilename, ldmFilename);
|
||||
ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
|
||||
if (ret) {
|
||||
printf("compress : failed with code %zu\n", ret);
|
||||
return ret;
|
||||
}
|
||||
printf("%s: %zu → %zu bytes, %.1f%%\n",
|
||||
inpFilename, sizeIn, sizeOut,
|
||||
(double)sizeOut / sizeIn * 100);
|
||||
printf("compress : done\n");
|
||||
|
||||
fclose(outFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
/* decompress */
|
||||
{
|
||||
FILE *inpFp = fopen(ldmFilename, "rb");
|
||||
FILE *outFp = fopen(decFilename, "wb");
|
||||
size_t ret;
|
||||
|
||||
printf("decompress : %s -> %s\n", ldmFilename, decFilename);
|
||||
ret = decompress_file(inpFp, outFp);
|
||||
if (ret) {
|
||||
printf("decompress : failed with code %zu\n", ret);
|
||||
return ret;
|
||||
}
|
||||
printf("decompress : done\n");
|
||||
|
||||
fclose(outFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
|
||||
/* verify */
|
||||
{
|
||||
FILE *inpFp = fopen(inpFilename, "rb");
|
||||
FILE *decFp = fopen(decFilename, "rb");
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
const int cmp = compare(inpFp, decFp);
|
||||
if(0 == cmp) {
|
||||
printf("verify : OK\n");
|
||||
} else {
|
||||
printf("verify : NG\n");
|
||||
}
|
||||
|
||||
fclose(decFp);
|
||||
fclose(inpFp);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -1,69 +0,0 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "util.h"
|
||||
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
|
||||
unsigned LDM_isLittleEndian(void) {
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 };
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
U16 LDM_read16(const void *memPtr) {
|
||||
U16 val;
|
||||
memcpy(&val, memPtr, sizeof(val));
|
||||
return val;
|
||||
}
|
||||
|
||||
U16 LDM_readLE16(const void *memPtr) {
|
||||
if (LDM_isLittleEndian()) {
|
||||
return LDM_read16(memPtr);
|
||||
} else {
|
||||
const BYTE *p = (const BYTE *)memPtr;
|
||||
return (U16)((U16)p[0] + (p[1] << 8));
|
||||
}
|
||||
}
|
||||
|
||||
void LDM_write16(void *memPtr, U16 value){
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
void LDM_write32(void *memPtr, U32 value) {
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
void LDM_writeLE16(void *memPtr, U16 value) {
|
||||
if (LDM_isLittleEndian()) {
|
||||
LDM_write16(memPtr, value);
|
||||
} else {
|
||||
BYTE* p = (BYTE *)memPtr;
|
||||
p[0] = (BYTE) value;
|
||||
p[1] = (BYTE)(value>>8);
|
||||
}
|
||||
}
|
||||
|
||||
U32 LDM_read32(const void *ptr) {
|
||||
return *(const U32 *)ptr;
|
||||
}
|
||||
|
||||
U64 LDM_read64(const void *ptr) {
|
||||
return *(const U64 *)ptr;
|
||||
}
|
||||
|
||||
void LDM_copy8(void *dst, const void *src) {
|
||||
memcpy(dst, src, 8);
|
||||
}
|
||||
|
||||
BYTE LDM_readByte(const void *memPtr) {
|
||||
BYTE val;
|
||||
memcpy(&val, memPtr, 1);
|
||||
return val;
|
||||
}
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
#ifndef LDM_UTIL_H
|
||||
#define LDM_UTIL_H
|
||||
|
||||
unsigned LDM_isLittleEndian(void);
|
||||
|
||||
uint16_t LDM_read16(const void *memPtr);
|
||||
|
||||
uint16_t LDM_readLE16(const void *memPtr);
|
||||
|
||||
void LDM_write16(void *memPtr, uint16_t value);
|
||||
|
||||
void LDM_write32(void *memPtr, uint32_t value);
|
||||
|
||||
void LDM_writeLE16(void *memPtr, uint16_t value);
|
||||
|
||||
uint32_t LDM_read32(const void *ptr);
|
||||
|
||||
uint64_t LDM_read64(const void *ptr);
|
||||
|
||||
void LDM_copy8(void *dst, const void *src);
|
||||
|
||||
uint8_t LDM_readByte(const void *ptr);
|
||||
|
||||
|
||||
#endif /* LDM_UTIL_H */
|
||||
@ -1,12 +1,3 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
# ################################################################
|
||||
|
||||
# This Makefile presumes libzstd is installed, using `sudo make install`
|
||||
|
||||
CFLAGS ?= -O3
|
||||
@ -26,15 +17,11 @@ default: all
|
||||
|
||||
all: main-ldm
|
||||
|
||||
|
||||
#main : ldm.c main.c
|
||||
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
main-ldm : util.c ldm.c main-ldm.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
|
||||
main main-ldm
|
||||
main-ldm
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
5
contrib/long_distance_matching/versions/v0.5/README
Normal file
5
contrib/long_distance_matching/versions/v0.5/README
Normal file
@ -0,0 +1,5 @@
|
||||
This version uses simple lz4-style compression with a rolling hash.
|
||||
- A rolling checksum based on rsync's Adler-32 style checksum is used.
|
||||
- The checksum is hashed using lz4's hash function.
|
||||
- Hash table replacement policy: direct overwrite.
|
||||
- The length of input to the hash function can be set with LDM_HASH_LENGTH.
|
||||
@ -19,8 +19,8 @@
|
||||
#define WINDOW_SIZE (1 << 20)
|
||||
|
||||
//These should be multiples of four.
|
||||
#define LDM_HASH_LENGTH 100
|
||||
#define MINMATCH 100
|
||||
#define LDM_HASH_LENGTH 4
|
||||
#define MINMATCH 4
|
||||
|
||||
#define ML_BITS 4
|
||||
#define ML_MASK ((1U<<ML_BITS)-1)
|
||||
@ -95,12 +95,12 @@ typedef struct LDM_CCtx {
|
||||
const BYTE *DEBUG_setNextHash;
|
||||
} LDM_CCtx;
|
||||
|
||||
#ifdef COMPUTE_STATS
|
||||
/**
|
||||
* Outputs compression statistics.
|
||||
*/
|
||||
static void printCompressStats(const LDM_CCtx *cctx) {
|
||||
const LDM_compressStats *stats = &(cctx->stats);
|
||||
#ifdef COMPUTE_STATS
|
||||
printf("=====================\n");
|
||||
printf("Compression statistics\n");
|
||||
printf("Total number of matches: %u\n", stats->numMatches);
|
||||
@ -131,8 +131,8 @@ static void printCompressStats(const LDM_CCtx *cctx) {
|
||||
}
|
||||
|
||||
printf("=====================\n");
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Checks whether the MINMATCH bytes from p are the same as the MINMATCH
|
||||
|
||||
Loading…
Reference in New Issue
Block a user