From b9836b2a030a3756b1f8cf1341875354c84dd207 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 14:01:57 -0400
Subject: [PATCH 01/54] Restore Framebench Tool

This reverts commit 70f14823a46719e81e808d9ed9df90f478bcfd3f.
---
 tests/Makefile     |   6 +-
 tests/framebench.c | 296 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 301 insertions(+), 1 deletion(-)
 create mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index 2b93c9f..f6a4ff3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest datagen framebench
 
 all32: CFLAGS+=-m32
 all32: all
@@ -99,6 +99,9 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -110,6 +113,7 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
+        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
new file mode 100644
index 0000000..18b5ff3
--- /dev/null
+++ b/tests/framebench.c
@@ -0,0 +1,296 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+#define LZ4_STATIC_LINKING_ONLY
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
+
+typedef struct {
+  size_t iter;
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  char *obuf;
+  size_t osize;
+  const char* ibuf;
+  size_t isize;
+  size_t num_ibuf;
+  const LZ4F_CDict* cdict;
+  LZ4F_preferences_t* prefs;
+  const LZ4F_compressOptions_t* options;
+} bench_params_t;
+
+size_t compress_frame(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressFrame_usingCDict(
+    cctx,
+    obuf,
+    osize,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    cdict,
+    prefs);
+  LZ4F_CHECK(oused);
+
+  return oused;
+}
+
+size_t compress_begin(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+  const LZ4F_compressOptions_t* options = p->options;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressUpdate(
+    cctx,
+    obuf,
+    oend - obuf,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    options);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
+  LZ4F_CHECK(oused);
+
+  return obuf - p->obuf;
+}
+
+size_t compress_default(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_stream_t *ctx = p->ctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+uint64_t bench(
+    size_t (*fun)(bench_params_t *),
+    uint64_t repetitions,
+    bench_params_t *params,
+    size_t *osizePtr
+) {
+  struct timespec start, end;
+  size_t i, osize = 0;
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
+
+  for (i = 0; i < repetitions; i++) {
+    size_t o;
+    params->iter = i;
+    o = fun(params);
+    if (!o) return 0;
+    osize += o;
+  }
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+  *osizePtr = osize / repetitions;
+  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+}
+
+int main(int argc, char *argv[]) {
+
+
+  struct stat st;
+  size_t bytes_read;
+
+  const char *dict_fn;
+  size_t dict_size;
+  char *dict_buf;
+  FILE *dict_file;
+
+  const char *in_fn;
+  size_t in_size;
+  size_t num_in_buf;
+  size_t cur_in_buf;
+  char *in_buf;
+  FILE *in_file;
+
+  size_t out_size;
+  char *out_buf;
+
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  LZ4F_CDict *cdict;
+  LZ4F_preferences_t prefs;
+  LZ4F_compressOptions_t options;
+
+  size_t out_used;
+
+  uint64_t time_taken;
+  uint64_t repetitions;
+
+  bench_params_t params;
+
+  if (argc != 3) return 1;
+  dict_fn = argv[1];
+  in_fn = argv[2];
+
+  if (stat(dict_fn, &st)) return 1;
+  dict_size = st.st_size;
+  dict_buf = (char *)malloc(dict_size);
+  if (!dict_buf) return 1;
+  dict_file = fopen(dict_fn, "r");
+  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
+  if (bytes_read != dict_size) return 1;
+
+  if (stat(in_fn, &st)) return 1;
+  in_size = st.st_size;
+  num_in_buf = 256 * 1024 * 1024 / in_size;
+  if (num_in_buf == 0) {
+    num_in_buf = 1;
+  }
+
+  in_buf = (char *)malloc(in_size * num_in_buf);
+  if (!in_buf) return 1;
+  in_file = fopen(in_fn, "r");
+  bytes_read = fread(in_buf, 1, in_size, in_file);
+  if (bytes_read != in_size) return 1;
+
+  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
+    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
+  }
+
+  if (in_size <= 1024) {
+    repetitions = 100000;
+  } else
+  if (in_size <= 16384) {
+    repetitions = 10000;
+  } else
+  if (in_size <= 131072) {
+    repetitions = 1000;
+  } else
+  if (in_size <= 1048576) {
+    repetitions = 100;
+  } else {
+    repetitions = 50;
+  }
+
+  memset(&prefs, 0, sizeof(prefs));
+  prefs.autoFlush = 1;
+  if (in_size < 64 * 1024)
+    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+  prefs.frameInfo.contentSize = in_size;
+
+  memset(&options, 0, sizeof(options));
+  options.stableSrc = 1;
+
+  out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  out_buf = (char *)malloc(out_size);
+  if (!out_buf) return 1;
+
+  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
+  ctx = LZ4_createStream();
+  if (ctx == NULL) return 1;
+
+  cdict = LZ4F_createCDict(dict_buf, dict_size);
+  if (!cdict) return 1;
+
+  fprintf(stderr, "dict  size: %zd\n", dict_size);
+  fprintf(stderr, "input size: %zd\n", in_size);
+
+  params.ctx = ctx;
+  params.cctx = cctx;
+  params.obuf = out_buf;
+  params.osize = out_size;
+  params.ibuf = in_buf;
+  params.isize = in_size;
+  params.num_ibuf = num_in_buf;
+  params.cdict = NULL;
+  params.prefs = &prefs;
+  params.options = &options;
+
+  time_taken = bench(compress_default, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  return 0;
+}
\ No newline at end of file

From 0bc13ab69c603346ff64050f7e3d3a1b3a5765a4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 15:00:59 -0400
Subject: [PATCH 02/54] Add HC Calls to Framebench

---
 tests/framebench.c | 103 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 84 insertions(+), 19 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 18b5ff3..e2bcf40 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -10,6 +10,7 @@
 
 #define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
+#include "lz4hc.h"
 #include "lz4frame.h"
 #include "lz4frame_static.h"
 
@@ -18,12 +19,14 @@
 typedef struct {
   size_t iter;
   LZ4_stream_t *ctx;
+  LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
   char *obuf;
   size_t osize;
   const char* ibuf;
   size_t isize;
   size_t num_ibuf;
+  int clevel;
   const LZ4F_CDict* cdict;
   LZ4F_preferences_t* prefs;
   const LZ4F_compressOptions_t* options;
@@ -117,11 +120,49 @@ size_t compress_extState(bench_params_t *p) {
   const char* ibuf = p->ibuf;
   size_t isize = p->isize;
   size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_hc(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_hc_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_streamHC_t *hcctx = p->hcctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -174,6 +215,7 @@ int main(int argc, char *argv[]) {
   char *out_buf;
 
   LZ4_stream_t *ctx;
+  LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
   LZ4F_CDict *cdict;
   LZ4F_preferences_t prefs;
@@ -216,18 +258,18 @@ int main(int argc, char *argv[]) {
   }
 
   if (in_size <= 1024) {
-    repetitions = 100000;
-  } else
-  if (in_size <= 16384) {
     repetitions = 10000;
   } else
-  if (in_size <= 131072) {
+  if (in_size <= 16384) {
     repetitions = 1000;
   } else
-  if (in_size <= 1048576) {
+  if (in_size <= 131072) {
     repetitions = 100;
+  } else
+  if (in_size <= 1048576) {
+    repetitions = 10;
   } else {
-    repetitions = 50;
+    repetitions = 5;
   }
 
   memset(&prefs, 0, sizeof(prefs));
@@ -249,6 +291,9 @@ int main(int argc, char *argv[]) {
   ctx = LZ4_createStream();
   if (ctx == NULL) return 1;
 
+  hcctx = LZ4_createStreamHC();
+  if (hcctx == NULL) return 1;
+
   cdict = LZ4F_createCDict(dict_buf, dict_size);
   if (!cdict) return 1;
 
@@ -256,41 +301,61 @@ int main(int argc, char *argv[]) {
   fprintf(stderr, "input size: %zd\n", in_size);
 
   params.ctx = ctx;
+  params.hcctx = hcctx;
   params.cctx = cctx;
   params.obuf = out_buf;
   params.osize = out_size;
   params.ibuf = in_buf;
   params.isize = in_size;
   params.num_ibuf = num_in_buf;
+  params.clevel = 1;
   params.cdict = NULL;
   params.prefs = &prefs;
   params.options = &options;
 
   time_taken = bench(compress_default, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_extState, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
   time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  params.cdict = NULL;
+  params.clevel = LZ4HC_CLEVEL_MIN;
+  params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
+
+  time_taken = bench(compress_hc, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_hc_extState, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   return 0;
-}
\ No newline at end of file
+}

From ffb2d8b0ed9615694c156ba1a3f51e9e789d3c4e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 16:26:44 -0400
Subject: [PATCH 03/54] Check Compressed Buffer is Correct in Frame Bench

---
 tests/framebench.c | 79 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 65 insertions(+), 14 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index e2bcf40..a802ce0 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -21,11 +21,16 @@ typedef struct {
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
+  LZ4F_dctx *dctx;
+  const char *dictbuf;
+  size_t dictsize;
   char *obuf;
   size_t osize;
   const char* ibuf;
   size_t isize;
   size_t num_ibuf;
+  char *checkbuf;
+  size_t checksize;
   int clevel;
   const LZ4F_CDict* cdict;
   LZ4F_preferences_t* prefs;
@@ -168,19 +173,46 @@ size_t compress_hc_extState(bench_params_t *p) {
   return obuf - p->obuf;
 }
 
+size_t check_lz4(bench_params_t *p, size_t csize) {
+  (void)csize;
+  memset(p->checkbuf, 0xFF, p->checksize);
+  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize)
+      && !memcmp(p->ibuf, p->checkbuf, p->isize);
+}
+
+size_t check_lz4f(bench_params_t *p, size_t csize) {
+  size_t cp = 0;
+  size_t dp = 0;
+  size_t dsize = p->checksize;
+  size_t cleft = csize;
+  size_t dleft = dsize;
+  size_t ret;
+  memset(p->checkbuf, 0xFF, p->checksize);
+  LZ4F_resetDecompressionContext(p->dctx);
+  do {
+    ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL);
+    cp += cleft;
+    dp += dleft;
+    cleft = csize - cp;
+    dleft = dsize - dp;
+    if (LZ4F_isError(ret)) return 0;
+  } while (cleft);
+  return !memcmp(p->ibuf, p->checkbuf, p->isize);
+}
+
 uint64_t bench(
     size_t (*fun)(bench_params_t *),
+    size_t (*checkfun)(bench_params_t *, size_t),
     uint64_t repetitions,
     bench_params_t *params,
     size_t *osizePtr
 ) {
   struct timespec start, end;
-  size_t i, osize = 0;
+  size_t i, osize = 0, o = 0;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
   for (i = 0; i < repetitions; i++) {
-    size_t o;
     params->iter = i;
     o = fun(params);
     if (!o) return 0;
@@ -189,6 +221,9 @@ uint64_t bench(
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
 
+  o = checkfun(params, o);
+  if (!o) return 0;
+
   *osizePtr = osize / repetitions;
   return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
 }
@@ -214,9 +249,13 @@ int main(int argc, char *argv[]) {
   size_t out_size;
   char *out_buf;
 
+  size_t check_size;
+  char *check_buf;
+
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
+  LZ4F_dctx *dctx;
   LZ4F_CDict *cdict;
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
@@ -257,6 +296,10 @@ int main(int argc, char *argv[]) {
     memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
   }
 
+  check_size = in_size;
+  check_buf = (char *)malloc(check_size);
+  if (!check_buf) return 1;
+
   if (in_size <= 1024) {
     repetitions = 10000;
   } else
@@ -288,6 +331,9 @@ int main(int argc, char *argv[]) {
   if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
   if (cctx == NULL) return 1;
 
+  if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
   ctx = LZ4_createStream();
   if (ctx == NULL) return 1;
 
@@ -303,58 +349,63 @@ int main(int argc, char *argv[]) {
   params.ctx = ctx;
   params.hcctx = hcctx;
   params.cctx = cctx;
+  params.dctx = dctx;
+  params.dictbuf = dict_buf;
+  params.dictsize = dict_size;
   params.obuf = out_buf;
   params.osize = out_size;
   params.ibuf = in_buf;
   params.isize = in_size;
   params.num_ibuf = num_in_buf;
+  params.checkbuf = check_buf;
+  params.checksize = check_size;
   params.clevel = 1;
   params.cdict = NULL;
   params.prefs = &prefs;
   params.options = &options;
 
-  time_taken = bench(compress_default, repetitions, &params, &out_used);
+  time_taken = bench(compress_default, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+  time_taken = bench(compress_extState, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = NULL;
   params.clevel = LZ4HC_CLEVEL_MIN;
   params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
 
-  time_taken = bench(compress_hc, repetitions, &params, &out_used);
+  time_taken = bench(compress_hc, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_hc_extState, repetitions, &params, &out_used);
+  time_taken = bench(compress_hc_extState, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   return 0;

From d4ee7554fc79fecd5db83913536f5c856765bbed Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 16:19:30 -0400
Subject: [PATCH 04/54] Print More Detailed Results Inside bench(), Add
 Compression Levels

---
 tests/framebench.c | 103 ++++++++++++++++++++++-----------------------
 1 file changed, 51 insertions(+), 52 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index a802ce0..6d99b1e 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -111,7 +111,9 @@ size_t compress_default(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  oused = LZ4_compress_default(
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -130,7 +132,10 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_fast_extState_fastReset(
+      ctx,
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -148,7 +153,9 @@ size_t compress_hc(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_HC(
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -167,7 +174,10 @@ size_t compress_hc_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_HC_extStateHC(
+      hcctx,
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -176,7 +186,8 @@ size_t compress_hc_extState(bench_params_t *p) {
 size_t check_lz4(bench_params_t *p, size_t csize) {
   (void)csize;
   memset(p->checkbuf, 0xFF, p->checksize);
-  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize)
+  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
+                                       p->dictbuf, p->dictsize)
       && !memcmp(p->ibuf, p->checkbuf, p->isize);
 }
 
@@ -190,7 +201,9 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
   memset(p->checkbuf, 0xFF, p->checksize);
   LZ4F_resetDecompressionContext(p->dctx);
   do {
-    ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL);
+    ret = LZ4F_decompress_usingDict(
+        p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft,
+        p->dictbuf, p->dictsize, NULL);
     cp += cleft;
     dp += dleft;
     cleft = csize - cp;
@@ -200,15 +213,17 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
   return !memcmp(p->ibuf, p->checkbuf, p->isize);
 }
 
+
 uint64_t bench(
+    char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
     uint64_t repetitions,
-    bench_params_t *params,
-    size_t *osizePtr
+    bench_params_t *params
 ) {
   struct timespec start, end;
   size_t i, osize = 0, o = 0;
+  size_t time_taken;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
@@ -224,8 +239,19 @@ uint64_t bench(
   o = checkfun(params, o);
   if (!o) return 0;
 
-  *osizePtr = osize / repetitions;
-  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+  time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
+               (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+
+  fprintf(
+      stderr,
+      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n",
+      bench_name, params->clevel,
+      params->isize, osize / repetitions,
+      repetitions, time_taken, time_taken / repetitions,
+      ((double) 1000 * params->isize * repetitions) / time_taken
+  );
+
+  return time_taken;
 }
 
 int main(int argc, char *argv[]) {
@@ -260,9 +286,8 @@ int main(int argc, char *argv[]) {
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
 
-  size_t out_used;
+  int clevels[] = {1, 2, 3, 6, 9, 10, 12};
 
-  uint64_t time_taken;
   uint64_t repetitions;
 
   bench_params_t params;
@@ -364,49 +389,23 @@ int main(int argc, char *argv[]) {
   params.prefs = &prefs;
   params.options = &options;
 
-  time_taken = bench(compress_default, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
+    params.clevel = clevels[clevelidx];
+    params.prefs->compressionLevel = clevels[clevelidx];
+    params.cdict = NULL;
 
-  time_taken = bench(compress_extState, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    bench("LZ4_compress_default"         , compress_default    , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , repetitions, &params);
+    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, repetitions, &params);
 
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    params.cdict = cdict;
 
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = NULL;
-  params.clevel = LZ4HC_CLEVEL_MIN;
-  params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
-
-  time_taken = bench(compress_hc, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_hc_extState, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, repetitions, &params);
+  }
 
   return 0;
 }

From a9a62321ffd077d1c4694b2762e84b60b2396c2d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 16:36:35 -0400
Subject: [PATCH 05/54] Auto-Calculate Appropriate Repetition Count

---
 tests/framebench.c | 67 ++++++++++++++++++++--------------------------
 1 file changed, 29 insertions(+), 38 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 6d99b1e..d5d2268 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -218,33 +218,41 @@ uint64_t bench(
     char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
-    uint64_t repetitions,
     bench_params_t *params
 ) {
   struct timespec start, end;
   size_t i, osize = 0, o = 0;
-  size_t time_taken;
+  size_t time_taken = 0;
+  uint64_t total_repetitions = 0;
+  uint64_t repetitions = 2;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
-  for (i = 0; i < repetitions; i++) {
-    params->iter = i;
-    o = fun(params);
-    if (!o) return 0;
-    osize += o;
-  }
+  while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms
+    if (total_repetitions) {
+      repetitions = total_repetitions; // double previous
+    }
 
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+    for (i = 0; i < repetitions; i++) {
+      params->iter = i;
+      o = fun(params);
+      if (!o) return 0;
+      osize += o;
+    }
+
+    if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+    time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
+                 (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+    total_repetitions += repetitions;
+  }
 
   o = checkfun(params, o);
   if (!o) return 0;
 
-  time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
-               (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-
   fprintf(
       stderr,
-      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n",
+      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
       bench_name, params->clevel,
       params->isize, osize / repetitions,
       repetitions, time_taken, time_taken / repetitions,
@@ -288,8 +296,6 @@ int main(int argc, char *argv[]) {
 
   int clevels[] = {1, 2, 3, 6, 9, 10, 12};
 
-  uint64_t repetitions;
-
   bench_params_t params;
 
   if (argc != 3) return 1;
@@ -325,21 +331,6 @@ int main(int argc, char *argv[]) {
   check_buf = (char *)malloc(check_size);
   if (!check_buf) return 1;
 
-  if (in_size <= 1024) {
-    repetitions = 10000;
-  } else
-  if (in_size <= 16384) {
-    repetitions = 1000;
-  } else
-  if (in_size <= 131072) {
-    repetitions = 100;
-  } else
-  if (in_size <= 1048576) {
-    repetitions = 10;
-  } else {
-    repetitions = 5;
-  }
-
   memset(&prefs, 0, sizeof(prefs));
   prefs.autoFlush = 1;
   if (in_size < 64 * 1024)
@@ -394,17 +385,17 @@ int main(int argc, char *argv[]) {
     params.prefs->compressionLevel = clevels[clevelidx];
     params.cdict = NULL;
 
-    bench("LZ4_compress_default"         , compress_default    , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , repetitions, &params);
-    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, repetitions, &params);
-    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, repetitions, &params);
+    bench("LZ4_compress_default"         , compress_default    , check_lz4 , &params);
+    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , &params);
+    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , &params);
+    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , &params);
+    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, &params);
+    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, &params);
 
     params.cdict = cdict;
 
-    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, repetitions, &params);
-    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, &params);
+    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, &params);
   }
 
   return 0;

From 09df7a05f9eb4471a5af7d6ca1bb37bea36ccf91 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 17:16:23 -0400
Subject: [PATCH 06/54] Add Run Name to Frame Bench Output

---
 tests/framebench.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index d5d2268..185d186 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -17,6 +17,7 @@
 #define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
 
 typedef struct {
+  const char *run_name;
   size_t iter;
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
@@ -252,8 +253,8 @@ uint64_t bench(
 
   fprintf(
       stderr,
-      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
-      bench_name, params->clevel,
+      "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
+      params->run_name, bench_name, params->clevel,
       params->isize, osize / repetitions,
       repetitions, time_taken, time_taken / repetitions,
       ((double) 1000 * params->isize * repetitions) / time_taken
@@ -263,7 +264,7 @@ uint64_t bench(
 }
 
 int main(int argc, char *argv[]) {
-
+  char *run_name;
 
   struct stat st;
   size_t bytes_read;
@@ -298,9 +299,10 @@ int main(int argc, char *argv[]) {
 
   bench_params_t params;
 
-  if (argc != 3) return 1;
-  dict_fn = argv[1];
-  in_fn = argv[2];
+  if (argc != 4) return 1;
+  run_name = argv[1];
+  dict_fn = argv[2];
+  in_fn = argv[3];
 
   if (stat(dict_fn, &st)) return 1;
   dict_size = st.st_size;
@@ -362,6 +364,7 @@ int main(int argc, char *argv[]) {
   fprintf(stderr, "dict  size: %zd\n", dict_size);
   fprintf(stderr, "input size: %zd\n", in_size);
 
+  params.run_name = run_name;
   params.ctx = ctx;
   params.hcctx = hcctx;
   params.cctx = cctx;

From 66f0c29aa4d81586a4a621a99c763ab17028c270 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 13:12:28 -0400
Subject: [PATCH 07/54] Fix Framebench Statistics

---
 tests/framebench.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 185d186..1060e99 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -255,9 +255,9 @@ uint64_t bench(
       stderr,
       "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
       params->run_name, bench_name, params->clevel,
-      params->isize, osize / repetitions,
-      repetitions, time_taken, time_taken / repetitions,
-      ((double) 1000 * params->isize * repetitions) / time_taken
+      params->isize, osize / total_repetitions,
+      total_repetitions, time_taken, time_taken / total_repetitions,
+      ((double) 1000 * params->isize * total_repetitions) / time_taken
   );
 
   return time_taken;

From 9d971fd5c3c1e18b225307ef88e529c9b3850fcb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 14:06:23 -0400
Subject: [PATCH 08/54] Switch to Unaligned Samples to Compress Different Blobs
 Each Time

---
 tests/framebench.c | 53 ++++++++++++++++++----------------------------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 1060e99..01e30b3 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -28,6 +28,7 @@ typedef struct {
   char *obuf;
   size_t osize;
   const char* ibuf;
+  const char* isample;
   size_t isize;
   size_t num_ibuf;
   char *checkbuf;
@@ -39,13 +40,11 @@ typedef struct {
 } bench_params_t;
 
 size_t compress_frame(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   const LZ4F_CDict* cdict = p->cdict;
   LZ4F_preferences_t* prefs = p->prefs;
 
@@ -57,7 +56,7 @@ size_t compress_frame(bench_params_t *p) {
     cctx,
     obuf,
     osize,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isample,
     isize,
     cdict,
     prefs);
@@ -67,13 +66,11 @@ size_t compress_frame(bench_params_t *p) {
 }
 
 size_t compress_begin(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   const LZ4F_CDict* cdict = p->cdict;
   LZ4F_preferences_t* prefs = p->prefs;
   const LZ4F_compressOptions_t* options = p->options;
@@ -90,7 +87,7 @@ size_t compress_begin(bench_params_t *p) {
     cctx,
     obuf,
     oend - obuf,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isample,
     isize,
     options);
   LZ4F_CHECK(oused);
@@ -102,74 +99,61 @@ size_t compress_begin(bench_params_t *p) {
 }
 
 size_t compress_default(bench_params_t *p) {
-  size_t iter = p->iter;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
 
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_default(
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf);
+  oused = LZ4_compress_default(isample, obuf, isize, oend - obuf);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_extState(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4_stream_t *ctx = p->ctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
   oused = LZ4_compress_fast_extState_fastReset(
-      ctx,
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf, clevel);
+      ctx, isample, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_hc(bench_params_t *p) {
-  size_t iter = p->iter;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
   oused = LZ4_compress_HC(
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf, clevel);
+      isample, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_hc_extState(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4_streamHC_t *hcctx = p->hcctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
@@ -177,7 +161,7 @@ size_t compress_hc_extState(bench_params_t *p) {
 
   oused = LZ4_compress_HC_extStateHC(
       hcctx,
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isample, obuf,
       isize, oend - obuf, clevel);
   obuf += oused;
 
@@ -189,7 +173,7 @@ size_t check_lz4(bench_params_t *p, size_t csize) {
   memset(p->checkbuf, 0xFF, p->checksize);
   return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
                                        p->dictbuf, p->dictsize)
-      && !memcmp(p->ibuf, p->checkbuf, p->isize);
+      && !memcmp(p->isample, p->checkbuf, p->isize);
 }
 
 size_t check_lz4f(bench_params_t *p, size_t csize) {
@@ -211,7 +195,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
     dleft = dsize - dp;
     if (LZ4F_isError(ret)) return 0;
   } while (cleft);
-  return !memcmp(p->ibuf, p->checkbuf, p->isize);
+  return !memcmp(p->isample, p->checkbuf, p->isize);
 }
 
 
@@ -236,6 +220,11 @@ uint64_t bench(
 
     for (i = 0; i < repetitions; i++) {
       params->iter = i;
+      if (params->num_ibuf == 1) {
+        params->isample = params->ibuf;
+      } else {
+        params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
+      }
       o = fun(params);
       if (!o) return 0;
       osize += o;
@@ -295,7 +284,7 @@ int main(int argc, char *argv[]) {
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
 
-  int clevels[] = {1, 2, 3, 6, 9, 10, 12};
+  int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
 
   bench_params_t params;
 

From f646c512e99032cf4545be1ffcfd2a17323fb586 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 18:08:13 -0400
Subject: [PATCH 09/54] Print Failure Message in Framebench

---
 tests/framebench.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 01e30b3..048df34 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -226,7 +226,14 @@ uint64_t bench(
         params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
       }
       o = fun(params);
-      if (!o) return 0;
+      if (!o) {
+        fprintf(
+            stderr,
+            "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n",
+            params->run_name, bench_name, params->clevel,
+            params->isize);
+        return 0;
+      }
       osize += o;
     }
 
@@ -238,7 +245,14 @@ uint64_t bench(
   }
 
   o = checkfun(params, o);
-  if (!o) return 0;
+  if (!o) {
+    fprintf(
+        stderr,
+        "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n",
+        params->run_name, bench_name, params->clevel,
+        params->isize);
+    return 0;
+  }
 
   fprintf(
       stderr,

From 9dae661b1fcc665a78a9de5341e651310d489f5c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 12:39:31 -0400
Subject: [PATCH 10/54] Fix Cast

---
 tests/framebench.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 048df34..70cfec5 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -200,7 +200,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
 
 
 uint64_t bench(
-    char *bench_name,
+    const char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
     bench_params_t *params
@@ -299,6 +299,7 @@ int main(int argc, char *argv[]) {
   LZ4F_compressOptions_t options;
 
   int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
+  unsigned int clevelidx;
 
   bench_params_t params;
 
@@ -386,7 +387,7 @@ int main(int argc, char *argv[]) {
   params.prefs = &prefs;
   params.options = &options;
 
-  for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
+  for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
     params.clevel = clevels[clevelidx];
     params.prefs->compressionLevel = clevels[clevelidx];
     params.cdict = NULL;

From e0d8add7917f5eafa4894979b5b68b0db1fea6f4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 15:38:41 -0400
Subject: [PATCH 11/54] Fix Framebench Output Buffer Sizing

---
 tests/framebench.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/framebench.c b/tests/framebench.c
index 70cfec5..9752f23 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -347,6 +347,9 @@ int main(int argc, char *argv[]) {
   options.stableSrc = 1;
 
   out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  if ((size_t)LZ4_compressBound(in_size) > out_size) {
+    out_size = LZ4_compressBound(in_size);
+  }
   out_buf = (char *)malloc(out_size);
   if (!out_buf) return 1;
 

From e75153f508073d87e7087e8ef472879773f116f8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 17:47:52 -0400
Subject: [PATCH 12/54] Add Debug Log Statements to HC

---
 lib/lz4frame.c | 21 +++++++++++++++++++++
 lib/lz4hc.c    | 18 ++++++++++++++++--
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 507e4fe..0000023 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -95,6 +95,18 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
+#  include <stdio.h>
+static int g_debuglog_enable = 1;
+#  define DEBUGLOG(l, ...) {                                  \
+                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
+                    fprintf(stderr, __FILE__ ": ");           \
+                    fprintf(stderr, __VA_ARGS__);             \
+                    fprintf(stderr, " \n");                   \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
 
 /*-************************************
 *  Basic Types
@@ -457,6 +469,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+    DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict);
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -479,6 +492,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 
 void LZ4F_freeCDict(LZ4F_CDict* cdict)
 {
+    DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict);
     if (cdict==NULL) return;  /* support free on NULL */
     FREEMEM(cdict->dictContent);
     LZ4_freeStream(cdict->fastCtx);
@@ -530,6 +544,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
+    DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict);
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
@@ -560,6 +575,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     BYTE* headerStart;
 
+    DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict);
+
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
@@ -778,6 +795,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
+    DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
@@ -916,6 +934,9 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
     BYTE* dstPtr = dstStart;
 
     size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
+
+    DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr);
+
     if (LZ4F_isError(flushSize)) return flushSize;
     dstPtr += flushSize;
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 111a09b..12d26c3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -86,6 +86,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 **************************************/
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
+    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
@@ -641,6 +642,8 @@ static int LZ4HC_compress_generic (
         { lz4opt,8192, LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
     };
 
+    DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr);
+
     if (limit == limitedDestSize && dstCapacity < 1) return 0;         /* Impossible to store anything */
     if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0;          /* Unsupported input size (too large or negative) */
 
@@ -706,8 +709,14 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 *  Streaming Functions
 **************************************/
 /* allocation */
-LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
-int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+LZ4_streamHC_t* LZ4_createStreamHC(void) {
+    LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
+    DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
+    return LZ4_streamHCPtr;
+}
+
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+    DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr);
     if (!LZ4_streamHCPtr) return 0;  /* support free on NULL */
     free(LZ4_streamHCPtr);
     return 0;
@@ -718,6 +727,7 @@ int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
 void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+    DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
@@ -732,6 +742,7 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize);
     if (dictSize > 64 KB) {
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
@@ -747,6 +758,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
 
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
+    DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
     if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
@@ -764,6 +776,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
                                             limitedOutput_directive limit)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr);
     /* auto-init if forgotten */
     if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
 
@@ -812,6 +825,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
 {
     LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
     int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+    DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
     if (dictSize > 64 KB) dictSize = 64 KB;
     if (dictSize < 4) dictSize = 0;
     if (dictSize > prefixSize) dictSize = prefixSize;

From f895b9a6c69a059e62de8baca39b878ed1031418 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 13:16:05 -0400
Subject: [PATCH 13/54] Add a Dictionary Context Pointer to the HC Context

---
 lib/lz4hc.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 7a25bee..1b08425 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -141,7 +141,8 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
 #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #include <stdint.h>
 
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
 {
     uint32_t   hashTable[LZ4HC_HASHTABLESIZE];
     uint16_t   chainTable[LZ4HC_MAXD];
@@ -153,11 +154,13 @@ typedef struct
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
     int        compressionLevel;
-} LZ4HC_CCtx_internal;
+    const LZ4HC_CCtx_internal* dictCtx;
+};
 
 #else
 
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
 {
     unsigned int   hashTable[LZ4HC_HASHTABLESIZE];
     unsigned short chainTable[LZ4HC_MAXD];
@@ -169,11 +172,12 @@ typedef struct
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
     int            compressionLevel;
-} LZ4HC_CCtx_internal;
+    const LZ4HC_CCtx_internal* dictCtx;
+};
 
 #endif
 
-#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
+#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */
 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
 union LZ4_streamHC_u {
     size_t table[LZ4_STREAMHCSIZE_SIZET];

From a992d11fc2204b7cb0a70962aa5911a58ed4918c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:21:07 -0400
Subject: [PATCH 14/54] Fully Bounds Check Hash Table Reads

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 12d26c3..6890613 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -222,7 +222,7 @@ LZ4HC_InsertAndGetWiderMatch (
     DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
                 matchIndex, lowLimit);
 
-    while ((matchIndex>=lowLimit) && (nbAttempts)) {
+    while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         if (matchIndex >= dictLimit) {
@@ -286,7 +286,7 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
     return longest;
 }

From fdeead0b09188213e51e037edcf9c9daf88d25c4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 14:49:47 -0400
Subject: [PATCH 15/54] Set dictCtx Rather than memcpy'ing Ctx

---
 lib/lz4frame.c | 9 +++------
 lib/lz4hc.c    | 1 +
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 0000023..821cb49 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -549,12 +549,9 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
-        if (cdict) {
-            memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
-            LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
-        } else {
-            LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level);
-        }
+        LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
+        LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level);
+        internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
     }
 }
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 6890613..16830c2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -729,6 +729,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 

From 6289ff4fb1a560115ffd2c5dff93389d917a6e4e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 16:43:15 -0400
Subject: [PATCH 16/54] Call LZ4F_applyCDict Even on NULL CDict

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 821cb49..ea6a668 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -739,8 +739,8 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
+    LZ4F_applyCDict(ctx, cdict, level);
     if (cdict) {
-        LZ4F_applyCDict(ctx, cdict, level);
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
     return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);

From 66d217e2400e07fd91753881890722b6dc28ee4b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 16:54:36 -0400
Subject: [PATCH 17/54] Perform Lookups into the Dictionary Context

---
 lib/lz4hc.c | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 16830c2..2775713 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -203,6 +203,7 @@ LZ4HC_InsertAndGetWiderMatch (
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
+    const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx;
     const BYTE* const base = hc4->base;
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
@@ -212,6 +213,7 @@ LZ4HC_InsertAndGetWiderMatch (
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
+    U32 dictMatchIndex;
     repeat_state_e repeat = rep_untested;
     size_t srcPatternLength = 0;
 
@@ -288,6 +290,39 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
+    if (dictCtx != NULL) {
+        ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
+        matchIndex = dictMatchIndex + dictIndexDelta;
+        while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+            const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
+
+            if (LZ4_read32(matchPtr) == pattern) {
+                int mlt;
+                int back = 0;
+                const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
+                if (vLimit > iHighLimit) vLimit = iHighLimit;
+                mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+                // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
+                //     mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
+                // }
+                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+                mlt -= back;
+                if (mlt > longest) {
+                    longest = mlt;
+                    *matchpos = base + matchIndex + back;
+                    *startpos = ip + back;
+                }
+            }
+
+            {
+                U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
+                dictMatchIndex -= nextOffset;
+                matchIndex -= nextOffset;
+            }
+        }
+    }
+
     return longest;
 }
 

From 595ea582890031c30ded483130116ec6a0179a5c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 16:59:50 -0400
Subject: [PATCH 18/54] Avoid Resetting Hash Table

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2775713..b6d9bf0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
     hc4->base = start - 64 KB;

From b6c35ed6422f60a7a542ed237cadfd9dfe76c219 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 17:00:04 -0400
Subject: [PATCH 19/54] Avoid Resetting Chain Table

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b6d9bf0..0caf001 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
     hc4->base = start - 64 KB;
     hc4->end = start;

From b88a0b4e8812777d1e715ab095738197f7f8eb96 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 26 Mar 2018 11:43:52 -0400
Subject: [PATCH 20/54] Only Perform Dict Lookup if Attempts Remain

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0caf001..e005cd7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -288,7 +288,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
-    if (dictCtx != NULL) {
+    if (dictCtx != NULL && nbAttempts) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + dictIndexDelta;

From 4f7b7a8ffa3032a64aa173f7d8bfb7c258316154 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 12:25:05 -0400
Subject: [PATCH 21/54] Clear Tables on Dict Load

---
 lib/lz4hc.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e005cd7..85419f3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -95,6 +95,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     hc4->lowLimit = 64 KB;
 }
 
+static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
+{
+    MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+    MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
+}
+
 
 /* Update chains up to ip (excluded) */
 LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
@@ -782,6 +788,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictSize = 64 KB;
     }
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;

From 22db704a73e38a869d846fa52ddcbb0a214a6c43 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 12:26:54 -0400
Subject: [PATCH 22/54] Shift Dict Limit Checks out of the Loop

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 85419f3..8ac650e 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -294,11 +294,11 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
-    if (dictCtx != NULL && nbAttempts) {
+    if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + dictIndexDelta;
-        while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {

From 895e76cc20f366ea9f2999abc9ff3e0dd11b4222 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 15:20:47 -0400
Subject: [PATCH 23/54] Push Previous Compression Offsets into the Past

---
 lib/lz4hc.c | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 8ac650e..c56d976 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -84,23 +84,29 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 /**************************************
 *  HC Compression
 **************************************/
-static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
-{
-    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    hc4->nextToUpdate = 64 KB;
-    hc4->base = start - 64 KB;
-    hc4->end = start;
-    hc4->dictBase = start - 64 KB;
-    hc4->dictLimit = 64 KB;
-    hc4->lowLimit = 64 KB;
-}
-
 static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 {
+    DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
 }
 
+static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
+{
+    U32 startingOffset = hc4->end - hc4->base + 64 KB;
+    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
+    if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
+        LZ4HC_clearTables(hc4);
+        startingOffset = 64 KB;
+    }
+    hc4->nextToUpdate = startingOffset;
+    hc4->base = start - startingOffset;
+    hc4->end = start;
+    hc4->dictBase = start - startingOffset;
+    hc4->dictLimit = startingOffset;
+    hc4->lowLimit = startingOffset;
+}
+
 
 /* Update chains up to ip (excluded) */
 LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
@@ -751,6 +757,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
     DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
+    LZ4_streamHCPtr->internal_donotuse.end = (void *)-1;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
 }
 
@@ -767,6 +776,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);

From bdd7af6f71c235cea53912d0eec3b94555e9ffde Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 15:59:00 -0400
Subject: [PATCH 24/54] Don't Bother Clearing Chain Table for Working Contexts

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c56d976..673b4b3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -96,7 +96,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     U32 startingOffset = hc4->end - hc4->base + 64 KB;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
-        LZ4HC_clearTables(hc4);
+        MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
         startingOffset = 64 KB;
     }
     hc4->nextToUpdate = startingOffset;

From a1beba13f7363df4b5d7ca9afe51963176df068d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 16:32:26 -0400
Subject: [PATCH 25/54] Reset Stream in LZ4_compress_HC

---
 lib/lz4hc.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 673b4b3..4a52f0c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -732,7 +732,10 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int cSize;
+    statePtr->internal_donotuse.end = (void *)-1;
+    LZ4_resetStreamHC(statePtr, compressionLevel);
+    cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif

From 221211d7d04478d74eec9fd76ccd98a73bd394ee Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 16:32:40 -0400
Subject: [PATCH 26/54] Fix Offset Math

---
 lib/lz4hc.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4a52f0c..5479159 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -93,12 +93,13 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
-    U32 startingOffset = hc4->end - hc4->base + 64 KB;
+    uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
         MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
-        startingOffset = 64 KB;
+        startingOffset = 0;
     }
+    startingOffset += MAX_DISTANCE;
     hc4->nextToUpdate = startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;

From 8f9a2db0e1ea492e836dd074dd0ea62720fb6169 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 6 Apr 2018 19:24:22 -0400
Subject: [PATCH 27/54] Fix Some Cast/Conversion Warnings

---
 lib/lz4hc.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5479159..11d820f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -100,12 +100,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
         startingOffset = 0;
     }
     startingOffset += MAX_DISTANCE;
-    hc4->nextToUpdate = startingOffset;
+    hc4->nextToUpdate = (U32) startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;
     hc4->dictBase = start - startingOffset;
-    hc4->dictLimit = startingOffset;
-    hc4->lowLimit = startingOffset;
+    hc4->dictLimit = (U32) startingOffset;
+    hc4->lowLimit = (U32) startingOffset;
 }
 
 
@@ -304,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch (
     if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + dictIndexDelta;
+        matchIndex = dictMatchIndex + (int)dictIndexDelta;
         while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
@@ -314,9 +314,11 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
-                //     mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
-                // }
+                /*
+                if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
+                    mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
+                }
+                */
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
@@ -734,7 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t* const statePtr = &state;
 #endif
     int cSize;
-    statePtr->internal_donotuse.end = (void *)-1;
+    statePtr->internal_donotuse.end = (const BYTE*)-1;
     LZ4_resetStreamHC(statePtr, compressionLevel);
     cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
@@ -761,7 +763,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
     DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
-    LZ4_streamHCPtr->internal_donotuse.end = (void *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;

From 8db291bc1dc95ab58904b6961a5ac0967b148952 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 17:41:15 -0400
Subject: [PATCH 28/54] Remove Match Upper Bounds Check

---
 lib/lz4hc.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 11d820f..fa2cbb7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -235,7 +235,7 @@ LZ4HC_InsertAndGetWiderMatch (
     DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
                 matchIndex, lowLimit);
 
-    while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) {
+    while ((matchIndex>=lowLimit) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         if (matchIndex >= dictLimit) {
@@ -299,7 +299,7 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
     if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
@@ -720,6 +720,7 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
+    LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel);
     LZ4HC_init (ctx, (const BYTE*)src);
     if (dstCapacity < LZ4_compressBound(srcSize))
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
@@ -735,10 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int cSize;
-    statePtr->internal_donotuse.end = (const BYTE*)-1;
-    LZ4_resetStreamHC(statePtr, compressionLevel);
-    cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif
@@ -750,6 +748,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
 int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+    LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel);
     LZ4HC_init(ctx, (const BYTE*) source);
     return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
 }
@@ -782,7 +781,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
-    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);

From 3591fe8ab8b4391d1f9104b38b9ac19deac2a3e8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 15:09:59 -0400
Subject: [PATCH 29/54] Add Fast Reset Paths

---
 lib/lz4hc.c | 21 +++++++++++++++++++--
 lib/lz4hc.h |  2 ++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index fa2cbb7..9deb90d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -716,11 +716,11 @@ static int LZ4HC_compress_generic (
 
 int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
 
-int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
-    LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel);
+    LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel);
     LZ4HC_init (ctx, (const BYTE*)src);
     if (dstCapacity < LZ4_compressBound(srcSize))
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
@@ -728,6 +728,13 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
 }
 
+int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+{
+    if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
+    LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel);
+    return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel);
+}
+
 int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
@@ -787,6 +794,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 
+void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+    LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+    DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
+}
+
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 1b08425..3b0d0d3 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -270,7 +270,9 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
 #endif   /* LZ4_HC_SLO_098092834 */
 #endif   /* LZ4_HC_STATIC_LINKING_ONLY */

From 61c7ceffede9607721ca496ac13d788625f9f668 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 15:25:48 -0400
Subject: [PATCH 30/54] Use Fast Reset API in LZ4F

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index ea6a668..d087a94 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -743,7 +743,7 @@ static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSi
     if (cdict) {
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
-    return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+    return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);
 }
 
 static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)

From 0a2abacd909d0e1e1be9a82de6c2a693264f8afa Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 20:23:19 -0400
Subject: [PATCH 31/54] Use Fast Reset in LZ4F Again

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d087a94..a43f595 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -550,7 +550,7 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
         LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
-        LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level);
+        LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
         internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
     }
 }

From 22e16d5b509c56ad350ae42664ddcd3da7b97f1f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 13:55:58 -0400
Subject: [PATCH 32/54] Split DictCtx-using Code Into Separate Inlining Chain

---
 lib/lz4hc.c | 94 +++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 74 insertions(+), 20 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9deb90d..5c2b001 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -79,6 +79,8 @@
 
 static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
 
+/*===   Enums   ===*/
+typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
 
 
 /**************************************
@@ -210,7 +212,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE** matchpos,
     const BYTE** startpos,
     const int maxNbAttempts,
-    const int patternAnalysis)
+    const int patternAnalysis,
+    const dictCtx_directive dict)
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
@@ -301,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
-    if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
+    if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
@@ -344,13 +347,14 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
                                  const BYTE* const ip, const BYTE* const iLimit,
                                  const BYTE** matchpos,
                                  const int maxNbAttempts,
-                                 const int patternAnalysis)
+                                 const int patternAnalysis,
+                                 const dictCtx_directive dict)
 {
     const BYTE* uselessPtr = ip;
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict);
 }
 
 
@@ -440,7 +444,8 @@ static int LZ4HC_compress_hashChain (
     int* srcSizePtr,
     int const maxOutputSize,
     unsigned maxNbAttempts,
-    limitedOutput_directive limit
+    const limitedOutput_directive limit,
+    const dictCtx_directive dict
     )
 {
     const int inputSize = *srcSizePtr;
@@ -472,7 +477,7 @@ static int LZ4HC_compress_hashChain (
 
     /* Main Loop */
     while (ip <= mflimit) {
-        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
+        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict);
         if (ml<MINMATCH) { ip++; continue; }
 
         /* saved, in case we would skip too much */
@@ -484,7 +489,7 @@ _Search2:
         if (ip+ml <= mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, patternAnalysis);
+                            maxNbAttempts, patternAnalysis, dict);
         else
             ml2 = ml;
 
@@ -531,7 +536,7 @@ _Search3:
         if (start2 + ml2 <= mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, patternAnalysis);
+                            maxNbAttempts, patternAnalysis, dict);
         else
             ml3 = ml2;
 
@@ -655,17 +660,19 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
     const char* const source, char* dst,
     int* srcSizePtr, int dstCapacity,
     int const nbSearches, size_t sufficient_len,
-    limitedOutput_directive limit, int const fullUpdate);
+    const limitedOutput_directive limit, int const fullUpdate,
+    const dictCtx_directive dict);
 
 
-static int LZ4HC_compress_generic (
+LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     LZ4HC_CCtx_internal* const ctx,
     const char* const src,
     char* const dst,
     int* const srcSizePtr,
     int const dstCapacity,
     int cLevel,
-    limitedOutput_directive limit
+    const limitedOutput_directive limit,
+    const dictCtx_directive dict
     )
 {
     typedef enum { lz4hc, lz4opt } lz4hc_strat_e;
@@ -704,12 +711,57 @@ static int LZ4HC_compress_generic (
         if (cParam.strat == lz4hc)
             return LZ4HC_compress_hashChain(ctx,
                                 src, dst, srcSizePtr, dstCapacity,
-                                cParam.nbSearches, limit);
+                                cParam.nbSearches, limit, dict);
         assert(cParam.strat == lz4opt);
         return LZ4HC_compress_optimal(ctx,
                             src, dst, srcSizePtr, dstCapacity,
                             cParam.nbSearches, cParam.targetLength, limit,
-                            cLevel == LZ4HC_CLEVEL_MAX);  /* ultra mode */
+                            cLevel == LZ4HC_CLEVEL_MAX, dict);  /* ultra mode */
+    }
+}
+
+static int LZ4HC_compress_generic_noDictCtx (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    assert(ctx->dictCtx == NULL);
+    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx);
+}
+
+static int LZ4HC_compress_generic_dictCtx (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    assert(ctx->dictCtx != NULL);
+    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+}
+
+static int LZ4HC_compress_generic (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    if (ctx->dictCtx == NULL) {
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else {
+        return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     }
 }
 
@@ -1019,7 +1071,8 @@ typedef struct {
 LZ4_FORCE_INLINE LZ4HC_match_t
 LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
                       const BYTE* ip, const BYTE* const iHighLimit,
-                      int minLen, int nbSearches)
+                      int minLen, int nbSearches,
+                      const dictCtx_directive dict)
 {
     LZ4HC_match_t match = { 0 , 0 };
     const BYTE* matchPtr = NULL;
@@ -1028,7 +1081,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
     int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
                                 ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */);
+                                nbSearches, 1 /* patternAnalysis */, dict);
     if (matchLength <= minLen) return match;
     match.len = matchLength;
     match.off = (int)(ip-matchPtr);
@@ -1044,8 +1097,9 @@ static int LZ4HC_compress_optimal (
     int dstCapacity,
     int const nbSearches,
     size_t sufficient_len,
-    limitedOutput_directive limit,
-    int const fullUpdate
+    const limitedOutput_directive limit,
+    int const fullUpdate,
+    const dictCtx_directive dict
     )
 {
 #define TRAILING_LITERALS 3
@@ -1073,7 +1127,7 @@ static int LZ4HC_compress_optimal (
          int best_mlen, best_off;
          int cur, last_match_pos = 0;
 
-         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
+         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict);
          if (firstMatch.len==0) { ip++; continue; }
 
          if ((size_t)firstMatch.len > sufficient_len) {
@@ -1143,10 +1197,10 @@ static int LZ4HC_compress_optimal (
 
              DEBUGLOG(7, "search at rPos:%u", cur);
              if (fullUpdate)
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict);
              else
                  /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict);
              if (!newMatch.len) continue;
 
              if ( ((size_t)newMatch.len > sufficient_len)

From b67de2a327644607c034250d105ba9374b0897e8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 15:52:04 -0400
Subject: [PATCH 33/54] Force Inline on HashChain

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5c2b001..c050c59 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -437,7 +437,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     return 0;
 }
 
-static int LZ4HC_compress_hashChain (
+LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     LZ4HC_CCtx_internal* const ctx,
     const char* const source,
     char* const dest,

From 0abc23f72e44dd9af86e7fb61a2497cee81ed320 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 13:02:55 -0400
Subject: [PATCH 34/54] Copy DictCtx into Working Context on Inputs Larger than
 4 KB

---
 lib/lz4hc.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c050c59..c201559 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -720,6 +720,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     }
 }
 
+static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock);
+
 static int LZ4HC_compress_generic_noDictCtx (
     LZ4HC_CCtx_internal* const ctx,
     const char* const src,
@@ -745,7 +747,16 @@ static int LZ4HC_compress_generic_dictCtx (
     )
 {
     assert(ctx->dictCtx != NULL);
-    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+    if (*srcSizePtr > 4 KB) {
+        memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
+        LZ4HC_setExternalDict(ctx, (const BYTE *)src);
+        ctx->compressionLevel = cLevel;
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else {
+        int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+        ctx->dictCtx = NULL;
+        return result;
+    }
 }
 
 static int LZ4HC_compress_generic (

From f4b13e17ea110498c41ba3ac8dc6df0438c37d29 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 17:51:10 -0400
Subject: [PATCH 35/54] Don't Clear the Dictionary Context Until No Longer
 Useful

---
 lib/lz4hc.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c201559..9a02787 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -746,15 +746,18 @@ static int LZ4HC_compress_generic_dictCtx (
     limitedOutput_directive limit
     )
 {
+    size_t position = ctx->end - ctx->base - ctx->lowLimit;
     assert(ctx->dictCtx != NULL);
-    if (*srcSizePtr > 4 KB) {
+    if (position >= 64 KB) {
+        ctx->dictCtx = NULL;
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else if (position == 0 && *srcSizePtr > 4 KB) {
         memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
         LZ4HC_setExternalDict(ctx, (const BYTE *)src);
         ctx->compressionLevel = cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
         int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
-        ctx->dictCtx = NULL;
         return result;
     }
 }

From 14c577d4c9332197dc0003833e043608b4b4b239 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 19:36:34 -0400
Subject: [PATCH 36/54] Fix Signedness of Comparison

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9a02787..20df8fa 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -308,7 +308,7 @@ LZ4HC_InsertAndGetWiderMatch (
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {

From 0064e8ebc7a475d27ba658ca92e40de75c9cac72 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 13:14:37 -0400
Subject: [PATCH 37/54] Remove Commented Out Support for Match Continuation
 over Segment Boundary

---
 lib/lz4hc.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 20df8fa..e09d8e0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -317,11 +317,6 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                /*
-                if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
-                    mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
-                }
-                */
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {

From 8f118cf6e9f3030f52414177d93ab447e8e24128 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:08:06 -0400
Subject: [PATCH 38/54] Remove inputBuffer from Context, Work Around its
 Absence

---
 lib/lz4hc.c | 13 +++++++------
 lib/lz4hc.h | 11 ++++++++---
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e09d8e0..dff42f0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -999,8 +999,8 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
 {
     LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1;   /* Error : pointer is not aligned for pointer (32 or 64 bits) */
+    LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel);
     LZ4HC_init(ctx, (const BYTE*)inputBuffer);
-    ctx->inputBuffer = inputBuffer;
     return 0;
 }
 
@@ -1008,9 +1008,8 @@ void* LZ4_createHC (const char* inputBuffer)
 {
     LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
+    LZ4_resetStreamHC(hc4, 0 /* compressionLevel */);
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
-    assert(sizeof(size_t) == sizeof(void*));
-    hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer;   /* ugly hack, circumvent -Wcast-qual */
     return hc4;
 }
 
@@ -1032,9 +1031,11 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c
 
 char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
 {
-    LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
-    int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
-    return (char*)(hc4->inputBuffer) + dictSize;
+    LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data;
+    const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit;
+    LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel);
+    /* avoid const char * -> char * conversion warning :( */
+    return (char *)(uptrval)bufferStart;
 }
 
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 3b0d0d3..cfc5d9e 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -149,7 +149,6 @@ struct LZ4HC_CCtx_internal
     const uint8_t* end;         /* next block here to continue on current prefix */
     const uint8_t* base;        /* All index relative to this position */
     const uint8_t* dictBase;    /* alternate base for extDict */
-    void*      inputBuffer;     /* deprecated */
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
@@ -167,7 +166,6 @@ struct LZ4HC_CCtx_internal
     const unsigned char* end;        /* next block here to continue on current prefix */
     const unsigned char* base;       /* All index relative to this position */
     const unsigned char* dictBase;   /* alternate base for extDict */
-    void*          inputBuffer;      /* deprecated */
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
@@ -210,7 +208,14 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
-/* Deprecated Streaming functions; should no longer be used */
+/* Obsolete streaming functions; degraded functionality; do not use!
+ *
+ * In order to perform streaming compression, these functions depended on data
+ * that is no longer tracked in the state. They have been preserved as well as
+ * possible: using them will still produce a correct output. However, use of
+ * LZ4_slideInputBufferHC() will truncate the history of the stream, rather
+ * than preserve a window-sized chunk of history.
+ */
 LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
 LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
 LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API   int   LZ4_freeHC (void* LZ4HC_Data);

From ca833f928f64d0b88fbfd2a8a8bc11b8333e487d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:16:27 -0400
Subject: [PATCH 39/54] Also Reset the Chain Table

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index dff42f0..124da86 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
-        MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+        LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
     startingOffset += MAX_DISTANCE;

From d7347f9eeaffe5351886b6174f9974bd541b9b7d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:52:13 -0400
Subject: [PATCH 40/54] Add API for Attaching Dictionaries

---
 lib/lz4frame.c |  3 +--
 lib/lz4hc.c    |  3 +++
 lib/lz4hc.h    | 27 +++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a43f595..d4d9397 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -549,9 +549,8 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
-        LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
         LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
-        internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
+        LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);
     }
 }
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 124da86..690015f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -887,6 +887,9 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
     return dictSize;
 }
 
+void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) {
+    working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+}
 
 /* compression */
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index cfc5d9e..fb8397e 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -279,5 +279,32 @@ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLeve
 
 int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
+/*! LZ4_attach_HC_dictionary() :
+ *  This is an experimental API that allows for the efficient use of a
+ *  static dictionary many times.
+ *
+ *  Rather than re-loading the dictionary buffer into a working context before
+ *  each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a
+ *  working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,
+ *  in which the working stream references the dictionary stream in-place.
+ *
+ *  Several assumptions are made about the state of the dictionary stream.
+ *  Currently, only streams which have been prepared by LZ4_loadDictHC() should
+ *  be expected to work.
+ *
+ *  Alternatively, the provided dictionary stream pointer may be NULL, in which
+ *  case any existing dictionary stream is unset.
+ *
+ *  A dictionary should only be attached to a stream without any history (i.e.,
+ *  a stream that has just been reset).
+ *
+ *  The dictionary will remain attached to the working stream only for the
+ *  current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the
+ *  dictionary context association from the working stream. The dictionary
+ *  stream (and source buffer) must remain in-place / accessible / unchanged
+ *  through the lifetime of the stream session.
+ */
+LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream);
+
 #endif   /* LZ4_HC_SLO_098092834 */
 #endif   /* LZ4_HC_STATIC_LINKING_ONLY */

From 3f087cf1cbcb0e0082ead149a621d259cb57e7ba Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:00:53 -0400
Subject: [PATCH 41/54] Add Comments on New Public APIs

---
 lib/lz4hc.h | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index fb8397e..a762d3f 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -275,8 +275,37 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_resetStreamHC_fast() :
+ *  When an LZ4_streamHC_t is known to be in a internally coherent state,
+ *  it can often be prepared for a new compression with almost no work, only
+ *  sometimes falling back to the full, expensive reset that is always required
+ *  when the stream is in an indeterminate state (i.e., the reset performed by
+ *  LZ4_resetStreamHC()).
+ *
+ *  LZ4_streamHCs are guaranteed to be in a valid state when:
+ *  - returned from LZ4_createStreamHC()
+ *  - reset by LZ4_resetStreamHC()
+ *  - memset(stream, 0, sizeof(LZ4_streamHC_t))
+ *  - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast()
+ *  - the stream was in a valid state and was then used in any compression call
+ *    that returned success
+ *  - the stream was in an indeterminate state and was used in a compression
+ *    call that fully reset the state (LZ4_compress_HC_extStateHC()) and that
+ *    returned success
+ */
 void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_compress_HC_extStateHC_fastReset() :
+ *  A variant of LZ4_compress_HC_extStateHC().
+ *
+ *  Using this variant avoids an expensive initialization step. It is only safe
+ *  to call if the state buffer is known to be correctly initialized already
+ *  (see above comment on LZ4_resetStreamHC_fast() for a definition of
+ *  "correctly initialized"). From a high level, the difference is that this
+ *  function initializes the provided state with a call to
+ *  LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a
+ *  call to LZ4_resetStreamHC().
+ */
 int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
 /*! LZ4_attach_HC_dictionary() :

From 209c9c29d1baf5b2f4fb1fc38b2612dd95d1b9a1 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:16:41 -0400
Subject: [PATCH 42/54] Add Some Simple Fuzzer Tests

---
 tests/fuzzer.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4f..8b21f8e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -446,7 +446,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test compression HC using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed")
+
+        /* Test compression HC using fast reset external state */
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()");
+        ret = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
 
         /* Test compression using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
@@ -810,6 +815,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
         }
 
+        /* Compress HC using external dictionary stream */
+        FUZ_DISPLAYTEST();
+        {
+            LZ4_streamHC_t LZ4_streamHC;
+
+            LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
+            LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST();
+            decodedBuffer[blockSize] = 0;
+            ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+            FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+            FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+            {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+                if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+                FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+            }
+        }
+
         /* Compress HC continue destSize */
         FUZ_DISPLAYTEST();
         {   int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;

From 7874cf06b3307a7ba5efdd141d068887480aaf11 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:30:08 -0400
Subject: [PATCH 43/54] Consts and Asserts and Other Minor Nits

---
 lib/lz4hc.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 690015f..e5eb11d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -305,10 +305,13 @@ LZ4HC_InsertAndGetWiderMatch (
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
     if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
-        ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        /* bounds check, since we need to downcast */
+        assert(dictIndexDelta <= 1 GB);
+        assert(dictIndexDelta >= -1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {
@@ -741,7 +744,7 @@ static int LZ4HC_compress_generic_dictCtx (
     limitedOutput_directive limit
     )
 {
-    size_t position = ctx->end - ctx->base - ctx->lowLimit;
+    const size_t position = ctx->end - ctx->base - ctx->lowLimit;
     assert(ctx->dictCtx != NULL);
     if (position >= 64 KB) {
         ctx->dictCtx = NULL;
@@ -752,8 +755,7 @@ static int LZ4HC_compress_generic_dictCtx (
         ctx->compressionLevel = cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
-        int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
-        return result;
+        return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
     }
 }
 
@@ -804,7 +806,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif

From 1d2500d44e97a46eb447745d02652e02435baadb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:10:47 -0400
Subject: [PATCH 44/54] Handle Index Underflows Safely

---
 lib/lz4hc.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e5eb11d..843b539 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -221,7 +221,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const base = hc4->base;
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE;
+    const U32 ipIndex = (U32)(ip - base);
+    const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
     int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
@@ -304,14 +305,12 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
-    if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
-        const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
-        /* bounds check, since we need to downcast */
-        assert(dictIndexDelta <= 1 GB);
-        assert(dictIndexDelta >= -1 GB);
+    if (dict == usingDictCtx && nbAttempts && ipIndex - lowLimit < MAX_DISTANCE) {
+        size_t const dictEndOffset = dictCtx->end - dictCtx->base;
+        assert(dictEndOffset <= 1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) {
+        matchIndex = dictMatchIndex + lowLimit - (U32)dictEndOffset;
+        while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {

From 86b381e40b6ec3394582a2113c7bc80f1d619bab Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:13:40 -0400
Subject: [PATCH 45/54] Fix Constant Value

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 843b539..48c42c6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -101,7 +101,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
-    startingOffset += MAX_DISTANCE;
+    startingOffset += 64 KB;
     hc4->nextToUpdate = (U32) startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;

From 756ed402da8161c7bc3749bebaac069259312d8b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:56:26 -0400
Subject: [PATCH 46/54] Sign-Extend -1 to Pointer Width

---
 lib/lz4hc.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 48c42c6..a973086 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -830,8 +830,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
@@ -850,7 +849,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);

From 85cac61dd8823886132b4a9c2ff4a43b237da917 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:35:51 -0400
Subject: [PATCH 47/54] Don't Segfault on Malloc Failure

---
 lib/lz4hc.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a973086..b0325a7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -830,9 +830,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-    LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    if (LZ4_streamHCPtr != NULL) {
+        LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+        LZ4_streamHCPtr->internal_donotuse.base = NULL;
+        LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    }
     return LZ4_streamHCPtr;
 }
 

From a8cb2feffdead7b49a09aad0dc7b13dcff206e5b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:37:07 -0400
Subject: [PATCH 48/54] Tolerate Base Pointer Underflow

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b0325a7..b70d03b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -97,7 +97,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
+    if (startingOffset > 1 GB) {
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
@@ -898,7 +898,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;

From fcc99d1f31bf8a913fa91a5455524b4a14b9a03d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:37:28 -0400
Subject: [PATCH 49/54] Simpler loadDict() Reset

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b70d03b..23d84f4 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -882,8 +882,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
+    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
-    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;

From 1895fa19a4f04b395f9ebb1fe38049ab4c909fc3 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:14:12 -0400
Subject: [PATCH 50/54] Remove Redundant Static Assert

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 23d84f4..f91ef4a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -859,7 +859,6 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 
 void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
-    LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;

From ee67f25576f111972bd5003c527d1ebee598dc71 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:18:30 -0400
Subject: [PATCH 51/54] Change vLimit Calculation

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f91ef4a..a12f298 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -316,7 +316,7 @@ LZ4HC_InsertAndGetWiderMatch (
             if (LZ4_read32(matchPtr) == pattern) {
                 int mlt;
                 int back = 0;
-                const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
+                const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;

From 13271a88d7c21bee4ee61f619ef359afc707f23c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 24 Apr 2018 11:55:53 -0400
Subject: [PATCH 52/54] Revert Stream Size Const to Correct Value

---
 lib/lz4hc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index a762d3f..28e2528 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -175,7 +175,7 @@ struct LZ4HC_CCtx_internal
 
 #endif
 
-#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */
+#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
 union LZ4_streamHC_u {
     size_t table[LZ4_STREAMHCSIZE_SIZET];

From db9deb7b747d630a5778ed588d88dae82328dc62 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 18:56:01 -0400
Subject: [PATCH 53/54] Remove the Framebench Tool

---
 tests/Makefile     |   6 +-
 tests/framebench.c | 412 ---------------------------------------------
 2 files changed, 1 insertion(+), 417 deletions(-)
 delete mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index f6a4ff3..2b93c9f 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen framebench
+all: fullbench fuzzer frametest datagen
 
 all32: CFLAGS+=-m32
 all32: all
@@ -99,9 +99,6 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -113,7 +110,6 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
deleted file mode 100644
index 9752f23..0000000
--- a/tests/framebench.c
+++ /dev/null
@@ -1,412 +0,0 @@
-#include <assert.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-
-#define LZ4_STATIC_LINKING_ONLY
-#include "lz4.h"
-#include "lz4hc.h"
-#include "lz4frame.h"
-#include "lz4frame_static.h"
-
-#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
-
-typedef struct {
-  const char *run_name;
-  size_t iter;
-  LZ4_stream_t *ctx;
-  LZ4_streamHC_t *hcctx;
-  LZ4F_cctx *cctx;
-  LZ4F_dctx *dctx;
-  const char *dictbuf;
-  size_t dictsize;
-  char *obuf;
-  size_t osize;
-  const char* ibuf;
-  const char* isample;
-  size_t isize;
-  size_t num_ibuf;
-  char *checkbuf;
-  size_t checksize;
-  int clevel;
-  const LZ4F_CDict* cdict;
-  LZ4F_preferences_t* prefs;
-  const LZ4F_compressOptions_t* options;
-} bench_params_t;
-
-size_t compress_frame(bench_params_t *p) {
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressFrame_usingCDict(
-    cctx,
-    obuf,
-    osize,
-    isample,
-    isize,
-    cdict,
-    prefs);
-  LZ4F_CHECK(oused);
-
-  return oused;
-}
-
-size_t compress_begin(bench_params_t *p) {
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-  const LZ4F_compressOptions_t* options = p->options;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressUpdate(
-    cctx,
-    obuf,
-    oend - obuf,
-    isample,
-    isize,
-    options);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
-  LZ4F_CHECK(oused);
-
-  return obuf - p->obuf;
-}
-
-size_t compress_default(bench_params_t *p) {
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_default(isample, obuf, isize, oend - obuf);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_extState(bench_params_t *p) {
-  LZ4_stream_t *ctx = p->ctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_fast_extState_fastReset(
-      ctx, isample, obuf, isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_hc(bench_params_t *p) {
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_HC(
-      isample, obuf, isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_hc_extState(bench_params_t *p) {
-  LZ4_streamHC_t *hcctx = p->hcctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_HC_extStateHC(
-      hcctx,
-      isample, obuf,
-      isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t check_lz4(bench_params_t *p, size_t csize) {
-  (void)csize;
-  memset(p->checkbuf, 0xFF, p->checksize);
-  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
-                                       p->dictbuf, p->dictsize)
-      && !memcmp(p->isample, p->checkbuf, p->isize);
-}
-
-size_t check_lz4f(bench_params_t *p, size_t csize) {
-  size_t cp = 0;
-  size_t dp = 0;
-  size_t dsize = p->checksize;
-  size_t cleft = csize;
-  size_t dleft = dsize;
-  size_t ret;
-  memset(p->checkbuf, 0xFF, p->checksize);
-  LZ4F_resetDecompressionContext(p->dctx);
-  do {
-    ret = LZ4F_decompress_usingDict(
-        p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft,
-        p->dictbuf, p->dictsize, NULL);
-    cp += cleft;
-    dp += dleft;
-    cleft = csize - cp;
-    dleft = dsize - dp;
-    if (LZ4F_isError(ret)) return 0;
-  } while (cleft);
-  return !memcmp(p->isample, p->checkbuf, p->isize);
-}
-
-
-uint64_t bench(
-    const char *bench_name,
-    size_t (*fun)(bench_params_t *),
-    size_t (*checkfun)(bench_params_t *, size_t),
-    bench_params_t *params
-) {
-  struct timespec start, end;
-  size_t i, osize = 0, o = 0;
-  size_t time_taken = 0;
-  uint64_t total_repetitions = 0;
-  uint64_t repetitions = 2;
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
-
-  while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms
-    if (total_repetitions) {
-      repetitions = total_repetitions; // double previous
-    }
-
-    for (i = 0; i < repetitions; i++) {
-      params->iter = i;
-      if (params->num_ibuf == 1) {
-        params->isample = params->ibuf;
-      } else {
-        params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
-      }
-      o = fun(params);
-      if (!o) {
-        fprintf(
-            stderr,
-            "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n",
-            params->run_name, bench_name, params->clevel,
-            params->isize);
-        return 0;
-      }
-      osize += o;
-    }
-
-    if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
-
-    time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
-                 (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-    total_repetitions += repetitions;
-  }
-
-  o = checkfun(params, o);
-  if (!o) {
-    fprintf(
-        stderr,
-        "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n",
-        params->run_name, bench_name, params->clevel,
-        params->isize);
-    return 0;
-  }
-
-  fprintf(
-      stderr,
-      "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
-      params->run_name, bench_name, params->clevel,
-      params->isize, osize / total_repetitions,
-      total_repetitions, time_taken, time_taken / total_repetitions,
-      ((double) 1000 * params->isize * total_repetitions) / time_taken
-  );
-
-  return time_taken;
-}
-
-int main(int argc, char *argv[]) {
-  char *run_name;
-
-  struct stat st;
-  size_t bytes_read;
-
-  const char *dict_fn;
-  size_t dict_size;
-  char *dict_buf;
-  FILE *dict_file;
-
-  const char *in_fn;
-  size_t in_size;
-  size_t num_in_buf;
-  size_t cur_in_buf;
-  char *in_buf;
-  FILE *in_file;
-
-  size_t out_size;
-  char *out_buf;
-
-  size_t check_size;
-  char *check_buf;
-
-  LZ4_stream_t *ctx;
-  LZ4_streamHC_t *hcctx;
-  LZ4F_cctx *cctx;
-  LZ4F_dctx *dctx;
-  LZ4F_CDict *cdict;
-  LZ4F_preferences_t prefs;
-  LZ4F_compressOptions_t options;
-
-  int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
-  unsigned int clevelidx;
-
-  bench_params_t params;
-
-  if (argc != 4) return 1;
-  run_name = argv[1];
-  dict_fn = argv[2];
-  in_fn = argv[3];
-
-  if (stat(dict_fn, &st)) return 1;
-  dict_size = st.st_size;
-  dict_buf = (char *)malloc(dict_size);
-  if (!dict_buf) return 1;
-  dict_file = fopen(dict_fn, "r");
-  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
-  if (bytes_read != dict_size) return 1;
-
-  if (stat(in_fn, &st)) return 1;
-  in_size = st.st_size;
-  num_in_buf = 256 * 1024 * 1024 / in_size;
-  if (num_in_buf == 0) {
-    num_in_buf = 1;
-  }
-
-  in_buf = (char *)malloc(in_size * num_in_buf);
-  if (!in_buf) return 1;
-  in_file = fopen(in_fn, "r");
-  bytes_read = fread(in_buf, 1, in_size, in_file);
-  if (bytes_read != in_size) return 1;
-
-  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
-    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
-  }
-
-  check_size = in_size;
-  check_buf = (char *)malloc(check_size);
-  if (!check_buf) return 1;
-
-  memset(&prefs, 0, sizeof(prefs));
-  prefs.autoFlush = 1;
-  if (in_size < 64 * 1024)
-    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
-  prefs.frameInfo.contentSize = in_size;
-
-  memset(&options, 0, sizeof(options));
-  options.stableSrc = 1;
-
-  out_size = LZ4F_compressFrameBound(in_size, &prefs);
-  if ((size_t)LZ4_compressBound(in_size) > out_size) {
-    out_size = LZ4_compressBound(in_size);
-  }
-  out_buf = (char *)malloc(out_size);
-  if (!out_buf) return 1;
-
-  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  ctx = LZ4_createStream();
-  if (ctx == NULL) return 1;
-
-  hcctx = LZ4_createStreamHC();
-  if (hcctx == NULL) return 1;
-
-  cdict = LZ4F_createCDict(dict_buf, dict_size);
-  if (!cdict) return 1;
-
-  fprintf(stderr, "dict  size: %zd\n", dict_size);
-  fprintf(stderr, "input size: %zd\n", in_size);
-
-  params.run_name = run_name;
-  params.ctx = ctx;
-  params.hcctx = hcctx;
-  params.cctx = cctx;
-  params.dctx = dctx;
-  params.dictbuf = dict_buf;
-  params.dictsize = dict_size;
-  params.obuf = out_buf;
-  params.osize = out_size;
-  params.ibuf = in_buf;
-  params.isize = in_size;
-  params.num_ibuf = num_in_buf;
-  params.checkbuf = check_buf;
-  params.checksize = check_size;
-  params.clevel = 1;
-  params.cdict = NULL;
-  params.prefs = &prefs;
-  params.options = &options;
-
-  for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
-    params.clevel = clevels[clevelidx];
-    params.prefs->compressionLevel = clevels[clevelidx];
-    params.cdict = NULL;
-
-    bench("LZ4_compress_default"         , compress_default    , check_lz4 , &params);
-    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , &params);
-    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , &params);
-    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , &params);
-    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, &params);
-    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, &params);
-
-    params.cdict = cdict;
-
-    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, &params);
-    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, &params);
-  }
-
-  return 0;
-}

From 5ed1463bf4668a0e668679ae1e458aa73aa9a6ec Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 21:00:19 -0400
Subject: [PATCH 54/54] Remove Debug Log Statements

---
 lib/lz4frame.c | 21 ---------------------
 lib/lz4hc.c    | 14 +++++---------
 2 files changed, 5 insertions(+), 30 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d4d9397..844c8d1 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -95,18 +95,6 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
-#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
-#  include <stdio.h>
-static int g_debuglog_enable = 1;
-#  define DEBUGLOG(l, ...) {                                  \
-                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
-                    fprintf(stderr, __FILE__ ": ");           \
-                    fprintf(stderr, __VA_ARGS__);             \
-                    fprintf(stderr, " \n");                   \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
 
 /*-************************************
 *  Basic Types
@@ -469,7 +457,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
-    DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict);
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -492,7 +479,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 
 void LZ4F_freeCDict(LZ4F_CDict* cdict)
 {
-    DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict);
     if (cdict==NULL) return;  /* support free on NULL */
     FREEMEM(cdict->dictContent);
     LZ4_freeStream(cdict->fastCtx);
@@ -544,7 +530,6 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
-    DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict);
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
@@ -571,8 +556,6 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     BYTE* headerStart;
 
-    DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict);
-
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
@@ -791,7 +774,6 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
-    DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
@@ -930,9 +912,6 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
     BYTE* dstPtr = dstStart;
 
     size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
-
-    DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr);
-
     if (LZ4F_isError(flushSize)) return flushSize;
     dstPtr += flushSize;
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a12f298..4126ef8 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -88,7 +88,6 @@ typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
 **************************************/
 static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 {
-    DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
 }
@@ -96,7 +95,6 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     uptrval startingOffset = hc4->end - hc4->base;
-    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB) {
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
@@ -830,11 +828,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    if (LZ4_streamHCPtr != NULL) {
-        LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-        LZ4_streamHCPtr->internal_donotuse.base = NULL;
-        LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
-    }
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
 }
 
@@ -881,8 +877,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
-    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
@@ -897,7 +893,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;