From 88f3d8641e55544fbf22f7226ee1793828c65983 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 5 Jul 2017 13:57:07 -0700
Subject: [PATCH 001/100] Initial long distance matcher commit

---
 contrib/long_distance_matching/Makefile |  27 +++
 contrib/long_distance_matching/ldm.c    |  43 +++++
 contrib/long_distance_matching/ldm.h    |  10 ++
 contrib/long_distance_matching/main.c   | 227 ++++++++++++++++++++++++
 contrib/long_distance_matching/main.h   |   7 +
 5 files changed, 314 insertions(+)
 create mode 100644 contrib/long_distance_matching/Makefile
 create mode 100644 contrib/long_distance_matching/ldm.c
 create mode 100644 contrib/long_distance_matching/ldm.h
 create mode 100644 contrib/long_distance_matching/main.c
 create mode 100644 contrib/long_distance_matching/main.h

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
new file mode 100644
index 00000000..bfe02ea2
--- /dev/null
+++ b/contrib/long_distance_matching/Makefile
@@ -0,0 +1,27 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+# This Makefile presumes libzstd is installed, using `sudo make install`
+
+
+.PHONY: default all clean
+
+default: all
+
+all: main
+
+
+main : ldm.c main.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+clean:
+	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
+	main
+	@echo Cleaning completed
+
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
new file mode 100644
index 00000000..34118c81
--- /dev/null
+++ b/contrib/long_distance_matching/ldm.c
@@ -0,0 +1,43 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "ldm.h"
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint64_t tag;
+
+struct hash_entry {
+  U64 offset;
+  tag t;
+};
+
+size_t LDM_compress(const char *source, char *dest, size_t source_size, size_t max_dest_size) {
+  // max_dest_size >= source_size
+
+
+  /**
+   * Loop:
+   *  Find match at position k (hash next n bytes, rolling hash)
+   *  Compute match length
+   *  Output literal length: k (sequences of 4 + (k-4) bytes)
+   *  Output match length
+   *  Output literals
+   *  Output offset
+   */
+
+  memcpy(dest, source, source_size);
+  return source_size;
+}
+
+size_t LDM_decompress(const char *source, char *dest, size_t compressed_size, size_t max_decompressed_size) {
+  memcpy(dest, source, compressed_size);
+  return compressed_size;
+}
+
+
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
new file mode 100644
index 00000000..d0151373
--- /dev/null
+++ b/contrib/long_distance_matching/ldm.h
@@ -0,0 +1,10 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+size_t LDM_compress(const char *source, char *dest, size_t source_size, size_t max_dest_size);
+
+size_t LDM_decompress(const char *source, char *dest, size_t compressed_size, size_t max_decompressed_size);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/main.c b/contrib/long_distance_matching/main.c
new file mode 100644
index 00000000..ddf5145f
--- /dev/null
+++ b/contrib/long_distance_matching/main.c
@@ -0,0 +1,227 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ldm.h"
+
+#define BUF_SIZE 16*1024  // Block size
+#define LDM_HEADER_SIZE 8
+
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) {
+      break;
+    }
+    if (0 == result) {
+      result = memcmp(b0, b1, r0);
+    }
+  }
+  return result;
+}
+
+int main(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+
+
+  return 0;
+}
+
+
diff --git a/contrib/long_distance_matching/main.h b/contrib/long_distance_matching/main.h
new file mode 100644
index 00000000..a0b03012
--- /dev/null
+++ b/contrib/long_distance_matching/main.h
@@ -0,0 +1,7 @@
+#ifndef _MAIN_H
+#define _MAIN_H
+
+void compress_file(FILE *in, FILE *out, int argc, char *argv[]);
+void decompress_file(FILE *in, FILE *out, int argc, char *argv[]);
+
+#endif  /* _MAIN_H */

From 8aa34a76086b32ae66119b50961e33c4777e2bbf Mon Sep 17 00:00:00 2001
From: Stella Lau <sl715@cam.ac.uk>
Date: Thu, 6 Jul 2017 07:30:49 -0700
Subject: [PATCH 002/100] Switch to mmapping files

---
 contrib/long_distance_matching/Makefile   |   9 +-
 contrib/long_distance_matching/main-ldm.c | 411 ++++++++++++++++++++++
 2 files changed, 418 insertions(+), 2 deletions(-)
 create mode 100644 contrib/long_distance_matching/main-ldm.c

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index bfe02ea2..0efae69b 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -10,18 +10,23 @@
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
 
+LDFLAGS += -lzstd
+
 .PHONY: default all clean
 
 default: all
 
-all: main
+all: main main-ldm
 
 
 main : ldm.c main.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+main-ldm : ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main
+	main main-ldm
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
new file mode 100644
index 00000000..bdcffb0f
--- /dev/null
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -0,0 +1,411 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+#define BUF_SIZE 16*1024  // Block size
+#define LDM_HEADER_SIZE 8
+#define DEBUG
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+#endif
+
+static size_t compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* open the input file */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* open the output file */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* find size of input file */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+ /* go to the location corresponding to the last byte */
+ if (lseek(fdout, statbuf.st_size - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+ }
+ 
+ /* write a dummy byte at the last location */
+ if (write(fdout, "", 1) != 1) {
+     perror("write error");
+     return 1;
+ }
+
+  /* mmap the input file */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, statbuf.st_size, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+  /* Copy input file to output file */
+//  memcpy(dst, src, statbuf.st_size);
+  size_t size_out = ZSTD_compress(dst, statbuf.st_size, 
+                                  src, statbuf.st_size, 1);
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname, 
+         (unsigned)statbuf.st_size, (unsigned)size_out, oname,
+         (double)size_out / (statbuf.st_size) * 100);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+static size_t decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* open the input file */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* open the output file */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* find size of input file */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* go to the location corresponding to the last byte */
+  if (lseek(fdout, statbuf.st_size - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+ 
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the input file */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, statbuf.st_size, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+  /* Copy input file to output file */
+//  memcpy(dst, src, statbuf.st_size);
+
+  size_t size_out = ZSTD_decompress(dst, statbuf.st_size, 
+                                    src, statbuf.st_size);
+
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) {
+      break;
+    }
+    if (0 == result) {
+      result = memcmp(b0, b1, r0);
+    }
+  }
+  return result;
+}
+
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+	const int cmp = compare(inpFp, decFp);
+	if(0 == cmp) {
+		printf("verify : OK\n");
+	} else {
+		printf("verify : NG\n");
+	}
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  if (compress(inpFilename, ldmFilename)) {
+      printf("Compress error");
+      return 1;
+  }
+
+  /* decompress */
+  if (decompress(ldmFilename, decFilename)) {
+      printf("Decompress error");
+      return 1;
+  }
+
+  /* verify */
+  verify(inpFilename, decFilename);
+}
+
+#if 0
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+

From b96ad327a48bf4a97574e3d11d2f94741d900616 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 6 Jul 2017 15:23:15 -0700
Subject: [PATCH 003/100] Add simple compress and decompress functions

---
 contrib/long_distance_matching/ldm.c      | 342 ++++++++++++++++++++--
 contrib/long_distance_matching/ldm.h      |   6 +-
 contrib/long_distance_matching/main-ldm.c |  44 ++-
 3 files changed, 364 insertions(+), 28 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 34118c81..c8051ea4 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -1,9 +1,25 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdint.h>
+#include <stdio.h>
 
 #include "ldm.h"
 
+#define LDM_MEMORY_USAGE 14
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
+
+#define WINDOW_SIZE (1 << 20)
+#define HASH_SIZE 4
+#define MINMATCH 4
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
 typedef  uint8_t BYTE;
 typedef uint16_t U16;
 typedef uint32_t U32;
@@ -12,31 +28,321 @@ typedef uint64_t U64;
 
 typedef uint64_t tag;
 
+static unsigned LDM_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+static U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+static U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+static void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_write16(void *memPtr, U16 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE*)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+
+
+static U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+static void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
+  BYTE *d = (BYTE *)dstPtr;
+  const BYTE *s = (const BYTE *)srcPtr;
+  BYTE * const e = (BYTE *)dstEnd;
+
+  do {
+    LDM_copy8(d, s);
+    d += 8;
+    s += 8;
+  } while (d < e);
+
+}
+
 struct hash_entry {
   U64 offset;
   tag t;
 };
 
-size_t LDM_compress(const char *source, char *dest, size_t source_size, size_t max_dest_size) {
-  // max_dest_size >= source_size
-
-
-  /**
-   * Loop:
-   *  Find match at position k (hash next n bytes, rolling hash)
-   *  Compute match length
-   *  Output literal length: k (sequences of 4 + (k-4) bytes)
-   *  Output match length
-   *  Output literals
-   *  Output offset
-   */
-
-  memcpy(dest, source, source_size);
-  return source_size;
+static U32 LDM_hash(U32 sequence) {
+  return ((sequence * 2654435761U) >> ((MINMATCH*8)-LDM_HASHLOG));
 }
 
-size_t LDM_decompress(const char *source, char *dest, size_t compressed_size, size_t max_decompressed_size) {
-  memcpy(dest, source, compressed_size);
+static U32 LDM_hash_position(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+
+static U64 find_best_match(tag t, U64 offset) {
+  return 0;
+}
+
+static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
+                                     const BYTE *srcBase) {
+  U32 *hashTable = (U32 *) tableBase;
+  hashTable[h] = (U32)(p - srcBase);
+}
+
+static void LDM_put_position(const BYTE *p, void *tableBase,
+                             const BYTE *srcBase) {
+  U32 const h = LDM_hash_position(p);
+  LDM_put_position_on_hash(p, h, tableBase, srcBase);
+}
+
+static const BYTE *LDM_get_position_on_hash(
+    U32 h, void *tableBase, const BYTE *srcBase) {
+  const U32 * const hashTable = (U32*)tableBase;
+  return hashTable[h] + srcBase;
+}
+
+static BYTE LDM_read_byte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
+static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
+                          const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+
+size_t LDM_compress(void const *source, void *dest, size_t source_size,
+                    size_t max_dest_size) {
+  const BYTE * const istart = (const BYTE*)source;
+  const BYTE *ip = istart;
+  const BYTE * const iend = istart + source_size;
+  const BYTE *ilimit = iend - HASH_SIZE;
+  const BYTE * const matchlimit = iend - HASH_SIZE;
+  BYTE *op = (BYTE*) dest;
+  U32 hashTable[LDM_HASHTABLESIZE_U32];
+  memset(hashTable, 0, sizeof(hashTable));
+
+  const BYTE *anchor = (const BYTE *)source;
+//  struct LDM_cctx cctx;
+  size_t output_size = 0;
+
+  U32 forwardH;
+
+  /* Hash first byte: put into hash table */
+
+  LDM_put_position(ip, hashTable, istart);
+  ip++;
+  forwardH = LDM_hash_position(ip);
+
+  while (ip < ilimit) {
+    const BYTE *match;
+    BYTE *token;
+    /* Find a match */
+    {
+      const BYTE *forwardIp = ip;
+      unsigned step = 1;
+
+      do {
+        U32 const h = forwardH;
+        ip = forwardIp;
+        forwardIp += step;
+
+        match = LDM_get_position_on_hash(h, hashTable, istart);
+
+        forwardH = LDM_hash_position(forwardIp);
+        LDM_put_position_on_hash(ip, h, hashTable, istart);
+      } while (ip - match > WINDOW_SIZE ||
+               LDM_read32(match) != LDM_read32(ip));
+    }
+
+    /* Encode literals */
+    {
+      unsigned const litLength = (unsigned)(ip - anchor);
+      token = op++;
+
+      printf("Cur position: %zu\n", anchor - istart);
+      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
+      /*
+      fwrite(match, 4, 1, stdout);
+      printf("\n");
+      */
+
+      if (litLength >= RUN_MASK) {
+        int len = (int)litLength - RUN_MASK;
+        *token = (RUN_MASK << ML_BITS);
+        for (; len >= 255; len -= 255) {
+          *op++ = (BYTE)len;
+        }
+      } else {
+        *token = (BYTE)(litLength << ML_BITS);
+      }
+
+      printf("Literals ");
+      fwrite(anchor, litLength, 1, stdout);
+      printf("\n");
+
+      LDM_wild_copy(op, anchor, op + litLength);
+      op += litLength;
+    }
+_next_match:
+    /* Encode offset */
+    {
+      LDM_writeLE16(op, (U16)(ip - match));
+      op += 2;
+    }
+
+    /* Encode Match Length */
+    {
+      unsigned matchCode;
+      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
+                            matchlimit);
+
+      printf("Match length %zu\n", matchCode + MINMATCH);
+      fwrite(ip, MINMATCH + matchCode, 1, stdout);
+      printf("\n");
+      ip += MINMATCH + matchCode;
+      if (matchCode >= ML_MASK) {
+        *token += ML_MASK;
+        matchCode -= ML_MASK;
+        LDM_write32(op, 0xFFFFFFFF);
+        while (matchCode >= 4*0xFF) {
+          op += 4;
+          LDM_write32(op, 0xffffffff);
+          matchCode -= 4*0xFF;
+        }
+        op += matchCode / 255;
+        *op++ = (BYTE)(matchCode % 255);
+      } else {
+        *token += (BYTE)(matchCode);
+      }
+      printf("\n");
+    }
+
+    anchor = ip;
+
+    LDM_put_position(ip, hashTable, istart);
+    forwardH = LDM_hash_position(++ip);
+  }
+    /* Encode last literals */
+  {
+    /*
+    size_t const lastRun = (size_t)(iend - anchor);
+    printf("last run length: %zu, %zu %zu %zu %zu\n", lastRun, iend-istart,
+           anchor-istart, ip-istart, ilimit-istart);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *op++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *op++ = 255;
+      }
+      *op++ = (BYTE) accumulator;
+    } else {
+      *op++ = (BYTE)(lastRun << ML_BITS);
+    }
+    fwrite(anchor, lastRun, 1, stdout);
+    printf("^last run\n");
+    memcpy(op, anchor, lastRun);
+    op += lastRun;
+
+//    memcpy(dest + (ip - istart), ip, 1);
+//    */
+  }
+  return (op - (BYTE *)dest);
+}
+
+size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
+                      size_t max_decompressed_size) {
+  const BYTE *ip = (const BYTE *)source;
+  const BYTE * const iend = ip + compressed_size;
+  BYTE *op = (BYTE *)dest;
+  BYTE * const oend = op + max_decompressed_size;
+  BYTE *cpy;
+
+  while (ip < iend) {
+    size_t length;
+    const BYTE *match;
+    size_t offset;
+
+    /* get literal length */
+    unsigned const token = *ip++;
+    if ((length=(token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+    printf("Literal length: %zu\n", length);
+
+    /* copy literals */
+    cpy = op + length;
+    LDM_wild_copy(op, ip, cpy);
+    ip += length;
+    op = cpy;
+
+    /* get offset */
+    offset = LDM_readLE16(ip);
+    printf("Offset: %zu\n", offset);
+    ip += 2;
+    match = op - offset;
+ //   LDM_write32(op, (U32)offset);
+
+    /* get matchlength */
+    length = token & ML_MASK;
+    printf("Match length: %zu\n", length);
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+
+    /* copy match */
+    cpy = op + length;
+
+
+
+  }
+
+//  memcpy(dest, source, compressed_size);
   return compressed_size;
 }
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index d0151373..0aab6aa3 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -3,8 +3,10 @@
 
 #include <stddef.h>   /* size_t */
 
-size_t LDM_compress(const char *source, char *dest, size_t source_size, size_t max_dest_size);
+size_t LDM_compress(void const *source, void *dest, size_t source_size,
+                    size_t max_dest_size);
 
-size_t LDM_decompress(const char *source, char *dest, size_t compressed_size, size_t max_decompressed_size);
+size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
+                      size_t max_decompressed_size);
 
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index bdcffb0f..8b97ce92 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -1,6 +1,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/time.h>
 #include <sys/types.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
@@ -13,6 +14,7 @@
 #define BUF_SIZE 16*1024  // Block size
 #define LDM_HEADER_SIZE 8
 #define DEBUG
+// #define ZSTD
 
 #if 0
 static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
@@ -163,7 +165,7 @@ static size_t compress(const char *fname, const char *oname) {
     perror("lseek error");
     return 1;
  }
- 
+
  /* write a dummy byte at the last location */
  if (write(fdout, "", 1) != 1) {
      perror("write error");
@@ -186,9 +188,15 @@ static size_t compress(const char *fname, const char *oname) {
 
   /* Copy input file to output file */
 //  memcpy(dst, src, statbuf.st_size);
-  size_t size_out = ZSTD_compress(dst, statbuf.st_size, 
-                                  src, statbuf.st_size, 1);
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname, 
+  #ifdef ZSTD
+    size_t size_out = ZSTD_compress(dst, statbuf.st_size,
+                                    src, statbuf.st_size, 1);
+  #else
+    size_t size_out = LDM_compress(src, dst, statbuf.st_size,
+                                   statbuf.st_size);
+  #endif
+  ftruncate(fdout, size_out);
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
          (unsigned)statbuf.st_size, (unsigned)size_out, oname,
          (double)size_out / (statbuf.st_size) * 100);
 
@@ -225,7 +233,7 @@ static size_t decompress(const char *fname, const char *oname) {
     perror("lseek error");
     return 1;
   }
- 
+
   /* write a dummy byte at the last location */
   if (write(fdout, "", 1) != 1) {
     perror("write error");
@@ -249,9 +257,14 @@ static size_t decompress(const char *fname, const char *oname) {
   /* Copy input file to output file */
 //  memcpy(dst, src, statbuf.st_size);
 
-  size_t size_out = ZSTD_decompress(dst, statbuf.st_size, 
-                                    src, statbuf.st_size);
-
+  #ifdef ZSTD
+    size_t size_out = ZSTD_decompress(dst, statbuf.st_size,
+                                      src, statbuf.st_size);
+  #else
+    size_t size_out = LDM_decompress(src, dst, statbuf.st_size,
+                                     statbuf.st_size);
+  #endif
+  ftruncate(fdout, size_out);
 
   close(fdin);
   close(fdout);
@@ -315,20 +328,35 @@ int main(int argc, const char *argv[]) {
 	printf("ldm = [%s]\n", ldmFilename);
 	printf("dec = [%s]\n", decFilename);
 
+  struct timeval tv1, tv2;
   /* compress */
+  {
+  gettimeofday(&tv1, NULL);
   if (compress(inpFilename, ldmFilename)) {
       printf("Compress error");
       return 1;
   }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+         (double) (tv2.tv_sec - tv1.tv_sec));
+  }
 
   /* decompress */
+
+  gettimeofday(&tv1, NULL);
   if (decompress(ldmFilename, decFilename)) {
       printf("Decompress error");
       return 1;
   }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+        (double) (tv2.tv_sec - tv1.tv_sec));
 
   /* verify */
   verify(inpFilename, decFilename);
+  return 0;
 }
 
 #if 0

From 3bbfa1249e8ef2f2b966f6b71a75d8cfbb53089a Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 6 Jul 2017 16:47:08 -0700
Subject: [PATCH 004/100] Update compressor and decompressor

---
 contrib/long_distance_matching/ldm.c      | 40 ++++++++++++++++++-----
 contrib/long_distance_matching/main-ldm.c |  4 +--
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index c8051ea4..908ac2ac 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -20,6 +20,8 @@
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
+//#define LDM_DEBUG
+
 typedef  uint8_t BYTE;
 typedef uint16_t U16;
 typedef uint32_t U32;
@@ -173,6 +175,7 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
   while (ip < ilimit) {
     const BYTE *match;
     BYTE *token;
+
     /* Find a match */
     {
       const BYTE *forwardIp = ip;
@@ -196,8 +199,10 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
       unsigned const litLength = (unsigned)(ip - anchor);
       token = op++;
 
+#ifdef LDM_DEBUG
       printf("Cur position: %zu\n", anchor - istart);
       printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
+#endif
       /*
       fwrite(match, 4, 1, stdout);
       printf("\n");
@@ -207,16 +212,17 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
         int len = (int)litLength - RUN_MASK;
         *token = (RUN_MASK << ML_BITS);
         for (; len >= 255; len -= 255) {
-          *op++ = (BYTE)len;
+          *op++ = 255;
         }
+        *op++ = (BYTE)len;
       } else {
         *token = (BYTE)(litLength << ML_BITS);
       }
-
+#ifdef LDM_DEBUG
       printf("Literals ");
       fwrite(anchor, litLength, 1, stdout);
       printf("\n");
-
+#endif
       LDM_wild_copy(op, anchor, op + litLength);
       op += litLength;
     }
@@ -232,10 +238,11 @@ _next_match:
       unsigned matchCode;
       matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
                             matchlimit);
-
+#ifdef LDM_DEBUG
       printf("Match length %zu\n", matchCode + MINMATCH);
       fwrite(ip, MINMATCH + matchCode, 1, stdout);
       printf("\n");
+#endif
       ip += MINMATCH + matchCode;
       if (matchCode >= ML_MASK) {
         *token += ML_MASK;
@@ -251,7 +258,9 @@ _next_match:
       } else {
         *token += (BYTE)(matchCode);
       }
+#ifdef LDM_DEBUG
       printf("\n");
+#endif
     }
 
     anchor = ip;
@@ -308,24 +317,33 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
         length += s;
       } while (s == 255);
     }
+#ifdef LDM_DEBUG
     printf("Literal length: %zu\n", length);
+#endif
 
     /* copy literals */
     cpy = op + length;
+#ifdef LDM_DEBUG
+    printf("Literals ");
+    fwrite(ip, length, 1, stdout);
+    printf("\n");
+#endif
     LDM_wild_copy(op, ip, cpy);
     ip += length;
     op = cpy;
 
     /* get offset */
     offset = LDM_readLE16(ip);
+
+#ifdef LDM_DEBUG
     printf("Offset: %zu\n", offset);
+#endif
     ip += 2;
     match = op - offset;
  //   LDM_write32(op, (U32)offset);
 
     /* get matchlength */
     length = token & ML_MASK;
-    printf("Match length: %zu\n", length);
     if (length == ML_MASK) {
       unsigned s;
       do {
@@ -334,16 +352,20 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
       } while (s == 255);
     }
     length += MINMATCH;
-
+#ifdef LDM_DEBUG
+    printf("Match length: %zu\n", length);
+#endif
     /* copy match */
     cpy = op + length;
 
-
-
+    // Inefficient for now
+    while (match < cpy - offset) {
+      *op++ = *match++;
+    }
   }
 
 //  memcpy(dest, source, compressed_size);
-  return compressed_size;
+  return op - (BYTE *)dest;
 }
 
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 8b97ce92..7f1abdab 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -229,7 +229,7 @@ static size_t decompress(const char *fname, const char *oname) {
   }
 
   /* go to the location corresponding to the last byte */
-  if (lseek(fdout, statbuf.st_size - 1, SEEK_SET) == -1) {
+  if (lseek(fdout, 2*statbuf.st_size - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
   }
@@ -264,7 +264,7 @@ static size_t decompress(const char *fname, const char *oname) {
     size_t size_out = LDM_decompress(src, dst, statbuf.st_size,
                                      statbuf.st_size);
   #endif
-  ftruncate(fdout, size_out);
+  //ftruncate(fdout, size_out);
 
   close(fdin);
   close(fdout);

From f791fc27e3faae15eef3dfa7ce768a05cd2773cb Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 7 Jul 2017 12:44:29 -0700
Subject: [PATCH 005/100] Add header with compress and decompress size

---
 contrib/long_distance_matching/Makefile   |  6 +--
 contrib/long_distance_matching/ldm.c      | 16 +++---
 contrib/long_distance_matching/ldm.h      |  7 +++
 contrib/long_distance_matching/main-ldm.c | 61 +++++++++++++++--------
 4 files changed, 57 insertions(+), 33 deletions(-)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 0efae69b..4e04fd6a 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -16,11 +16,11 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main main-ldm
+all: main-ldm
 
 
-main : ldm.c main.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+#main : ldm.c main.c
+#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 main-ldm : ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 908ac2ac..cb90efec 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -69,8 +69,6 @@ static void LDM_writeLE16(void *memPtr, U16 value) {
   }
 }
 
-
-
 static U32 LDM_read32(const void *ptr) {
   return *(const U32 *)ptr;
 }
@@ -98,17 +96,13 @@ struct hash_entry {
 };
 
 static U32 LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((MINMATCH*8)-LDM_HASHLOG));
+  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
 }
 
 static U32 LDM_hash_position(const void * const p) {
   return LDM_hash(LDM_read32(p));
 }
 
-static U64 find_best_match(tag t, U64 offset) {
-  return 0;
-}
-
 static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
                                      const BYTE *srcBase) {
   U32 *hashTable = (U32 *) tableBase;
@@ -148,6 +142,12 @@ static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
   return (unsigned)(pIn - pStart);
 }
 
+void LDM_read_header(void const *source, size_t *compressed_size,
+                     size_t *decompressed_size) {
+  U32 *ip = (U32 *)source;
+  *compressed_size = *ip++;
+  *decompressed_size = *ip;
+}
 
 size_t LDM_compress(void const *source, void *dest, size_t source_size,
                     size_t max_dest_size) {
@@ -359,7 +359,7 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
     cpy = op + length;
 
     // Inefficient for now
-    while (match < cpy - offset) {
+    while (match < cpy - offset && op < oend) {
       *op++ = *match++;
     }
   }
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 0aab6aa3..f4ca25a3 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -3,10 +3,17 @@
 
 #include <stddef.h>   /* size_t */
 
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
 size_t LDM_compress(void const *source, void *dest, size_t source_size,
                     size_t max_dest_size);
 
 size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
                       size_t max_decompressed_size);
 
+void LDM_read_header(void const *source, size_t *compressed_size,
+                     size_t *decompressed_size);
+
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 7f1abdab..4d54ef6d 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -11,10 +11,10 @@
 #include <fcntl.h>
 #include "ldm.h"
 
-#define BUF_SIZE 16*1024  // Block size
-#define LDM_HEADER_SIZE 8
+// #define BUF_SIZE 16*1024  // Block size
 #define DEBUG
-// #define ZSTD
+
+//#define ZSTD
 
 #if 0
 static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
@@ -159,9 +159,10 @@ static size_t compress(const char *fname, const char *oname) {
     perror("Fstat error");
     return 1;
   }
+  size_t size_in = statbuf.st_size;
 
  /* go to the location corresponding to the last byte */
- if (lseek(fdout, statbuf.st_size - 1, SEEK_SET) == -1) {
+ if (lseek(fdout, size_in + LDM_HEADER_SIZE - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
  }
@@ -178,24 +179,31 @@ static size_t compress(const char *fname, const char *oname) {
       perror("mmap error for input");
       return 1;
   }
+  size_t out_size = statbuf.st_size + LDM_HEADER_SIZE;
 
   /* mmap the output file */
-  if ((dst = mmap(0, statbuf.st_size, PROT_READ | PROT_WRITE,
+  if ((dst = mmap(0, out_size, PROT_READ | PROT_WRITE,
                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
       perror("mmap error for output");
       return 1;
   }
 
-  /* Copy input file to output file */
-//  memcpy(dst, src, statbuf.st_size);
   #ifdef ZSTD
     size_t size_out = ZSTD_compress(dst, statbuf.st_size,
                                     src, statbuf.st_size, 1);
   #else
-    size_t size_out = LDM_compress(src, dst, statbuf.st_size,
+    size_t size_out = LDM_compress(src, dst + LDM_HEADER_SIZE, statbuf.st_size,
                                    statbuf.st_size);
+    size_out += LDM_HEADER_SIZE;
+
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+    memcpy(dst, &size_out, 4);
+    memcpy(dst + 4, &(statbuf.st_size), 4);
+    printf("Compressed size: %zu\n", size_out);
+    printf("Decompressed size: %zu\n", statbuf.st_size);
   #endif
   ftruncate(fdout, size_out);
+
   printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
          (unsigned)statbuf.st_size, (unsigned)size_out, oname,
          (double)size_out / (statbuf.st_size) * 100);
@@ -228,8 +236,22 @@ static size_t decompress(const char *fname, const char *oname) {
     return 1;
   }
 
+  /* mmap the input file */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* read header */
+  size_t compressed_size, decompressed_size;
+  LDM_read_header(src, &compressed_size, &decompressed_size);
+
+  printf("Size, compressed_size, decompressed_size: %zu %zu %zu\n",
+         statbuf.st_size, compressed_size, decompressed_size);
+
   /* go to the location corresponding to the last byte */
-  if (lseek(fdout, 2*statbuf.st_size - 1, SEEK_SET) == -1) {
+  if (lseek(fdout, decompressed_size - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
   }
@@ -240,15 +262,8 @@ static size_t decompress(const char *fname, const char *oname) {
     return 1;
   }
 
-  /* mmap the input file */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
   /* mmap the output file */
-  if ((dst = mmap(0, statbuf.st_size, PROT_READ | PROT_WRITE,
+  if ((dst = mmap(0, decompressed_size, PROT_READ | PROT_WRITE,
                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
       perror("mmap error for output");
       return 1;
@@ -258,13 +273,15 @@ static size_t decompress(const char *fname, const char *oname) {
 //  memcpy(dst, src, statbuf.st_size);
 
   #ifdef ZSTD
-    size_t size_out = ZSTD_decompress(dst, statbuf.st_size,
-                                      src, statbuf.st_size);
+    size_t size_out = ZSTD_decompress(dst, decomrpessed_size,
+                                      src + LDM_HEADER_SIZE,
+                                      statbuf.st_size - LDM_HEADER_SIZE);
   #else
-    size_t size_out = LDM_decompress(src, dst, statbuf.st_size,
-                                     statbuf.st_size);
+    size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
+                                     statbuf.st_size - LDM_HEADER_SIZE,
+                                     decompressed_size);
   #endif
-  //ftruncate(fdout, size_out);
+  ftruncate(fdout, size_out);
 
   close(fdin);
   close(fdout);

From 7945f9ee4757f045d4006813cd1f3e1ddfafa85c Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 7 Jul 2017 14:14:01 -0700
Subject: [PATCH 006/100] Fix offset overflow bug

---
 contrib/long_distance_matching/ldm.c      | 36 ++++++++++++++---------
 contrib/long_distance_matching/main-ldm.c |  1 +
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index cb90efec..b02869fe 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -11,7 +11,7 @@
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
 
-#define WINDOW_SIZE (1 << 20)
+#define WINDOW_SIZE (1 << 15)
 #define HASH_SIZE 4
 #define MINMATCH 4
 
@@ -144,7 +144,7 @@ static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
 
 void LDM_read_header(void const *source, size_t *compressed_size,
                      size_t *decompressed_size) {
-  U32 *ip = (U32 *)source;
+  const U32 *ip = (const U32 *)source;
   *compressed_size = *ip++;
   *decompressed_size = *ip;
 }
@@ -156,6 +156,7 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
   const BYTE * const iend = istart + source_size;
   const BYTE *ilimit = iend - HASH_SIZE;
   const BYTE * const matchlimit = iend - HASH_SIZE;
+  const BYTE * const mflimit = iend - MINMATCH;
   BYTE *op = (BYTE*) dest;
   U32 hashTable[LDM_HASHTABLESIZE_U32];
   memset(hashTable, 0, sizeof(hashTable));
@@ -172,6 +173,7 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
   ip++;
   forwardH = LDM_hash_position(ip);
 
+  //TODO Loop terminates before ip>=ilimit.
   while (ip < ilimit) {
     const BYTE *match;
     BYTE *token;
@@ -186,6 +188,10 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
         ip = forwardIp;
         forwardIp += step;
 
+        if (forwardIp > mflimit) {
+          goto _last_literals;
+        }
+
         match = LDM_get_position_on_hash(h, hashTable, istart);
 
         forwardH = LDM_hash_position(forwardIp);
@@ -194,6 +200,12 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
                LDM_read32(match) != LDM_read32(ip));
     }
 
+    // TODO catchup
+    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
+      ip--;
+      match--;
+    }
+
     /* Encode literals */
     {
       unsigned const litLength = (unsigned)(ip - anchor);
@@ -223,7 +235,8 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
       fwrite(anchor, litLength, 1, stdout);
       printf("\n");
 #endif
-      LDM_wild_copy(op, anchor, op + litLength);
+      memcpy(op, anchor, litLength);
+      //LDM_wild_copy(op, anchor, op + litLength);
       op += litLength;
     }
 _next_match:
@@ -268,29 +281,22 @@ _next_match:
     LDM_put_position(ip, hashTable, istart);
     forwardH = LDM_hash_position(++ip);
   }
+_last_literals:
     /* Encode last literals */
   {
-    /*
     size_t const lastRun = (size_t)(iend - anchor);
-    printf("last run length: %zu, %zu %zu %zu %zu\n", lastRun, iend-istart,
-           anchor-istart, ip-istart, ilimit-istart);
     if (lastRun >= RUN_MASK) {
       size_t accumulator = lastRun - RUN_MASK;
       *op++ = RUN_MASK << ML_BITS;
       for(; accumulator >= 255; accumulator -= 255) {
         *op++ = 255;
       }
-      *op++ = (BYTE) accumulator;
+      *op++ = (BYTE)accumulator;
     } else {
       *op++ = (BYTE)(lastRun << ML_BITS);
     }
-    fwrite(anchor, lastRun, 1, stdout);
-    printf("^last run\n");
     memcpy(op, anchor, lastRun);
     op += lastRun;
-
-//    memcpy(dest + (ip - istart), ip, 1);
-//    */
   }
   return (op - (BYTE *)dest);
 }
@@ -328,7 +334,8 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
     fwrite(ip, length, 1, stdout);
     printf("\n");
 #endif
-    LDM_wild_copy(op, ip, cpy);
+    memcpy(op, ip, length);
+//    LDM_wild_copy(op, ip, cpy);
     ip += length;
     op = cpy;
 
@@ -358,12 +365,13 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
     /* copy match */
     cpy = op + length;
 
+//    printf("TMP_PREV: %zu\n", op - (BYTE *)dest);
     // Inefficient for now
     while (match < cpy - offset && op < oend) {
       *op++ = *match++;
     }
+//    printf("TMP: %zu\n", op - (BYTE *)dest);
   }
-
 //  memcpy(dest, source, compressed_size);
   return op - (BYTE *)dest;
 }
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 4d54ef6d..26db1e94 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -280,6 +280,7 @@ static size_t decompress(const char *fname, const char *oname) {
     size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
                                      statbuf.st_size - LDM_HEADER_SIZE,
                                      decompressed_size);
+    printf("Ret size out: %zu\n", size_out);
   #endif
   ftruncate(fdout, size_out);
 

From 4076be09ec17c53680927396ffd85e02e10160ad Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 7 Jul 2017 14:52:40 -0700
Subject: [PATCH 007/100] [ldm] Update to hash every position

---
 contrib/long_distance_matching/ldm.c | 48 ++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index b02869fe..b03e4368 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -11,7 +11,8 @@
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
 
-#define WINDOW_SIZE (1 << 15)
+#define WINDOW_SIZE (1 << 20)
+#define MAX_WINDOW_SIZE 31
 #define HASH_SIZE 4
 #define MINMATCH 4
 
@@ -73,6 +74,11 @@ static U32 LDM_read32(const void *ptr) {
   return *(const U32 *)ptr;
 }
 
+static U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+
 static void LDM_copy8(void *dst, const void *src) {
   memcpy(dst, src, 8);
 }
@@ -87,7 +93,6 @@ static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
     d += 8;
     s += 8;
   } while (d < e);
-
 }
 
 struct hash_entry {
@@ -99,12 +104,23 @@ static U32 LDM_hash(U32 sequence) {
   return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
 }
 
+static U32 LDM_hash5(U64 sequence) {
+  static const U64 prime5bytes = 889523592379ULL;
+  static const U64 prime8bytes = 11400714785074694791ULL;
+  const U32 hashLog = LDM_HASHLOG;
+  if (LDM_isLittleEndian())
+    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
+  else
+    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+}
+
 static U32 LDM_hash_position(const void * const p) {
   return LDM_hash(LDM_read32(p));
 }
 
 static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
                                      const BYTE *srcBase) {
+//  printf("Hashing: %zu\n", p - srcBase);
   U32 *hashTable = (U32 *) tableBase;
   hashTable[h] = (U32)(p - srcBase);
 }
@@ -170,6 +186,7 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
   /* Hash first byte: put into hash table */
 
   LDM_put_position(ip, hashTable, istart);
+  const BYTE *lastHash = ip;
   ip++;
   forwardH = LDM_hash_position(ip);
 
@@ -196,8 +213,9 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
 
         forwardH = LDM_hash_position(forwardIp);
         LDM_put_position_on_hash(ip, h, hashTable, istart);
+        lastHash = ip;
       } while (ip - match > WINDOW_SIZE ||
-               LDM_read32(match) != LDM_read32(ip));
+               LDM_read64(match) != LDM_read64(ip));
     }
 
     // TODO catchup
@@ -215,10 +233,6 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
       printf("Cur position: %zu\n", anchor - istart);
       printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
 #endif
-      /*
-      fwrite(match, 4, 1, stdout);
-      printf("\n");
-      */
 
       if (litLength >= RUN_MASK) {
         int len = (int)litLength - RUN_MASK;
@@ -242,8 +256,8 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
 _next_match:
     /* Encode offset */
     {
-      LDM_writeLE16(op, (U16)(ip - match));
-      op += 2;
+      LDM_write32(op, ip - match);
+      op += 4;
     }
 
     /* Encode Match Length */
@@ -256,7 +270,13 @@ _next_match:
       fwrite(ip, MINMATCH + matchCode, 1, stdout);
       printf("\n");
 #endif
-      ip += MINMATCH + matchCode;
+
+      unsigned ctr = 1;
+      ip++;
+      for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
+        LDM_put_position(ip, hashTable, istart);
+      }
+//      ip += MINMATCH + matchCode;
       if (matchCode >= ML_MASK) {
         *token += ML_MASK;
         matchCode -= ML_MASK;
@@ -280,6 +300,7 @@ _next_match:
 
     LDM_put_position(ip, hashTable, istart);
     forwardH = LDM_hash_position(++ip);
+    lastHash = ip;
   }
 _last_literals:
     /* Encode last literals */
@@ -340,12 +361,12 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
     op = cpy;
 
     /* get offset */
-    offset = LDM_readLE16(ip);
+    offset = LDM_read32(ip);
 
 #ifdef LDM_DEBUG
     printf("Offset: %zu\n", offset);
 #endif
-    ip += 2;
+    ip += 4;
     match = op - offset;
  //   LDM_write32(op, (U32)offset);
 
@@ -365,12 +386,11 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
     /* copy match */
     cpy = op + length;
 
-//    printf("TMP_PREV: %zu\n", op - (BYTE *)dest);
     // Inefficient for now
+
     while (match < cpy - offset && op < oend) {
       *op++ = *match++;
     }
-//    printf("TMP: %zu\n", op - (BYTE *)dest);
   }
 //  memcpy(dest, source, compressed_size);
   return op - (BYTE *)dest;

From acdeb9f30211b018460c8f9e71595b49d553e555 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 7 Jul 2017 17:09:28 -0700
Subject: [PATCH 008/100] Add compression statistics

---
 contrib/long_distance_matching/ldm.c      | 57 ++++++++++++++++++++---
 contrib/long_distance_matching/main-ldm.c |  2 +
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index b03e4368..f8061f53 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -5,6 +5,8 @@
 
 #include "ldm.h"
 
+#define HASH_EVERY 7
+
 #define LDM_MEMORY_USAGE 14
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
@@ -13,8 +15,8 @@
 
 #define WINDOW_SIZE (1 << 20)
 #define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 4
-#define MINMATCH 4
+#define HASH_SIZE 8
+#define MINMATCH 8
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
@@ -95,6 +97,26 @@ static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
   } while (d < e);
 }
 
+typedef struct compress_stats {
+  U32 num_matches;
+  U32 total_match_length;
+  U32 total_literal_length;
+  U64 total_offset;
+} compress_stats;
+
+static void print_compress_stats(const compress_stats *stats) {
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Total number of matches: %u\n", stats->num_matches);
+  printf("Average match length: %.1f\n", ((double)stats->total_match_length) /
+                                         (double)stats->num_matches);
+  printf("Average literal length: %.1f\n",
+         ((double)stats->total_literal_length) / (double)stats->num_matches);
+  printf("Average offset length: %.1f\n",
+         ((double)stats->total_offset) / (double)stats->num_matches);
+  printf("=====================\n");
+}
+
 struct hash_entry {
   U64 offset;
   tag t;
@@ -121,12 +143,19 @@ static U32 LDM_hash_position(const void * const p) {
 static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
                                      const BYTE *srcBase) {
 //  printf("Hashing: %zu\n", p - srcBase);
+  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+
   U32 *hashTable = (U32 *) tableBase;
   hashTable[h] = (U32)(p - srcBase);
 }
 
 static void LDM_put_position(const BYTE *p, void *tableBase,
                              const BYTE *srcBase) {
+  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
   U32 const h = LDM_hash_position(p);
   LDM_put_position_on_hash(p, h, tableBase, srcBase);
 }
@@ -174,6 +203,9 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
   const BYTE * const matchlimit = iend - HASH_SIZE;
   const BYTE * const mflimit = iend - MINMATCH;
   BYTE *op = (BYTE*) dest;
+
+  compress_stats compressStats = { 0 };
+
   U32 hashTable[LDM_HASHTABLESIZE_U32];
   memset(hashTable, 0, sizeof(hashTable));
 
@@ -217,8 +249,9 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
       } while (ip - match > WINDOW_SIZE ||
                LDM_read64(match) != LDM_read64(ip));
     }
+    compressStats.num_matches++;
 
-    // TODO catchup
+    /* Catchup: look back to extend match from found match */
     while (ip > anchor && match > istart && ip[-1] == match[-1]) {
       ip--;
       match--;
@@ -229,6 +262,8 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
       unsigned const litLength = (unsigned)(ip - anchor);
       token = op++;
 
+      compressStats.total_literal_length += litLength;
+
 #ifdef LDM_DEBUG
       printf("Cur position: %zu\n", anchor - istart);
       printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
@@ -256,8 +291,13 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
 _next_match:
     /* Encode offset */
     {
+      /*
+      LDM_writeLE16(op, ip-match);
+      op += 2;
+      */
       LDM_write32(op, ip - match);
       op += 4;
+      compressStats.total_offset += (ip - match);
     }
 
     /* Encode Match Length */
@@ -270,7 +310,7 @@ _next_match:
       fwrite(ip, MINMATCH + matchCode, 1, stdout);
       printf("\n");
 #endif
-
+      compressStats.total_match_length += matchCode + MINMATCH;
       unsigned ctr = 1;
       ip++;
       for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
@@ -293,6 +333,7 @@ _next_match:
       }
 #ifdef LDM_DEBUG
       printf("\n");
+
 #endif
     }
 
@@ -319,6 +360,7 @@ _last_literals:
     memcpy(op, anchor, lastRun);
     op += lastRun;
   }
+  print_compress_stats(&compressStats);
   return (op - (BYTE *)dest);
 }
 
@@ -361,12 +403,15 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
     op = cpy;
 
     /* get offset */
+    /*
+    offset = LDM_readLE16(ip);
+    ip += 2;
+    */
     offset = LDM_read32(ip);
-
+    ip += 4;
 #ifdef LDM_DEBUG
     printf("Offset: %zu\n", offset);
 #endif
-    ip += 4;
     match = op - offset;
  //   LDM_write32(op, (U32)offset);
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 26db1e94..10869cce 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -1,3 +1,5 @@
+// TODO: file size must fit into a U32
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

From 719ccdc5a58efc9531a9b0d35ee358da7650cef3 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Sun, 9 Jul 2017 22:45:54 -0700
Subject: [PATCH 009/100] Update mainfile

---
 contrib/long_distance_matching/main.c | 29 +++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/contrib/long_distance_matching/main.c b/contrib/long_distance_matching/main.c
index ddf5145f..67144166 100644
--- a/contrib/long_distance_matching/main.c
+++ b/contrib/long_distance_matching/main.c
@@ -1,12 +1,31 @@
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
 
 #include "ldm.h"
 
 #define BUF_SIZE 16*1024  // Block size
 #define LDM_HEADER_SIZE 8
 
+/*
+static size_t compress_file_mmap(FILE *in, FILE *out, size_t *size_in,
+                                 size_t *size_out) {
+  char *src, *dst;
+  struct stat statbuf;
+
+  if (fstat(in, &statbuf) < 0) {
+    printf("fstat error\n");
+    return 1;
+  }
+
+
+  return 0;
+}
+*/
+
 static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
                             size_t *size_out) {
   char *src, *buf = NULL;
@@ -26,7 +45,6 @@ static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
     goto cleanup;
   }
 
-
   for (;;) {
     k = fread(src, 1, BUF_SIZE, in);
     if (k == 0)
@@ -37,10 +55,8 @@ static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
 
     // n = k;
     // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
+    offset = n;
+    count_out += n;
 
     k = fwrite(buf, 1, offset, out);
     if (k < offset) {
@@ -94,8 +110,6 @@ static size_t decompress_file(FILE *in, FILE *out) {
       }
     }
 
-    // TODO
-
     /* Decompress:
      * Continue while there is more input to read.
      */
@@ -220,7 +234,6 @@ int main(int argc, char *argv[]) {
 		fclose(inpFp);
 	}
 
-
   return 0;
 }
 

From eb280cd5685ae960ff75ff3c5381507ebaaef403 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 10 Jul 2017 06:32:05 -0700
Subject: [PATCH 010/100] Add folder for old versions

---
 .../long_distance_matching/versions/v1/ldm.c  | 394 +++++++++++++++
 .../long_distance_matching/versions/v1/ldm.h  |  19 +
 .../versions/v1/main-ldm.c                    | 459 ++++++++++++++++++
 3 files changed, 872 insertions(+)
 create mode 100644 contrib/long_distance_matching/versions/v1/ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v1/ldm.h
 create mode 100644 contrib/long_distance_matching/versions/v1/main-ldm.c

diff --git a/contrib/long_distance_matching/versions/v1/ldm.c b/contrib/long_distance_matching/versions/v1/ldm.c
new file mode 100644
index 00000000..266425f8
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v1/ldm.c
@@ -0,0 +1,394 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "ldm.h"
+
+#define LDM_MEMORY_USAGE 14
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
+
+#define WINDOW_SIZE (1 << 20)
+#define MAX_WINDOW_SIZE 31
+#define HASH_SIZE 4
+#define MINMATCH 4
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+//#define LDM_DEBUG
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint64_t tag;
+
+static unsigned LDM_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+static U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+static U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+static void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_write16(void *memPtr, U16 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE*)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+static U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+static U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+
+static void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
+  BYTE *d = (BYTE *)dstPtr;
+  const BYTE *s = (const BYTE *)srcPtr;
+  BYTE * const e = (BYTE *)dstEnd;
+
+  do {
+    LDM_copy8(d, s);
+    d += 8;
+    s += 8;
+  } while (d < e);
+
+}
+
+struct hash_entry {
+  U64 offset;
+  tag t;
+};
+
+static U32 LDM_hash(U32 sequence) {
+  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
+}
+
+static U32 LDM_hash5(U64 sequence) {
+  static const U64 prime5bytes = 889523592379ULL;
+  static const U64 prime8bytes = 11400714785074694791ULL;
+  const U32 hashLog = LDM_HASHLOG;
+  if (LDM_isLittleEndian())
+    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
+  else
+    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+}
+
+static U32 LDM_hash_position(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+
+static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
+                                     const BYTE *srcBase) {
+  U32 *hashTable = (U32 *) tableBase;
+  hashTable[h] = (U32)(p - srcBase);
+}
+
+static void LDM_put_position(const BYTE *p, void *tableBase,
+                             const BYTE *srcBase) {
+  U32 const h = LDM_hash_position(p);
+  LDM_put_position_on_hash(p, h, tableBase, srcBase);
+}
+
+static const BYTE *LDM_get_position_on_hash(
+    U32 h, void *tableBase, const BYTE *srcBase) {
+  const U32 * const hashTable = (U32*)tableBase;
+  return hashTable[h] + srcBase;
+}
+
+static BYTE LDM_read_byte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
+static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
+                          const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+void LDM_read_header(void const *source, size_t *compressed_size,
+                     size_t *decompressed_size) {
+  const U32 *ip = (const U32 *)source;
+  *compressed_size = *ip++;
+  *decompressed_size = *ip;
+}
+
+size_t LDM_compress(void const *source, void *dest, size_t source_size,
+                    size_t max_dest_size) {
+  const BYTE * const istart = (const BYTE*)source;
+  const BYTE *ip = istart;
+  const BYTE * const iend = istart + source_size;
+  const BYTE *ilimit = iend - HASH_SIZE;
+  const BYTE * const matchlimit = iend - HASH_SIZE;
+  const BYTE * const mflimit = iend - MINMATCH;
+  BYTE *op = (BYTE*) dest;
+  U32 hashTable[LDM_HASHTABLESIZE_U32];
+  memset(hashTable, 0, sizeof(hashTable));
+
+  const BYTE *anchor = (const BYTE *)source;
+//  struct LDM_cctx cctx;
+  size_t output_size = 0;
+
+  U32 forwardH;
+
+  /* Hash first byte: put into hash table */
+
+  LDM_put_position(ip, hashTable, istart);
+  ip++;
+  forwardH = LDM_hash_position(ip);
+
+  //TODO Loop terminates before ip>=ilimit.
+  while (ip < ilimit) {
+    const BYTE *match;
+    BYTE *token;
+
+    /* Find a match */
+    {
+      const BYTE *forwardIp = ip;
+      unsigned step = 1;
+
+      do {
+        U32 const h = forwardH;
+        ip = forwardIp;
+        forwardIp += step;
+
+        if (forwardIp > mflimit) {
+          goto _last_literals;
+        }
+
+        match = LDM_get_position_on_hash(h, hashTable, istart);
+
+        forwardH = LDM_hash_position(forwardIp);
+        LDM_put_position_on_hash(ip, h, hashTable, istart);
+      } while (ip - match > WINDOW_SIZE ||
+               LDM_read64(match) != LDM_read64(ip));
+    }
+
+    // TODO catchup
+    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
+      ip--;
+      match--;
+    }
+
+    /* Encode literals */
+    {
+      unsigned const litLength = (unsigned)(ip - anchor);
+      token = op++;
+
+#ifdef LDM_DEBUG
+      printf("Cur position: %zu\n", anchor - istart);
+      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
+#endif
+      /*
+      fwrite(match, 4, 1, stdout);
+      printf("\n");
+      */
+
+      if (litLength >= RUN_MASK) {
+        int len = (int)litLength - RUN_MASK;
+        *token = (RUN_MASK << ML_BITS);
+        for (; len >= 255; len -= 255) {
+          *op++ = 255;
+        }
+        *op++ = (BYTE)len;
+      } else {
+        *token = (BYTE)(litLength << ML_BITS);
+      }
+#ifdef LDM_DEBUG
+      printf("Literals ");
+      fwrite(anchor, litLength, 1, stdout);
+      printf("\n");
+#endif
+      memcpy(op, anchor, litLength);
+      //LDM_wild_copy(op, anchor, op + litLength);
+      op += litLength;
+    }
+_next_match:
+    /* Encode offset */
+    {
+      LDM_write32(op, ip - match);
+      op += 4;
+    }
+
+    /* Encode Match Length */
+    {
+      unsigned matchCode;
+      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
+                            matchlimit);
+#ifdef LDM_DEBUG
+      printf("Match length %zu\n", matchCode + MINMATCH);
+      fwrite(ip, MINMATCH + matchCode, 1, stdout);
+      printf("\n");
+#endif
+      ip += MINMATCH + matchCode;
+      if (matchCode >= ML_MASK) {
+        *token += ML_MASK;
+        matchCode -= ML_MASK;
+        LDM_write32(op, 0xFFFFFFFF);
+        while (matchCode >= 4*0xFF) {
+          op += 4;
+          LDM_write32(op, 0xffffffff);
+          matchCode -= 4*0xFF;
+        }
+        op += matchCode / 255;
+        *op++ = (BYTE)(matchCode % 255);
+      } else {
+        *token += (BYTE)(matchCode);
+      }
+#ifdef LDM_DEBUG
+      printf("\n");
+#endif
+    }
+
+    anchor = ip;
+
+    LDM_put_position(ip, hashTable, istart);
+    forwardH = LDM_hash_position(++ip);
+  }
+_last_literals:
+    /* Encode last literals */
+  {
+    size_t const lastRun = (size_t)(iend - anchor);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *op++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *op++ = 255;
+      }
+      *op++ = (BYTE)accumulator;
+    } else {
+      *op++ = (BYTE)(lastRun << ML_BITS);
+    }
+    memcpy(op, anchor, lastRun);
+    op += lastRun;
+  }
+  return (op - (BYTE *)dest);
+}
+
+size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
+                      size_t max_decompressed_size) {
+  const BYTE *ip = (const BYTE *)source;
+  const BYTE * const iend = ip + compressed_size;
+  BYTE *op = (BYTE *)dest;
+  BYTE * const oend = op + max_decompressed_size;
+  BYTE *cpy;
+
+  while (ip < iend) {
+    size_t length;
+    const BYTE *match;
+    size_t offset;
+
+    /* get literal length */
+    unsigned const token = *ip++;
+    if ((length=(token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+#ifdef LDM_DEBUG
+    printf("Literal length: %zu\n", length);
+#endif
+
+    /* copy literals */
+    cpy = op + length;
+#ifdef LDM_DEBUG
+    printf("Literals ");
+    fwrite(ip, length, 1, stdout);
+    printf("\n");
+#endif
+    memcpy(op, ip, length);
+//    LDM_wild_copy(op, ip, cpy);
+    ip += length;
+    op = cpy;
+
+    /* get offset */
+    offset = LDM_read32(ip);
+
+#ifdef LDM_DEBUG
+    printf("Offset: %zu\n", offset);
+#endif
+    ip += 4;
+    match = op - offset;
+ //   LDM_write32(op, (U32)offset);
+
+    /* get matchlength */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+#ifdef LDM_DEBUG
+    printf("Match length: %zu\n", length);
+#endif
+    /* copy match */
+    cpy = op + length;
+
+    // Inefficient for now
+
+    while (match < cpy - offset && op < oend) {
+      *op++ = *match++;
+    }
+  }
+//  memcpy(dest, source, compressed_size);
+  return op - (BYTE *)dest;
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v1/ldm.h b/contrib/long_distance_matching/versions/v1/ldm.h
new file mode 100644
index 00000000..f4ca25a3
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v1/ldm.h
@@ -0,0 +1,19 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
+size_t LDM_compress(void const *source, void *dest, size_t source_size,
+                    size_t max_dest_size);
+
+size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
+                      size_t max_decompressed_size);
+
+void LDM_read_header(void const *source, size_t *compressed_size,
+                     size_t *decompressed_size);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v1/main-ldm.c b/contrib/long_distance_matching/versions/v1/main-ldm.c
new file mode 100644
index 00000000..10869cce
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v1/main-ldm.c
@@ -0,0 +1,459 @@
+// TODO: file size must fit into a U32
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+// #define BUF_SIZE 16*1024  // Block size
+#define DEBUG
+
+//#define ZSTD
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+#endif
+
+static size_t compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* open the input file */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* open the output file */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* find size of input file */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+  size_t size_in = statbuf.st_size;
+
+ /* go to the location corresponding to the last byte */
+ if (lseek(fdout, size_in + LDM_HEADER_SIZE - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+ }
+
+ /* write a dummy byte at the last location */
+ if (write(fdout, "", 1) != 1) {
+     perror("write error");
+     return 1;
+ }
+
+  /* mmap the input file */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+  size_t out_size = statbuf.st_size + LDM_HEADER_SIZE;
+
+  /* mmap the output file */
+  if ((dst = mmap(0, out_size, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+  #ifdef ZSTD
+    size_t size_out = ZSTD_compress(dst, statbuf.st_size,
+                                    src, statbuf.st_size, 1);
+  #else
+    size_t size_out = LDM_compress(src, dst + LDM_HEADER_SIZE, statbuf.st_size,
+                                   statbuf.st_size);
+    size_out += LDM_HEADER_SIZE;
+
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+    memcpy(dst, &size_out, 4);
+    memcpy(dst + 4, &(statbuf.st_size), 4);
+    printf("Compressed size: %zu\n", size_out);
+    printf("Decompressed size: %zu\n", statbuf.st_size);
+  #endif
+  ftruncate(fdout, size_out);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)size_out, oname,
+         (double)size_out / (statbuf.st_size) * 100);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+static size_t decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* open the input file */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* open the output file */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* find size of input file */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* read header */
+  size_t compressed_size, decompressed_size;
+  LDM_read_header(src, &compressed_size, &decompressed_size);
+
+  printf("Size, compressed_size, decompressed_size: %zu %zu %zu\n",
+         statbuf.st_size, compressed_size, decompressed_size);
+
+  /* go to the location corresponding to the last byte */
+  if (lseek(fdout, decompressed_size - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressed_size, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+  /* Copy input file to output file */
+//  memcpy(dst, src, statbuf.st_size);
+
+  #ifdef ZSTD
+    size_t size_out = ZSTD_decompress(dst, decomrpessed_size,
+                                      src + LDM_HEADER_SIZE,
+                                      statbuf.st_size - LDM_HEADER_SIZE);
+  #else
+    size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
+                                     statbuf.st_size - LDM_HEADER_SIZE,
+                                     decompressed_size);
+    printf("Ret size out: %zu\n", size_out);
+  #endif
+  ftruncate(fdout, size_out);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) {
+      break;
+    }
+    if (0 == result) {
+      result = memcmp(b0, b1, r0);
+    }
+  }
+  return result;
+}
+
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+	const int cmp = compare(inpFp, decFp);
+	if(0 == cmp) {
+		printf("verify : OK\n");
+	} else {
+		printf("verify : NG\n");
+	}
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  struct timeval tv1, tv2;
+  /* compress */
+  {
+  gettimeofday(&tv1, NULL);
+  if (compress(inpFilename, ldmFilename)) {
+      printf("Compress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+         (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+
+  /* decompress */
+
+  gettimeofday(&tv1, NULL);
+  if (decompress(ldmFilename, decFilename)) {
+      printf("Decompress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+        (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+#if 0
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+

From 474e06ac5bd166b4771dd8dc2640ce8f0440aa77 Mon Sep 17 00:00:00 2001
From: Stella Lau <sl715@cam.ac.uk>
Date: Mon, 10 Jul 2017 06:32:29 -0700
Subject: [PATCH 011/100] Minor refactoring

---
 contrib/long_distance_matching/ldm.c      |  31 +-
 contrib/long_distance_matching/ldm.h      |  12 +-
 contrib/long_distance_matching/main-ldm.c | 499 +++++++++++-----------
 3 files changed, 279 insertions(+), 263 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index f8061f53..aeef4a33 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -187,29 +187,30 @@ static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
   return (unsigned)(pIn - pStart);
 }
 
-void LDM_read_header(void const *source, size_t *compressed_size,
+void LDM_read_header(const void *src, size_t *compressed_size,
                      size_t *decompressed_size) {
-  const U32 *ip = (const U32 *)source;
+  const U32 *ip = (const U32 *)src;
   *compressed_size = *ip++;
   *decompressed_size = *ip;
 }
 
-size_t LDM_compress(void const *source, void *dest, size_t source_size,
-                    size_t max_dest_size) {
-  const BYTE * const istart = (const BYTE*)source;
+// TODO: maxDstSize is unused
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  const BYTE * const istart = (const BYTE*)src;
   const BYTE *ip = istart;
-  const BYTE * const iend = istart + source_size;
+  const BYTE * const iend = istart + srcSize;
   const BYTE *ilimit = iend - HASH_SIZE;
   const BYTE * const matchlimit = iend - HASH_SIZE;
   const BYTE * const mflimit = iend - MINMATCH;
-  BYTE *op = (BYTE*) dest;
+  BYTE *op = (BYTE*) dst;
 
   compress_stats compressStats = { 0 };
 
   U32 hashTable[LDM_HASHTABLESIZE_U32];
   memset(hashTable, 0, sizeof(hashTable));
 
-  const BYTE *anchor = (const BYTE *)source;
+  const BYTE *anchor = (const BYTE *)src;
 //  struct LDM_cctx cctx;
   size_t output_size = 0;
 
@@ -361,14 +362,14 @@ _last_literals:
     op += lastRun;
   }
   print_compress_stats(&compressStats);
-  return (op - (BYTE *)dest);
+  return (op - (BYTE *)dst);
 }
 
-size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
-                      size_t max_decompressed_size) {
-  const BYTE *ip = (const BYTE *)source;
+size_t LDM_decompress(const void *src, size_t compressed_size,
+                      void *dst, size_t max_decompressed_size) {
+  const BYTE *ip = (const BYTE *)src;
   const BYTE * const iend = ip + compressed_size;
-  BYTE *op = (BYTE *)dest;
+  BYTE *op = (BYTE *)dst;
   BYTE * const oend = op + max_decompressed_size;
   BYTE *cpy;
 
@@ -437,8 +438,8 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
       *op++ = *match++;
     }
   }
-//  memcpy(dest, source, compressed_size);
-  return op - (BYTE *)dest;
+//  memcpy(dst, src, compressed_size);
+  return op - (BYTE *)dst;
 }
 
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index f4ca25a3..0ac7b2ec 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -7,13 +7,13 @@
 #define LDM_DECOMPRESS_SIZE 4
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
 
-size_t LDM_compress(void const *source, void *dest, size_t source_size,
-                    size_t max_dest_size);
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize);
 
-size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
-                      size_t max_decompressed_size);
+size_t LDM_decompress(const void *src, size_t srcSize,
+                      void *dst, size_t maxDstSize);
 
-void LDM_read_header(void const *source, size_t *compressed_size,
-                     size_t *decompressed_size);
+void LDM_read_header(const void *src, size_t *compressSize,
+                     size_t *decompressSize);
 
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 10869cce..0017335b 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -18,6 +18,263 @@
 
 //#define ZSTD
 
+/* Compress file given by fname and output to oname.
+ * Returns 0 if successful, error code otherwise.
+ */
+static int compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+
+ /* Go to the location corresponding to the last byte. */
+ /* TODO: fallocate? */ 
+  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+ /* Write a dummy byte at the last location. */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
+                                  src, statbuf.st_size, 1);
+#else
+  size_t compressSize = LDM_HEADER_SIZE + 
+      LDM_compress(src, statbuf.st_size,
+                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+
+    // Write compress and decompress size to header 
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  memcpy(dst, &compressSize, 4);
+  memcpy(dst + 4, &(statbuf.st_size), 4);
+
+#ifdef DEBUG
+  printf("Compressed size: %zu\n", compressSize);
+  printf("Decompressed size: %zu\n", statbuf.st_size);
+#endif
+#endif
+
+  // Truncate file to compressSize.
+  ftruncate(fdout, compressSize);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
+         (double)compressSize / (statbuf.st_size) * 100);
+
+  // Close files.
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Decompress file compressed using LDM_compress.
+ * The input file should have the LDM_HEADER followed by payload.
+ * Returns 0 if succesful, and an error code otherwise. 
+ */
+static int decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* Read the header. */
+  size_t compressSize, decompressSize;
+  LDM_read_header(src, &compressSize, &decompressSize);
+
+#ifdef DEBUG
+  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
+         statbuf.st_size, compressSize, decompressSize);
+#endif
+
+  /* Go to the location corresponding to the last byte. */
+  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
+                                  src + LDM_HEADER_SIZE,
+                                  statbuf.st_size - LDM_HEADER_SIZE);
+#else
+  size_t outSize = LDM_decompress(
+      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
+      dst, decompressSize);
+
+  printf("Ret size out: %zu\n", outSize);
+  #endif
+  ftruncate(fdout, outSize);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Compare two files.
+ * Returns 0 iff they are the same.
+ */
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) break;
+
+    if (0 == result) result = memcmp(b0, b1, r0);
+  }
+  return result;
+}
+
+/* Verify the input file is the same as the decompressed file. */
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+	const int cmp = compare(inpFp, decFp);
+	if(0 == cmp) {
+		printf("verify : OK\n");
+	} else {
+		printf("verify : NG\n");
+	}
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  struct timeval tv1, tv2;
+
+  /* Compress */
+
+  gettimeofday(&tv1, NULL);
+  if (compress(inpFilename, ldmFilename)) {
+      printf("Compress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+         (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* Decompress */
+
+  gettimeofday(&tv1, NULL);
+  if (decompress(ldmFilename, decFilename)) {
+      printf("Decompress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+        (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+
 #if 0
 static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
                             size_t *size_out) {
@@ -137,249 +394,7 @@ static size_t decompress_file(FILE *in, FILE *out) {
 
   return ret;
 }
-#endif
 
-static size_t compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* open the input file */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* open the output file */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* find size of input file */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-  size_t size_in = statbuf.st_size;
-
- /* go to the location corresponding to the last byte */
- if (lseek(fdout, size_in + LDM_HEADER_SIZE - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
- }
-
- /* write a dummy byte at the last location */
- if (write(fdout, "", 1) != 1) {
-     perror("write error");
-     return 1;
- }
-
-  /* mmap the input file */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-  size_t out_size = statbuf.st_size + LDM_HEADER_SIZE;
-
-  /* mmap the output file */
-  if ((dst = mmap(0, out_size, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  #ifdef ZSTD
-    size_t size_out = ZSTD_compress(dst, statbuf.st_size,
-                                    src, statbuf.st_size, 1);
-  #else
-    size_t size_out = LDM_compress(src, dst + LDM_HEADER_SIZE, statbuf.st_size,
-                                   statbuf.st_size);
-    size_out += LDM_HEADER_SIZE;
-
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-    memcpy(dst, &size_out, 4);
-    memcpy(dst + 4, &(statbuf.st_size), 4);
-    printf("Compressed size: %zu\n", size_out);
-    printf("Decompressed size: %zu\n", statbuf.st_size);
-  #endif
-  ftruncate(fdout, size_out);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)size_out, oname,
-         (double)size_out / (statbuf.st_size) * 100);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-static size_t decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* open the input file */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* open the output file */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* find size of input file */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* read header */
-  size_t compressed_size, decompressed_size;
-  LDM_read_header(src, &compressed_size, &decompressed_size);
-
-  printf("Size, compressed_size, decompressed_size: %zu %zu %zu\n",
-         statbuf.st_size, compressed_size, decompressed_size);
-
-  /* go to the location corresponding to the last byte */
-  if (lseek(fdout, decompressed_size - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressed_size, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  /* Copy input file to output file */
-//  memcpy(dst, src, statbuf.st_size);
-
-  #ifdef ZSTD
-    size_t size_out = ZSTD_decompress(dst, decomrpessed_size,
-                                      src + LDM_HEADER_SIZE,
-                                      statbuf.st_size - LDM_HEADER_SIZE);
-  #else
-    size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
-                                     statbuf.st_size - LDM_HEADER_SIZE,
-                                     decompressed_size);
-    printf("Ret size out: %zu\n", size_out);
-  #endif
-  ftruncate(fdout, size_out);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) {
-      break;
-    }
-    if (0 == result) {
-      result = memcmp(b0, b1, r0);
-    }
-  }
-  return result;
-}
-
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-	const int cmp = compare(inpFp, decFp);
-	if(0 == cmp) {
-		printf("verify : OK\n");
-	} else {
-		printf("verify : NG\n");
-	}
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  struct timeval tv1, tv2;
-  /* compress */
-  {
-  gettimeofday(&tv1, NULL);
-  if (compress(inpFilename, ldmFilename)) {
-      printf("Compress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-         (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-
-  /* decompress */
-
-  gettimeofday(&tv1, NULL);
-  if (decompress(ldmFilename, decFilename)) {
-      printf("Decompress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-        (double) (tv2.tv_sec - tv1.tv_sec));
-
-  /* verify */
-  verify(inpFilename, decFilename);
-  return 0;
-}
-
-#if 0
 int main2(int argc, char *argv[]) {
   char inpFilename[256] = { 0 };
   char ldmFilename[256] = { 0 };

From 5432214ee31b0dc56b4311e786e2420548afb7c7 Mon Sep 17 00:00:00 2001
From: Stella Lau <sl715@cam.ac.uk>
Date: Mon, 10 Jul 2017 06:50:49 -0700
Subject: [PATCH 012/100] Minor refactoring

---
 contrib/long_distance_matching/ldm.c          |  47 +-
 .../versions/v2/Makefile                      |  32 ++
 .../long_distance_matching/versions/v2/ldm.c  | 436 ++++++++++++++++
 .../long_distance_matching/versions/v2/ldm.h  |  19 +
 .../versions/v2/main-ldm.c                    | 474 ++++++++++++++++++
 5 files changed, 980 insertions(+), 28 deletions(-)
 create mode 100644 contrib/long_distance_matching/versions/v2/Makefile
 create mode 100644 contrib/long_distance_matching/versions/v2/ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v2/ldm.h
 create mode 100644 contrib/long_distance_matching/versions/v2/main-ldm.c

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index aeef4a33..9081d136 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -33,8 +33,7 @@ typedef uint64_t U64;
 
 typedef uint64_t tag;
 
-static unsigned LDM_isLittleEndian(void)
-{
+static unsigned LDM_isLittleEndian(void) {
     const union { U32 u; BYTE c[4]; } one = { 1 };
     return one.c[0];
 }
@@ -54,11 +53,11 @@ static U16 LDM_readLE16(const void *memPtr) {
   }
 }
 
-static void LDM_write32(void *memPtr, U32 value) {
+static void LDM_write16(void *memPtr, U16 value){
   memcpy(memPtr, &value, sizeof(value));
 }
 
-static void LDM_write16(void *memPtr, U16 value) {
+static void LDM_write32(void *memPtr, U32 value) {
   memcpy(memPtr, &value, sizeof(value));
 }
 
@@ -80,23 +79,10 @@ static U64 LDM_read64(const void *ptr) {
   return *(const U64 *)ptr;
 }
 
-
 static void LDM_copy8(void *dst, const void *src) {
   memcpy(dst, src, 8);
 }
 
-static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
-  BYTE *d = (BYTE *)dstPtr;
-  const BYTE *s = (const BYTE *)srcPtr;
-  BYTE * const e = (BYTE *)dstEnd;
-
-  do {
-    LDM_copy8(d, s);
-    d += 8;
-    s += 8;
-  } while (d < e);
-}
-
 typedef struct compress_stats {
   U32 num_matches;
   U32 total_match_length;
@@ -104,7 +90,7 @@ typedef struct compress_stats {
   U64 total_offset;
 } compress_stats;
 
-static void print_compress_stats(const compress_stats *stats) {
+static void LDM_printCompressStats(const compress_stats *stats) {
   printf("=====================\n");
   printf("Compression statistics\n");
   printf("Total number of matches: %u\n", stats->num_matches);
@@ -117,6 +103,7 @@ static void print_compress_stats(const compress_stats *stats) {
   printf("=====================\n");
 }
 
+// TODO: unused.
 struct hash_entry {
   U64 offset;
   tag t;
@@ -142,7 +129,6 @@ static U32 LDM_hash_position(const void * const p) {
 
 static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
                                      const BYTE *srcBase) {
-//  printf("Hashing: %zu\n", p - srcBase);
   if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
     return;
   }
@@ -187,11 +173,11 @@ static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
   return (unsigned)(pIn - pStart);
 }
 
-void LDM_read_header(const void *src, size_t *compressed_size,
-                     size_t *decompressed_size) {
+void LDM_read_header(const void *src, size_t *compressSize,
+                     size_t *decompressSize) {
   const U32 *ip = (const U32 *)src;
-  *compressed_size = *ip++;
-  *decompressed_size = *ip;
+  *compressSize = *ip++;
+  *decompressSize = *ip;
 }
 
 // TODO: maxDstSize is unused
@@ -286,7 +272,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
       printf("\n");
 #endif
       memcpy(op, anchor, litLength);
-      //LDM_wild_copy(op, anchor, op + litLength);
       op += litLength;
     }
 _next_match:
@@ -361,10 +346,19 @@ _last_literals:
     memcpy(op, anchor, lastRun);
     op += lastRun;
   }
-  print_compress_stats(&compressStats);
+  LDM_printCompressStats(&compressStats);
   return (op - (BYTE *)dst);
 }
 
+typedef struct LDM_DCtx {
+  const BYTE * const ibase;   /* Pointer to base of input */
+  const BYTE *ip;             /* Pointer to current input position */
+  const BYTE *iend;           /* End of source */
+  BYTE *op;                   /* Pointer to output */
+  const BYTE * const oend;    /* Pointer to end of output */
+
+} LDM_DCtx;
+
 size_t LDM_decompress(const void *src, size_t compressed_size,
                       void *dst, size_t max_decompressed_size) {
   const BYTE *ip = (const BYTE *)src;
@@ -399,7 +393,6 @@ size_t LDM_decompress(const void *src, size_t compressed_size,
     printf("\n");
 #endif
     memcpy(op, ip, length);
-//    LDM_wild_copy(op, ip, cpy);
     ip += length;
     op = cpy;
 
@@ -433,12 +426,10 @@ size_t LDM_decompress(const void *src, size_t compressed_size,
     cpy = op + length;
 
     // Inefficient for now
-
     while (match < cpy - offset && op < oend) {
       *op++ = *match++;
     }
   }
-//  memcpy(dst, src, compressed_size);
   return op - (BYTE *)dst;
 }
 
diff --git a/contrib/long_distance_matching/versions/v2/Makefile b/contrib/long_distance_matching/versions/v2/Makefile
new file mode 100644
index 00000000..4e04fd6a
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v2/Makefile
@@ -0,0 +1,32 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+# This Makefile presumes libzstd is installed, using `sudo make install`
+
+
+LDFLAGS += -lzstd
+
+.PHONY: default all clean
+
+default: all
+
+all: main-ldm
+
+
+#main : ldm.c main.c
+#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+main-ldm : ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+clean:
+	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
+	main main-ldm
+	@echo Cleaning completed
+
diff --git a/contrib/long_distance_matching/versions/v2/ldm.c b/contrib/long_distance_matching/versions/v2/ldm.c
new file mode 100644
index 00000000..9081d136
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v2/ldm.c
@@ -0,0 +1,436 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "ldm.h"
+
+#define HASH_EVERY 7
+
+#define LDM_MEMORY_USAGE 14
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
+
+#define WINDOW_SIZE (1 << 20)
+#define MAX_WINDOW_SIZE 31
+#define HASH_SIZE 8
+#define MINMATCH 8
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+//#define LDM_DEBUG
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint64_t tag;
+
+static unsigned LDM_isLittleEndian(void) {
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+static U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+static U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+static void LDM_write16(void *memPtr, U16 value){
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE*)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+static U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+static U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+static void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+typedef struct compress_stats {
+  U32 num_matches;
+  U32 total_match_length;
+  U32 total_literal_length;
+  U64 total_offset;
+} compress_stats;
+
+static void LDM_printCompressStats(const compress_stats *stats) {
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Total number of matches: %u\n", stats->num_matches);
+  printf("Average match length: %.1f\n", ((double)stats->total_match_length) /
+                                         (double)stats->num_matches);
+  printf("Average literal length: %.1f\n",
+         ((double)stats->total_literal_length) / (double)stats->num_matches);
+  printf("Average offset length: %.1f\n",
+         ((double)stats->total_offset) / (double)stats->num_matches);
+  printf("=====================\n");
+}
+
+// TODO: unused.
+struct hash_entry {
+  U64 offset;
+  tag t;
+};
+
+static U32 LDM_hash(U32 sequence) {
+  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
+}
+
+static U32 LDM_hash5(U64 sequence) {
+  static const U64 prime5bytes = 889523592379ULL;
+  static const U64 prime8bytes = 11400714785074694791ULL;
+  const U32 hashLog = LDM_HASHLOG;
+  if (LDM_isLittleEndian())
+    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
+  else
+    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+}
+
+static U32 LDM_hash_position(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+
+static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
+                                     const BYTE *srcBase) {
+  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+
+  U32 *hashTable = (U32 *) tableBase;
+  hashTable[h] = (U32)(p - srcBase);
+}
+
+static void LDM_put_position(const BYTE *p, void *tableBase,
+                             const BYTE *srcBase) {
+  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  U32 const h = LDM_hash_position(p);
+  LDM_put_position_on_hash(p, h, tableBase, srcBase);
+}
+
+static const BYTE *LDM_get_position_on_hash(
+    U32 h, void *tableBase, const BYTE *srcBase) {
+  const U32 * const hashTable = (U32*)tableBase;
+  return hashTable[h] + srcBase;
+}
+
+static BYTE LDM_read_byte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
+static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
+                          const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+void LDM_read_header(const void *src, size_t *compressSize,
+                     size_t *decompressSize) {
+  const U32 *ip = (const U32 *)src;
+  *compressSize = *ip++;
+  *decompressSize = *ip;
+}
+
+// TODO: maxDstSize is unused
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  const BYTE * const istart = (const BYTE*)src;
+  const BYTE *ip = istart;
+  const BYTE * const iend = istart + srcSize;
+  const BYTE *ilimit = iend - HASH_SIZE;
+  const BYTE * const matchlimit = iend - HASH_SIZE;
+  const BYTE * const mflimit = iend - MINMATCH;
+  BYTE *op = (BYTE*) dst;
+
+  compress_stats compressStats = { 0 };
+
+  U32 hashTable[LDM_HASHTABLESIZE_U32];
+  memset(hashTable, 0, sizeof(hashTable));
+
+  const BYTE *anchor = (const BYTE *)src;
+//  struct LDM_cctx cctx;
+  size_t output_size = 0;
+
+  U32 forwardH;
+
+  /* Hash first byte: put into hash table */
+
+  LDM_put_position(ip, hashTable, istart);
+  const BYTE *lastHash = ip;
+  ip++;
+  forwardH = LDM_hash_position(ip);
+
+  //TODO Loop terminates before ip>=ilimit.
+  while (ip < ilimit) {
+    const BYTE *match;
+    BYTE *token;
+
+    /* Find a match */
+    {
+      const BYTE *forwardIp = ip;
+      unsigned step = 1;
+
+      do {
+        U32 const h = forwardH;
+        ip = forwardIp;
+        forwardIp += step;
+
+        if (forwardIp > mflimit) {
+          goto _last_literals;
+        }
+
+        match = LDM_get_position_on_hash(h, hashTable, istart);
+
+        forwardH = LDM_hash_position(forwardIp);
+        LDM_put_position_on_hash(ip, h, hashTable, istart);
+        lastHash = ip;
+      } while (ip - match > WINDOW_SIZE ||
+               LDM_read64(match) != LDM_read64(ip));
+    }
+    compressStats.num_matches++;
+
+    /* Catchup: look back to extend match from found match */
+    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
+      ip--;
+      match--;
+    }
+
+    /* Encode literals */
+    {
+      unsigned const litLength = (unsigned)(ip - anchor);
+      token = op++;
+
+      compressStats.total_literal_length += litLength;
+
+#ifdef LDM_DEBUG
+      printf("Cur position: %zu\n", anchor - istart);
+      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
+#endif
+
+      if (litLength >= RUN_MASK) {
+        int len = (int)litLength - RUN_MASK;
+        *token = (RUN_MASK << ML_BITS);
+        for (; len >= 255; len -= 255) {
+          *op++ = 255;
+        }
+        *op++ = (BYTE)len;
+      } else {
+        *token = (BYTE)(litLength << ML_BITS);
+      }
+#ifdef LDM_DEBUG
+      printf("Literals ");
+      fwrite(anchor, litLength, 1, stdout);
+      printf("\n");
+#endif
+      memcpy(op, anchor, litLength);
+      op += litLength;
+    }
+_next_match:
+    /* Encode offset */
+    {
+      /*
+      LDM_writeLE16(op, ip-match);
+      op += 2;
+      */
+      LDM_write32(op, ip - match);
+      op += 4;
+      compressStats.total_offset += (ip - match);
+    }
+
+    /* Encode Match Length */
+    {
+      unsigned matchCode;
+      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
+                            matchlimit);
+#ifdef LDM_DEBUG
+      printf("Match length %zu\n", matchCode + MINMATCH);
+      fwrite(ip, MINMATCH + matchCode, 1, stdout);
+      printf("\n");
+#endif
+      compressStats.total_match_length += matchCode + MINMATCH;
+      unsigned ctr = 1;
+      ip++;
+      for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
+        LDM_put_position(ip, hashTable, istart);
+      }
+//      ip += MINMATCH + matchCode;
+      if (matchCode >= ML_MASK) {
+        *token += ML_MASK;
+        matchCode -= ML_MASK;
+        LDM_write32(op, 0xFFFFFFFF);
+        while (matchCode >= 4*0xFF) {
+          op += 4;
+          LDM_write32(op, 0xffffffff);
+          matchCode -= 4*0xFF;
+        }
+        op += matchCode / 255;
+        *op++ = (BYTE)(matchCode % 255);
+      } else {
+        *token += (BYTE)(matchCode);
+      }
+#ifdef LDM_DEBUG
+      printf("\n");
+
+#endif
+    }
+
+    anchor = ip;
+
+    LDM_put_position(ip, hashTable, istart);
+    forwardH = LDM_hash_position(++ip);
+    lastHash = ip;
+  }
+_last_literals:
+    /* Encode last literals */
+  {
+    size_t const lastRun = (size_t)(iend - anchor);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *op++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *op++ = 255;
+      }
+      *op++ = (BYTE)accumulator;
+    } else {
+      *op++ = (BYTE)(lastRun << ML_BITS);
+    }
+    memcpy(op, anchor, lastRun);
+    op += lastRun;
+  }
+  LDM_printCompressStats(&compressStats);
+  return (op - (BYTE *)dst);
+}
+
+typedef struct LDM_DCtx {
+  const BYTE * const ibase;   /* Pointer to base of input */
+  const BYTE *ip;             /* Pointer to current input position */
+  const BYTE *iend;           /* End of source */
+  BYTE *op;                   /* Pointer to output */
+  const BYTE * const oend;    /* Pointer to end of output */
+
+} LDM_DCtx;
+
+size_t LDM_decompress(const void *src, size_t compressed_size,
+                      void *dst, size_t max_decompressed_size) {
+  const BYTE *ip = (const BYTE *)src;
+  const BYTE * const iend = ip + compressed_size;
+  BYTE *op = (BYTE *)dst;
+  BYTE * const oend = op + max_decompressed_size;
+  BYTE *cpy;
+
+  while (ip < iend) {
+    size_t length;
+    const BYTE *match;
+    size_t offset;
+
+    /* get literal length */
+    unsigned const token = *ip++;
+    if ((length=(token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+#ifdef LDM_DEBUG
+    printf("Literal length: %zu\n", length);
+#endif
+
+    /* copy literals */
+    cpy = op + length;
+#ifdef LDM_DEBUG
+    printf("Literals ");
+    fwrite(ip, length, 1, stdout);
+    printf("\n");
+#endif
+    memcpy(op, ip, length);
+    ip += length;
+    op = cpy;
+
+    /* get offset */
+    /*
+    offset = LDM_readLE16(ip);
+    ip += 2;
+    */
+    offset = LDM_read32(ip);
+    ip += 4;
+#ifdef LDM_DEBUG
+    printf("Offset: %zu\n", offset);
+#endif
+    match = op - offset;
+ //   LDM_write32(op, (U32)offset);
+
+    /* get matchlength */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+#ifdef LDM_DEBUG
+    printf("Match length: %zu\n", length);
+#endif
+    /* copy match */
+    cpy = op + length;
+
+    // Inefficient for now
+    while (match < cpy - offset && op < oend) {
+      *op++ = *match++;
+    }
+  }
+  return op - (BYTE *)dst;
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v2/ldm.h b/contrib/long_distance_matching/versions/v2/ldm.h
new file mode 100644
index 00000000..0ac7b2ec
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v2/ldm.h
@@ -0,0 +1,19 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize);
+
+size_t LDM_decompress(const void *src, size_t srcSize,
+                      void *dst, size_t maxDstSize);
+
+void LDM_read_header(const void *src, size_t *compressSize,
+                     size_t *decompressSize);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v2/main-ldm.c b/contrib/long_distance_matching/versions/v2/main-ldm.c
new file mode 100644
index 00000000..0017335b
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v2/main-ldm.c
@@ -0,0 +1,474 @@
+// TODO: file size must fit into a U32
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+// #define BUF_SIZE 16*1024  // Block size
+#define DEBUG
+
+//#define ZSTD
+
+/* Compress file given by fname and output to oname.
+ * Returns 0 if successful, error code otherwise.
+ */
+static int compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+
+ /* Go to the location corresponding to the last byte. */
+ /* TODO: fallocate? */ 
+  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+ /* Write a dummy byte at the last location. */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
+                                  src, statbuf.st_size, 1);
+#else
+  size_t compressSize = LDM_HEADER_SIZE + 
+      LDM_compress(src, statbuf.st_size,
+                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+
+    // Write compress and decompress size to header 
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  memcpy(dst, &compressSize, 4);
+  memcpy(dst + 4, &(statbuf.st_size), 4);
+
+#ifdef DEBUG
+  printf("Compressed size: %zu\n", compressSize);
+  printf("Decompressed size: %zu\n", statbuf.st_size);
+#endif
+#endif
+
+  // Truncate file to compressSize.
+  ftruncate(fdout, compressSize);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
+         (double)compressSize / (statbuf.st_size) * 100);
+
+  // Close files.
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Decompress file compressed using LDM_compress.
+ * The input file should have the LDM_HEADER followed by payload.
+ * Returns 0 if succesful, and an error code otherwise. 
+ */
+static int decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* Read the header. */
+  size_t compressSize, decompressSize;
+  LDM_read_header(src, &compressSize, &decompressSize);
+
+#ifdef DEBUG
+  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
+         statbuf.st_size, compressSize, decompressSize);
+#endif
+
+  /* Go to the location corresponding to the last byte. */
+  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
+                                  src + LDM_HEADER_SIZE,
+                                  statbuf.st_size - LDM_HEADER_SIZE);
+#else
+  size_t outSize = LDM_decompress(
+      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
+      dst, decompressSize);
+
+  printf("Ret size out: %zu\n", outSize);
+  #endif
+  ftruncate(fdout, outSize);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Compare two files.
+ * Returns 0 iff they are the same.
+ */
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) break;
+
+    if (0 == result) result = memcmp(b0, b1, r0);
+  }
+  return result;
+}
+
+/* Verify the input file is the same as the decompressed file. */
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+	const int cmp = compare(inpFp, decFp);
+	if(0 == cmp) {
+		printf("verify : OK\n");
+	} else {
+		printf("verify : NG\n");
+	}
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  struct timeval tv1, tv2;
+
+  /* Compress */
+
+  gettimeofday(&tv1, NULL);
+  if (compress(inpFilename, ldmFilename)) {
+      printf("Compress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+         (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* Decompress */
+
+  gettimeofday(&tv1, NULL);
+  if (decompress(ldmFilename, decFilename)) {
+      printf("Decompress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+        (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+

From ae9cf235d62bb7482496309294852f51bdfd1f1d Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 10 Jul 2017 07:38:09 -0700
Subject: [PATCH 013/100] Add LDM_DCtx

---
 contrib/long_distance_matching/ldm.c      | 74 ++++++++++++++---------
 contrib/long_distance_matching/main-ldm.c |  4 +-
 2 files changed, 47 insertions(+), 31 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 9081d136..b18ed3d4 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -351,33 +351,49 @@ _last_literals:
 }
 
 typedef struct LDM_DCtx {
-  const BYTE * const ibase;   /* Pointer to base of input */
-  const BYTE *ip;             /* Pointer to current input position */
-  const BYTE *iend;           /* End of source */
-  BYTE *op;                   /* Pointer to output */
-  const BYTE * const oend;    /* Pointer to end of output */
+  const BYTE *ibase;   /* Pointer to base of input */
+  const BYTE *ip;      /* Pointer to current input position */
+  const BYTE *iend;    /* End of source */
 
+  const BYTE *obase;   /* Pointer to base of output */ 
+  BYTE *op;            /* Pointer to output */
+  const BYTE *oend;    /* Pointer to end of output */
+
+  size_t compressSize;
+  size_t maxDecompressSize;
 } LDM_DCtx;
 
-size_t LDM_decompress(const void *src, size_t compressed_size,
-                      void *dst, size_t max_decompressed_size) {
-  const BYTE *ip = (const BYTE *)src;
-  const BYTE * const iend = ip + compressed_size;
-  BYTE *op = (BYTE *)dst;
-  BYTE * const oend = op + max_decompressed_size;
+static void LDM_initializeDCtx(LDM_DCtx *dctx,
+                               const void *src, size_t compressSize,
+                               void *dst, size_t maxDecompressSize) {
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + compressSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + maxDecompressSize;
+ 
+  dctx->compressSize = compressSize;
+  dctx->maxDecompressSize = maxDecompressSize;
+}
+
+size_t LDM_decompress(const void *src, size_t compressSize,
+                      void *dst, size_t maxDecompressSize) {
+  LDM_DCtx dctx;
+  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
+
   BYTE *cpy;
 
-  while (ip < iend) {
+  while (dctx.ip < dctx.iend) {
     size_t length;
     const BYTE *match;
     size_t offset;
 
     /* get literal length */
-    unsigned const token = *ip++;
+    unsigned const token = *(dctx.ip)++;
     if ((length=(token >> ML_BITS)) == RUN_MASK) {
       unsigned s;
       do {
-        s = *ip++;
+        s = *(dctx.ip)++;
         length += s;
       } while (s == 255);
     }
@@ -386,27 +402,27 @@ size_t LDM_decompress(const void *src, size_t compressed_size,
 #endif
 
     /* copy literals */
-    cpy = op + length;
+    cpy = dctx.op + length;
 #ifdef LDM_DEBUG
     printf("Literals ");
-    fwrite(ip, length, 1, stdout);
+    fwrite(dctx.ip, length, 1, stdout);
     printf("\n");
 #endif
-    memcpy(op, ip, length);
-    ip += length;
-    op = cpy;
+    memcpy(dctx.op, dctx.ip, length);
+    dctx.ip += length;
+    dctx.op = cpy;
 
     /* get offset */
     /*
-    offset = LDM_readLE16(ip);
-    ip += 2;
+    offset = LDM_readLE16(dctx.ip);
+    dctx.ip += 2;
     */
-    offset = LDM_read32(ip);
-    ip += 4;
+    offset = LDM_read32(dctx.ip);
+    dctx.ip += 4;
 #ifdef LDM_DEBUG
     printf("Offset: %zu\n", offset);
 #endif
-    match = op - offset;
+    match = dctx.op - offset;
  //   LDM_write32(op, (U32)offset);
 
     /* get matchlength */
@@ -414,7 +430,7 @@ size_t LDM_decompress(const void *src, size_t compressed_size,
     if (length == ML_MASK) {
       unsigned s;
       do {
-        s = *ip++;
+        s = *(dctx.ip)++;
         length += s;
       } while (s == 255);
     }
@@ -423,14 +439,14 @@ size_t LDM_decompress(const void *src, size_t compressed_size,
     printf("Match length: %zu\n", length);
 #endif
     /* copy match */
-    cpy = op + length;
+    cpy = dctx.op + length;
 
     // Inefficient for now
-    while (match < cpy - offset && op < oend) {
-      *op++ = *match++;
+    while (match < cpy - offset && dctx.op < dctx.oend) {
+      *(dctx.op)++ = *match++;
     }
   }
-  return op - (BYTE *)dst;
+  return dctx.op - (BYTE *)dst;
 }
 
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 0017335b..b529201f 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -88,7 +88,7 @@ static int compress(const char *fname, const char *oname) {
 
 #ifdef DEBUG
   printf("Compressed size: %zu\n", compressSize);
-  printf("Decompressed size: %zu\n", statbuf.st_size);
+  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
 #endif
 #endif
 
@@ -145,7 +145,7 @@ static int decompress(const char *fname, const char *oname) {
 
 #ifdef DEBUG
   printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
-         statbuf.st_size, compressSize, decompressSize);
+         (size_t)statbuf.st_size, compressSize, decompressSize);
 #endif
 
   /* Go to the location corresponding to the last byte. */

From 89190ef07dc87d3c292ad98ff1f5d20f0238c19e Mon Sep 17 00:00:00 2001
From: Paul Cruz <paulcruz74@fb.com>
Date: Mon, 10 Jul 2017 11:32:30 -0700
Subject: [PATCH 014/100] renamed pool.c to poolTests.c

---
 tests/Makefile                | 10 +++++-----
 tests/{pool.c => poolTests.c} |  0
 2 files changed, 5 insertions(+), 5 deletions(-)
 rename tests/{pool.c => poolTests.c} (100%)

diff --git a/tests/Makefile b/tests/Makefile
index 345e0e8e..cd89a940 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -79,7 +79,7 @@ all32: fullbench32 fuzzer32 zstreamtest32
 
 allnothread: fullbench fuzzer paramgrill datagen  decodecorpus
 
-dll: fuzzer-dll zstreamtest-dll 
+dll: fuzzer-dll zstreamtest-dll
 
 zstd:
 	$(MAKE) -C $(PRGDIR) $@
@@ -192,7 +192,7 @@ else
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so
 endif
 
-pool  : pool.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c
+poolTests  : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c
 	$(CC)    $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
 
 namespaceTest:
@@ -213,7 +213,7 @@ clean:
         fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
         zstreamtest$(EXT) zstreamtest32$(EXT) \
         datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
-        symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT) \
+        symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
 	decodecorpus$(EXT)
 	@echo Cleaning completed
 
@@ -375,7 +375,7 @@ test-decodecorpus-cli: decodecorpus
 	cd ..
 	@rm -rf testdir
 
-test-pool: pool
-	$(QEMU_SYS) ./pool
+test-pool: poolTests
+	$(QEMU_SYS) ./poolTests
 
 endif
diff --git a/tests/pool.c b/tests/poolTests.c
similarity index 100%
rename from tests/pool.c
rename to tests/poolTests.c

From e32fb0c1fe92cc225ed9585efd2a3bcee3aab053 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 10 Jul 2017 12:29:57 -0700
Subject: [PATCH 015/100] added ZSTD_sizeof_CCtx() test

---
 lib/compress/zstdmt_compress.c |  4 ++--
 tests/fuzzer.c                 | 18 ++++++++++++++----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 0cee01ea..d5607adf 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -763,10 +763,10 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
         zcs->inBuff.filled = 0;
         zcs->dictSize = 0;
         zcs->frameEnded = 1;
-        if (zcs->nextJobID == 0)
+        if (zcs->nextJobID == 0) {
             /* single chunk exception : checksum is calculated directly within worker thread */
             zcs->params.fParams.checksumFlag = 0;
-    }
+    }   }
 
     DEBUGLOG(4, "posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)",
                 zcs->nextJobID,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 4d88893a..c9461ee9 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -136,10 +136,20 @@ static int basicUnitTests(U32 seed, double compressibility)
 
 
     DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
-    CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
-                               CNBuffer, CNBuffSize, 1),
-              cSize=r );
-    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+    {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        if (cctx==NULL) goto _output_error;
+        CHECKPLUS(r, ZSTD_compressCCtx(cctx,
+                            compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                            CNBuffer, CNBuffSize, 1),
+                  cSize=r );
+        DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+        DISPLAYLEVEL(4, "test%3i : size of cctx for level 1 : ", testNb++);
+        {   size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
+            DISPLAYLEVEL(4, "%u bytes \n", (U32)cctxSize);
+        }
+        ZSTD_freeCCtx(cctx);
+    }
 
 
     DISPLAYLEVEL(4, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);

From 10a71d9f1c31539b79c66c90750025e26094621c Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 10 Jul 2017 12:38:27 -0700
Subject: [PATCH 016/100] Add compression context

---
 contrib/long_distance_matching/ldm.c | 287 ++++++++++++++++-----------
 1 file changed, 167 insertions(+), 120 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index b18ed3d4..7bf26781 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -31,7 +31,10 @@ typedef uint32_t U32;
 typedef  int32_t S32;
 typedef uint64_t U64;
 
-typedef uint64_t tag;
+typedef uint32_t offset_t;
+typedef uint32_t hash_t;
+
+// typedef uint64_t tag;
 
 static unsigned LDM_isLittleEndian(void) {
     const union { U32 u; BYTE c[4]; } one = { 1 };
@@ -65,7 +68,7 @@ static void LDM_writeLE16(void *memPtr, U16 value) {
   if (LDM_isLittleEndian()) {
     LDM_write16(memPtr, value);
   } else {
-    BYTE* p = (BYTE*)memPtr;
+    BYTE* p = (BYTE *)memPtr;
     p[0] = (BYTE) value;
     p[1] = (BYTE)(value>>8);
   }
@@ -82,15 +85,18 @@ static U64 LDM_read64(const void *ptr) {
 static void LDM_copy8(void *dst, const void *src) {
   memcpy(dst, src, 8);
 }
+typedef struct LDM_hashEntry {
+  offset_t offset;
+} LDM_hashEntry;
 
-typedef struct compress_stats {
+typedef struct LDM_compressStats {
   U32 num_matches;
   U32 total_match_length;
   U32 total_literal_length;
   U64 total_offset;
-} compress_stats;
+} LDM_compressStats;
 
-static void LDM_printCompressStats(const compress_stats *stats) {
+static void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("=====================\n");
   printf("Compression statistics\n");
   printf("Total number of matches: %u\n", stats->num_matches);
@@ -103,53 +109,83 @@ static void LDM_printCompressStats(const compress_stats *stats) {
   printf("=====================\n");
 }
 
-// TODO: unused.
-struct hash_entry {
-  U64 offset;
-  tag t;
-};
+typedef struct LDM_CCtx {
+  size_t isize;             /* Input size */
+  size_t maxOSize;          /* Maximum output size */
 
-static U32 LDM_hash(U32 sequence) {
+  const BYTE *ibase;        /* Base of input */
+  const BYTE *ip;           /* Current input position */
+  const BYTE *iend;         /* End of input */
+
+  // Maximum input position such that hashing at the position does not exceed
+  // end of input.
+  const BYTE *ihashLimit;
+
+  // Maximum input position such that finding a match of at least the minimum
+  // match length does not exceed end of input.
+  const BYTE *imatchLimit;
+
+  const BYTE *obase;        /* Base of output */
+  BYTE *op;                 /* Output */
+
+  const BYTE *anchor;       /* Anchor to start of current (match) block */
+
+  LDM_compressStats stats;            /* Compression statistics */
+
+  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+
+  const BYTE *lastPosHashed;          /* Last position hashed */
+  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
+
+} LDM_CCtx;
+
+
+static hash_t LDM_hash(U32 sequence) {
   return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
 }
 
-static U32 LDM_hash5(U64 sequence) {
+static hash_t LDM_hash5(U64 sequence) {
   static const U64 prime5bytes = 889523592379ULL;
   static const U64 prime8bytes = 11400714785074694791ULL;
   const U32 hashLog = LDM_HASHLOG;
   if (LDM_isLittleEndian())
-    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
+    return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
   else
-    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+    return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
 }
 
-static U32 LDM_hash_position(const void * const p) {
+static hash_t LDM_hash_position(const void * const p) {
   return LDM_hash(LDM_read32(p));
 }
 
-static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
-                                     const BYTE *srcBase) {
+static void LDM_put_position_on_hash(const BYTE *p, hash_t h,
+                                     void *tableBase, const BYTE *srcBase) {
   if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
     return;
   }
 
-  U32 *hashTable = (U32 *) tableBase;
-  hashTable[h] = (U32)(p - srcBase);
+  LDM_hashEntry *hashTable = (LDM_hashEntry *) tableBase;
+  hashTable[h] = (LDM_hashEntry) { (hash_t )(p - srcBase) };
 }
 
-static void LDM_put_position(const BYTE *p, void *tableBase,
+static void LDM_putPosition(const BYTE *p, void *tableBase,
                              const BYTE *srcBase) {
   if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
     return;
   }
-  U32 const h = LDM_hash_position(p);
+  hash_t const h = LDM_hash_position(p);
   LDM_put_position_on_hash(p, h, tableBase, srcBase);
 }
 
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *const cctx) {
+  LDM_putPosition(cctx->ip, cctx->hashTable, cctx->ibase);
+}
+
+
 static const BYTE *LDM_get_position_on_hash(
-    U32 h, void *tableBase, const BYTE *srcBase) {
-  const U32 * const hashTable = (U32*)tableBase;
-  return hashTable[h] + srcBase;
+    hash_t h, void *tableBase, const BYTE *srcBase) {
+  const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
+  return hashTable[h].offset + srcBase;
 }
 
 static BYTE LDM_read_byte(const void *memPtr) {
@@ -180,140 +216,149 @@ void LDM_read_header(const void *src, size_t *compressSize,
   *decompressSize = *ip;
 }
 
-// TODO: maxDstSize is unused
+static void LDM_initializeCCtx(LDM_CCtx *cctx,
+                               const void *src, size_t srcSize,
+                               void *dst, size_t maxDstSize) {
+  cctx->isize = srcSize;
+  cctx->maxOSize = maxDstSize;
+
+  cctx->ibase = (const BYTE *)src;
+  cctx->ip = cctx->ibase;
+  cctx->iend = cctx->ibase + srcSize;
+
+  cctx->ihashLimit = cctx->iend - HASH_SIZE;
+  cctx->imatchLimit = cctx->iend - MINMATCH;
+
+  cctx->obase = (BYTE *)dst;
+  cctx->op = (BYTE *)cctx->obase;
+
+  cctx->anchor = cctx->ibase;
+
+  memset(&(cctx->stats), 0, sizeof(cctx->stats));
+  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+
+  cctx->lastPosHashed = NULL;
+}
+
+// TODO: srcSize and maxDstSize is unused
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
-  const BYTE * const istart = (const BYTE*)src;
-  const BYTE *ip = istart;
-  const BYTE * const iend = istart + srcSize;
-  const BYTE *ilimit = iend - HASH_SIZE;
-  const BYTE * const matchlimit = iend - HASH_SIZE;
-  const BYTE * const mflimit = iend - MINMATCH;
-  BYTE *op = (BYTE*) dst;
-
-  compress_stats compressStats = { 0 };
-
-  U32 hashTable[LDM_HASHTABLESIZE_U32];
-  memset(hashTable, 0, sizeof(hashTable));
-
-  const BYTE *anchor = (const BYTE *)src;
-//  struct LDM_cctx cctx;
-  size_t output_size = 0;
+  LDM_CCtx cctx;
+  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   U32 forwardH;
 
-  /* Hash first byte: put into hash table */
+  /* Hash the first position and put it into the hash table. */
+  LDM_putHashOfCurrentPosition(&cctx);
+  const BYTE *lastHash = cctx.ip;
+  cctx.ip++;
+  forwardH = LDM_hash_position(cctx.ip);
 
-  LDM_put_position(ip, hashTable, istart);
-  const BYTE *lastHash = ip;
-  ip++;
-  forwardH = LDM_hash_position(ip);
-
-  //TODO Loop terminates before ip>=ilimit.
-  while (ip < ilimit) {
+  // TODO: loop condition is not accurate.
+  while (1) {
     const BYTE *match;
     BYTE *token;
 
     /* Find a match */
     {
-      const BYTE *forwardIp = ip;
+      const BYTE *forwardIp = cctx.ip;
       unsigned step = 1;
 
       do {
         U32 const h = forwardH;
-        ip = forwardIp;
+        cctx.ip = forwardIp;
         forwardIp += step;
 
-        if (forwardIp > mflimit) {
+        if (forwardIp > cctx.imatchLimit) {
           goto _last_literals;
         }
 
-        match = LDM_get_position_on_hash(h, hashTable, istart);
+        match = LDM_get_position_on_hash(h, cctx.hashTable, cctx.ibase);
 
         forwardH = LDM_hash_position(forwardIp);
-        LDM_put_position_on_hash(ip, h, hashTable, istart);
-        lastHash = ip;
-      } while (ip - match > WINDOW_SIZE ||
-               LDM_read64(match) != LDM_read64(ip));
+        LDM_put_position_on_hash(cctx.ip, h, cctx.hashTable, cctx.ibase);
+        lastHash = cctx.ip;
+      } while (cctx.ip - match > WINDOW_SIZE ||
+               LDM_read64(match) != LDM_read64(cctx.ip));
     }
-    compressStats.num_matches++;
+    cctx.stats.num_matches++;
 
     /* Catchup: look back to extend match from found match */
-    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
-      ip--;
+    while (cctx.ip > cctx.anchor && match > cctx.ibase && cctx.ip[-1] == match[-1]) {
+      cctx.ip--;
       match--;
     }
 
     /* Encode literals */
     {
-      unsigned const litLength = (unsigned)(ip - anchor);
-      token = op++;
+      unsigned const litLength = (unsigned)(cctx.ip - cctx.anchor);
+      token = cctx.op++;
 
-      compressStats.total_literal_length += litLength;
+      cctx.stats.total_literal_length += litLength;
 
 #ifdef LDM_DEBUG
-      printf("Cur position: %zu\n", anchor - istart);
-      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
+      printf("Cur position: %zu\n", cctx.anchor - cctx.ibase);
+      printf("LitLength %zu. (Match offset). %zu\n", litLength, cctx.ip - match);
 #endif
 
       if (litLength >= RUN_MASK) {
         int len = (int)litLength - RUN_MASK;
         *token = (RUN_MASK << ML_BITS);
         for (; len >= 255; len -= 255) {
-          *op++ = 255;
+          *(cctx.op)++ = 255;
         }
-        *op++ = (BYTE)len;
+        *(cctx.op)++ = (BYTE)len;
       } else {
         *token = (BYTE)(litLength << ML_BITS);
       }
 #ifdef LDM_DEBUG
       printf("Literals ");
-      fwrite(anchor, litLength, 1, stdout);
+      fwrite(cctx.anchor, litLength, 1, stdout);
       printf("\n");
 #endif
-      memcpy(op, anchor, litLength);
-      op += litLength;
+      memcpy(cctx.op, cctx.anchor, litLength);
+      cctx.op += litLength;
     }
 _next_match:
     /* Encode offset */
     {
       /*
-      LDM_writeLE16(op, ip-match);
-      op += 2;
+      LDM_writeLE16(cctx.op, cctx.ip-match);
+      cctx.op += 2;
       */
-      LDM_write32(op, ip - match);
-      op += 4;
-      compressStats.total_offset += (ip - match);
+      LDM_write32(cctx.op, cctx.ip - match);
+      cctx.op += 4;
+      cctx.stats.total_offset += (cctx.ip - match);
     }
 
     /* Encode Match Length */
     {
       unsigned matchCode;
-      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
-                            matchlimit);
+      matchCode = LDM_count(cctx.ip + MINMATCH, match + MINMATCH,
+                            cctx.ihashLimit);
 #ifdef LDM_DEBUG
       printf("Match length %zu\n", matchCode + MINMATCH);
-      fwrite(ip, MINMATCH + matchCode, 1, stdout);
+      fwrite(cctx.ip, MINMATCH + matchCode, 1, stdout);
       printf("\n");
 #endif
-      compressStats.total_match_length += matchCode + MINMATCH;
+      cctx.stats.total_match_length += matchCode + MINMATCH;
       unsigned ctr = 1;
-      ip++;
-      for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
-        LDM_put_position(ip, hashTable, istart);
+      cctx.ip++;
+      for (; ctr < MINMATCH + matchCode; cctx.ip++, ctr++) {
+        LDM_putHashOfCurrentPosition(&cctx);
       }
-//      ip += MINMATCH + matchCode;
+//      cctx.ip += MINMATCH + matchCode;
       if (matchCode >= ML_MASK) {
         *token += ML_MASK;
         matchCode -= ML_MASK;
-        LDM_write32(op, 0xFFFFFFFF);
+        LDM_write32(cctx.op, 0xFFFFFFFF);
         while (matchCode >= 4*0xFF) {
-          op += 4;
-          LDM_write32(op, 0xffffffff);
+          cctx.op += 4;
+          LDM_write32(cctx.op, 0xffffffff);
           matchCode -= 4*0xFF;
         }
-        op += matchCode / 255;
-        *op++ = (BYTE)(matchCode % 255);
+        cctx.op += matchCode / 255;
+        *(cctx.op)++ = (BYTE)(matchCode % 255);
       } else {
         *token += (BYTE)(matchCode);
       }
@@ -323,57 +368,58 @@ _next_match:
 #endif
     }
 
-    anchor = ip;
+    cctx.anchor = cctx.ip;
 
-    LDM_put_position(ip, hashTable, istart);
-    forwardH = LDM_hash_position(++ip);
-    lastHash = ip;
+    LDM_putPosition(cctx.ip, cctx.hashTable, cctx.ibase);
+    forwardH = LDM_hash_position(++cctx.ip);
+    lastHash = cctx.ip;
   }
 _last_literals:
     /* Encode last literals */
   {
-    size_t const lastRun = (size_t)(iend - anchor);
+    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
     if (lastRun >= RUN_MASK) {
       size_t accumulator = lastRun - RUN_MASK;
-      *op++ = RUN_MASK << ML_BITS;
+      *(cctx.op)++ = RUN_MASK << ML_BITS;
       for(; accumulator >= 255; accumulator -= 255) {
-        *op++ = 255;
+        *(cctx.op)++ = 255;
       }
-      *op++ = (BYTE)accumulator;
+      *(cctx.op)++ = (BYTE)accumulator;
     } else {
-      *op++ = (BYTE)(lastRun << ML_BITS);
+      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
     }
-    memcpy(op, anchor, lastRun);
-    op += lastRun;
+    memcpy(cctx.op, cctx.anchor, lastRun);
+    cctx.op += lastRun;
   }
-  LDM_printCompressStats(&compressStats);
-  return (op - (BYTE *)dst);
+  LDM_printCompressStats(&cctx.stats);
+  return (cctx.op - (BYTE *)cctx.obase);
 }
 
 typedef struct LDM_DCtx {
-  const BYTE *ibase;   /* Pointer to base of input */
-  const BYTE *ip;      /* Pointer to current input position */
-  const BYTE *iend;    /* End of source */
-
-  const BYTE *obase;   /* Pointer to base of output */ 
-  BYTE *op;            /* Pointer to output */
-  const BYTE *oend;    /* Pointer to end of output */
-
   size_t compressSize;
   size_t maxDecompressSize;
+
+  const BYTE *ibase;   /* Base of input */
+  const BYTE *ip;      /* Current input position */
+  const BYTE *iend;    /* End of source */
+
+  const BYTE *obase;   /* Base of output */
+  BYTE *op;            /* Current output position */
+  const BYTE *oend;    /* End of output */
 } LDM_DCtx;
 
 static void LDM_initializeDCtx(LDM_DCtx *dctx,
                                const void *src, size_t compressSize,
                                void *dst, size_t maxDecompressSize) {
-  dctx->ibase = src;
-  dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + compressSize;
-  dctx->op = dst;
-  dctx->oend = dctx->op + maxDecompressSize;
- 
   dctx->compressSize = compressSize;
   dctx->maxDecompressSize = maxDecompressSize;
+
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + dctx->compressSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + dctx->maxDecompressSize;
+
 }
 
 size_t LDM_decompress(const void *src, size_t compressSize,
@@ -382,15 +428,14 @@ size_t LDM_decompress(const void *src, size_t compressSize,
   LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
 
   BYTE *cpy;
+  size_t length;
+  const BYTE *match;
+  size_t offset;
 
   while (dctx.ip < dctx.iend) {
-    size_t length;
-    const BYTE *match;
-    size_t offset;
-
     /* get literal length */
     unsigned const token = *(dctx.ip)++;
-    if ((length=(token >> ML_BITS)) == RUN_MASK) {
+    if ((length = (token >> ML_BITS)) == RUN_MASK) {
       unsigned s;
       do {
         s = *(dctx.ip)++;
@@ -417,6 +462,8 @@ size_t LDM_decompress(const void *src, size_t compressSize,
     offset = LDM_readLE16(dctx.ip);
     dctx.ip += 2;
     */
+
+    //TODO : dynamic offset size
     offset = LDM_read32(dctx.ip);
     dctx.ip += 4;
 #ifdef LDM_DEBUG

From e4155b11d749c2c5f6faac9cabbc6e0dc1262e14 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 10 Jul 2017 13:08:19 -0700
Subject: [PATCH 017/100] Add warning flags to makefile and clean up code to
 remove warnings

---
 contrib/long_distance_matching/Makefile   | 10 ++-
 contrib/long_distance_matching/ldm.c      | 57 +++++++++--------
 contrib/long_distance_matching/ldm.h      |  4 +-
 contrib/long_distance_matching/main-ldm.c | 77 ++++++++++++-----------
 contrib/long_distance_matching/util.c     | 64 +++++++++++++++++++
 contrib/long_distance_matching/util.h     | 23 +++++++
 6 files changed, 171 insertions(+), 64 deletions(-)
 create mode 100644 contrib/long_distance_matching/util.c
 create mode 100644 contrib/long_distance_matching/util.h

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 4e04fd6a..5ffd4eaf 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -9,6 +9,14 @@
 
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
+CFLAGS  ?= -O3
+DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
 LDFLAGS += -lzstd
 
@@ -22,7 +30,7 @@ all: main-ldm
 #main : ldm.c main.c
 #	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-ldm : ldm.c main-ldm.c
+main-ldm : util.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 7bf26781..12cffc40 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -4,6 +4,7 @@
 #include <stdio.h>
 
 #include "ldm.h"
+#include "util.h"
 
 #define HASH_EVERY 7
 
@@ -36,6 +37,7 @@ typedef uint32_t hash_t;
 
 // typedef uint64_t tag;
 
+/*
 static unsigned LDM_isLittleEndian(void) {
     const union { U32 u; BYTE c[4]; } one = { 1 };
     return one.c[0];
@@ -85,6 +87,8 @@ static U64 LDM_read64(const void *ptr) {
 static void LDM_copy8(void *dst, const void *src) {
   memcpy(dst, src, 8);
 }
+
+*/
 typedef struct LDM_hashEntry {
   offset_t offset;
 } LDM_hashEntry;
@@ -144,6 +148,7 @@ static hash_t LDM_hash(U32 sequence) {
   return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
 }
 
+/*
 static hash_t LDM_hash5(U64 sequence) {
   static const U64 prime5bytes = 889523592379ULL;
   static const U64 prime8bytes = 11400714785074694791ULL;
@@ -153,35 +158,40 @@ static hash_t LDM_hash5(U64 sequence) {
   else
     return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
 }
+*/
 
 static hash_t LDM_hash_position(const void * const p) {
   return LDM_hash(LDM_read32(p));
 }
 
-static void LDM_put_position_on_hash(const BYTE *p, hash_t h,
-                                     void *tableBase, const BYTE *srcBase) {
+static void LDM_putHashOfPosition(const BYTE *p, hash_t h,
+                                  void *tableBase, const BYTE *srcBase) {
+  LDM_hashEntry *hashTable;
   if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
     return;
   }
 
-  LDM_hashEntry *hashTable = (LDM_hashEntry *) tableBase;
-  hashTable[h] = (LDM_hashEntry) { (hash_t )(p - srcBase) };
+  hashTable = (LDM_hashEntry *) tableBase;
+  hashTable[h] = (LDM_hashEntry) { (hash_t)(p - srcBase) };
 }
 
 static void LDM_putPosition(const BYTE *p, void *tableBase,
-                             const BYTE *srcBase) {
+                            const BYTE *srcBase) {
+  hash_t hash;
   if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
     return;
   }
-  hash_t const h = LDM_hash_position(p);
-  LDM_put_position_on_hash(p, h, tableBase, srcBase);
+  hash = LDM_hash_position(p);
+  LDM_putHashOfPosition(p, hash, tableBase, srcBase);
 }
 
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *const cctx) {
-  LDM_putPosition(cctx->ip, cctx->hashTable, cctx->ibase);
+  hash_t hash = LDM_hash_position(cctx->ip);
+  LDM_putHashOfPosition(cctx->ip, hash, cctx->hashTable, cctx->ibase);
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
 }
 
-
 static const BYTE *LDM_get_position_on_hash(
     hash_t h, void *tableBase, const BYTE *srcBase) {
   const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
@@ -209,8 +219,8 @@ static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
   return (unsigned)(pIn - pStart);
 }
 
-void LDM_read_header(const void *src, size_t *compressSize,
-                     size_t *decompressSize) {
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize) {
   const U32 *ip = (const U32 *)src;
   *compressSize = *ip++;
   *decompressSize = *ip;
@@ -230,7 +240,7 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->imatchLimit = cctx->iend - MINMATCH;
 
   cctx->obase = (BYTE *)dst;
-  cctx->op = (BYTE *)cctx->obase;
+  cctx->op = (BYTE *)dst;
 
   cctx->anchor = cctx->ibase;
 
@@ -244,13 +254,12 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
+  U32 forwardH;
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
-  U32 forwardH;
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
-  const BYTE *lastHash = cctx.ip;
   cctx.ip++;
   forwardH = LDM_hash_position(cctx.ip);
 
@@ -276,8 +285,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
         match = LDM_get_position_on_hash(h, cctx.hashTable, cctx.ibase);
 
         forwardH = LDM_hash_position(forwardIp);
-        LDM_put_position_on_hash(cctx.ip, h, cctx.hashTable, cctx.ibase);
-        lastHash = cctx.ip;
+        LDM_putHashOfPosition(cctx.ip, h, cctx.hashTable, cctx.ibase);
       } while (cctx.ip - match > WINDOW_SIZE ||
                LDM_read64(match) != LDM_read64(cctx.ip));
     }
@@ -319,7 +327,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
       memcpy(cctx.op, cctx.anchor, litLength);
       cctx.op += litLength;
     }
-_next_match:
+
     /* Encode offset */
     {
       /*
@@ -334,6 +342,7 @@ _next_match:
     /* Encode Match Length */
     {
       unsigned matchCode;
+      unsigned ctr = 1;
       matchCode = LDM_count(cctx.ip + MINMATCH, match + MINMATCH,
                             cctx.ihashLimit);
 #ifdef LDM_DEBUG
@@ -342,7 +351,6 @@ _next_match:
       printf("\n");
 #endif
       cctx.stats.total_match_length += matchCode + MINMATCH;
-      unsigned ctr = 1;
       cctx.ip++;
       for (; ctr < MINMATCH + matchCode; cctx.ip++, ctr++) {
         LDM_putHashOfCurrentPosition(&cctx);
@@ -372,7 +380,6 @@ _next_match:
 
     LDM_putPosition(cctx.ip, cctx.hashTable, cctx.ibase);
     forwardH = LDM_hash_position(++cctx.ip);
-    lastHash = cctx.ip;
   }
 _last_literals:
     /* Encode last literals */
@@ -392,7 +399,7 @@ _last_literals:
     cctx.op += lastRun;
   }
   LDM_printCompressStats(&cctx.stats);
-  return (cctx.op - (BYTE *)cctx.obase);
+  return (cctx.op - (const BYTE *)cctx.obase);
 }
 
 typedef struct LDM_DCtx {
@@ -427,12 +434,12 @@ size_t LDM_decompress(const void *src, size_t compressSize,
   LDM_DCtx dctx;
   LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
 
-  BYTE *cpy;
-  size_t length;
-  const BYTE *match;
-  size_t offset;
-
   while (dctx.ip < dctx.iend) {
+    BYTE *cpy;
+    size_t length;
+    const BYTE *match;
+    size_t offset;
+
     /* get literal length */
     unsigned const token = *(dctx.ip)++;
     if ((length = (token >> ML_BITS)) == RUN_MASK) {
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 0ac7b2ec..287d444d 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -13,7 +13,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
 size_t LDM_decompress(const void *src, size_t srcSize,
                       void *dst, size_t maxDstSize);
 
-void LDM_read_header(const void *src, size_t *compressSize,
-                     size_t *decompressSize);
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize);
 
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index b529201f..724d735d 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -25,6 +25,7 @@ static int compress(const char *fname, const char *oname) {
   int fdin, fdout;
   struct stat statbuf;
   char *src, *dst;
+  size_t maxCompressSize, compressSize;
 
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
@@ -44,10 +45,10 @@ static int compress(const char *fname, const char *oname) {
     return 1;
   }
 
-  size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
 
  /* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */ 
+ /* TODO: fallocate? */
   if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
@@ -74,14 +75,14 @@ static int compress(const char *fname, const char *oname) {
   }
 
 #ifdef ZSTD
-  size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
+  compressSize = ZSTD_compress(dst, statbuf.st_size,
                                   src, statbuf.st_size, 1);
 #else
-  size_t compressSize = LDM_HEADER_SIZE + 
+  compressSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
                    dst + LDM_HEADER_SIZE, statbuf.st_size);
 
-    // Write compress and decompress size to header 
+    // Write compress and decompress size to header
     // TODO: should depend on LDM_DECOMPRESS_SIZE write32
   memcpy(dst, &compressSize, 4);
   memcpy(dst + 4, &(statbuf.st_size), 4);
@@ -107,12 +108,13 @@ static int compress(const char *fname, const char *oname) {
 
 /* Decompress file compressed using LDM_compress.
  * The input file should have the LDM_HEADER followed by payload.
- * Returns 0 if succesful, and an error code otherwise. 
+ * Returns 0 if succesful, and an error code otherwise.
  */
 static int decompress(const char *fname, const char *oname) {
   int fdin, fdout;
   struct stat statbuf;
   char *src, *dst;
+  size_t compressSize, decompressSize, outSize;
 
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
@@ -140,8 +142,7 @@ static int decompress(const char *fname, const char *oname) {
   }
 
   /* Read the header. */
-  size_t compressSize, decompressSize;
-  LDM_read_header(src, &compressSize, &decompressSize);
+  LDM_readHeader(src, &compressSize, &decompressSize);
 
 #ifdef DEBUG
   printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
@@ -168,11 +169,11 @@ static int decompress(const char *fname, const char *oname) {
   }
 
 #ifdef ZSTD
-  size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
+  outSize = ZSTD_decompress(dst, decomrpessed_size,
                                   src + LDM_HEADER_SIZE,
                                   statbuf.st_size - LDM_HEADER_SIZE);
 #else
-  size_t outSize = LDM_decompress(
+  outSize = LDM_decompress(
       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
       dst, decompressSize);
 
@@ -211,12 +212,14 @@ static void verify(const char *inpFilename, const char *decFilename) {
   FILE *decFp = fopen(decFilename, "rb");
 
   printf("verify : %s <-> %s\n", inpFilename, decFilename);
-	const int cmp = compare(inpFp, decFp);
-	if(0 == cmp) {
-		printf("verify : OK\n");
-	} else {
-		printf("verify : NG\n");
-	}
+  {
+    const int cmp = compare(inpFp, decFp);
+    if(0 == cmp) {
+      printf("verify : OK\n");
+    } else {
+      printf("verify : NG\n");
+    }
+  }
 
 	fclose(decFp);
 	fclose(inpFp);
@@ -243,32 +246,34 @@ int main(int argc, const char *argv[]) {
 	printf("ldm = [%s]\n", ldmFilename);
 	printf("dec = [%s]\n", decFilename);
 
-  struct timeval tv1, tv2;
 
   /* Compress */
-
-  gettimeofday(&tv1, NULL);
-  if (compress(inpFilename, ldmFilename)) {
-      printf("Compress error");
-      return 1;
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (compress(inpFilename, ldmFilename)) {
+        printf("Compress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total time = %f seconds\n",
+           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+           (double) (tv2.tv_sec - tv1.tv_sec));
   }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-         (double) (tv2.tv_sec - tv1.tv_sec));
 
   /* Decompress */
-
-  gettimeofday(&tv1, NULL);
-  if (decompress(ldmFilename, decFilename)) {
-      printf("Decompress error");
-      return 1;
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (decompress(ldmFilename, decFilename)) {
+        printf("Decompress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total time = %f seconds\n",
+          (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+          (double) (tv2.tv_sec - tv1.tv_sec));
   }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-        (double) (tv2.tv_sec - tv1.tv_sec));
-
   /* verify */
   verify(inpFilename, decFilename);
   return 0;
diff --git a/contrib/long_distance_matching/util.c b/contrib/long_distance_matching/util.c
new file mode 100644
index 00000000..9ea4ca1e
--- /dev/null
+++ b/contrib/long_distance_matching/util.c
@@ -0,0 +1,64 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "util.h"
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+unsigned LDM_isLittleEndian(void) {
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+void LDM_write16(void *memPtr, U16 value){
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE *)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+
diff --git a/contrib/long_distance_matching/util.h b/contrib/long_distance_matching/util.h
new file mode 100644
index 00000000..90726412
--- /dev/null
+++ b/contrib/long_distance_matching/util.h
@@ -0,0 +1,23 @@
+#ifndef LDM_UTIL_H
+#define LDM_UTIL_H
+
+unsigned LDM_isLittleEndian(void);
+
+uint16_t LDM_read16(const void *memPtr);
+
+uint16_t LDM_readLE16(const void *memPtr);
+
+void LDM_write16(void *memPtr, uint16_t value);
+
+void LDM_write32(void *memPtr, uint32_t value);
+
+void LDM_writeLE16(void *memPtr, uint16_t value);
+
+uint32_t LDM_read32(const void *ptr);
+
+uint64_t LDM_read64(const void *ptr);
+
+void LDM_copy8(void *dst, const void *src);
+
+
+#endif /* LDM_UTIL_H */

From f9524cf366af289d2c24e673360c8b05f70b6a83 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 10 Jul 2017 13:48:41 -0700
Subject: [PATCH 018/100] added --memtest to fuzzer

---
 tests/fuzzer.c | 193 +++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 179 insertions(+), 14 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c9461ee9..6586fc12 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -51,14 +51,14 @@ static const U32 nbTestsDefault = 30000;
 /*-************************************
 *  Display Macros
 **************************************/
-#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
+#define DISPLAY(...)          fprintf(stdout, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static U32 g_displayLevel = 2;
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
             if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
             { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stderr); } }
+            if (g_displayLevel>=4) fflush(stdout); } }
 static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
 static clock_t g_displayClock = 0;
 
@@ -97,7 +97,161 @@ static unsigned FUZ_highbit32(U32 v32)
 
 
 /*=============================================
-*   Basic Unit tests
+*   Memory Tests
+=============================================*/
+#if defined(__APPLE__) && defined(__MACH__)
+
+#include <malloc/malloc.h>    /* malloc_size */
+
+typedef struct {
+    unsigned long long totalMalloc;
+    size_t peakMalloc;
+    unsigned nbMalloc;
+    unsigned nbFree;
+} mallocCounter_t;
+
+static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0 };
+
+static void* FUZ_mallocDebug(void* counter, size_t size)
+{
+    mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
+    void* const ptr = malloc(size);
+    if (ptr==NULL) return NULL;
+    mcPtr->totalMalloc += size;
+    mcPtr->peakMalloc += size;
+    mcPtr->nbMalloc += 1;
+    return ptr;
+}
+
+static void FUZ_freeDebug(void* counter, void* address)
+{
+    mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
+    free(address);
+    mcPtr->nbFree += 1;
+    mcPtr->peakMalloc -= malloc_size(address);  /* OS-X specific */
+}
+
+static void FUZ_displayMallocStats(mallocCounter_t count)
+{
+    DISPLAYLEVEL(3, "peak:%u KB,  nbMallocs:%u, total:%u KB \n",
+        (U32)(count.peakMalloc >> 10),
+        count.nbMalloc,
+        (U32)(count.totalMalloc >> 10));
+}
+
+static int FUZ_mallocTests(unsigned seed, double compressibility)
+{
+    size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
+    size_t const outSize = ZSTD_compressBound(inSize);
+    void* const inBuffer = malloc(inSize);
+    void* const outBuffer = malloc(outSize);
+
+    /* test only played in verbose mode, as they are long */
+    if (g_displayLevel<3) return 0;
+
+    /* Create compressible noise */
+    if (!inBuffer || !outBuffer) {
+        DISPLAY("Not enough memory, aborting\n");
+        exit(1);
+    }
+    RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
+
+    /* simple compression tests */
+    {   int compressionLevel;
+        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
+        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
+            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
+            ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel);
+            ZSTD_freeCCtx(cctx);
+            DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
+            FUZ_displayMallocStats(malcount);
+            malcount = INIT_MALLOC_COUNTER;
+    }   }
+
+    /* streaming compression tests */
+    {   int compressionLevel;
+        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
+        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
+            ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
+            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
+            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
+            ZSTD_initCStream(cstream, compressionLevel);
+            ZSTD_compressStream(cstream, &out, &in);
+            ZSTD_endStream(cstream, &out);
+            ZSTD_freeCStream(cstream);
+            DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
+            FUZ_displayMallocStats(malcount);
+            malcount = INIT_MALLOC_COUNTER;
+    }   }
+
+    /* advanced API test */
+    {   int compressionLevel;
+        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
+        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
+            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
+            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
+            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
+            ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
+            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+            ZSTD_freeCCtx(cctx);
+            DISPLAYLEVEL(3, "compress_generic,end level %i : ", compressionLevel);
+            FUZ_displayMallocStats(malcount);
+            malcount = INIT_MALLOC_COUNTER;
+    }   }
+
+    /* advanced MT API test */
+    {   int compressionLevel;
+        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
+        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
+            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
+            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
+            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
+            ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
+            ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, 2);
+            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+            ZSTD_freeCCtx(cctx);
+            DISPLAYLEVEL(3, "compress_generic,-T2,end level %i : ", compressionLevel);
+            FUZ_displayMallocStats(malcount);
+            malcount = INIT_MALLOC_COUNTER;
+    }   }
+
+    /* advanced MT streaming API test */
+    {   int compressionLevel;
+        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
+        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
+            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
+            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
+            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
+            ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
+            ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, 2);
+            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
+            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+            ZSTD_freeCCtx(cctx);
+            DISPLAYLEVEL(3, "compress_generic,-T2,continue level %i : ", compressionLevel);
+            FUZ_displayMallocStats(malcount);
+            malcount = INIT_MALLOC_COUNTER;
+    }   }
+
+    return 0;
+}
+
+#else
+
+static int FUZ_mallocTests(unsigned seed, double compressibility)
+{
+    (void)seed; (void)compressibility;
+    return 0;
+}
+
+#endif
+
+/*=============================================
+*   Unit tests
 =============================================*/
 
 #define CHECK_V(var, fn)  size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
@@ -108,7 +262,8 @@ static int basicUnitTests(U32 seed, double compressibility)
 {
     size_t const CNBuffSize = 5 MB;
     void* const CNBuffer = malloc(CNBuffSize);
-    void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
+    size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
+    void* const compressedBuffer = malloc(compressedBufferSize);
     void* const decodedBuffer = malloc(CNBuffSize);
     ZSTD_DCtx* dctx = ZSTD_createDCtx();
     int testResult = 0;
@@ -123,6 +278,9 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
 
+    /* memory tests */
+    FUZ_mallocTests(seed, compressibility);
+
     /* Basic tests */
     DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++);
     {   const char* errorString = ZSTD_getErrorName(0);
@@ -139,7 +297,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
         if (cctx==NULL) goto _output_error;
         CHECKPLUS(r, ZSTD_compressCCtx(cctx,
-                            compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                            compressedBuffer, compressedBufferSize,
                             CNBuffer, CNBuffSize, 1),
                   cSize=r );
         DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
@@ -226,7 +384,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
             DISPLAYLEVEL(4, "test%3i : simple compression test with static CCtx : ", testNb++);
             CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx,
-                            compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                            compressedBuffer, compressedBufferSize,
                             CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
                       cSize=r );
             DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n",
@@ -295,7 +453,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize);
         CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx,
-                                compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                                compressedBuffer, compressedBufferSize,
                                 CNBuffer, CNBuffSize,
                                 1),
                   cSize=r );
@@ -382,7 +540,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
         cSize = 0;
-        CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+        CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, compressedBufferSize,
                                            (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
                   cSize += r);
         DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
@@ -398,7 +556,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
         {   size_t const cSizeOrig = cSize;
             cSize = 0;
-            CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+            CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, compressedBufferSize,
                                                (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
                       cSize += r);
             if (cSize != cSizeOrig) goto _output_error;   /* should be identical ==> same size */
@@ -483,7 +641,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK : %u \n", dictID);
 
         DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
-        cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+        cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
                                         CNBuffer, CNBuffSize,
                                         dictBuffer, dictSize, 4);
         if (ZSTD_isError(cSize)) goto _output_error;
@@ -521,7 +679,7 @@ static int basicUnitTests(U32 seed, double compressibility)
                                             1 /* byReference */, ZSTD_dm_auto,
                                             cParams, ZSTD_defaultCMem);
             DISPLAYLEVEL(4, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict));
-            cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+            cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
                                                  CNBuffer, CNBuffSize, cdict);
             ZSTD_freeCDict(cdict);
             if (ZSTD_isError(cSize)) goto _output_error;
@@ -556,7 +714,7 @@ static int basicUnitTests(U32 seed, double compressibility)
                     goto _output_error;
                 }
                 cSize = ZSTD_compress_usingCDict(cctx,
-                                compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                                compressedBuffer, compressedBufferSize,
                                 CNBuffer, CNBuffSize, cdict);
                 if (ZSTD_isError(cSize)) {
                     DISPLAY("ZSTD_compress_usingCDict failed ");
@@ -570,7 +728,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         {   ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
             ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
-            cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+            cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize,
                                                  CNBuffer, CNBuffSize, cdict, fParams);
             ZSTD_freeCDict(cdict);
             if (ZSTD_isError(cSize)) goto _output_error;
@@ -594,7 +752,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
         {   ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
             p.fParams.noDictIDFlag = 1;
-            cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+            cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
                                            CNBuffer, CNBuffSize,
                                            dictBuffer, dictSize, p);
             if (ZSTD_isError(cSize)) goto _output_error;
@@ -1245,6 +1403,7 @@ int main(int argc, const char** argv)
     U32 mainPause = 0;
     U32 maxDuration = 0;
     int bigTests = 1;
+    U32 memTestsOnly = 0;
     const char* const programName = argv[0];
 
     /* Check command line */
@@ -1255,6 +1414,7 @@ int main(int argc, const char** argv)
         /* Handle commands. Aggregated commands are allowed */
         if (argument[0]=='-') {
 
+            if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
             if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
 
             argument++;
@@ -1326,6 +1486,11 @@ int main(int argc, const char** argv)
     DISPLAY("Seed = %u\n", seed);
     if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %u%%\n", proba);
 
+    if (memTestsOnly) {
+        g_displayLevel=3;
+        return FUZ_mallocTests(seed, ((double)proba) / 100);
+    }
+
     if (nbTests < testNb) nbTests = testNb;
 
     if (testNb==0)

From 88da8f181660890c369bbee4aafd6eaa8f6bb830 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 10 Jul 2017 14:02:33 -0700
Subject: [PATCH 019/100] fix : propagate custom allocator to ZSTDMT though
 ZSTD_CCtx_setParameter()

also : compile fuzzer with MT enabled
---
 lib/compress/zstd_compress.c |  2 +-
 tests/Makefile               | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9300357f..c17645de 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -377,7 +377,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
                 return ERROR(compressionParameter_unsupported);
             ZSTDMT_freeCCtx(cctx->mtctx);
             cctx->nbThreads = 1;
-            cctx->mtctx = ZSTDMT_createCCtx(value);
+            cctx->mtctx = ZSTDMT_createCCtx_advanced(value, cctx->customMem);
             if (cctx->mtctx == NULL) return ERROR(memory_allocation);
         }
         cctx->nbThreads = value;
diff --git a/tests/Makefile b/tests/Makefile
index 345e0e8e..4784136e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -79,7 +79,7 @@ all32: fullbench32 fuzzer32 zstreamtest32
 
 allnothread: fullbench fuzzer paramgrill datagen  decodecorpus
 
-dll: fuzzer-dll zstreamtest-dll 
+dll: fuzzer-dll zstreamtest-dll
 
 zstd:
 	$(MAKE) -C $(PRGDIR) $@
@@ -108,11 +108,11 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 
-fuzzer   : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
-
-fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
-	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+fuzzer : CPPFLAGS += $(MULTITHREAD_CPP)
+fuzzer : LDFLAGS += $(MULTITHREAD_LD)
+fuzzer32: CFLAGS += -m32
+fuzzer fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
 fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c

From ee3423d70950f2ec41ff884a6cdacb463118f935 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 10 Jul 2017 14:09:16 -0700
Subject: [PATCH 020/100] extended fuzzer MT memory tests

---
 tests/fuzzer.c | 88 ++++++++++++++++++++++----------------------------
 1 file changed, 39 insertions(+), 49 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 6586fc12..c05ed5ae 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -160,7 +160,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
     {   int compressionLevel;
         mallocCounter_t malcount = INIT_MALLOC_COUNTER;
         ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
-        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
+        for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
             ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
             ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel);
             ZSTD_freeCCtx(cctx);
@@ -173,7 +173,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
     {   int compressionLevel;
         mallocCounter_t malcount = INIT_MALLOC_COUNTER;
         ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
-        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
+        for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
             ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
             ZSTD_outBuffer out = { outBuffer, outSize, 0 };
             ZSTD_inBuffer in = { inBuffer, inSize, 0 };
@@ -186,56 +186,46 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
             malcount = INIT_MALLOC_COUNTER;
     }   }
 
-    /* advanced API test */
-    {   int compressionLevel;
-        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
-        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
-        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
-            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
-            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
-            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
-            ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
-            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
-            ZSTD_freeCCtx(cctx);
-            DISPLAYLEVEL(3, "compress_generic,end level %i : ", compressionLevel);
-            FUZ_displayMallocStats(malcount);
-            malcount = INIT_MALLOC_COUNTER;
-    }   }
-
     /* advanced MT API test */
-    {   int compressionLevel;
-        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
-        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
-        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
-            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
-            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
-            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
-            ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
-            ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, 2);
-            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
-            ZSTD_freeCCtx(cctx);
-            DISPLAYLEVEL(3, "compress_generic,-T2,end level %i : ", compressionLevel);
-            FUZ_displayMallocStats(malcount);
-            malcount = INIT_MALLOC_COUNTER;
-    }   }
+    {   U32 nbThreads;
+        for (nbThreads=1; nbThreads<=4; nbThreads++) {
+            int compressionLevel;
+            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
+            for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
+                ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
+                ZSTD_outBuffer out = { outBuffer, outSize, 0 };
+                ZSTD_inBuffer in = { inBuffer, inSize, 0 };
+                ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
+                ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, nbThreads);
+                ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+                ZSTD_freeCCtx(cctx);
+                DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
+                                nbThreads, compressionLevel);
+                FUZ_displayMallocStats(malcount);
+                malcount = INIT_MALLOC_COUNTER;
+    }   }   }
 
     /* advanced MT streaming API test */
-    {   int compressionLevel;
-        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
-        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
-        for (compressionLevel=1; compressionLevel<=5; compressionLevel++) {
-            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
-            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
-            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
-            ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
-            ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, 2);
-            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
-            ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
-            ZSTD_freeCCtx(cctx);
-            DISPLAYLEVEL(3, "compress_generic,-T2,continue level %i : ", compressionLevel);
-            FUZ_displayMallocStats(malcount);
-            malcount = INIT_MALLOC_COUNTER;
-    }   }
+    {   U32 nbThreads;
+        for (nbThreads=1; nbThreads<=4; nbThreads++) {
+            int compressionLevel;
+            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
+            for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
+                ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
+                ZSTD_outBuffer out = { outBuffer, outSize, 0 };
+                ZSTD_inBuffer in = { inBuffer, inSize, 0 };
+                ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
+                ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, nbThreads);
+                ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
+                ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+                ZSTD_freeCCtx(cctx);
+                DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ",
+                                nbThreads, compressionLevel);
+                FUZ_displayMallocStats(malcount);
+                malcount = INIT_MALLOC_COUNTER;
+    }   }   }
 
     return 0;
 }

From 3510efb02dc0f5a3d90b31546d425963811b4c90 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 10 Jul 2017 14:21:40 -0700
Subject: [PATCH 021/100] fix : custom allocator correctly propagated to child
 contexts

---
 lib/compress/zstdmt_compress.c |  2 +-
 tests/fuzzer.c                 | 20 ++++++++------------
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index d5607adf..bad2db9c 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -215,7 +215,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
         pool->availCCtx--;
         return pool->cctx[pool->availCCtx];
     }
-    return ZSTD_createCCtx();   /* note : can be NULL, when creation fails ! */
+    return ZSTD_createCCtx_advanced(pool->cMem);   /* note : can be NULL, when creation fails ! */
 }
 
 static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c05ed5ae..aa1ebd48 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -158,22 +158,21 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
 
     /* simple compression tests */
     {   int compressionLevel;
-        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
-        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
         for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
+            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
             ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
             ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel);
             ZSTD_freeCCtx(cctx);
             DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
             FUZ_displayMallocStats(malcount);
-            malcount = INIT_MALLOC_COUNTER;
     }   }
 
     /* streaming compression tests */
     {   int compressionLevel;
-        mallocCounter_t malcount = INIT_MALLOC_COUNTER;
-        ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
         for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
+            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
             ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
             ZSTD_outBuffer out = { outBuffer, outSize, 0 };
             ZSTD_inBuffer in = { inBuffer, inSize, 0 };
@@ -183,16 +182,15 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
             ZSTD_freeCStream(cstream);
             DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
             FUZ_displayMallocStats(malcount);
-            malcount = INIT_MALLOC_COUNTER;
     }   }
 
     /* advanced MT API test */
     {   U32 nbThreads;
         for (nbThreads=1; nbThreads<=4; nbThreads++) {
             int compressionLevel;
-            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
-            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
             for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
+                mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+                ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
                 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
                 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
                 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
@@ -203,16 +201,15 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
                 DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
                                 nbThreads, compressionLevel);
                 FUZ_displayMallocStats(malcount);
-                malcount = INIT_MALLOC_COUNTER;
     }   }   }
 
     /* advanced MT streaming API test */
     {   U32 nbThreads;
         for (nbThreads=1; nbThreads<=4; nbThreads++) {
             int compressionLevel;
-            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
-            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
             for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
+                mallocCounter_t malcount = INIT_MALLOC_COUNTER;
+                ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
                 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
                 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
                 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
@@ -224,7 +221,6 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
                 DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ",
                                 nbThreads, compressionLevel);
                 FUZ_displayMallocStats(malcount);
-                malcount = INIT_MALLOC_COUNTER;
     }   }   }
 
     return 0;

From ef2b72831636d56d4e18168f5abaa16965e4a1a8 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 10 Jul 2017 15:48:47 -0700
Subject: [PATCH 022/100] Clean up and refactor compress function

---
 contrib/long_distance_matching/ldm.c | 317 ++++++++++-----------------
 1 file changed, 115 insertions(+), 202 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 12cffc40..a1d4449e 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -14,6 +14,8 @@
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
 
+#define LDM_OFFSET_SIZE 4
+
 #define WINDOW_SIZE (1 << 20)
 #define MAX_WINDOW_SIZE 31
 #define HASH_SIZE 8
@@ -35,81 +37,27 @@ typedef uint64_t U64;
 typedef uint32_t offset_t;
 typedef uint32_t hash_t;
 
-// typedef uint64_t tag;
-
-/*
-static unsigned LDM_isLittleEndian(void) {
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-static U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-static U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-static void LDM_write16(void *memPtr, U16 value){
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE *)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-static U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-static U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-static void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-*/
 typedef struct LDM_hashEntry {
   offset_t offset;
 } LDM_hashEntry;
 
 typedef struct LDM_compressStats {
-  U32 num_matches;
-  U32 total_match_length;
-  U32 total_literal_length;
-  U64 total_offset;
+  U32 numMatches;
+  U32 totalMatchLength;
+  U32 totalLiteralLength;
+  U64 totalOffset;
 } LDM_compressStats;
 
 static void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("=====================\n");
   printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->num_matches);
-  printf("Average match length: %.1f\n", ((double)stats->total_match_length) /
-                                         (double)stats->num_matches);
+  printf("Total number of matches: %u\n", stats->numMatches);
+  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
+                                         (double)stats->numMatches);
   printf("Average literal length: %.1f\n",
-         ((double)stats->total_literal_length) / (double)stats->num_matches);
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
   printf("Average offset length: %.1f\n",
-         ((double)stats->total_offset) / (double)stats->num_matches);
+         ((double)stats->totalOffset) / (double)stats->numMatches);
   printf("=====================\n");
 }
 
@@ -140,6 +88,10 @@ typedef struct LDM_CCtx {
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
+  const BYTE *forwardIp;
+  hash_t forwardHash;
+
+  unsigned step;
 
 } LDM_CCtx;
 
@@ -160,38 +112,25 @@ static hash_t LDM_hash5(U64 sequence) {
 }
 */
 
-static hash_t LDM_hash_position(const void * const p) {
+static hash_t LDM_hashPosition(const void * const p) {
   return LDM_hash(LDM_read32(p));
 }
 
-static void LDM_putHashOfPosition(const BYTE *p, hash_t h,
-                                  void *tableBase, const BYTE *srcBase) {
-  LDM_hashEntry *hashTable;
-  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
+static void LDM_putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash) {
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
     return;
   }
-
-  hashTable = (LDM_hashEntry *) tableBase;
-  hashTable[h] = (LDM_hashEntry) { (hash_t)(p - srcBase) };
-}
-
-static void LDM_putPosition(const BYTE *p, void *tableBase,
-                            const BYTE *srcBase) {
-  hash_t hash;
-  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  hash = LDM_hash_position(p);
-  LDM_putHashOfPosition(p, hash, tableBase, srcBase);
-}
-
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *const cctx) {
-  hash_t hash = LDM_hash_position(cctx->ip);
-  LDM_putHashOfPosition(cctx->ip, hash, cctx->hashTable, cctx->ibase);
+  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
   cctx->lastPosHashed = cctx->ip;
   cctx->lastHash = hash;
 }
 
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  hash_t hash = LDM_hashPosition(cctx->ip);
+  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
+}
+
 static const BYTE *LDM_get_position_on_hash(
     hash_t h, void *tableBase, const BYTE *srcBase) {
   const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
@@ -248,141 +187,136 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
 
   cctx->lastPosHashed = NULL;
+  cctx->forwardIp = NULL;
+
+  cctx->step = 1;
+}
+
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+  cctx->forwardIp = cctx->ip;
+
+  do {
+    hash_t const h = cctx->forwardHash;
+    cctx->ip = cctx->forwardIp;
+    cctx->forwardIp += cctx->step;
+
+    if (cctx->forwardIp > cctx->imatchLimit) {
+      return 1;
+    }
+
+    *match = LDM_get_position_on_hash(h, cctx->hashTable, cctx->ibase);
+
+    cctx->forwardHash = LDM_hashPosition(cctx->forwardIp);
+    LDM_putHashOfCurrentPositionFromHash(cctx, h);
+  } while (cctx->ip - *match > WINDOW_SIZE ||
+           LDM_read64(*match) != LDM_read64(cctx->ip));
+  return 0;
 }
 
 // TODO: srcSize and maxDstSize is unused
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
-  U32 forwardH;
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
-
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
   cctx.ip++;
-  forwardH = LDM_hash_position(cctx.ip);
+  cctx.forwardHash = LDM_hashPosition(cctx.ip);
 
   // TODO: loop condition is not accurate.
   while (1) {
     const BYTE *match;
-    BYTE *token;
 
-    /* Find a match */
-    {
-      const BYTE *forwardIp = cctx.ip;
-      unsigned step = 1;
-
-      do {
-        U32 const h = forwardH;
-        cctx.ip = forwardIp;
-        forwardIp += step;
-
-        if (forwardIp > cctx.imatchLimit) {
-          goto _last_literals;
-        }
-
-        match = LDM_get_position_on_hash(h, cctx.hashTable, cctx.ibase);
-
-        forwardH = LDM_hash_position(forwardIp);
-        LDM_putHashOfPosition(cctx.ip, h, cctx.hashTable, cctx.ibase);
-      } while (cctx.ip - match > WINDOW_SIZE ||
-               LDM_read64(match) != LDM_read64(cctx.ip));
+    /**
+     * Find a match.
+     * If no more matches can be found (i.e. the length of the remaining input
+     * is less than the minimum match length), then stop searching for matches
+     * and encode the final literals.
+     */
+    if (LDM_findBestMatch(&cctx, &match) != 0) {
+      goto _last_literals;
     }
-    cctx.stats.num_matches++;
 
-    /* Catchup: look back to extend match from found match */
-    while (cctx.ip > cctx.anchor && match > cctx.ibase && cctx.ip[-1] == match[-1]) {
+    cctx.stats.numMatches++;
+
+    /**
+     * Catchup: look back to extend the match backwards from the found match.
+     */
+    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
+           cctx.ip[-1] == match[-1]) {
       cctx.ip--;
       match--;
     }
 
-    /* Encode literals */
+    /**
+     * Write current block (literals, literal length, match offset, match
+     * length) and update pointers and hashes.
+     */
     {
-      unsigned const litLength = (unsigned)(cctx.ip - cctx.anchor);
-      token = cctx.op++;
+      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
+      unsigned const offset = cctx.ip - match;
+      unsigned const matchLength = LDM_count(
+          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
+      BYTE *token = cctx.op++;
 
-      cctx.stats.total_literal_length += litLength;
+      cctx.stats.totalLiteralLength += literalLength;
+      cctx.stats.totalOffset += offset;
+      cctx.stats.totalMatchLength += matchLength + MINMATCH;
 
-#ifdef LDM_DEBUG
-      printf("Cur position: %zu\n", cctx.anchor - cctx.ibase);
-      printf("LitLength %zu. (Match offset). %zu\n", litLength, cctx.ip - match);
-#endif
-
-      if (litLength >= RUN_MASK) {
-        int len = (int)litLength - RUN_MASK;
+      /* Encode the literal length. */
+      if (literalLength >= RUN_MASK) {
+        int len = (int)literalLength - RUN_MASK;
         *token = (RUN_MASK << ML_BITS);
         for (; len >= 255; len -= 255) {
           *(cctx.op)++ = 255;
         }
         *(cctx.op)++ = (BYTE)len;
       } else {
-        *token = (BYTE)(litLength << ML_BITS);
+        *token = (BYTE)(literalLength << ML_BITS);
       }
-#ifdef LDM_DEBUG
-      printf("Literals ");
-      fwrite(cctx.anchor, litLength, 1, stdout);
-      printf("\n");
-#endif
-      memcpy(cctx.op, cctx.anchor, litLength);
-      cctx.op += litLength;
-    }
 
-    /* Encode offset */
-    {
-      /*
-      LDM_writeLE16(cctx.op, cctx.ip-match);
-      cctx.op += 2;
-      */
-      LDM_write32(cctx.op, cctx.ip - match);
-      cctx.op += 4;
-      cctx.stats.total_offset += (cctx.ip - match);
-    }
+      /* Encode the literals. */
+      memcpy(cctx.op, cctx.anchor, literalLength);
+      cctx.op += literalLength;
 
-    /* Encode Match Length */
-    {
-      unsigned matchCode;
-      unsigned ctr = 1;
-      matchCode = LDM_count(cctx.ip + MINMATCH, match + MINMATCH,
-                            cctx.ihashLimit);
-#ifdef LDM_DEBUG
-      printf("Match length %zu\n", matchCode + MINMATCH);
-      fwrite(cctx.ip, MINMATCH + matchCode, 1, stdout);
-      printf("\n");
-#endif
-      cctx.stats.total_match_length += matchCode + MINMATCH;
-      cctx.ip++;
-      for (; ctr < MINMATCH + matchCode; cctx.ip++, ctr++) {
-        LDM_putHashOfCurrentPosition(&cctx);
-      }
-//      cctx.ip += MINMATCH + matchCode;
-      if (matchCode >= ML_MASK) {
+      /* Encode the offset. */
+      LDM_write32(cctx.op, offset);
+      cctx.op += LDM_OFFSET_SIZE;
+
+      /* Encode match length */
+      if (matchLength >= ML_MASK) {
+        unsigned matchLengthRemaining = matchLength;
         *token += ML_MASK;
-        matchCode -= ML_MASK;
+        matchLengthRemaining -= ML_MASK;
         LDM_write32(cctx.op, 0xFFFFFFFF);
-        while (matchCode >= 4*0xFF) {
+        while (matchLengthRemaining >= 4*0xFF) {
           cctx.op += 4;
           LDM_write32(cctx.op, 0xffffffff);
-          matchCode -= 4*0xFF;
+          matchLengthRemaining -= 4*0xFF;
         }
-        cctx.op += matchCode / 255;
-        *(cctx.op)++ = (BYTE)(matchCode % 255);
+        cctx.op += matchLengthRemaining / 255;
+        *(cctx.op)++ = (BYTE)(matchLengthRemaining % 255);
       } else {
-        *token += (BYTE)(matchCode);
+        *token += (BYTE)(matchLength);
       }
-#ifdef LDM_DEBUG
-      printf("\n");
 
-#endif
+      /* Update input pointer, inserting hashes into hash table along the
+       * way.
+       */
+      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
+        LDM_putHashOfCurrentPosition(&cctx);
+        cctx.ip++;
+      }
     }
 
+    // Set start of next block to current input pointer.
     cctx.anchor = cctx.ip;
-
-    LDM_putPosition(cctx.ip, cctx.hashTable, cctx.ibase);
-    forwardH = LDM_hash_position(++cctx.ip);
+    LDM_putHashOfCurrentPosition(&cctx);
+    cctx.forwardHash = LDM_hashPosition(++cctx.ip);
   }
 _last_literals:
-    /* Encode last literals */
+  /* Encode the last literals (no more matches). */
   {
     size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
     if (lastRun >= RUN_MASK) {
@@ -436,11 +370,10 @@ size_t LDM_decompress(const void *src, size_t compressSize,
 
   while (dctx.ip < dctx.iend) {
     BYTE *cpy;
-    size_t length;
     const BYTE *match;
-    size_t offset;
+    size_t length, offset;
 
-    /* get literal length */
+    /* Get the literal length. */
     unsigned const token = *(dctx.ip)++;
     if ((length = (token >> ML_BITS)) == RUN_MASK) {
       unsigned s;
@@ -449,37 +382,19 @@ size_t LDM_decompress(const void *src, size_t compressSize,
         length += s;
       } while (s == 255);
     }
-#ifdef LDM_DEBUG
-    printf("Literal length: %zu\n", length);
-#endif
 
-    /* copy literals */
+    /* Copy literals. */
     cpy = dctx.op + length;
-#ifdef LDM_DEBUG
-    printf("Literals ");
-    fwrite(dctx.ip, length, 1, stdout);
-    printf("\n");
-#endif
     memcpy(dctx.op, dctx.ip, length);
     dctx.ip += length;
     dctx.op = cpy;
 
-    /* get offset */
-    /*
-    offset = LDM_readLE16(dctx.ip);
-    dctx.ip += 2;
-    */
-
     //TODO : dynamic offset size
     offset = LDM_read32(dctx.ip);
-    dctx.ip += 4;
-#ifdef LDM_DEBUG
-    printf("Offset: %zu\n", offset);
-#endif
+    dctx.ip += LDM_OFFSET_SIZE;
     match = dctx.op - offset;
- //   LDM_write32(op, (U32)offset);
 
-    /* get matchlength */
+    /* Get the match length. */
     length = token & ML_MASK;
     if (length == ML_MASK) {
       unsigned s;
@@ -489,10 +404,8 @@ size_t LDM_decompress(const void *src, size_t compressSize,
       } while (s == 255);
     }
     length += MINMATCH;
-#ifdef LDM_DEBUG
-    printf("Match length: %zu\n", length);
-#endif
-    /* copy match */
+
+    /* Copy match. */
     cpy = dctx.op + length;
 
     // Inefficient for now

From 670b1fc547424897aa324736d47a1c79bf61e355 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 10 Jul 2017 16:30:55 -0700
Subject: [PATCH 023/100] optimized memory usage for ZSTDMT_compress()

Previously, each job would reserve a CCtx right before being posted.
The CCtx would be "part of the job description",
and only released when the job is completed (aka flushed).
For ZSTDMT_compress(), which creates all jobs first and only join at the end,
that meant one CCtx per job.
The nb of jobs used to be == nb of threads,
but since latest modification,
which reduces the size of jobs in order to spread the load of difficult areas,
it also increases the nb of jobs for large sources / small compression level.
This resulted in many more CCtx being created.

In this new version, CCtx are reserved within the worker thread.
It guaranteea there cannot be more CCtx reserved than workers (<= nb threads).

To do that, it required to make the CCtx Pool multi-threading-safe :
it can now be called from multiple threads in parallel.
---
 lib/compress/zstdmt_compress.c | 97 ++++++++++++++++++++--------------
 tests/fuzzer.c                 | 45 ++++++++++------
 2 files changed, 86 insertions(+), 56 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index bad2db9c..a176c3ee 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -126,6 +126,7 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
  *  assumption : invocation from main thread only ! */
 static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
 {
+    DEBUGLOG(2, "ZSTDMT_getBuffer");
     if (pool->nbBuffers) {   /* try to use an existing buffer */
         buffer_t const buf = pool->bTable[--(pool->nbBuffers)];
         size_t const availBufferSize = buf.size;
@@ -160,21 +161,23 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
 
 /* =====   CCtx Pool   ===== */
 
+/* a single cctxPool can be called from multiple threads in parallel */
+
 typedef struct {
+    pthread_mutex_t poolMutex;
     unsigned totalCCtx;
     unsigned availCCtx;
     ZSTD_customMem cMem;
     ZSTD_CCtx* cctx[1];   /* variable size */
 } ZSTDMT_CCtxPool;
 
-/* assumption : CCtxPool invocation only from main thread */
-
 /* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
 static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
 {
     unsigned u;
     for (u=0; u<pool->totalCCtx; u++)
         ZSTD_freeCCtx(pool->cctx[u]);  /* note : compatible with free on NULL */
+    pthread_mutex_destroy(&pool->poolMutex);
     ZSTD_free(pool, pool->cMem);
 }
 
@@ -186,6 +189,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
     ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
         sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
     if (!cctxPool) return NULL;
+    pthread_mutex_init(&cctxPool->poolMutex, NULL);
     cctxPool->cMem = cMem;
     cctxPool->totalCCtx = nbThreads;
     cctxPool->availCCtx = 1;   /* at least one cctx for single-thread mode */
@@ -198,34 +202,47 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
 /* only works during initialization phase, not during compression */
 static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
 {
-    unsigned const nbThreads = cctxPool->totalCCtx;
-    size_t const poolSize = sizeof(*cctxPool)
-                            + (nbThreads-1)*sizeof(ZSTD_CCtx*);
-    unsigned u;
-    size_t totalCCtxSize = 0;
-    for (u=0; u<nbThreads; u++)
-        totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
-
-    return poolSize + totalCCtxSize;
+    pthread_mutex_lock(&cctxPool->poolMutex);
+    {   unsigned const nbThreads = cctxPool->totalCCtx;
+        size_t const poolSize = sizeof(*cctxPool)
+                                + (nbThreads-1)*sizeof(ZSTD_CCtx*);
+        unsigned u;
+        size_t totalCCtxSize = 0;
+        for (u=0; u<nbThreads; u++) {
+            totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
+        }
+        pthread_mutex_unlock(&cctxPool->poolMutex);
+        return poolSize + totalCCtxSize;
+    }
 }
 
-static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
+static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
 {
-    if (pool->availCCtx) {
-        pool->availCCtx--;
-        return pool->cctx[pool->availCCtx];
-    }
-    return ZSTD_createCCtx_advanced(pool->cMem);   /* note : can be NULL, when creation fails ! */
+    DEBUGLOG(5, "ZSTDMT_getCCtx");
+    pthread_mutex_lock(&cctxPool->poolMutex);
+    if (cctxPool->availCCtx) {
+        cctxPool->availCCtx--;
+        {   ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx];
+            pthread_mutex_unlock(&cctxPool->poolMutex);
+            return cctx;
+    }   }
+    pthread_mutex_unlock(&cctxPool->poolMutex);
+    DEBUGLOG(5, "create one more CCtx");
+    return ZSTD_createCCtx_advanced(cctxPool->cMem);   /* note : can be NULL, when creation fails ! */
 }
 
 static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return;   /* compatibility with release on NULL */
+    pthread_mutex_lock(&pool->poolMutex);
     if (pool->availCCtx < pool->totalCCtx)
         pool->cctx[pool->availCCtx++] = cctx;
-    else
+    else {
         /* pool overflow : should not happen, since totalCCtx==nbThreads */
+        DEBUGLOG(5, "CCtx pool overflow : free cctx");
         ZSTD_freeCCtx(cctx);
+    }
+    pthread_mutex_unlock(&pool->poolMutex);
 }
 
 
@@ -237,7 +254,6 @@ typedef struct {
 } inBuff_t;
 
 typedef struct {
-    ZSTD_CCtx* cctx;
     buffer_t src;
     const void* srcStart;
     size_t   srcSize;
@@ -253,6 +269,7 @@ typedef struct {
     pthread_cond_t* jobCompleted_cond;
     ZSTD_parameters params;
     const ZSTD_CDict* cdict;
+    ZSTDMT_CCtxPool* cctxPool;
     unsigned long long fullFrameSize;
 } ZSTDMT_jobDescription;
 
@@ -260,37 +277,45 @@ typedef struct {
 void ZSTDMT_compressChunk(void* jobDescription)
 {
     ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
+    ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
     const void* const src = (const char*)job->srcStart + job->dictSize;
     buffer_t const dstBuff = job->dstBuff;
     DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
                  job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
+
+    if (cctx==NULL) {
+        job->cSize = ERROR(memory_allocation);
+        goto _endJob;
+    }
+
     if (job->cdict) {  /* should only happen for first segment */
-        size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
+        size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
         DEBUGLOG(5, "using CDict");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
         if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
-        {   size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1);  /* Force loading dictionary in "content-only" mode (no header analysis) */
-            size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
+        {   size_t const dictModeError = ZSTD_setCCtxParameter(cctx, ZSTD_p_forceRawDict, 1);  /* Force loading dictionary in "content-only" mode (no header analysis) */
+            size_t const initError = ZSTD_compressBegin_advanced(cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
             if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; }
-            ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
+            ZSTD_setCCtxParameter(cctx, ZSTD_p_forceWindow, 1);
     }   }
     if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
-        size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
+        size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
         if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
-        ZSTD_invalidateRepCodes(job->cctx);
+        ZSTD_invalidateRepCodes(cctx);
     }
 
     DEBUGLOG(5, "Compressing : ");
     DEBUG_PRINTHEX(4, job->srcStart, 12);
     job->cSize = (job->lastChunk) ?
-                 ZSTD_compressEnd     (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
-                 ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
+                 ZSTD_compressEnd     (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
+                 ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
     DEBUGLOG(5, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
                 (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
     DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
 
 _endJob:
+    ZSTDMT_releaseCCtx(job->cctxPool, cctx);
     PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
     job->jobCompleted = 1;
     job->jobScanned = 0;
@@ -390,8 +415,6 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
         mtctx->jobs[jobID].dstBuff = g_nullBuffer;
         ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].src);
         mtctx->jobs[jobID].src = g_nullBuffer;
-        ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[jobID].cctx);
-        mtctx->jobs[jobID].cctx = NULL;
     }
     memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
     ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->inBuff.buffer);
@@ -497,10 +520,9 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
             size_t const dstBufferCapacity = ZSTD_compressBound(chunkSize);
             buffer_t const dstAsBuffer = { (char*)dst + dstBufferPos, dstBufferCapacity };
             buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity);
-            ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(mtctx->cctxPool);
             size_t dictSize = u ? overlapSize : 0;
 
-            if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+            if (dstBuffer.start==NULL) {
                 mtctx->jobs[u].cSize = ERROR(memory_allocation);   /* job result */
                 mtctx->jobs[u].jobCompleted = 1;
                 nbChunks = u+1;   /* only wait and free u jobs, instead of initially expected nbChunks ones */
@@ -516,7 +538,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
             /* do not calculate checksum within sections, but write it in header for first section */
             if (u!=0) mtctx->jobs[u].params.fParams.checksumFlag = 0;
             mtctx->jobs[u].dstBuff = dstBuffer;
-            mtctx->jobs[u].cctx = cctx;
+            mtctx->jobs[u].cctxPool = mtctx->cctxPool;
             mtctx->jobs[u].firstChunk = (u==0);
             mtctx->jobs[u].lastChunk = (u==nbChunks-1);
             mtctx->jobs[u].jobCompleted = 0;
@@ -545,8 +567,6 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
             pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
             DEBUGLOG(5, "ready to write chunk %u ", chunkID);
 
-            ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx);
-            mtctx->jobs[chunkID].cctx = NULL;
             mtctx->jobs[chunkID].srcStart = NULL;
             {   size_t const cSize = mtctx->jobs[chunkID].cSize;
                 if (ZSTD_isError(cSize)) error = cSize;
@@ -703,10 +723,9 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
 {
     size_t const dstBufferCapacity = ZSTD_compressBound(srcSize);
     buffer_t const dstBuffer = ZSTDMT_getBuffer(zcs->buffPool, dstBufferCapacity);
-    ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(zcs->cctxPool);
     unsigned const jobID = zcs->nextJobID & zcs->jobIDMask;
 
-    if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+    if (dstBuffer.start==NULL) {
         zcs->jobs[jobID].jobCompleted = 1;
         zcs->nextJobID++;
         ZSTDMT_waitForAllJobsCompleted(zcs);
@@ -727,7 +746,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
     zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL;
     zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize;
     zcs->jobs[jobID].dstBuff = dstBuffer;
-    zcs->jobs[jobID].cctx = cctx;
+    zcs->jobs[jobID].cctxPool = zcs->cctxPool;
     zcs->jobs[jobID].firstChunk = (zcs->nextJobID==0);
     zcs->jobs[jobID].lastChunk = endFrame;
     zcs->jobs[jobID].jobCompleted = 0;
@@ -804,8 +823,6 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
                 ZSTDMT_releaseAllJobResources(zcs);
                 return job.cSize;
             }
-            ZSTDMT_releaseCCtx(zcs->cctxPool, job.cctx);
-            zcs->jobs[wJobID].cctx = NULL;
             DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag);
             if (zcs->params.fParams.checksumFlag) {
                 XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
@@ -884,7 +901,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
     /* fill input buffer */
     if ((input->src) && (mtctx->inBuff.buffer.start)) {   /* support NULL input */
         size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
-        DEBUGLOG(2, "inBuff:%08X;  inBuffSize=%u;  ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
+        DEBUGLOG(5, "inBuff:%08X;  inBuffSize=%u;  ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
         memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
         input->pos += toLoad;
         mtctx->inBuff.filled += toLoad;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index aa1ebd48..667de08c 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -105,12 +105,13 @@ static unsigned FUZ_highbit32(U32 v32)
 
 typedef struct {
     unsigned long long totalMalloc;
+    size_t currentMalloc;
     size_t peakMalloc;
     unsigned nbMalloc;
     unsigned nbFree;
 } mallocCounter_t;
 
-static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0 };
+static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
 
 static void* FUZ_mallocDebug(void* counter, size_t size)
 {
@@ -118,7 +119,9 @@ static void* FUZ_mallocDebug(void* counter, size_t size)
     void* const ptr = malloc(size);
     if (ptr==NULL) return NULL;
     mcPtr->totalMalloc += size;
-    mcPtr->peakMalloc += size;
+    mcPtr->currentMalloc += size;
+    if (mcPtr->currentMalloc > mcPtr->peakMalloc)
+        mcPtr->peakMalloc = mcPtr->currentMalloc;
     mcPtr->nbMalloc += 1;
     return ptr;
 }
@@ -126,9 +129,10 @@ static void* FUZ_mallocDebug(void* counter, size_t size)
 static void FUZ_freeDebug(void* counter, void* address)
 {
     mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
-    free(address);
+    DISPLAYLEVEL(4, "releasing %u KB \n", (U32)(malloc_size(address) >> 10));
     mcPtr->nbFree += 1;
-    mcPtr->peakMalloc -= malloc_size(address);  /* OS-X specific */
+    mcPtr->currentMalloc -= malloc_size(address);  /* OS-X specific */
+    free(address);
 }
 
 static void FUZ_displayMallocStats(mallocCounter_t count)
@@ -139,6 +143,14 @@ static void FUZ_displayMallocStats(mallocCounter_t count)
         (U32)(count.totalMalloc >> 10));
 }
 
+#define CHECK_Z(f) {                               \
+    size_t const err = f;                          \
+    if (ZSTD_isError(err)) {                       \
+        DISPLAY("Error => %s : %s ",               \
+                #f, ZSTD_getErrorName(err));       \
+        exit(1);                                   \
+}   }
+
 static int FUZ_mallocTests(unsigned seed, double compressibility)
 {
     size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
@@ -162,7 +174,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
             mallocCounter_t malcount = INIT_MALLOC_COUNTER;
             ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
             ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
-            ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel);
+            CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
             ZSTD_freeCCtx(cctx);
             DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
             FUZ_displayMallocStats(malcount);
@@ -176,9 +188,9 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
             ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
             ZSTD_outBuffer out = { outBuffer, outSize, 0 };
             ZSTD_inBuffer in = { inBuffer, inSize, 0 };
-            ZSTD_initCStream(cstream, compressionLevel);
-            ZSTD_compressStream(cstream, &out, &in);
-            ZSTD_endStream(cstream, &out);
+            CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
+            CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
+            CHECK_Z( ZSTD_endStream(cstream, &out) );
             ZSTD_freeCStream(cstream);
             DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
             FUZ_displayMallocStats(malcount);
@@ -194,9 +206,9 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
                 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
                 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
                 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
-                ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
-                ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, nbThreads);
-                ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) );
+                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, nbThreads) );
+                while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
                 ZSTD_freeCCtx(cctx);
                 DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
                                 nbThreads, compressionLevel);
@@ -213,10 +225,10 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
                 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
                 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
                 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
-                ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel);
-                ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, nbThreads);
-                ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
-                ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
+                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) );
+                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, nbThreads) );
+                CHECK_Z( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue) );
+                while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
                 ZSTD_freeCCtx(cctx);
                 DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ",
                                 nbThreads, compressionLevel);
@@ -1046,6 +1058,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
         goto _output_error;                                   \
 }   }
 
+#undef CHECK_Z
 #define CHECK_Z(f) {                                          \
     size_t const err = f;                                     \
     if (ZSTD_isError(err)) {                                  \
@@ -1473,7 +1486,7 @@ int main(int argc, const char** argv)
     if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %u%%\n", proba);
 
     if (memTestsOnly) {
-        g_displayLevel=3;
+        g_displayLevel = MAX(3, g_displayLevel);
         return FUZ_mallocTests(seed, ((double)proba) / 100);
     }
 

From 4616fad18b2f8950144cdbcffc081b74cc115fb1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 10 Jul 2017 17:16:41 -0700
Subject: [PATCH 024/100] improved ZSTDMT_compress() memory usage

does not need the input buffer for streaming operations

also : reduced a few tests time length
---
 lib/compress/zstdmt_compress.c | 31 +++++++++++++++++++------------
 tests/fuzzer.c                 |  2 +-
 tests/playTests.sh             | 31 +++++++++++++++----------------
 tests/zstreamtest.c            | 19 +------------------
 4 files changed, 36 insertions(+), 47 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index a176c3ee..fbb86b00 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -134,9 +134,11 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
             /* large enough, but not too much */
             return buf;
         /* size conditions not respected : scratch this buffer, create new one */
+        DEBUGLOG(2, "existing buffer does not meet size conditions => freeing");
         ZSTD_free(buf.start, pool->cMem);
     }
     /* create new buffer */
+    DEBUGLOG(2, "create a new buffer");
     {   buffer_t buffer;
         void* const start = ZSTD_malloc(bSize, pool->cMem);
         if (start==NULL) bSize = 0;
@@ -149,12 +151,14 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
 /* store buffer for later re-use, up to pool capacity */
 static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
 {
+    DEBUGLOG(2, "ZSTDMT_releaseBuffer");
     if (buf.start == NULL) return;   /* release on NULL */
     if (pool->nbBuffers < pool->totalBuffers) {
         pool->bTable[pool->nbBuffers++] = buf;   /* store for later re-use */
         return;
     }
     /* Reached bufferPool capacity (should not happen) */
+    DEBUGLOG(2, "buffer pool capacity reached => freeing ");
     ZSTD_free(buf.start, pool->cMem);
 }
 
@@ -635,8 +639,8 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     if (zcs->nbThreads==1) {
         DEBUGLOG(4, "single thread mode");
         return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0],
-                                dict, dictSize, cdict,
-                                params, pledgedSrcSize);
+                                        dict, dictSize, cdict,
+                                        params, pledgedSrcSize);
     }
 
     if (zcs->allJobsCompleted == 0) {   /* previous compression not correctly finished */
@@ -671,9 +675,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
     zcs->marginSize = zcs->targetSectionSize >> 2;
     zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
-    zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
-    if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
-    zcs->inBuff.filled = 0;
+    zcs->inBuff.buffer = g_nullBuffer;
     zcs->dictSize = 0;
     zcs->doneJobID = 0;
     zcs->nextJobID = 0;
@@ -899,13 +901,18 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
     }
 
     /* fill input buffer */
-    if ((input->src) && (mtctx->inBuff.buffer.start)) {   /* support NULL input */
-        size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
-        DEBUGLOG(5, "inBuff:%08X;  inBuffSize=%u;  ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
-        memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
-        input->pos += toLoad;
-        mtctx->inBuff.filled += toLoad;
-    }
+    if (input->src) {   /* support NULL input */
+        if (mtctx->inBuff.buffer.start == NULL) {
+            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->buffPool, mtctx->inBuffSize);
+            if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
+            mtctx->inBuff.filled = 0;
+        }
+        {   size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
+            DEBUGLOG(5, "inBuff:%08X;  inBuffSize=%u;  ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
+            memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
+            input->pos += toLoad;
+            mtctx->inBuff.filled += toLoad;
+    }   }
 
     if ( (mtctx->inBuff.filled >= newJobThreshold)  /* filled enough : let's compress */
       && (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) {   /* avoid overwriting job round buffer */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 667de08c..1e25383f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -129,7 +129,7 @@ static void* FUZ_mallocDebug(void* counter, size_t size)
 static void FUZ_freeDebug(void* counter, void* address)
 {
     mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
-    DISPLAYLEVEL(4, "releasing %u KB \n", (U32)(malloc_size(address) >> 10));
+    DISPLAYLEVEL(4, "freeing %u KB \n", (U32)(malloc_size(address) >> 10));
     mcPtr->nbFree += 1;
     mcPtr->currentMalloc -= malloc_size(address);  /* OS-X specific */
     free(address);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 2e1cc682..88a1c2ab 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -7,17 +7,17 @@ die() {
 
 roundTripTest() {
     if [ -n "$3" ]; then
-        local_c="$3"
-        local_p="$2"
+        cLevel="$3"
+        proba="$2"
     else
-        local_c="$2"
-        local_p=""
+        cLevel="$2"
+        proba=""
     fi
 
     rm -f tmp1 tmp2
-    $ECHO "roundTripTest: ./datagen $1 $local_p | $ZSTD -v$local_c | $ZSTD -d"
-    ./datagen $1 $local_p | $MD5SUM > tmp1
-    ./datagen $1 $local_p | $ZSTD --ultra -v$local_c | $ZSTD -d  | $MD5SUM > tmp2
+    $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
+    ./datagen $1 $proba | $MD5SUM > tmp1
+    ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d  | $MD5SUM > tmp2
     $DIFF -q tmp1 tmp2
 }
 
@@ -625,16 +625,15 @@ roundTripTest -g35000000 -P75 10
 roundTripTest -g35000000 -P75 11
 roundTripTest -g35000000 -P75 12
 
-roundTripTest -g18000000 -P80 13
-roundTripTest -g18000000 -P80 14
-roundTripTest -g18000000 -P80 15
-roundTripTest -g18000000 -P80 16
-roundTripTest -g18000000 -P80 17
+roundTripTest -g18000013 -P80 13
+roundTripTest -g18000014 -P80 14
+roundTripTest -g18000015 -P80 15
+roundTripTest -g18000016 -P80 16
+roundTripTest -g18000017 -P80 17
+roundTripTest -g18000018 -P94 18
+roundTripTest -g18000019 -P94 19
 
-roundTripTest -g50000000 -P94 18
-roundTripTest -g50000000 -P94 19
-
-roundTripTest -g99000000 -P99 20
+roundTripTest -g68000020 -P99 20
 roundTripTest -g6000000000 -P99 1
 
 fileRoundTripTest -g4193M -P99 1
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 9b2b8eaf..8b84400d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -95,19 +95,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
     return rand32 >> 5;
 }
 
-static void* allocFunction(void* opaque, size_t size)
-{
-    void* address = malloc(size);
-    (void)opaque;
-    return address;
-}
-
-static void freeFunction(void* opaque, void* address)
-{
-    (void)opaque;
-    free(address);
-}
-
 
 /*======================================================
 *   Basic Unit tests
@@ -1543,7 +1530,6 @@ int main(int argc, const char** argv)
     int bigTests = (sizeof(size_t) == 8);
     e_api selected_api = simple_api;
     const char* const programName = argv[0];
-    ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
     ZSTD_customMem const customNULL = ZSTD_defaultCMem;
 
     /* Check command line */
@@ -1657,10 +1643,7 @@ int main(int argc, const char** argv)
 
     if (testNb==0) {
         result = basicUnitTests(0, ((double)proba) / 100, customNULL);  /* constant seed for predictability */
-        if (!result) {
-            DISPLAYLEVEL(3, "Unit tests using customMem :\n")
-            result = basicUnitTests(0, ((double)proba) / 100, customMem);  /* use custom memory allocation functions */
-    }   }
+    }
 
     if (!result) {
         switch(selected_api)

From 6c3673f4c388d2a3b0d8af696e1e69c9c8f68b5b Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 10 Jul 2017 22:27:43 -0700
Subject: [PATCH 025/100] Add rolling hash

---
 contrib/long_distance_matching/ldm.c | 87 +++++++++++++++++++++-------
 1 file changed, 66 insertions(+), 21 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index a1d4449e..42a4affd 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -3,12 +3,13 @@
 #include <stdint.h>
 #include <stdio.h>
 
+
 #include "ldm.h"
 #include "util.h"
 
 #define HASH_EVERY 7
 
-#define LDM_MEMORY_USAGE 14
+#define LDM_MEMORY_USAGE 20
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
@@ -16,16 +17,18 @@
 
 #define LDM_OFFSET_SIZE 4
 
-#define WINDOW_SIZE (1 << 20)
+#define WINDOW_SIZE (1 << 24)
 #define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 8
-#define MINMATCH 8
+#define HASH_SIZE 4
+#define LDM_HASH_LENGTH 4
+#define MINMATCH 4
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
+#define LDM_ROLLING_HASH
 //#define LDM_DEBUG
 
 typedef  uint8_t BYTE;
@@ -36,6 +39,7 @@ typedef uint64_t U64;
 
 typedef uint32_t offset_t;
 typedef uint32_t hash_t;
+typedef signed char schar;
 
 typedef struct LDM_hashEntry {
   offset_t offset;
@@ -88,18 +92,63 @@ typedef struct LDM_CCtx {
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  const BYTE *forwardIp;
-  hash_t forwardHash;
+  const BYTE *nextIp;
+  hash_t nextHash;                    /* Hash corresponding to nextIp */
 
   unsigned step;
-
 } LDM_CCtx;
 
+#ifdef LDM_ROLLING_HASH
+/**
+ * Convert a sum computed from LDM_getRollingHash to a hash value in the range
+ * of the hash table.
+ */
+static hash_t LDM_sumToHash(U32 sum) {
+  return sum % (LDM_HASHTABLESIZE >> 2);
+//  return sum & (LDM_HASHTABLESIZE - 1);
+}
 
+static U32 LDM_getRollingHash(const char *data, U32 len) {
+  U32 i;
+  U32 s1, s2;
+  const schar *buf = (const schar *)data;
+
+  s1 = s2 = 0;
+  for (i = 0; i < (len - 4); i += 4) {
+    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
+          (2 * buf[i + 2]) + (buf[i + 3]);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
+  }
+  for(; i < len; i++) {
+    s1 += buf[i];
+    s2 += s1;
+  }
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
+}
+
+typedef struct LDM_sumStruct {
+  U16 s1, s2;
+} LDM_sumStruct;
+
+static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
+  sumStruct->s1 = sum & 0xffff;
+  sumStruct->s2 = sum >> 16;
+}
+
+#else
 static hash_t LDM_hash(U32 sequence) {
   return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
 }
 
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+#endif
+
 /*
 static hash_t LDM_hash5(U64 sequence) {
   static const U64 prime5bytes = 889523592379ULL;
@@ -112,10 +161,6 @@ static hash_t LDM_hash5(U64 sequence) {
 }
 */
 
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-
 static void LDM_putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash) {
   if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
@@ -187,26 +232,26 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
 
   cctx->lastPosHashed = NULL;
-  cctx->forwardIp = NULL;
+  cctx->nextIp = NULL;
 
   cctx->step = 1;
 }
 
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
-  cctx->forwardIp = cctx->ip;
+  cctx->nextIp = cctx->ip;
 
   do {
-    hash_t const h = cctx->forwardHash;
-    cctx->ip = cctx->forwardIp;
-    cctx->forwardIp += cctx->step;
+    hash_t const h = cctx->nextHash;
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
 
-    if (cctx->forwardIp > cctx->imatchLimit) {
+    if (cctx->nextIp > cctx->imatchLimit) {
       return 1;
     }
 
     *match = LDM_get_position_on_hash(h, cctx->hashTable, cctx->ibase);
 
-    cctx->forwardHash = LDM_hashPosition(cctx->forwardIp);
+    cctx->nextHash = LDM_hashPosition(cctx->nextIp);
     LDM_putHashOfCurrentPositionFromHash(cctx, h);
   } while (cctx->ip - *match > WINDOW_SIZE ||
            LDM_read64(*match) != LDM_read64(cctx->ip));
@@ -222,7 +267,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
   cctx.ip++;
-  cctx.forwardHash = LDM_hashPosition(cctx.ip);
+  cctx.nextHash = LDM_hashPosition(cctx.ip);
 
   // TODO: loop condition is not accurate.
   while (1) {
@@ -241,7 +286,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
     cctx.stats.numMatches++;
 
     /**
-     * Catchup: look back to extend the match backwards from the found match.
+     * Catch up: look back to extend the match backwards from the found match.
      */
     while (cctx.ip > cctx.anchor && match > cctx.ibase &&
            cctx.ip[-1] == match[-1]) {
@@ -313,7 +358,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
     // Set start of next block to current input pointer.
     cctx.anchor = cctx.ip;
     LDM_putHashOfCurrentPosition(&cctx);
-    cctx.forwardHash = LDM_hashPosition(++cctx.ip);
+    cctx.nextHash = LDM_hashPosition(++cctx.ip);
   }
 _last_literals:
   /* Encode the last literals (no more matches). */

From ef0ff7fe7fd90a94721c2626d1588b053ffc76ce Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 11 Jul 2017 08:54:29 -0700
Subject: [PATCH 026/100] zstdmt: removed margin for improved memory usage

---
 lib/compress/zstdmt_compress.c | 12 +++++-------
 tests/fuzzer.c                 |  2 ++
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index fbb86b00..255b9c7e 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -340,7 +340,6 @@ struct ZSTDMT_CCtx_s {
     pthread_mutex_t jobCompleted_mutex;
     pthread_cond_t jobCompleted_cond;
     size_t targetSectionSize;
-    size_t marginSize;
     size_t inBuffSize;
     size_t dictSize;
     size_t targetDictSize;
@@ -673,8 +672,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
     zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
     DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
-    zcs->marginSize = zcs->targetSectionSize >> 2;
-    zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
+    zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize;
     zcs->inBuff.buffer = g_nullBuffer;
     zcs->dictSize = 0;
     zcs->doneJobID = 0;
@@ -871,18 +869,18 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
                                      ZSTD_inBuffer* input,
                                      ZSTD_EndDirective endOp)
 {
-    size_t const newJobThreshold = mtctx->dictSize + mtctx->targetSectionSize + mtctx->marginSize;
+    size_t const newJobThreshold = mtctx->dictSize + mtctx->targetSectionSize;
     assert(output->pos <= output->size);
     assert(input->pos  <= input->size);
     if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
         /* current frame being ended. Only flush/end are allowed. Or start new frame with init */
         return ERROR(stage_wrong);
     }
-    if (mtctx->nbThreads==1) {
+    if (mtctx->nbThreads==1) {  /* delegate to single-thread (synchronous) */
         return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp);
     }
 
-    /* single-pass shortcut (note : this is blocking-mode) */
+    /* single-pass shortcut (note : this is synchronous-mode) */
     if ( (mtctx->nextJobID==0)      /* just started */
       && (mtctx->inBuff.filled==0)  /* nothing buffered */
       && (endOp==ZSTD_e_end)        /* end order */
@@ -901,7 +899,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
     }
 
     /* fill input buffer */
-    if (input->src) {   /* support NULL input */
+    if (input->size > input->pos) {   /* support NULL input */
         if (mtctx->inBuff.buffer.start == NULL) {
             mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->buffPool, mtctx->inBuffSize);
             if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 1e25383f..6bed9ec5 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -118,6 +118,8 @@ static void* FUZ_mallocDebug(void* counter, size_t size)
     mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
     void* const ptr = malloc(size);
     if (ptr==NULL) return NULL;
+    DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
+        (U32)(size >> 10), (U32)(malloc_size(ptr) >> 10));  /* OS-X specific */
     mcPtr->totalMalloc += size;
     mcPtr->currentMalloc += size;
     if (mcPtr->currentMalloc > mcPtr->peakMalloc)

From f6c5d07fe295997d9f01461d2c40714e10aa5827 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Tue, 11 Jul 2017 09:23:44 -0700
Subject: [PATCH 027/100] Save v3

---
 contrib/long_distance_matching/ldm.c          |   8 +-
 .../versions/v3/Makefile                      |  40 ++
 .../long_distance_matching/versions/v3/ldm.c  | 464 +++++++++++++++++
 .../long_distance_matching/versions/v3/ldm.h  |  19 +
 .../versions/v3/main-ldm.c                    | 479 ++++++++++++++++++
 .../long_distance_matching/versions/v3/util.c |  64 +++
 .../long_distance_matching/versions/v3/util.h |  23 +
 7 files changed, 1093 insertions(+), 4 deletions(-)
 create mode 100644 contrib/long_distance_matching/versions/v3/Makefile
 create mode 100644 contrib/long_distance_matching/versions/v3/ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v3/ldm.h
 create mode 100644 contrib/long_distance_matching/versions/v3/main-ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v3/util.c
 create mode 100644 contrib/long_distance_matching/versions/v3/util.h

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 42a4affd..1dedf5c3 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -7,9 +7,9 @@
 #include "ldm.h"
 #include "util.h"
 
-#define HASH_EVERY 7
+#define HASH_EVERY 1
 
-#define LDM_MEMORY_USAGE 20
+#define LDM_MEMORY_USAGE 16
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
@@ -17,7 +17,7 @@
 
 #define LDM_OFFSET_SIZE 4
 
-#define WINDOW_SIZE (1 << 24)
+#define WINDOW_SIZE (1 << 20)
 #define MAX_WINDOW_SIZE 31
 #define HASH_SIZE 4
 #define LDM_HASH_LENGTH 4
@@ -28,7 +28,7 @@
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
-#define LDM_ROLLING_HASH
+//#define LDM_ROLLING_HASH
 //#define LDM_DEBUG
 
 typedef  uint8_t BYTE;
diff --git a/contrib/long_distance_matching/versions/v3/Makefile b/contrib/long_distance_matching/versions/v3/Makefile
new file mode 100644
index 00000000..5ffd4eaf
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v3/Makefile
@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+# This Makefile presumes libzstd is installed, using `sudo make install`
+
+CFLAGS  ?= -O3
+DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS)
+
+LDFLAGS += -lzstd
+
+.PHONY: default all clean
+
+default: all
+
+all: main-ldm
+
+
+#main : ldm.c main.c
+#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+main-ldm : util.c ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+clean:
+	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
+	main main-ldm
+	@echo Cleaning completed
+
diff --git a/contrib/long_distance_matching/versions/v3/ldm.c b/contrib/long_distance_matching/versions/v3/ldm.c
new file mode 100644
index 00000000..1dedf5c3
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v3/ldm.c
@@ -0,0 +1,464 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+
+#include "ldm.h"
+#include "util.h"
+
+#define HASH_EVERY 1
+
+#define LDM_MEMORY_USAGE 16
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
+
+#define LDM_OFFSET_SIZE 4
+
+#define WINDOW_SIZE (1 << 20)
+#define MAX_WINDOW_SIZE 31
+#define HASH_SIZE 4
+#define LDM_HASH_LENGTH 4
+#define MINMATCH 4
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+//#define LDM_ROLLING_HASH
+//#define LDM_DEBUG
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint32_t offset_t;
+typedef uint32_t hash_t;
+typedef signed char schar;
+
+typedef struct LDM_hashEntry {
+  offset_t offset;
+} LDM_hashEntry;
+
+typedef struct LDM_compressStats {
+  U32 numMatches;
+  U32 totalMatchLength;
+  U32 totalLiteralLength;
+  U64 totalOffset;
+} LDM_compressStats;
+
+static void LDM_printCompressStats(const LDM_compressStats *stats) {
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Total number of matches: %u\n", stats->numMatches);
+  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
+                                         (double)stats->numMatches);
+  printf("Average literal length: %.1f\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
+  printf("Average offset length: %.1f\n",
+         ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("=====================\n");
+}
+
+typedef struct LDM_CCtx {
+  size_t isize;             /* Input size */
+  size_t maxOSize;          /* Maximum output size */
+
+  const BYTE *ibase;        /* Base of input */
+  const BYTE *ip;           /* Current input position */
+  const BYTE *iend;         /* End of input */
+
+  // Maximum input position such that hashing at the position does not exceed
+  // end of input.
+  const BYTE *ihashLimit;
+
+  // Maximum input position such that finding a match of at least the minimum
+  // match length does not exceed end of input.
+  const BYTE *imatchLimit;
+
+  const BYTE *obase;        /* Base of output */
+  BYTE *op;                 /* Output */
+
+  const BYTE *anchor;       /* Anchor to start of current (match) block */
+
+  LDM_compressStats stats;            /* Compression statistics */
+
+  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+
+  const BYTE *lastPosHashed;          /* Last position hashed */
+  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
+  const BYTE *nextIp;
+  hash_t nextHash;                    /* Hash corresponding to nextIp */
+
+  unsigned step;
+} LDM_CCtx;
+
+#ifdef LDM_ROLLING_HASH
+/**
+ * Convert a sum computed from LDM_getRollingHash to a hash value in the range
+ * of the hash table.
+ */
+static hash_t LDM_sumToHash(U32 sum) {
+  return sum % (LDM_HASHTABLESIZE >> 2);
+//  return sum & (LDM_HASHTABLESIZE - 1);
+}
+
+static U32 LDM_getRollingHash(const char *data, U32 len) {
+  U32 i;
+  U32 s1, s2;
+  const schar *buf = (const schar *)data;
+
+  s1 = s2 = 0;
+  for (i = 0; i < (len - 4); i += 4) {
+    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
+          (2 * buf[i + 2]) + (buf[i + 3]);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
+  }
+  for(; i < len; i++) {
+    s1 += buf[i];
+    s2 += s1;
+  }
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
+}
+
+typedef struct LDM_sumStruct {
+  U16 s1, s2;
+} LDM_sumStruct;
+
+static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
+  sumStruct->s1 = sum & 0xffff;
+  sumStruct->s2 = sum >> 16;
+}
+
+#else
+static hash_t LDM_hash(U32 sequence) {
+  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
+}
+
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+#endif
+
+/*
+static hash_t LDM_hash5(U64 sequence) {
+  static const U64 prime5bytes = 889523592379ULL;
+  static const U64 prime8bytes = 11400714785074694791ULL;
+  const U32 hashLog = LDM_HASHLOG;
+  if (LDM_isLittleEndian())
+    return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
+  else
+    return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+}
+*/
+
+static void LDM_putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash) {
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+}
+
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  hash_t hash = LDM_hashPosition(cctx->ip);
+  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
+}
+
+static const BYTE *LDM_get_position_on_hash(
+    hash_t h, void *tableBase, const BYTE *srcBase) {
+  const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
+  return hashTable[h].offset + srcBase;
+}
+
+static BYTE LDM_read_byte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
+static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
+                          const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize) {
+  const U32 *ip = (const U32 *)src;
+  *compressSize = *ip++;
+  *decompressSize = *ip;
+}
+
+static void LDM_initializeCCtx(LDM_CCtx *cctx,
+                               const void *src, size_t srcSize,
+                               void *dst, size_t maxDstSize) {
+  cctx->isize = srcSize;
+  cctx->maxOSize = maxDstSize;
+
+  cctx->ibase = (const BYTE *)src;
+  cctx->ip = cctx->ibase;
+  cctx->iend = cctx->ibase + srcSize;
+
+  cctx->ihashLimit = cctx->iend - HASH_SIZE;
+  cctx->imatchLimit = cctx->iend - MINMATCH;
+
+  cctx->obase = (BYTE *)dst;
+  cctx->op = (BYTE *)dst;
+
+  cctx->anchor = cctx->ibase;
+
+  memset(&(cctx->stats), 0, sizeof(cctx->stats));
+  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+
+  cctx->lastPosHashed = NULL;
+  cctx->nextIp = NULL;
+
+  cctx->step = 1;
+}
+
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+  cctx->nextIp = cctx->ip;
+
+  do {
+    hash_t const h = cctx->nextHash;
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
+
+    if (cctx->nextIp > cctx->imatchLimit) {
+      return 1;
+    }
+
+    *match = LDM_get_position_on_hash(h, cctx->hashTable, cctx->ibase);
+
+    cctx->nextHash = LDM_hashPosition(cctx->nextIp);
+    LDM_putHashOfCurrentPositionFromHash(cctx, h);
+  } while (cctx->ip - *match > WINDOW_SIZE ||
+           LDM_read64(*match) != LDM_read64(cctx->ip));
+  return 0;
+}
+
+// TODO: srcSize and maxDstSize is unused
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  LDM_CCtx cctx;
+  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+
+  /* Hash the first position and put it into the hash table. */
+  LDM_putHashOfCurrentPosition(&cctx);
+  cctx.ip++;
+  cctx.nextHash = LDM_hashPosition(cctx.ip);
+
+  // TODO: loop condition is not accurate.
+  while (1) {
+    const BYTE *match;
+
+    /**
+     * Find a match.
+     * If no more matches can be found (i.e. the length of the remaining input
+     * is less than the minimum match length), then stop searching for matches
+     * and encode the final literals.
+     */
+    if (LDM_findBestMatch(&cctx, &match) != 0) {
+      goto _last_literals;
+    }
+
+    cctx.stats.numMatches++;
+
+    /**
+     * Catch up: look back to extend the match backwards from the found match.
+     */
+    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
+           cctx.ip[-1] == match[-1]) {
+      cctx.ip--;
+      match--;
+    }
+
+    /**
+     * Write current block (literals, literal length, match offset, match
+     * length) and update pointers and hashes.
+     */
+    {
+      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
+      unsigned const offset = cctx.ip - match;
+      unsigned const matchLength = LDM_count(
+          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
+      BYTE *token = cctx.op++;
+
+      cctx.stats.totalLiteralLength += literalLength;
+      cctx.stats.totalOffset += offset;
+      cctx.stats.totalMatchLength += matchLength + MINMATCH;
+
+      /* Encode the literal length. */
+      if (literalLength >= RUN_MASK) {
+        int len = (int)literalLength - RUN_MASK;
+        *token = (RUN_MASK << ML_BITS);
+        for (; len >= 255; len -= 255) {
+          *(cctx.op)++ = 255;
+        }
+        *(cctx.op)++ = (BYTE)len;
+      } else {
+        *token = (BYTE)(literalLength << ML_BITS);
+      }
+
+      /* Encode the literals. */
+      memcpy(cctx.op, cctx.anchor, literalLength);
+      cctx.op += literalLength;
+
+      /* Encode the offset. */
+      LDM_write32(cctx.op, offset);
+      cctx.op += LDM_OFFSET_SIZE;
+
+      /* Encode match length */
+      if (matchLength >= ML_MASK) {
+        unsigned matchLengthRemaining = matchLength;
+        *token += ML_MASK;
+        matchLengthRemaining -= ML_MASK;
+        LDM_write32(cctx.op, 0xFFFFFFFF);
+        while (matchLengthRemaining >= 4*0xFF) {
+          cctx.op += 4;
+          LDM_write32(cctx.op, 0xffffffff);
+          matchLengthRemaining -= 4*0xFF;
+        }
+        cctx.op += matchLengthRemaining / 255;
+        *(cctx.op)++ = (BYTE)(matchLengthRemaining % 255);
+      } else {
+        *token += (BYTE)(matchLength);
+      }
+
+      /* Update input pointer, inserting hashes into hash table along the
+       * way.
+       */
+      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
+        LDM_putHashOfCurrentPosition(&cctx);
+        cctx.ip++;
+      }
+    }
+
+    // Set start of next block to current input pointer.
+    cctx.anchor = cctx.ip;
+    LDM_putHashOfCurrentPosition(&cctx);
+    cctx.nextHash = LDM_hashPosition(++cctx.ip);
+  }
+_last_literals:
+  /* Encode the last literals (no more matches). */
+  {
+    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *(cctx.op)++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *(cctx.op)++ = 255;
+      }
+      *(cctx.op)++ = (BYTE)accumulator;
+    } else {
+      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
+    }
+    memcpy(cctx.op, cctx.anchor, lastRun);
+    cctx.op += lastRun;
+  }
+  LDM_printCompressStats(&cctx.stats);
+  return (cctx.op - (const BYTE *)cctx.obase);
+}
+
+typedef struct LDM_DCtx {
+  size_t compressSize;
+  size_t maxDecompressSize;
+
+  const BYTE *ibase;   /* Base of input */
+  const BYTE *ip;      /* Current input position */
+  const BYTE *iend;    /* End of source */
+
+  const BYTE *obase;   /* Base of output */
+  BYTE *op;            /* Current output position */
+  const BYTE *oend;    /* End of output */
+} LDM_DCtx;
+
+static void LDM_initializeDCtx(LDM_DCtx *dctx,
+                               const void *src, size_t compressSize,
+                               void *dst, size_t maxDecompressSize) {
+  dctx->compressSize = compressSize;
+  dctx->maxDecompressSize = maxDecompressSize;
+
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + dctx->compressSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + dctx->maxDecompressSize;
+
+}
+
+size_t LDM_decompress(const void *src, size_t compressSize,
+                      void *dst, size_t maxDecompressSize) {
+  LDM_DCtx dctx;
+  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
+
+  while (dctx.ip < dctx.iend) {
+    BYTE *cpy;
+    const BYTE *match;
+    size_t length, offset;
+
+    /* Get the literal length. */
+    unsigned const token = *(dctx.ip)++;
+    if ((length = (token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+
+    /* Copy literals. */
+    cpy = dctx.op + length;
+    memcpy(dctx.op, dctx.ip, length);
+    dctx.ip += length;
+    dctx.op = cpy;
+
+    //TODO : dynamic offset size
+    offset = LDM_read32(dctx.ip);
+    dctx.ip += LDM_OFFSET_SIZE;
+    match = dctx.op - offset;
+
+    /* Get the match length. */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+
+    /* Copy match. */
+    cpy = dctx.op + length;
+
+    // Inefficient for now
+    while (match < cpy - offset && dctx.op < dctx.oend) {
+      *(dctx.op)++ = *match++;
+    }
+  }
+  return dctx.op - (BYTE *)dst;
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v3/ldm.h b/contrib/long_distance_matching/versions/v3/ldm.h
new file mode 100644
index 00000000..287d444d
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v3/ldm.h
@@ -0,0 +1,19 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize);
+
+size_t LDM_decompress(const void *src, size_t srcSize,
+                      void *dst, size_t maxDstSize);
+
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v3/main-ldm.c b/contrib/long_distance_matching/versions/v3/main-ldm.c
new file mode 100644
index 00000000..724d735d
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v3/main-ldm.c
@@ -0,0 +1,479 @@
+// TODO: file size must fit into a U32
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+// #define BUF_SIZE 16*1024  // Block size
+#define DEBUG
+
+//#define ZSTD
+
+/* Compress file given by fname and output to oname.
+ * Returns 0 if successful, error code otherwise.
+ */
+static int compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+  size_t maxCompressSize, compressSize;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+
+ /* Go to the location corresponding to the last byte. */
+ /* TODO: fallocate? */
+  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+ /* Write a dummy byte at the last location. */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  compressSize = ZSTD_compress(dst, statbuf.st_size,
+                                  src, statbuf.st_size, 1);
+#else
+  compressSize = LDM_HEADER_SIZE +
+      LDM_compress(src, statbuf.st_size,
+                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+
+    // Write compress and decompress size to header
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  memcpy(dst, &compressSize, 4);
+  memcpy(dst + 4, &(statbuf.st_size), 4);
+
+#ifdef DEBUG
+  printf("Compressed size: %zu\n", compressSize);
+  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
+#endif
+#endif
+
+  // Truncate file to compressSize.
+  ftruncate(fdout, compressSize);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
+         (double)compressSize / (statbuf.st_size) * 100);
+
+  // Close files.
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Decompress file compressed using LDM_compress.
+ * The input file should have the LDM_HEADER followed by payload.
+ * Returns 0 if succesful, and an error code otherwise.
+ */
+static int decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+  size_t compressSize, decompressSize, outSize;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* Read the header. */
+  LDM_readHeader(src, &compressSize, &decompressSize);
+
+#ifdef DEBUG
+  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
+         (size_t)statbuf.st_size, compressSize, decompressSize);
+#endif
+
+  /* Go to the location corresponding to the last byte. */
+  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  outSize = ZSTD_decompress(dst, decomrpessed_size,
+                                  src + LDM_HEADER_SIZE,
+                                  statbuf.st_size - LDM_HEADER_SIZE);
+#else
+  outSize = LDM_decompress(
+      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
+      dst, decompressSize);
+
+  printf("Ret size out: %zu\n", outSize);
+  #endif
+  ftruncate(fdout, outSize);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Compare two files.
+ * Returns 0 iff they are the same.
+ */
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) break;
+
+    if (0 == result) result = memcmp(b0, b1, r0);
+  }
+  return result;
+}
+
+/* Verify the input file is the same as the decompressed file. */
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+  {
+    const int cmp = compare(inpFp, decFp);
+    if(0 == cmp) {
+      printf("verify : OK\n");
+    } else {
+      printf("verify : NG\n");
+    }
+  }
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+
+  /* Compress */
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (compress(inpFilename, ldmFilename)) {
+        printf("Compress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total time = %f seconds\n",
+           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+           (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+
+  /* Decompress */
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (decompress(ldmFilename, decFilename)) {
+        printf("Decompress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total time = %f seconds\n",
+          (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+          (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+
diff --git a/contrib/long_distance_matching/versions/v3/util.c b/contrib/long_distance_matching/versions/v3/util.c
new file mode 100644
index 00000000..9ea4ca1e
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v3/util.c
@@ -0,0 +1,64 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "util.h"
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+unsigned LDM_isLittleEndian(void) {
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+void LDM_write16(void *memPtr, U16 value){
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE *)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v3/util.h b/contrib/long_distance_matching/versions/v3/util.h
new file mode 100644
index 00000000..90726412
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v3/util.h
@@ -0,0 +1,23 @@
+#ifndef LDM_UTIL_H
+#define LDM_UTIL_H
+
+unsigned LDM_isLittleEndian(void);
+
+uint16_t LDM_read16(const void *memPtr);
+
+uint16_t LDM_readLE16(const void *memPtr);
+
+void LDM_write16(void *memPtr, uint16_t value);
+
+void LDM_write32(void *memPtr, uint32_t value);
+
+void LDM_writeLE16(void *memPtr, uint16_t value);
+
+uint32_t LDM_read32(const void *ptr);
+
+uint64_t LDM_read64(const void *ptr);
+
+void LDM_copy8(void *dst, const void *src);
+
+
+#endif /* LDM_UTIL_H */

From c325c8db82808801a3586694327c7e64bff585f7 Mon Sep 17 00:00:00 2001
From: Jacques Germishuys <jacquesg@striata.com>
Date: Tue, 11 Jul 2017 19:25:14 +0200
Subject: [PATCH 028/100] fix missing symbol 'nanosleep' for Solaris

---
 build/cmake/programs/CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index 98ca6cca..13dd3157 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -31,6 +31,9 @@ ENDIF (MSVC)
 
 ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
 TARGET_LINK_LIBRARIES(zstd libzstd_static)
+IF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
+    TARGET_LINK_LIBRARIES(zstd rt)
+ENDIF (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
 INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
 
 IF (UNIX)

From 16261e6951910e56f0c447f012900d7e06aba733 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 11 Jul 2017 14:14:07 -0700
Subject: [PATCH 029/100] buffer pool can be invoked from multiple threads

---
 lib/common/pool.c              | 20 +++++++++---------
 lib/common/pool.h              | 10 ++++-----
 lib/compress/zstdmt_compress.c | 38 ++++++++++++++++++++++------------
 3 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index 749fa4f2..06d8a5f5 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -92,7 +92,7 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
      * and full queues.
      */
     ctx->queueSize = queueSize + 1;
-    ctx->queue = (POOL_job *)malloc(ctx->queueSize * sizeof(POOL_job));
+    ctx->queue = (POOL_job*) malloc(ctx->queueSize * sizeof(POOL_job));
     ctx->queueHead = 0;
     ctx->queueTail = 0;
     pthread_mutex_init(&ctx->queueMutex, NULL);
@@ -100,7 +100,7 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
     pthread_cond_init(&ctx->queuePopCond, NULL);
     ctx->shutdown = 0;
     /* Allocate space for the thread handles */
-    ctx->threads = (pthread_t *)malloc(numThreads * sizeof(pthread_t));
+    ctx->threads = (pthread_t*)malloc(numThreads * sizeof(pthread_t));
     ctx->numThreads = 0;
     /* Check for errors */
     if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
@@ -153,8 +153,8 @@ size_t POOL_sizeof(POOL_ctx *ctx) {
         + ctx->numThreads * sizeof(pthread_t);
 }
 
-void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
-    POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
+void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
+    POOL_ctx* const ctx = (POOL_ctx*)ctxVoid;
     if (!ctx) { return; }
 
     pthread_mutex_lock(&ctx->queueMutex);
@@ -183,22 +183,22 @@ struct POOL_ctx_s {
   int data;
 };
 
-POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
   (void)numThreads;
   (void)queueSize;
-  return (POOL_ctx *)malloc(sizeof(POOL_ctx));
+  return (POOL_ctx*)malloc(sizeof(POOL_ctx));
 }
 
-void POOL_free(POOL_ctx *ctx) {
-  if (ctx) free(ctx);
+void POOL_free(POOL_ctx* ctx) {
+    free(ctx);
 }
 
-void POOL_add(void *ctx, POOL_function function, void *opaque) {
+void POOL_add(void* ctx, POOL_function function, void* opaque) {
   (void)ctx;
   function(opaque);
 }
 
-size_t POOL_sizeof(POOL_ctx *ctx) {
+size_t POOL_sizeof(POOL_ctx* ctx) {
     if (ctx==NULL) return 0;  /* supports sizeof NULL */
     return sizeof(*ctx);
 }
diff --git a/lib/common/pool.h b/lib/common/pool.h
index 386cd674..957100f4 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -19,11 +19,11 @@ extern "C" {
 typedef struct POOL_ctx_s POOL_ctx;
 
 /*! POOL_create() :
-    Create a thread pool with at most `numThreads` threads.
-    `numThreads` must be at least 1.
-    The maximum number of queued jobs before blocking is `queueSize`.
-    `queueSize` must be at least 1.
-    @return : The POOL_ctx pointer on success else NULL.
+ *  Create a thread pool with at most `numThreads` threads.
+ * `numThreads` must be at least 1.
+ *  The maximum number of queued jobs before blocking is `queueSize`.
+ * `queueSize` must be at least 1.
+ * @return : POOL_ctx pointer on success, else NULL.
 */
 POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 255b9c7e..c4547c81 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -73,6 +73,7 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
 
 
 /* =====   Buffer Pool   ===== */
+/* a single Buffer Pool can be invoked from multiple threads in parallel */
 
 typedef struct buffer_s {
     void* start;
@@ -82,6 +83,7 @@ typedef struct buffer_s {
 static const buffer_t g_nullBuffer = { NULL, 0 };
 
 typedef struct ZSTDMT_bufferPool_s {
+    pthread_mutex_t poolMutex;
     unsigned totalBuffers;
     unsigned nbBuffers;
     ZSTD_customMem cMem;
@@ -94,6 +96,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_custo
     ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
         sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
     if (bufPool==NULL) return NULL;
+    pthread_mutex_init(&bufPool->poolMutex, NULL);
     bufPool->totalBuffers = maxNbBuffers;
     bufPool->nbBuffers = 0;
     bufPool->cMem = cMem;
@@ -106,6 +109,7 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
     if (!bufPool) return;   /* compatibility with free on NULL */
     for (u=0; u<bufPool->totalBuffers; u++)
         ZSTD_free(bufPool->bTable[u].start, bufPool->cMem);
+    pthread_mutex_destroy(&bufPool->poolMutex);
     ZSTD_free(bufPool, bufPool->cMem);
 }
 
@@ -116,31 +120,37 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
                             + (bufPool->totalBuffers - 1) * sizeof(buffer_t);
     unsigned u;
     size_t totalBufferSize = 0;
+    pthread_mutex_lock(&bufPool->poolMutex);
     for (u=0; u<bufPool->totalBuffers; u++)
         totalBufferSize += bufPool->bTable[u].size;
+    pthread_mutex_unlock(&bufPool->poolMutex);
 
     return poolSize + totalBufferSize;
 }
 
 /** ZSTDMT_getBuffer() :
  *  assumption : invocation from main thread only ! */
-static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
+static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool, size_t bSize)
 {
     DEBUGLOG(2, "ZSTDMT_getBuffer");
-    if (pool->nbBuffers) {   /* try to use an existing buffer */
-        buffer_t const buf = pool->bTable[--(pool->nbBuffers)];
+    pthread_mutex_lock(&bufPool->poolMutex);
+    if (bufPool->nbBuffers) {   /* try to use an existing buffer */
+        buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)];
         size_t const availBufferSize = buf.size;
-        if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize))
+        if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) {
             /* large enough, but not too much */
+            pthread_mutex_unlock(&bufPool->poolMutex);
             return buf;
+        }
         /* size conditions not respected : scratch this buffer, create new one */
         DEBUGLOG(2, "existing buffer does not meet size conditions => freeing");
-        ZSTD_free(buf.start, pool->cMem);
+        ZSTD_free(buf.start, bufPool->cMem);
     }
+    pthread_mutex_unlock(&bufPool->poolMutex);
     /* create new buffer */
     DEBUGLOG(2, "create a new buffer");
     {   buffer_t buffer;
-        void* const start = ZSTD_malloc(bSize, pool->cMem);
+        void* const start = ZSTD_malloc(bSize, bufPool->cMem);
         if (start==NULL) bSize = 0;
         buffer.start = start;   /* note : start can be NULL if malloc fails ! */
         buffer.size = bSize;
@@ -149,23 +159,25 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
 }
 
 /* store buffer for later re-use, up to pool capacity */
-static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
+static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
 {
     DEBUGLOG(2, "ZSTDMT_releaseBuffer");
     if (buf.start == NULL) return;   /* release on NULL */
-    if (pool->nbBuffers < pool->totalBuffers) {
-        pool->bTable[pool->nbBuffers++] = buf;   /* store for later re-use */
+    pthread_mutex_lock(&bufPool->poolMutex);
+    if (bufPool->nbBuffers < bufPool->totalBuffers) {
+        bufPool->bTable[bufPool->nbBuffers++] = buf;   /* store for later re-use */
+        pthread_mutex_unlock(&bufPool->poolMutex);
         return;
     }
+    pthread_mutex_unlock(&bufPool->poolMutex);
     /* Reached bufferPool capacity (should not happen) */
     DEBUGLOG(2, "buffer pool capacity reached => freeing ");
-    ZSTD_free(buf.start, pool->cMem);
+    ZSTD_free(buf.start, bufPool->cMem);
 }
 
 
 /* =====   CCtx Pool   ===== */
-
-/* a single cctxPool can be called from multiple threads in parallel */
+/* a single CCtx Pool can be invoked from multiple threads in parallel */
 
 typedef struct {
     pthread_mutex_t poolMutex;
@@ -314,7 +326,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
     job->cSize = (job->lastChunk) ?
                  ZSTD_compressEnd     (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
                  ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
-    DEBUGLOG(5, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
+    DEBUGLOG(2, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
                 (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
     DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
 

From 34b2b956314581f2e191697ce3b18d170cfe3d42 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 11 Jul 2017 14:59:10 -0700
Subject: [PATCH 030/100] zstdmt : intermediate outBuffer allocated from within
 worker

reduces total amount of memory needed,
since jobs in queue do not have an outBuffer pre-reserved now
---
 lib/compress/zstdmt_compress.c | 81 ++++++++++++++++------------------
 1 file changed, 39 insertions(+), 42 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index c4547c81..9f30d318 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -286,6 +286,7 @@ typedef struct {
     ZSTD_parameters params;
     const ZSTD_CDict* cdict;
     ZSTDMT_CCtxPool* cctxPool;
+    ZSTDMT_bufferPool* bufPool;
     unsigned long long fullFrameSize;
 } ZSTDMT_jobDescription;
 
@@ -295,7 +296,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
     ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
     ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
     const void* const src = (const char*)job->srcStart + job->dictSize;
-    buffer_t const dstBuff = job->dstBuff;
+    buffer_t dstBuff = job->dstBuff;
     DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
                  job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
 
@@ -304,6 +305,16 @@ void ZSTDMT_compressChunk(void* jobDescription)
         goto _endJob;
     }
 
+    if (dstBuff.start == NULL) {
+        size_t const dstCapacity = ZSTD_compressBound(job->srcSize);
+        dstBuff = ZSTDMT_getBuffer(job->bufPool, dstCapacity);
+        if (dstBuff.start==NULL) {
+            job->cSize = ERROR(memory_allocation);
+            goto _endJob;
+        }
+        job->dstBuff = dstBuff;
+    }
+
     if (job->cdict) {  /* should only happen for first segment */
         size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
         DEBUGLOG(5, "using CDict");
@@ -347,7 +358,7 @@ _endJob:
 struct ZSTDMT_CCtx_s {
     POOL_ctx* factory;
     ZSTDMT_jobDescription* jobs;
-    ZSTDMT_bufferPool* buffPool;
+    ZSTDMT_bufferPool* bufPool;
     ZSTDMT_CCtxPool* cctxPool;
     pthread_mutex_t jobCompleted_mutex;
     pthread_cond_t jobCompleted_cond;
@@ -402,9 +413,9 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
     mtctx->factory = POOL_create(nbThreads, 1);
     mtctx->jobs = ZSTDMT_allocJobsTable(&nbJobs, cMem);
     mtctx->jobIDMask = nbJobs - 1;
-    mtctx->buffPool = ZSTDMT_createBufferPool(nbThreads, cMem);
+    mtctx->bufPool = ZSTDMT_createBufferPool(nbThreads, cMem);
     mtctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads, cMem);
-    if (!mtctx->factory | !mtctx->jobs | !mtctx->buffPool | !mtctx->cctxPool) {
+    if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool) {
         ZSTDMT_freeCCtx(mtctx);
         return NULL;
     }
@@ -426,13 +437,13 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
     unsigned jobID;
     DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
     for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
-        ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].dstBuff);
+        ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
         mtctx->jobs[jobID].dstBuff = g_nullBuffer;
-        ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].src);
+        ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].src);
         mtctx->jobs[jobID].src = g_nullBuffer;
     }
     memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
-    ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->inBuff.buffer);
+    ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->inBuff.buffer);
     mtctx->inBuff.buffer = g_nullBuffer;
     mtctx->allJobsCompleted = 1;
 }
@@ -442,7 +453,7 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     if (mtctx==NULL) return 0;   /* compatible with free on NULL */
     POOL_free(mtctx->factory);
     if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
-    ZSTDMT_freeBufferPool(mtctx->buffPool);  /* release job resources into pools first */
+    ZSTDMT_freeBufferPool(mtctx->bufPool);  /* release job resources into pools first */
     ZSTD_free(mtctx->jobs, mtctx->cMem);
     ZSTDMT_freeCCtxPool(mtctx->cctxPool);
     ZSTD_freeCDict(mtctx->cdictLocal);
@@ -456,11 +467,11 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
 {
     if (mtctx == NULL) return 0;   /* supports sizeof NULL */
     return sizeof(*mtctx)
-        + POOL_sizeof(mtctx->factory)
-        + ZSTDMT_sizeof_bufferPool(mtctx->buffPool)
-        + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription)
-        + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool)
-        + ZSTD_sizeof_CDict(mtctx->cdictLocal);
+            + POOL_sizeof(mtctx->factory)
+            + ZSTDMT_sizeof_bufferPool(mtctx->bufPool)
+            + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription)
+            + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool)
+            + ZSTD_sizeof_CDict(mtctx->cdictLocal);
 }
 
 size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
@@ -534,16 +545,9 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
             size_t const chunkSize = MIN(remainingSrcSize, avgChunkSize);
             size_t const dstBufferCapacity = ZSTD_compressBound(chunkSize);
             buffer_t const dstAsBuffer = { (char*)dst + dstBufferPos, dstBufferCapacity };
-            buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity);
+            buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : g_nullBuffer;
             size_t dictSize = u ? overlapSize : 0;
 
-            if (dstBuffer.start==NULL) {
-                mtctx->jobs[u].cSize = ERROR(memory_allocation);   /* job result */
-                mtctx->jobs[u].jobCompleted = 1;
-                nbChunks = u+1;   /* only wait and free u jobs, instead of initially expected nbChunks ones */
-                break;   /* let's wait for previous jobs to complete, but don't start new ones */
-            }
-
             mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
             mtctx->jobs[u].dictSize = dictSize;
             mtctx->jobs[u].srcSize = chunkSize;
@@ -554,6 +558,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
             if (u!=0) mtctx->jobs[u].params.fParams.checksumFlag = 0;
             mtctx->jobs[u].dstBuff = dstBuffer;
             mtctx->jobs[u].cctxPool = mtctx->cctxPool;
+            mtctx->jobs[u].bufPool = mtctx->bufPool;
             mtctx->jobs[u].firstChunk = (u==0);
             mtctx->jobs[u].lastChunk = (u==nbChunks-1);
             mtctx->jobs[u].jobCompleted = 0;
@@ -591,13 +596,13 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
                         memmove((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize);  /* may overlap when chunk compressed within dst */
                     if (chunkID >= compressWithinDst) {  /* chunk compressed into its own buffer, which must be released */
                         DEBUGLOG(5, "releasing buffer %u>=%u", chunkID, compressWithinDst);
-                        ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff);
+                        ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[chunkID].dstBuff);
                     }
                     mtctx->jobs[chunkID].dstBuff = g_nullBuffer;
                 }
                 dstPos += cSize ;
             }
-        }
+        }  /* for (chunkID=0; chunkID<nbChunks; chunkID++) */
         if (!error) DEBUGLOG(4, "compressed size : %u  ", (U32)dstPos);
         return error ? error : dstPos;
     }
@@ -696,8 +701,9 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
 }
 
 size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
-                                const void* dict, size_t dictSize,
-                                ZSTD_parameters params, unsigned long long pledgedSrcSize)
+                             const void* dict, size_t dictSize,
+                                   ZSTD_parameters params,
+                                   unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(5, "ZSTDMT_initCStream_advanced");
     return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, NULL, params, pledgedSrcSize);
@@ -733,18 +739,8 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
 
 static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsigned endFrame)
 {
-    size_t const dstBufferCapacity = ZSTD_compressBound(srcSize);
-    buffer_t const dstBuffer = ZSTDMT_getBuffer(zcs->buffPool, dstBufferCapacity);
     unsigned const jobID = zcs->nextJobID & zcs->jobIDMask;
 
-    if (dstBuffer.start==NULL) {
-        zcs->jobs[jobID].jobCompleted = 1;
-        zcs->nextJobID++;
-        ZSTDMT_waitForAllJobsCompleted(zcs);
-        ZSTDMT_releaseAllJobResources(zcs);
-        return ERROR(memory_allocation);
-    }
-
     DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ",
                 zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
     zcs->jobs[jobID].src = zcs->inBuff.buffer;
@@ -757,8 +753,9 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
     if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0;
     zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL;
     zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize;
-    zcs->jobs[jobID].dstBuff = dstBuffer;
+    zcs->jobs[jobID].dstBuff = g_nullBuffer;
     zcs->jobs[jobID].cctxPool = zcs->cctxPool;
+    zcs->jobs[jobID].bufPool = zcs->bufPool;
     zcs->jobs[jobID].firstChunk = (zcs->nextJobID==0);
     zcs->jobs[jobID].lastChunk = endFrame;
     zcs->jobs[jobID].jobCompleted = 0;
@@ -770,7 +767,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
     if (!endFrame) {
         size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
         DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
-        zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+        zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->bufPool, zcs->inBuffSize);
         if (zcs->inBuff.buffer.start == NULL) {   /* not enough memory to allocate next input buffer */
             zcs->jobs[jobID].jobCompleted = 1;
             zcs->nextJobID++;
@@ -845,19 +842,19 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
                     job.cSize += 4;
                     zcs->jobs[wJobID].cSize += 4;
             }   }
-            ZSTDMT_releaseBuffer(zcs->buffPool, job.src);
+            ZSTDMT_releaseBuffer(zcs->bufPool, job.src);
             zcs->jobs[wJobID].srcStart = NULL;
             zcs->jobs[wJobID].src = g_nullBuffer;
             zcs->jobs[wJobID].jobScanned = 1;
         }
         {   size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
-            DEBUGLOG(5, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
+            DEBUGLOG(2, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
             memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
             output->pos += toWrite;
             job.dstFlushed += toWrite;
         }
         if (job.dstFlushed == job.cSize) {   /* output buffer fully flushed => move to next one */
-            ZSTDMT_releaseBuffer(zcs->buffPool, job.dstBuff);
+            ZSTDMT_releaseBuffer(zcs->bufPool, job.dstBuff);
             zcs->jobs[wJobID].dstBuff = g_nullBuffer;
             zcs->jobs[wJobID].jobCompleted = 0;
             zcs->doneJobID++;
@@ -904,7 +901,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
         if (ZSTD_isError(cSize)) return cSize;
         input->pos = input->size;
         output->pos += cSize;
-        ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->inBuff.buffer);  /* was allocated in initStream */
+        ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->inBuff.buffer);  /* was allocated in initStream */
         mtctx->allJobsCompleted = 1;
         mtctx->frameEnded = 1;
         return 0;
@@ -913,7 +910,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
     /* fill input buffer */
     if (input->size > input->pos) {   /* support NULL input */
         if (mtctx->inBuff.buffer.start == NULL) {
-            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->buffPool, mtctx->inBuffSize);
+            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->bufPool, mtctx->inBuffSize);
             if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
             mtctx->inBuff.filled = 0;
         }

From 57236184afb9dad253bee28d9a34f31f0491dca7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 11 Jul 2017 15:17:25 -0700
Subject: [PATCH 031/100] buffer pool : all buffers have same size

to reduce memory fragmentation.
They can be used for in or out, interchangeably.
---
 lib/compress/zstdmt_compress.c | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 9f30d318..703d25e3 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -84,6 +84,7 @@ static const buffer_t g_nullBuffer = { NULL, 0 };
 
 typedef struct ZSTDMT_bufferPool_s {
     pthread_mutex_t poolMutex;
+    size_t bufferSize;
     unsigned totalBuffers;
     unsigned nbBuffers;
     ZSTD_customMem cMem;
@@ -97,6 +98,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_custo
         sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
     if (bufPool==NULL) return NULL;
     pthread_mutex_init(&bufPool->poolMutex, NULL);
+    bufPool->bufferSize = 64 KB;
     bufPool->totalBuffers = maxNbBuffers;
     bufPool->nbBuffers = 0;
     bufPool->cMem = cMem;
@@ -128,10 +130,16 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
     return poolSize + totalBufferSize;
 }
 
-/** ZSTDMT_getBuffer() :
- *  assumption : invocation from main thread only ! */
-static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool, size_t bSize)
+static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize)
 {
+    bufPool->bufferSize = bSize;
+}
+
+/** ZSTDMT_getBuffer() :
+ *  assumption : bufPool must be valid */
+static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
+{
+    size_t const bSize = bufPool->bufferSize;
     DEBUGLOG(2, "ZSTDMT_getBuffer");
     pthread_mutex_lock(&bufPool->poolMutex);
     if (bufPool->nbBuffers) {   /* try to use an existing buffer */
@@ -151,9 +159,8 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool, size_t bSize)
     DEBUGLOG(2, "create a new buffer");
     {   buffer_t buffer;
         void* const start = ZSTD_malloc(bSize, bufPool->cMem);
-        if (start==NULL) bSize = 0;
         buffer.start = start;   /* note : start can be NULL if malloc fails ! */
-        buffer.size = bSize;
+        buffer.size = (start==NULL) ? 0 : bSize;
         return buffer;
     }
 }
@@ -306,8 +313,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
     }
 
     if (dstBuff.start == NULL) {
-        size_t const dstCapacity = ZSTD_compressBound(job->srcSize);
-        dstBuff = ZSTDMT_getBuffer(job->bufPool, dstCapacity);
+        dstBuff = ZSTDMT_getBuffer(job->bufPool);
         if (dstBuff.start==NULL) {
             job->cSize = ERROR(memory_allocation);
             goto _endJob;
@@ -530,6 +536,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
         return ZSTD_compress_advanced(cctx, dst, dstCapacity, src, srcSize, NULL, 0, params);
     }
     assert(avgChunkSize >= 256 KB);  /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), which is useful to avoid allocating extra buffers */
+    ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgChunkSize) );
 
     if (nbChunks > mtctx->jobIDMask+1) {  /* enlarge job table */
         U32 nbJobs = nbChunks;
@@ -690,6 +697,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
     DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
     zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize;
+    ZSTDMT_setBufferSize(zcs->bufPool, MAX(zcs->inBuffSize, ZSTD_compressBound(zcs->targetSectionSize)) );
     zcs->inBuff.buffer = g_nullBuffer;
     zcs->dictSize = 0;
     zcs->doneJobID = 0;
@@ -767,7 +775,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
     if (!endFrame) {
         size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
         DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
-        zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->bufPool, zcs->inBuffSize);
+        zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->bufPool);
         if (zcs->inBuff.buffer.start == NULL) {   /* not enough memory to allocate next input buffer */
             zcs->jobs[jobID].jobCompleted = 1;
             zcs->nextJobID++;
@@ -910,7 +918,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
     /* fill input buffer */
     if (input->size > input->pos) {   /* support NULL input */
         if (mtctx->inBuff.buffer.start == NULL) {
-            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->bufPool, mtctx->inBuffSize);
+            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->bufPool);
             if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
             mtctx->inBuff.filled = 0;
         }

From 2a62f48bf4df42f3ca389ee891ff76da56623802 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 11 Jul 2017 15:56:40 -0700
Subject: [PATCH 032/100] release input buffers from inside worker thread

buffers are released sooner, which makes them available faster for next job.
=> decreases total nb of buffers necessary
---
 lib/compress/zstdmt_compress.c | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 703d25e3..7cf637f5 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -169,7 +169,7 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
 static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
 {
     DEBUGLOG(2, "ZSTDMT_releaseBuffer");
-    if (buf.start == NULL) return;   /* release on NULL */
+    if (buf.start == NULL) return;   /* compatible with release on NULL */
     pthread_mutex_lock(&bufPool->poolMutex);
     if (bufPool->nbBuffers < bufPool->totalBuffers) {
         bufPool->bTable[bufPool->nbBuffers++] = buf;   /* store for later re-use */
@@ -271,16 +271,11 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
 
 /* =====   Thread worker   ===== */
 
-typedef struct {
-    buffer_t buffer;
-    size_t filled;
-} inBuff_t;
-
 typedef struct {
     buffer_t src;
     const void* srcStart;
-    size_t   srcSize;
     size_t   dictSize;
+    size_t   srcSize;
     buffer_t dstBuff;
     size_t   cSize;
     size_t   dstFlushed;
@@ -349,6 +344,8 @@ void ZSTDMT_compressChunk(void* jobDescription)
 
 _endJob:
     ZSTDMT_releaseCCtx(job->cctxPool, cctx);
+    ZSTDMT_releaseBuffer(job->bufPool, job->src);
+    job->src = g_nullBuffer; job->srcStart = NULL;
     PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
     job->jobCompleted = 1;
     job->jobScanned = 0;
@@ -361,6 +358,11 @@ _endJob:
 /* =====   Multi-threaded compression   ===== */
 /* ------------------------------------------ */
 
+typedef struct {
+    buffer_t buffer;
+    size_t filled;
+} inBuff_t;
+
 struct ZSTDMT_CCtx_s {
     POOL_ctx* factory;
     ZSTDMT_jobDescription* jobs;
@@ -513,6 +515,7 @@ static unsigned computeNbChunks(size_t srcSize, unsigned windowLog, unsigned nbT
 }
 
 
+/* Note : missing checksum at the end ! */
 size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
                            void* dst, size_t dstCapacity,
                      const void* src, size_t srcSize,
@@ -555,6 +558,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
             buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : g_nullBuffer;
             size_t dictSize = u ? overlapSize : 0;
 
+            mtctx->jobs[u].src = g_nullBuffer;
             mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
             mtctx->jobs[u].dictSize = dictSize;
             mtctx->jobs[u].srcSize = chunkSize;
@@ -771,10 +775,12 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
     zcs->jobs[jobID].jobCompleted_mutex = &zcs->jobCompleted_mutex;
     zcs->jobs[jobID].jobCompleted_cond = &zcs->jobCompleted_cond;
 
+    if (zcs->params.fParams.checksumFlag)
+        XXH64_update(&zcs->xxhState, (const char*)zcs->inBuff.buffer.start + zcs->dictSize, srcSize);
+
     /* get a new buffer for next input */
     if (!endFrame) {
         size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
-        DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
         zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->bufPool);
         if (zcs->inBuff.buffer.start == NULL) {   /* not enough memory to allocate next input buffer */
             zcs->jobs[jobID].jobCompleted = 1;
@@ -783,18 +789,12 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
             ZSTDMT_releaseAllJobResources(zcs);
             return ERROR(memory_allocation);
         }
-        DEBUGLOG(5, "inBuff currently filled to %u", (U32)zcs->inBuff.filled);
         zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize;
-        DEBUGLOG(5, "new job : inBuff filled to %u, with %u dict and %u src",
-                    (U32)zcs->inBuff.filled, (U32)newDictSize,
-                    (U32)(zcs->inBuff.filled - newDictSize));
         memmove(zcs->inBuff.buffer.start,
             (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize,
             zcs->inBuff.filled);
-        DEBUGLOG(5, "new inBuff pre-filled");
         zcs->dictSize = newDictSize;
     } else {   /* if (endFrame==1) */
-        DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
         zcs->inBuff.buffer = g_nullBuffer;
         zcs->inBuff.filled = 0;
         zcs->dictSize = 0;
@@ -842,7 +842,6 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
             }
             DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag);
             if (zcs->params.fParams.checksumFlag) {
-                XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
                 if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) {  /* write checksum at end of last section */
                     U32 const checksum = (U32)XXH64_digest(&zcs->xxhState);
                     DEBUGLOG(5, "writing checksum : %08X \n", checksum);
@@ -850,9 +849,6 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
                     job.cSize += 4;
                     zcs->jobs[wJobID].cSize += 4;
             }   }
-            ZSTDMT_releaseBuffer(zcs->bufPool, job.src);
-            zcs->jobs[wJobID].srcStart = NULL;
-            zcs->jobs[wJobID].src = g_nullBuffer;
             zcs->jobs[wJobID].jobScanned = 1;
         }
         {   size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);

From 052a95f77c4a7c0c13c1c94e7b805616b5924df7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 11 Jul 2017 17:18:26 -0700
Subject: [PATCH 033/100] fix : ZSTDMT_compress_advanced() correctly generates
 checksum

when params.fParams.checksumFlag==1.
This use case used to be impossible when only ZSTD_compress() was available
---
 lib/compress/zstdmt_compress.c | 49 +++++++++++++++++++++++-----------
 lib/compress/zstdmt_compress.h |  6 ++---
 tests/fuzzer.c                 | 20 +++++++++++---
 tests/zstreamtest.c            |  7 +++--
 4 files changed, 56 insertions(+), 26 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 7cf637f5..3fdfe9e1 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -140,7 +140,7 @@ static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize)
 static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
 {
     size_t const bSize = bufPool->bufferSize;
-    DEBUGLOG(2, "ZSTDMT_getBuffer");
+    DEBUGLOG(5, "ZSTDMT_getBuffer");
     pthread_mutex_lock(&bufPool->poolMutex);
     if (bufPool->nbBuffers) {   /* try to use an existing buffer */
         buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)];
@@ -151,12 +151,12 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
             return buf;
         }
         /* size conditions not respected : scratch this buffer, create new one */
-        DEBUGLOG(2, "existing buffer does not meet size conditions => freeing");
+        DEBUGLOG(5, "existing buffer does not meet size conditions => freeing");
         ZSTD_free(buf.start, bufPool->cMem);
     }
     pthread_mutex_unlock(&bufPool->poolMutex);
     /* create new buffer */
-    DEBUGLOG(2, "create a new buffer");
+    DEBUGLOG(5, "create a new buffer");
     {   buffer_t buffer;
         void* const start = ZSTD_malloc(bSize, bufPool->cMem);
         buffer.start = start;   /* note : start can be NULL if malloc fails ! */
@@ -168,17 +168,17 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
 /* store buffer for later re-use, up to pool capacity */
 static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
 {
-    DEBUGLOG(2, "ZSTDMT_releaseBuffer");
     if (buf.start == NULL) return;   /* compatible with release on NULL */
+    DEBUGLOG(5, "ZSTDMT_releaseBuffer");
     pthread_mutex_lock(&bufPool->poolMutex);
     if (bufPool->nbBuffers < bufPool->totalBuffers) {
-        bufPool->bTable[bufPool->nbBuffers++] = buf;   /* store for later re-use */
+        bufPool->bTable[bufPool->nbBuffers++] = buf;  /* stored for later use */
         pthread_mutex_unlock(&bufPool->poolMutex);
         return;
     }
     pthread_mutex_unlock(&bufPool->poolMutex);
     /* Reached bufferPool capacity (should not happen) */
-    DEBUGLOG(2, "buffer pool capacity reached => freeing ");
+    DEBUGLOG(5, "buffer pool capacity reached => freeing ");
     ZSTD_free(buf.start, bufPool->cMem);
 }
 
@@ -338,7 +338,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
     job->cSize = (job->lastChunk) ?
                  ZSTD_compressEnd     (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
                  ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
-    DEBUGLOG(2, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
+    DEBUGLOG(5, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
                 (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
     DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
 
@@ -515,13 +515,12 @@ static unsigned computeNbChunks(size_t srcSize, unsigned windowLog, unsigned nbT
 }
 
 
-/* Note : missing checksum at the end ! */
 size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
-                           void* dst, size_t dstCapacity,
-                     const void* src, size_t srcSize,
-                     const ZSTD_CDict* cdict,
-                           ZSTD_parameters const params,
-                           unsigned overlapRLog)
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const ZSTD_CDict* cdict,
+                               ZSTD_parameters const params,
+                               unsigned overlapRLog)
 {
     size_t const overlapSize = (overlapRLog>=9) ? 0 : (size_t)1 << (params.cParams.windowLog - overlapRLog);
     unsigned nbChunks = computeNbChunks(srcSize, params.cParams.windowLog, mtctx->nbThreads);
@@ -531,6 +530,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
     size_t remainingSrcSize = srcSize;
     unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize));  /* presumes avgChunkSize >= 256 KB, which should be the case */
     size_t frameStartPos = 0, dstBufferPos = 0;
+    XXH64_state_t xxh64;
 
     DEBUGLOG(4, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
     if (nbChunks==1) {   /* fallback to single-thread mode */
@@ -540,6 +540,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
     }
     assert(avgChunkSize >= 256 KB);  /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), which is useful to avoid allocating extra buffers */
     ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgChunkSize) );
+    XXH64_reset(&xxh64, 0);
 
     if (nbChunks > mtctx->jobIDMask+1) {  /* enlarge job table */
         U32 nbJobs = nbChunks;
@@ -576,6 +577,10 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
             mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex;
             mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond;
 
+            if (params.fParams.checksumFlag) {
+                XXH64_update(&xxh64, srcStart + frameStartPos, chunkSize);
+            }
+
             DEBUGLOG(5, "posting job %u   (%u bytes)", u, (U32)chunkSize);
             DEBUG_PRINTHEX(6, mtctx->jobs[u].srcStart, 12);
             POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);
@@ -586,8 +591,8 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
     }   }
 
     /* collect result */
-    {   unsigned chunkID;
-        size_t error = 0, dstPos = 0;
+    {   size_t error = 0, dstPos = 0;
+        unsigned chunkID;
         for (chunkID=0; chunkID<nbChunks; chunkID++) {
             DEBUGLOG(5, "waiting for chunk %u ", chunkID);
             PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
@@ -614,6 +619,18 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
                 dstPos += cSize ;
             }
         }  /* for (chunkID=0; chunkID<nbChunks; chunkID++) */
+
+        DEBUGLOG(4, "checksumFlag : %u ", params.fParams.checksumFlag);
+        if (params.fParams.checksumFlag) {
+            U32 const checksum = (U32)XXH64_digest(&xxh64);
+            if (dstPos + 4 > dstCapacity) {
+                error = ERROR(dstSize_tooSmall);
+            } else {
+                DEBUGLOG(4, "writing checksum : %08X \n", checksum);
+                MEM_writeLE32((char*)dst + dstPos, checksum);
+                dstPos += 4;
+        }   }
+
         if (!error) DEBUGLOG(4, "compressed size : %u  ", (U32)dstPos);
         return error ? error : dstPos;
     }
@@ -852,7 +869,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
             zcs->jobs[wJobID].jobScanned = 1;
         }
         {   size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
-            DEBUGLOG(2, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
+            DEBUGLOG(5, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
             memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
             output->pos += toWrite;
             job.dstFlushed += toWrite;
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index a6e1759b..7584007f 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -16,8 +16,8 @@
 
 
 /* Note : This is an internal API.
- *        Some methods are still exposed (ZSTDLIB_API), because for some time,
- *        it used to be the only way to invoke MT compression.
+ *        Some methods are still exposed (ZSTDLIB_API),
+ *        because it used to be the only way to invoke MT compression.
  *        Now, it's recommended to use ZSTD_compress_generic() instead.
  *        These methods will stop being exposed in a future version */
 
@@ -68,7 +68,7 @@ ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
                                      const void* src, size_t srcSize,
                                      const ZSTD_CDict* cdict,
                                            ZSTD_parameters const params,
-                                           unsigned overlapRLog);
+                                           unsigned overlapRLog);            /* overlapRLog = 9 - overlapLog */
 
 ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
                                         const void* dict, size_t dictSize,   /* dict can be released after init, a local copy is preserved within zcs */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 6bed9ec5..904ce6fd 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -278,9 +278,6 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
 
-    /* memory tests */
-    FUZ_mallocTests(seed, compressibility);
-
     /* Basic tests */
     DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++);
     {   const char* errorString = ZSTD_getErrorName(0);
@@ -479,6 +476,23 @@ static int basicUnitTests(U32 seed, double compressibility)
         }   }
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : compress -T2 with checksum : ", testNb++);
+        {   ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, 0);
+            params.fParams.checksumFlag = 1;
+            params.fParams.contentSizeFlag = 1;
+            CHECKPLUS(r, ZSTDMT_compress_advanced(mtctx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize,
+                                    NULL, params, 3 /*overlapRLog*/),
+                      cSize=r );
+        }
+        DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+        DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
+        { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+          if (r != CNBuffSize) goto _output_error; }
+        DISPLAYLEVEL(4, "OK \n");
+
         ZSTDMT_freeCCtx(mtctx);
     }
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8b84400d..3e551e33 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1377,13 +1377,12 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
         /* multi-segments compression test */
         XXH64_reset(&xxhState, 0);
         {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
-            U32 n;
-            for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
+            for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
                 /* compress random chunks into randomly sized dst buffers */
                 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
                 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
                 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
-                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
                 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
                 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
                 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
@@ -1402,7 +1401,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
             {   size_t remainingToFlush = (size_t)(-1);
                 while (remainingToFlush) {
                     ZSTD_inBuffer inBuff = { NULL, 0, 0 };
-                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
                     size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
                     outBuff.size = outBuff.pos + adjustedDstSize;
                     DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);

From 583dda17a811a9f4aab42ed4178d296ca5551447 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Tue, 11 Jul 2017 18:13:26 -0700
Subject: [PATCH 034/100] Update rolling hash

---
 contrib/long_distance_matching/ldm.c          | 435 ++++++++++++----
 contrib/long_distance_matching/ldm.h          |   3 +
 contrib/long_distance_matching/main-ldm.c     |  15 +-
 contrib/long_distance_matching/util.c         |   5 +
 contrib/long_distance_matching/util.h         |   2 +
 .../versions/v0.1/ldm.c                       | 394 ++++++++++++++
 .../versions/v0.1/ldm.h                       |  19 +
 .../versions/v0.1/main-ldm.c                  | 459 +++++++++++++++++
 .../versions/v0.2/Makefile                    |  32 ++
 .../versions/v0.2/ldm.c                       | 436 ++++++++++++++++
 .../versions/v0.2/ldm.h                       |  19 +
 .../versions/v0.2/main-ldm.c                  | 474 +++++++++++++++++
 .../versions/v0.3/Makefile                    |  40 ++
 .../versions/v0.3/ldm.c                       | 464 +++++++++++++++++
 .../versions/v0.3/ldm.h                       |  19 +
 .../versions/v0.3/main-ldm.c                  | 479 ++++++++++++++++++
 .../versions/v0.3/util.c                      |  64 +++
 .../versions/v0.3/util.h                      |  23 +
 18 files changed, 3282 insertions(+), 100 deletions(-)
 create mode 100644 contrib/long_distance_matching/versions/v0.1/ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.1/ldm.h
 create mode 100644 contrib/long_distance_matching/versions/v0.1/main-ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.2/Makefile
 create mode 100644 contrib/long_distance_matching/versions/v0.2/ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.2/ldm.h
 create mode 100644 contrib/long_distance_matching/versions/v0.2/main-ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.3/Makefile
 create mode 100644 contrib/long_distance_matching/versions/v0.3/ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.3/ldm.h
 create mode 100644 contrib/long_distance_matching/versions/v0.3/main-ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.3/util.c
 create mode 100644 contrib/long_distance_matching/versions/v0.3/util.h

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 1dedf5c3..ca4f0f2c 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -9,7 +9,7 @@
 
 #define HASH_EVERY 1
 
-#define LDM_MEMORY_USAGE 16
+#define LDM_MEMORY_USAGE 22
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
@@ -17,10 +17,12 @@
 
 #define LDM_OFFSET_SIZE 4
 
-#define WINDOW_SIZE (1 << 20)
+#define WINDOW_SIZE (1 << 23)
 #define MAX_WINDOW_SIZE 31
 #define HASH_SIZE 4
-#define LDM_HASH_LENGTH 4
+#define LDM_HASH_LENGTH 100
+
+// Should be multiple of four
 #define MINMATCH 4
 
 #define ML_BITS 4
@@ -28,7 +30,9 @@
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
-//#define LDM_ROLLING_HASH
+#define LDM_ROLLING_HASH
+#define COMPUTE_STATS
+//#define RUN_CHECKS
 //#define LDM_DEBUG
 
 typedef  uint8_t BYTE;
@@ -41,6 +45,7 @@ typedef uint32_t offset_t;
 typedef uint32_t hash_t;
 typedef signed char schar;
 
+
 typedef struct LDM_hashEntry {
   offset_t offset;
 } LDM_hashEntry;
@@ -50,6 +55,9 @@ typedef struct LDM_compressStats {
   U32 totalMatchLength;
   U32 totalLiteralLength;
   U64 totalOffset;
+
+  U32 numCollisions;
+  U32 numHashInserts;
 } LDM_compressStats;
 
 static void LDM_printCompressStats(const LDM_compressStats *stats) {
@@ -62,6 +70,11 @@ static void LDM_printCompressStats(const LDM_compressStats *stats) {
          ((double)stats->totalLiteralLength) / (double)stats->numMatches);
   printf("Average offset length: %.1f\n",
          ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
+         stats->numCollisions, stats->numHashInserts,
+         stats->numHashInserts == 0 ?
+            1.0 : (100.0 * (double)stats->numCollisions) /
+                  (double)stats->numHashInserts);
   printf("=====================\n");
 }
 
@@ -95,17 +108,43 @@ typedef struct LDM_CCtx {
   const BYTE *nextIp;
   hash_t nextHash;                    /* Hash corresponding to nextIp */
 
+  // Members for rolling hash.
+  U32 lastSum;
+  U32 nextSum;
+
   unsigned step;
+
+  // DEBUG
+  const BYTE *DEBUG_setNextHash;
 } LDM_CCtx;
 
+static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
+  U16 lengthLeft = MINMATCH;
+  const BYTE *curP = p;
+  const BYTE *curMatch = match;
+
+  for (; lengthLeft >= 8; lengthLeft -= 8) {
+    if (LDM_read64(curP) != LDM_read64(curMatch)) {
+      return 0;
+    }
+    curP += 8;
+    curMatch += 8;
+  }
+  if (lengthLeft > 0) {
+    return LDM_read32(curP) == LDM_read32(curMatch);
+  }
+  return 1;
+}
+
+
+
 #ifdef LDM_ROLLING_HASH
 /**
  * Convert a sum computed from LDM_getRollingHash to a hash value in the range
  * of the hash table.
  */
 static hash_t LDM_sumToHash(U32 sum) {
-  return sum % (LDM_HASHTABLESIZE >> 2);
-//  return sum & (LDM_HASHTABLESIZE - 1);
+  return sum & (LDM_HASH_SIZE_U32 - 1);
 }
 
 static U32 LDM_getRollingHash(const char *data, U32 len) {
@@ -126,18 +165,102 @@ static U32 LDM_getRollingHash(const char *data, U32 len) {
   return (s1 & 0xffff) + (s2 << 16);
 }
 
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
-}
-
 typedef struct LDM_sumStruct {
   U16 s1, s2;
 } LDM_sumStruct;
 
+static U32 LDM_updateRollingHash(U32 sum, U32 len,
+                                 schar toRemove, schar toAdd) {
+  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
+  U32 s2 = (sum >> 16) - (toRemove * len) + s1;
+
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+
+/*
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
+}
+*/
+
+/*
 static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
   sumStruct->s1 = sum & 0xffff;
   sumStruct->s2 = sum >> 16;
 }
+*/
+
+static void LDM_setNextHash(LDM_CCtx *cctx) {
+  U32 check;
+
+#ifdef RUN_CHECKS
+  if ((cctx->nextIp - cctx->ibase != 1) &&
+      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
+    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
+            cctx->DEBUG_setNextHash - cctx->ibase);
+  }
+
+  cctx->DEBUG_setNextHash = cctx->nextIp;
+#endif
+
+  cctx->nextSum = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  /*
+  check = LDM_updateRollingHash(
+      cctx->lastSum, LDM_HASH_LENGTH,
+      (schar)((cctx->lastPosHashed)[0]),
+      (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
+      */
+
+#ifdef RUN_CHECKS
+  if (check != cctx->nextSum) {
+    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
+//    printf("INFO: %u %u %u\n", LDM_read32(cctx->nextIp),
+  } else {
+//    printf("CHECK: setNextHash passed\n");
+  }
+#endif
+  cctx->nextHash = LDM_sumToHash(cctx->nextSum);
+
+#ifdef RUN_CHECKS
+  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
+    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
+            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
+            cctx->ip - cctx->ibase);
+  }
+#endif
+
+}
+
+static void LDM_putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash, U32 sum) {
+  /*
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  */
+#ifdef COMPUTE_STATS
+  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
+    offset_t offset = (cctx->hashTable)[hash].offset;
+    cctx->stats.numHashInserts++;
+    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+      cctx->stats.numCollisions++;
+    }
+  }
+#endif
+  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+  cctx->lastSum = sum;
+}
+
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  U32 sum = LDM_getRollingHash((const char *)cctx->ip, LDM_HASH_LENGTH);
+  hash_t hash = LDM_sumToHash(sum);
+//  hash_t hash = LDM_hashPosition(cctx->ip);
+  LDM_putHashOfCurrentPositionFromHash(cctx, hash, sum);
+//  printf("Offset %zu\n", cctx->ip - cctx->ibase);
+}
 
 #else
 static hash_t LDM_hash(U32 sequence) {
@@ -147,6 +270,39 @@ static hash_t LDM_hash(U32 sequence) {
 static hash_t LDM_hashPosition(const void * const p) {
   return LDM_hash(LDM_read32(p));
 }
+
+static void LDM_putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash) {
+  /*
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  */
+#ifdef COMPUTE_STATS
+  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
+    offset_t offset = (cctx->hashTable)[hash].offset;
+    cctx->stats.numHashInserts++;
+    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+      cctx->stats.numCollisions++;
+    }
+  }
+#endif
+
+  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
+#ifdef RUN_CHECKS
+  if (cctx->ip - cctx->lastPosHashed != 1) {
+    printf("putHashError\n");
+  }
+#endif
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+}
+
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  hash_t hash = LDM_hashPosition(cctx->ip);
+  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
+}
+
 #endif
 
 /*
@@ -161,38 +317,19 @@ static hash_t LDM_hash5(U64 sequence) {
 }
 */
 
-static void LDM_putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash) {
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-}
 
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  hash_t hash = LDM_hashPosition(cctx->ip);
-  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
-}
-
-static const BYTE *LDM_get_position_on_hash(
+static const BYTE *LDM_getPositionOnHash(
     hash_t h, void *tableBase, const BYTE *srcBase) {
   const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
   return hashTable[h].offset + srcBase;
 }
 
-static BYTE LDM_read_byte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
 
 static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
                           const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
   while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
     if (!diff) {
       pIn++;
       pMatch++;
@@ -220,7 +357,11 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->ip = cctx->ibase;
   cctx->iend = cctx->ibase + srcSize;
 
+#ifdef LDM_ROLLING_HASH
+  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
+#else
   cctx->ihashLimit = cctx->iend - HASH_SIZE;
+#endif
   cctx->imatchLimit = cctx->iend - MINMATCH;
 
   cctx->obase = (BYTE *)dst;
@@ -232,11 +373,46 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
 
   cctx->lastPosHashed = NULL;
-  cctx->nextIp = NULL;
 
   cctx->step = 1;
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  cctx->DEBUG_setNextHash = 0;
 }
 
+#ifdef LDM_ROLLING_HASH
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  do {
+    hash_t h;
+    U32 sum;
+//    printf("Call A\n");
+    LDM_setNextHash(cctx);
+//    printf("End call a\n");
+    h = cctx->nextHash;
+    sum = cctx->nextSum;
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
+
+    if (cctx->ip > cctx->imatchLimit) {
+      return 1;
+    }
+
+    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
+
+//    // Compute cctx->nextSum and cctx->nextHash from cctx->nextIp.
+//    LDM_setNextHash(cctx);
+    LDM_putHashOfCurrentPositionFromHash(cctx, h, sum);
+
+//    printf("%u %u\n", cctx->lastHash, cctx->nextHash);
+  } while (cctx->ip - *match > WINDOW_SIZE ||
+           !LDM_isValidMatch(cctx->ip, *match));
+//           LDM_read64(*match) != LDM_read64(cctx->ip));
+  LDM_setNextHash(cctx);
+  return 0;
+}
+#else
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
   cctx->nextIp = cctx->ip;
 
@@ -245,33 +421,131 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
     cctx->ip = cctx->nextIp;
     cctx->nextIp += cctx->step;
 
-    if (cctx->nextIp > cctx->imatchLimit) {
+    if (cctx->ip > cctx->imatchLimit) {
       return 1;
     }
 
-    *match = LDM_get_position_on_hash(h, cctx->hashTable, cctx->ibase);
+    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
 
     cctx->nextHash = LDM_hashPosition(cctx->nextIp);
     LDM_putHashOfCurrentPositionFromHash(cctx, h);
+
   } while (cctx->ip - *match > WINDOW_SIZE ||
-           LDM_read64(*match) != LDM_read64(cctx->ip));
+           !LDM_isValidMatch(cctx->ip, *match));
   return 0;
 }
 
+#endif
+
+/**
+ * Write current block (literals, literal length, match offset,
+ * match length).
+ *
+ * Update input pointer, inserting hashes into hash table along the
+ * way.
+ */
+static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
+  unsigned const literalLength = (unsigned)(cctx->ip - cctx->anchor);
+  unsigned const offset = cctx->ip - match;
+  unsigned const matchLength = LDM_count(
+      cctx->ip + MINMATCH, match + MINMATCH, cctx->ihashLimit);
+  BYTE *token = cctx->op++;
+
+  cctx->stats.totalLiteralLength += literalLength;
+  cctx->stats.totalOffset += offset;
+  cctx->stats.totalMatchLength += matchLength + MINMATCH;
+
+  /* Encode the literal length. */
+  if (literalLength >= RUN_MASK) {
+    int len = (int)literalLength - RUN_MASK;
+    *token = (RUN_MASK << ML_BITS);
+    for (; len >= 255; len -= 255) {
+      *(cctx->op)++ = 255;
+    }
+    *(cctx->op)++ = (BYTE)len;
+  } else {
+    *token = (BYTE)(literalLength << ML_BITS);
+  }
+
+  /* Encode the literals. */
+  memcpy(cctx->op, cctx->anchor, literalLength);
+  cctx->op += literalLength;
+
+  /* Encode the offset. */
+  LDM_write32(cctx->op, offset);
+  cctx->op += LDM_OFFSET_SIZE;
+
+  /* Encode match length */
+  if (matchLength >= ML_MASK) {
+    unsigned matchLengthRemaining = matchLength;
+    *token += ML_MASK;
+    matchLengthRemaining -= ML_MASK;
+    LDM_write32(cctx->op, 0xFFFFFFFF);
+    while (matchLengthRemaining >= 4*0xFF) {
+      cctx->op += 4;
+      LDM_write32(cctx->op, 0xffffffff);
+      matchLengthRemaining -= 4*0xFF;
+    }
+    cctx->op += matchLengthRemaining / 255;
+    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
+  } else {
+    *token += (BYTE)(matchLength);
+  }
+
+//  LDM_setNextHash(cctx);
+//  cctx->ip = cctx->lastPosHashed + 1;
+//  cctx->nextIp = cctx->ip + cctx->step;
+//  printf("HERE: %zu %zu %zu\n", cctx->ip - cctx->ibase,
+//         cctx->lastPosHashed - cctx->ibase, cctx->nextIp - cctx->ibase);
+
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  while (cctx->ip < cctx->anchor + MINMATCH + matchLength + literalLength) {
+//    printf("Loop\n");
+    if (cctx->ip > cctx->lastPosHashed) {
+      LDM_putHashOfCurrentPosition(cctx);
+#ifdef LDM_ROLLING_HASH
+      LDM_setNextHash(cctx);
+#endif
+    }
+    /*
+    printf("Call b %zu %zu %zu\n",
+            cctx->lastPosHashed - cctx->ibase,
+            cctx->nextIp - cctx->ibase,
+            cctx->ip - cctx->ibase);
+            */
+//    printf("end call b\n");
+    cctx->ip++;
+    cctx->nextIp++;
+  }
+
+//  printf("There: %zu %zu\n", cctx->ip - cctx->ibase, cctx->lastPosHashed - cctx->ibase);
+}
+
 // TODO: srcSize and maxDstSize is unused
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
+  U32 tmp_hash;
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
+#ifdef LDM_ROLLING_HASH
+//  LDM_setNextHash(&cctx);
+//  tmp_hash = LDM_updateRollingHash(cctx.lastSum, LDM_HASH_LENGTH,
+//                                   cctx.ip[0], cctx.ip[LDM_HASH_LENGTH]);
+//  printf("Update test: %u %u\n", tmp_hash, cctx.nextSum);
+//  cctx.ip++;
+#else
   cctx.ip++;
   cctx.nextHash = LDM_hashPosition(cctx.ip);
+#endif
 
   // TODO: loop condition is not accurate.
   while (1) {
     const BYTE *match;
+//    printf("Start of loop\n");
 
     /**
      * Find a match.
@@ -282,6 +556,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
     if (LDM_findBestMatch(&cctx, &match) != 0) {
       goto _last_literals;
     }
+//    printf("End of match finding\n");
 
     cctx.stats.numMatches++;
 
@@ -290,6 +565,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
      */
     while (cctx.ip > cctx.anchor && match > cctx.ibase &&
            cctx.ip[-1] == match[-1]) {
+//      printf("Catch up\n");
       cctx.ip--;
       match--;
     }
@@ -298,67 +574,24 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * Write current block (literals, literal length, match offset, match
      * length) and update pointers and hashes.
      */
-    {
-      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
-      unsigned const offset = cctx.ip - match;
-      unsigned const matchLength = LDM_count(
-          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
-      BYTE *token = cctx.op++;
-
-      cctx.stats.totalLiteralLength += literalLength;
-      cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + MINMATCH;
-
-      /* Encode the literal length. */
-      if (literalLength >= RUN_MASK) {
-        int len = (int)literalLength - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for (; len >= 255; len -= 255) {
-          *(cctx.op)++ = 255;
-        }
-        *(cctx.op)++ = (BYTE)len;
-      } else {
-        *token = (BYTE)(literalLength << ML_BITS);
-      }
-
-      /* Encode the literals. */
-      memcpy(cctx.op, cctx.anchor, literalLength);
-      cctx.op += literalLength;
-
-      /* Encode the offset. */
-      LDM_write32(cctx.op, offset);
-      cctx.op += LDM_OFFSET_SIZE;
-
-      /* Encode match length */
-      if (matchLength >= ML_MASK) {
-        unsigned matchLengthRemaining = matchLength;
-        *token += ML_MASK;
-        matchLengthRemaining -= ML_MASK;
-        LDM_write32(cctx.op, 0xFFFFFFFF);
-        while (matchLengthRemaining >= 4*0xFF) {
-          cctx.op += 4;
-          LDM_write32(cctx.op, 0xffffffff);
-          matchLengthRemaining -= 4*0xFF;
-        }
-        cctx.op += matchLengthRemaining / 255;
-        *(cctx.op)++ = (BYTE)(matchLengthRemaining % 255);
-      } else {
-        *token += (BYTE)(matchLength);
-      }
-
-      /* Update input pointer, inserting hashes into hash table along the
-       * way.
-       */
-      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
-        LDM_putHashOfCurrentPosition(&cctx);
-        cctx.ip++;
-      }
-    }
+    LDM_outputBlock(&cctx, match);
+//    printf("End of loop\n");
 
     // Set start of next block to current input pointer.
     cctx.anchor = cctx.ip;
     LDM_putHashOfCurrentPosition(&cctx);
-    cctx.nextHash = LDM_hashPosition(++cctx.ip);
+#ifndef LDM_ROLLING_HASH
+    cctx.ip++;
+#endif
+
+    /*
+    LDM_putHashOfCurrentPosition(&cctx);
+    printf("Call c\n");
+    LDM_setNextHash(&cctx);
+    printf("End call c\n");
+    cctx.ip++;
+    cctx.nextIp++;
+    */
   }
 _last_literals:
   /* Encode the last literals (no more matches). */
@@ -453,7 +686,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
     /* Copy match. */
     cpy = dctx.op + length;
 
-    // Inefficient for now
+    // Inefficient for now.
     while (match < cpy - offset && dctx.op < dctx.oend) {
       *(dctx.op)++ = *match++;
     }
@@ -461,4 +694,20 @@ size_t LDM_decompress(const void *src, size_t compressSize,
   return dctx.op - (BYTE *)dst;
 }
 
+void LDM_test(const void *src, size_t srcSize,
+              void *dst, size_t maxDstSize) {
+#ifdef LDM_ROLLING_HASH
+  const BYTE *ip = (const BYTE *)src + 1125;
+  U32 sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
+  U32 sum2;
+  ++ip;
+  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
+    sum2 = LDM_updateRollingHash(sum, LDM_HASH_LENGTH,
+                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
+    sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
+    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
+  }
+#endif
+}
+
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 287d444d..a34faac4 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -16,4 +16,7 @@ size_t LDM_decompress(const void *src, size_t srcSize,
 void LDM_readHeader(const void *src, size_t *compressSize,
                     size_t *decompressSize);
 
+void LDM_test(const void *src, size_t srcSize,
+              void *dst, size_t maxDstSize);
+
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 724d735d..f8ae5469 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -15,6 +15,7 @@
 
 // #define BUF_SIZE 16*1024  // Block size
 #define DEBUG
+//#define TEST
 
 //#define ZSTD
 
@@ -74,6 +75,11 @@ static int compress(const char *fname, const char *oname) {
       return 1;
   }
 
+#ifdef TEST
+  LDM_test(src, statbuf.st_size,
+           dst + LDM_HEADER_SIZE, statbuf.st_size);
+#endif
+
 #ifdef ZSTD
   compressSize = ZSTD_compress(dst, statbuf.st_size,
                                   src, statbuf.st_size, 1);
@@ -144,11 +150,6 @@ static int decompress(const char *fname, const char *oname) {
   /* Read the header. */
   LDM_readHeader(src, &compressSize, &decompressSize);
 
-#ifdef DEBUG
-  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
-         (size_t)statbuf.st_size, compressSize, decompressSize);
-#endif
-
   /* Go to the location corresponding to the last byte. */
   if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
     perror("lseek error");
@@ -256,7 +257,7 @@ int main(int argc, const char *argv[]) {
         return 1;
     }
     gettimeofday(&tv2, NULL);
-    printf("Total time = %f seconds\n",
+    printf("Total compress time = %f seconds\n",
            (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
            (double) (tv2.tv_sec - tv1.tv_sec));
   }
@@ -270,7 +271,7 @@ int main(int argc, const char *argv[]) {
         return 1;
     }
     gettimeofday(&tv2, NULL);
-    printf("Total time = %f seconds\n",
+    printf("Total decompress time = %f seconds\n",
           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
           (double) (tv2.tv_sec - tv1.tv_sec));
   }
diff --git a/contrib/long_distance_matching/util.c b/contrib/long_distance_matching/util.c
index 9ea4ca1e..70fcbc2c 100644
--- a/contrib/long_distance_matching/util.c
+++ b/contrib/long_distance_matching/util.c
@@ -61,4 +61,9 @@ void LDM_copy8(void *dst, const void *src) {
   memcpy(dst, src, 8);
 }
 
+BYTE LDM_readByte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
 
diff --git a/contrib/long_distance_matching/util.h b/contrib/long_distance_matching/util.h
index 90726412..d1c3c999 100644
--- a/contrib/long_distance_matching/util.h
+++ b/contrib/long_distance_matching/util.h
@@ -19,5 +19,7 @@ uint64_t LDM_read64(const void *ptr);
 
 void LDM_copy8(void *dst, const void *src);
 
+uint8_t LDM_readByte(const void *ptr);
+
 
 #endif /* LDM_UTIL_H */
diff --git a/contrib/long_distance_matching/versions/v0.1/ldm.c b/contrib/long_distance_matching/versions/v0.1/ldm.c
new file mode 100644
index 00000000..266425f8
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.1/ldm.c
@@ -0,0 +1,394 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "ldm.h"
+
+#define LDM_MEMORY_USAGE 14
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
+
+#define WINDOW_SIZE (1 << 20)
+#define MAX_WINDOW_SIZE 31
+#define HASH_SIZE 4
+#define MINMATCH 4
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+//#define LDM_DEBUG
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint64_t tag;
+
+static unsigned LDM_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+static U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+static U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+static void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_write16(void *memPtr, U16 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE*)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+static U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+static U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+
+static void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
+  BYTE *d = (BYTE *)dstPtr;
+  const BYTE *s = (const BYTE *)srcPtr;
+  BYTE * const e = (BYTE *)dstEnd;
+
+  do {
+    LDM_copy8(d, s);
+    d += 8;
+    s += 8;
+  } while (d < e);
+
+}
+
+struct hash_entry {
+  U64 offset;
+  tag t;
+};
+
+static U32 LDM_hash(U32 sequence) {
+  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
+}
+
+static U32 LDM_hash5(U64 sequence) {
+  static const U64 prime5bytes = 889523592379ULL;
+  static const U64 prime8bytes = 11400714785074694791ULL;
+  const U32 hashLog = LDM_HASHLOG;
+  if (LDM_isLittleEndian())
+    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
+  else
+    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+}
+
+static U32 LDM_hash_position(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+
+static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
+                                     const BYTE *srcBase) {
+  U32 *hashTable = (U32 *) tableBase;
+  hashTable[h] = (U32)(p - srcBase);
+}
+
+static void LDM_put_position(const BYTE *p, void *tableBase,
+                             const BYTE *srcBase) {
+  U32 const h = LDM_hash_position(p);
+  LDM_put_position_on_hash(p, h, tableBase, srcBase);
+}
+
+static const BYTE *LDM_get_position_on_hash(
+    U32 h, void *tableBase, const BYTE *srcBase) {
+  const U32 * const hashTable = (U32*)tableBase;
+  return hashTable[h] + srcBase;
+}
+
+static BYTE LDM_read_byte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
+static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
+                          const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+void LDM_read_header(void const *source, size_t *compressed_size,
+                     size_t *decompressed_size) {
+  const U32 *ip = (const U32 *)source;
+  *compressed_size = *ip++;
+  *decompressed_size = *ip;
+}
+
+size_t LDM_compress(void const *source, void *dest, size_t source_size,
+                    size_t max_dest_size) {
+  const BYTE * const istart = (const BYTE*)source;
+  const BYTE *ip = istart;
+  const BYTE * const iend = istart + source_size;
+  const BYTE *ilimit = iend - HASH_SIZE;
+  const BYTE * const matchlimit = iend - HASH_SIZE;
+  const BYTE * const mflimit = iend - MINMATCH;
+  BYTE *op = (BYTE*) dest;
+  U32 hashTable[LDM_HASHTABLESIZE_U32];
+  memset(hashTable, 0, sizeof(hashTable));
+
+  const BYTE *anchor = (const BYTE *)source;
+//  struct LDM_cctx cctx;
+  size_t output_size = 0;
+
+  U32 forwardH;
+
+  /* Hash first byte: put into hash table */
+
+  LDM_put_position(ip, hashTable, istart);
+  ip++;
+  forwardH = LDM_hash_position(ip);
+
+  //TODO Loop terminates before ip>=ilimit.
+  while (ip < ilimit) {
+    const BYTE *match;
+    BYTE *token;
+
+    /* Find a match */
+    {
+      const BYTE *forwardIp = ip;
+      unsigned step = 1;
+
+      do {
+        U32 const h = forwardH;
+        ip = forwardIp;
+        forwardIp += step;
+
+        if (forwardIp > mflimit) {
+          goto _last_literals;
+        }
+
+        match = LDM_get_position_on_hash(h, hashTable, istart);
+
+        forwardH = LDM_hash_position(forwardIp);
+        LDM_put_position_on_hash(ip, h, hashTable, istart);
+      } while (ip - match > WINDOW_SIZE ||
+               LDM_read64(match) != LDM_read64(ip));
+    }
+
+    // TODO catchup
+    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
+      ip--;
+      match--;
+    }
+
+    /* Encode literals */
+    {
+      unsigned const litLength = (unsigned)(ip - anchor);
+      token = op++;
+
+#ifdef LDM_DEBUG
+      printf("Cur position: %zu\n", anchor - istart);
+      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
+#endif
+      /*
+      fwrite(match, 4, 1, stdout);
+      printf("\n");
+      */
+
+      if (litLength >= RUN_MASK) {
+        int len = (int)litLength - RUN_MASK;
+        *token = (RUN_MASK << ML_BITS);
+        for (; len >= 255; len -= 255) {
+          *op++ = 255;
+        }
+        *op++ = (BYTE)len;
+      } else {
+        *token = (BYTE)(litLength << ML_BITS);
+      }
+#ifdef LDM_DEBUG
+      printf("Literals ");
+      fwrite(anchor, litLength, 1, stdout);
+      printf("\n");
+#endif
+      memcpy(op, anchor, litLength);
+      //LDM_wild_copy(op, anchor, op + litLength);
+      op += litLength;
+    }
+_next_match:
+    /* Encode offset */
+    {
+      LDM_write32(op, ip - match);
+      op += 4;
+    }
+
+    /* Encode Match Length */
+    {
+      unsigned matchCode;
+      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
+                            matchlimit);
+#ifdef LDM_DEBUG
+      printf("Match length %zu\n", matchCode + MINMATCH);
+      fwrite(ip, MINMATCH + matchCode, 1, stdout);
+      printf("\n");
+#endif
+      ip += MINMATCH + matchCode;
+      if (matchCode >= ML_MASK) {
+        *token += ML_MASK;
+        matchCode -= ML_MASK;
+        LDM_write32(op, 0xFFFFFFFF);
+        while (matchCode >= 4*0xFF) {
+          op += 4;
+          LDM_write32(op, 0xffffffff);
+          matchCode -= 4*0xFF;
+        }
+        op += matchCode / 255;
+        *op++ = (BYTE)(matchCode % 255);
+      } else {
+        *token += (BYTE)(matchCode);
+      }
+#ifdef LDM_DEBUG
+      printf("\n");
+#endif
+    }
+
+    anchor = ip;
+
+    LDM_put_position(ip, hashTable, istart);
+    forwardH = LDM_hash_position(++ip);
+  }
+_last_literals:
+    /* Encode last literals */
+  {
+    size_t const lastRun = (size_t)(iend - anchor);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *op++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *op++ = 255;
+      }
+      *op++ = (BYTE)accumulator;
+    } else {
+      *op++ = (BYTE)(lastRun << ML_BITS);
+    }
+    memcpy(op, anchor, lastRun);
+    op += lastRun;
+  }
+  return (op - (BYTE *)dest);
+}
+
+size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
+                      size_t max_decompressed_size) {
+  const BYTE *ip = (const BYTE *)source;
+  const BYTE * const iend = ip + compressed_size;
+  BYTE *op = (BYTE *)dest;
+  BYTE * const oend = op + max_decompressed_size;
+  BYTE *cpy;
+
+  while (ip < iend) {
+    size_t length;
+    const BYTE *match;
+    size_t offset;
+
+    /* get literal length */
+    unsigned const token = *ip++;
+    if ((length=(token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+#ifdef LDM_DEBUG
+    printf("Literal length: %zu\n", length);
+#endif
+
+    /* copy literals */
+    cpy = op + length;
+#ifdef LDM_DEBUG
+    printf("Literals ");
+    fwrite(ip, length, 1, stdout);
+    printf("\n");
+#endif
+    memcpy(op, ip, length);
+//    LDM_wild_copy(op, ip, cpy);
+    ip += length;
+    op = cpy;
+
+    /* get offset */
+    offset = LDM_read32(ip);
+
+#ifdef LDM_DEBUG
+    printf("Offset: %zu\n", offset);
+#endif
+    ip += 4;
+    match = op - offset;
+ //   LDM_write32(op, (U32)offset);
+
+    /* get matchlength */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+#ifdef LDM_DEBUG
+    printf("Match length: %zu\n", length);
+#endif
+    /* copy match */
+    cpy = op + length;
+
+    // Inefficient for now
+
+    while (match < cpy - offset && op < oend) {
+      *op++ = *match++;
+    }
+  }
+//  memcpy(dest, source, compressed_size);
+  return op - (BYTE *)dest;
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v0.1/ldm.h b/contrib/long_distance_matching/versions/v0.1/ldm.h
new file mode 100644
index 00000000..f4ca25a3
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.1/ldm.h
@@ -0,0 +1,19 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
+size_t LDM_compress(void const *source, void *dest, size_t source_size,
+                    size_t max_dest_size);
+
+size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
+                      size_t max_decompressed_size);
+
+void LDM_read_header(void const *source, size_t *compressed_size,
+                     size_t *decompressed_size);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.1/main-ldm.c b/contrib/long_distance_matching/versions/v0.1/main-ldm.c
new file mode 100644
index 00000000..10869cce
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.1/main-ldm.c
@@ -0,0 +1,459 @@
+// TODO: file size must fit into a U32
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+// #define BUF_SIZE 16*1024  // Block size
+#define DEBUG
+
+//#define ZSTD
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+#endif
+
+static size_t compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* open the input file */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* open the output file */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* find size of input file */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+  size_t size_in = statbuf.st_size;
+
+ /* go to the location corresponding to the last byte */
+ if (lseek(fdout, size_in + LDM_HEADER_SIZE - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+ }
+
+ /* write a dummy byte at the last location */
+ if (write(fdout, "", 1) != 1) {
+     perror("write error");
+     return 1;
+ }
+
+  /* mmap the input file */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+  size_t out_size = statbuf.st_size + LDM_HEADER_SIZE;
+
+  /* mmap the output file */
+  if ((dst = mmap(0, out_size, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+  #ifdef ZSTD
+    size_t size_out = ZSTD_compress(dst, statbuf.st_size,
+                                    src, statbuf.st_size, 1);
+  #else
+    size_t size_out = LDM_compress(src, dst + LDM_HEADER_SIZE, statbuf.st_size,
+                                   statbuf.st_size);
+    size_out += LDM_HEADER_SIZE;
+
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+    memcpy(dst, &size_out, 4);
+    memcpy(dst + 4, &(statbuf.st_size), 4);
+    printf("Compressed size: %zu\n", size_out);
+    printf("Decompressed size: %zu\n", statbuf.st_size);
+  #endif
+  ftruncate(fdout, size_out);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)size_out, oname,
+         (double)size_out / (statbuf.st_size) * 100);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+static size_t decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* open the input file */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* open the output file */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* find size of input file */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* read header */
+  size_t compressed_size, decompressed_size;
+  LDM_read_header(src, &compressed_size, &decompressed_size);
+
+  printf("Size, compressed_size, decompressed_size: %zu %zu %zu\n",
+         statbuf.st_size, compressed_size, decompressed_size);
+
+  /* go to the location corresponding to the last byte */
+  if (lseek(fdout, decompressed_size - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressed_size, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+  /* Copy input file to output file */
+//  memcpy(dst, src, statbuf.st_size);
+
+  #ifdef ZSTD
+    size_t size_out = ZSTD_decompress(dst, decomrpessed_size,
+                                      src + LDM_HEADER_SIZE,
+                                      statbuf.st_size - LDM_HEADER_SIZE);
+  #else
+    size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
+                                     statbuf.st_size - LDM_HEADER_SIZE,
+                                     decompressed_size);
+    printf("Ret size out: %zu\n", size_out);
+  #endif
+  ftruncate(fdout, size_out);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) {
+      break;
+    }
+    if (0 == result) {
+      result = memcmp(b0, b1, r0);
+    }
+  }
+  return result;
+}
+
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+	const int cmp = compare(inpFp, decFp);
+	if(0 == cmp) {
+		printf("verify : OK\n");
+	} else {
+		printf("verify : NG\n");
+	}
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  struct timeval tv1, tv2;
+  /* compress */
+  {
+  gettimeofday(&tv1, NULL);
+  if (compress(inpFilename, ldmFilename)) {
+      printf("Compress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+         (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+
+  /* decompress */
+
+  gettimeofday(&tv1, NULL);
+  if (decompress(ldmFilename, decFilename)) {
+      printf("Decompress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+        (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+#if 0
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+
diff --git a/contrib/long_distance_matching/versions/v0.2/Makefile b/contrib/long_distance_matching/versions/v0.2/Makefile
new file mode 100644
index 00000000..4e04fd6a
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.2/Makefile
@@ -0,0 +1,32 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+# This Makefile presumes libzstd is installed, using `sudo make install`
+
+
+LDFLAGS += -lzstd
+
+.PHONY: default all clean
+
+default: all
+
+all: main-ldm
+
+
+#main : ldm.c main.c
+#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+main-ldm : ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+clean:
+	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
+	main main-ldm
+	@echo Cleaning completed
+
diff --git a/contrib/long_distance_matching/versions/v0.2/ldm.c b/contrib/long_distance_matching/versions/v0.2/ldm.c
new file mode 100644
index 00000000..9081d136
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.2/ldm.c
@@ -0,0 +1,436 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "ldm.h"
+
+#define HASH_EVERY 7
+
+#define LDM_MEMORY_USAGE 14
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
+
+#define WINDOW_SIZE (1 << 20)
+#define MAX_WINDOW_SIZE 31
+#define HASH_SIZE 8
+#define MINMATCH 8
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+//#define LDM_DEBUG
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint64_t tag;
+
+static unsigned LDM_isLittleEndian(void) {
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+static U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+static U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+static void LDM_write16(void *memPtr, U16 value){
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+static void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE*)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+static U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+static U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+static void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+typedef struct compress_stats {
+  U32 num_matches;
+  U32 total_match_length;
+  U32 total_literal_length;
+  U64 total_offset;
+} compress_stats;
+
+static void LDM_printCompressStats(const compress_stats *stats) {
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Total number of matches: %u\n", stats->num_matches);
+  printf("Average match length: %.1f\n", ((double)stats->total_match_length) /
+                                         (double)stats->num_matches);
+  printf("Average literal length: %.1f\n",
+         ((double)stats->total_literal_length) / (double)stats->num_matches);
+  printf("Average offset length: %.1f\n",
+         ((double)stats->total_offset) / (double)stats->num_matches);
+  printf("=====================\n");
+}
+
+// TODO: unused.
+struct hash_entry {
+  U64 offset;
+  tag t;
+};
+
+static U32 LDM_hash(U32 sequence) {
+  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
+}
+
+static U32 LDM_hash5(U64 sequence) {
+  static const U64 prime5bytes = 889523592379ULL;
+  static const U64 prime8bytes = 11400714785074694791ULL;
+  const U32 hashLog = LDM_HASHLOG;
+  if (LDM_isLittleEndian())
+    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
+  else
+    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+}
+
+static U32 LDM_hash_position(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+
+static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
+                                     const BYTE *srcBase) {
+  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+
+  U32 *hashTable = (U32 *) tableBase;
+  hashTable[h] = (U32)(p - srcBase);
+}
+
+static void LDM_put_position(const BYTE *p, void *tableBase,
+                             const BYTE *srcBase) {
+  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  U32 const h = LDM_hash_position(p);
+  LDM_put_position_on_hash(p, h, tableBase, srcBase);
+}
+
+static const BYTE *LDM_get_position_on_hash(
+    U32 h, void *tableBase, const BYTE *srcBase) {
+  const U32 * const hashTable = (U32*)tableBase;
+  return hashTable[h] + srcBase;
+}
+
+static BYTE LDM_read_byte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
+static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
+                          const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+void LDM_read_header(const void *src, size_t *compressSize,
+                     size_t *decompressSize) {
+  const U32 *ip = (const U32 *)src;
+  *compressSize = *ip++;
+  *decompressSize = *ip;
+}
+
+// TODO: maxDstSize is unused
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  const BYTE * const istart = (const BYTE*)src;
+  const BYTE *ip = istart;
+  const BYTE * const iend = istart + srcSize;
+  const BYTE *ilimit = iend - HASH_SIZE;
+  const BYTE * const matchlimit = iend - HASH_SIZE;
+  const BYTE * const mflimit = iend - MINMATCH;
+  BYTE *op = (BYTE*) dst;
+
+  compress_stats compressStats = { 0 };
+
+  U32 hashTable[LDM_HASHTABLESIZE_U32];
+  memset(hashTable, 0, sizeof(hashTable));
+
+  const BYTE *anchor = (const BYTE *)src;
+//  struct LDM_cctx cctx;
+  size_t output_size = 0;
+
+  U32 forwardH;
+
+  /* Hash first byte: put into hash table */
+
+  LDM_put_position(ip, hashTable, istart);
+  const BYTE *lastHash = ip;
+  ip++;
+  forwardH = LDM_hash_position(ip);
+
+  //TODO Loop terminates before ip>=ilimit.
+  while (ip < ilimit) {
+    const BYTE *match;
+    BYTE *token;
+
+    /* Find a match */
+    {
+      const BYTE *forwardIp = ip;
+      unsigned step = 1;
+
+      do {
+        U32 const h = forwardH;
+        ip = forwardIp;
+        forwardIp += step;
+
+        if (forwardIp > mflimit) {
+          goto _last_literals;
+        }
+
+        match = LDM_get_position_on_hash(h, hashTable, istart);
+
+        forwardH = LDM_hash_position(forwardIp);
+        LDM_put_position_on_hash(ip, h, hashTable, istart);
+        lastHash = ip;
+      } while (ip - match > WINDOW_SIZE ||
+               LDM_read64(match) != LDM_read64(ip));
+    }
+    compressStats.num_matches++;
+
+    /* Catchup: look back to extend match from found match */
+    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
+      ip--;
+      match--;
+    }
+
+    /* Encode literals */
+    {
+      unsigned const litLength = (unsigned)(ip - anchor);
+      token = op++;
+
+      compressStats.total_literal_length += litLength;
+
+#ifdef LDM_DEBUG
+      printf("Cur position: %zu\n", anchor - istart);
+      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
+#endif
+
+      if (litLength >= RUN_MASK) {
+        int len = (int)litLength - RUN_MASK;
+        *token = (RUN_MASK << ML_BITS);
+        for (; len >= 255; len -= 255) {
+          *op++ = 255;
+        }
+        *op++ = (BYTE)len;
+      } else {
+        *token = (BYTE)(litLength << ML_BITS);
+      }
+#ifdef LDM_DEBUG
+      printf("Literals ");
+      fwrite(anchor, litLength, 1, stdout);
+      printf("\n");
+#endif
+      memcpy(op, anchor, litLength);
+      op += litLength;
+    }
+_next_match:
+    /* Encode offset */
+    {
+      /*
+      LDM_writeLE16(op, ip-match);
+      op += 2;
+      */
+      LDM_write32(op, ip - match);
+      op += 4;
+      compressStats.total_offset += (ip - match);
+    }
+
+    /* Encode Match Length */
+    {
+      unsigned matchCode;
+      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
+                            matchlimit);
+#ifdef LDM_DEBUG
+      printf("Match length %zu\n", matchCode + MINMATCH);
+      fwrite(ip, MINMATCH + matchCode, 1, stdout);
+      printf("\n");
+#endif
+      compressStats.total_match_length += matchCode + MINMATCH;
+      unsigned ctr = 1;
+      ip++;
+      for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
+        LDM_put_position(ip, hashTable, istart);
+      }
+//      ip += MINMATCH + matchCode;
+      if (matchCode >= ML_MASK) {
+        *token += ML_MASK;
+        matchCode -= ML_MASK;
+        LDM_write32(op, 0xFFFFFFFF);
+        while (matchCode >= 4*0xFF) {
+          op += 4;
+          LDM_write32(op, 0xffffffff);
+          matchCode -= 4*0xFF;
+        }
+        op += matchCode / 255;
+        *op++ = (BYTE)(matchCode % 255);
+      } else {
+        *token += (BYTE)(matchCode);
+      }
+#ifdef LDM_DEBUG
+      printf("\n");
+
+#endif
+    }
+
+    anchor = ip;
+
+    LDM_put_position(ip, hashTable, istart);
+    forwardH = LDM_hash_position(++ip);
+    lastHash = ip;
+  }
+_last_literals:
+    /* Encode last literals */
+  {
+    size_t const lastRun = (size_t)(iend - anchor);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *op++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *op++ = 255;
+      }
+      *op++ = (BYTE)accumulator;
+    } else {
+      *op++ = (BYTE)(lastRun << ML_BITS);
+    }
+    memcpy(op, anchor, lastRun);
+    op += lastRun;
+  }
+  LDM_printCompressStats(&compressStats);
+  return (op - (BYTE *)dst);
+}
+
+typedef struct LDM_DCtx {
+  const BYTE * const ibase;   /* Pointer to base of input */
+  const BYTE *ip;             /* Pointer to current input position */
+  const BYTE *iend;           /* End of source */
+  BYTE *op;                   /* Pointer to output */
+  const BYTE * const oend;    /* Pointer to end of output */
+
+} LDM_DCtx;
+
+size_t LDM_decompress(const void *src, size_t compressed_size,
+                      void *dst, size_t max_decompressed_size) {
+  const BYTE *ip = (const BYTE *)src;
+  const BYTE * const iend = ip + compressed_size;
+  BYTE *op = (BYTE *)dst;
+  BYTE * const oend = op + max_decompressed_size;
+  BYTE *cpy;
+
+  while (ip < iend) {
+    size_t length;
+    const BYTE *match;
+    size_t offset;
+
+    /* get literal length */
+    unsigned const token = *ip++;
+    if ((length=(token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+#ifdef LDM_DEBUG
+    printf("Literal length: %zu\n", length);
+#endif
+
+    /* copy literals */
+    cpy = op + length;
+#ifdef LDM_DEBUG
+    printf("Literals ");
+    fwrite(ip, length, 1, stdout);
+    printf("\n");
+#endif
+    memcpy(op, ip, length);
+    ip += length;
+    op = cpy;
+
+    /* get offset */
+    /*
+    offset = LDM_readLE16(ip);
+    ip += 2;
+    */
+    offset = LDM_read32(ip);
+    ip += 4;
+#ifdef LDM_DEBUG
+    printf("Offset: %zu\n", offset);
+#endif
+    match = op - offset;
+ //   LDM_write32(op, (U32)offset);
+
+    /* get matchlength */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *ip++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+#ifdef LDM_DEBUG
+    printf("Match length: %zu\n", length);
+#endif
+    /* copy match */
+    cpy = op + length;
+
+    // Inefficient for now
+    while (match < cpy - offset && op < oend) {
+      *op++ = *match++;
+    }
+  }
+  return op - (BYTE *)dst;
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v0.2/ldm.h b/contrib/long_distance_matching/versions/v0.2/ldm.h
new file mode 100644
index 00000000..0ac7b2ec
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.2/ldm.h
@@ -0,0 +1,19 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize);
+
+size_t LDM_decompress(const void *src, size_t srcSize,
+                      void *dst, size_t maxDstSize);
+
+void LDM_read_header(const void *src, size_t *compressSize,
+                     size_t *decompressSize);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.2/main-ldm.c b/contrib/long_distance_matching/versions/v0.2/main-ldm.c
new file mode 100644
index 00000000..0017335b
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.2/main-ldm.c
@@ -0,0 +1,474 @@
+// TODO: file size must fit into a U32
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+// #define BUF_SIZE 16*1024  // Block size
+#define DEBUG
+
+//#define ZSTD
+
+/* Compress file given by fname and output to oname.
+ * Returns 0 if successful, error code otherwise.
+ */
+static int compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+
+ /* Go to the location corresponding to the last byte. */
+ /* TODO: fallocate? */ 
+  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+ /* Write a dummy byte at the last location. */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
+                                  src, statbuf.st_size, 1);
+#else
+  size_t compressSize = LDM_HEADER_SIZE + 
+      LDM_compress(src, statbuf.st_size,
+                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+
+    // Write compress and decompress size to header 
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  memcpy(dst, &compressSize, 4);
+  memcpy(dst + 4, &(statbuf.st_size), 4);
+
+#ifdef DEBUG
+  printf("Compressed size: %zu\n", compressSize);
+  printf("Decompressed size: %zu\n", statbuf.st_size);
+#endif
+#endif
+
+  // Truncate file to compressSize.
+  ftruncate(fdout, compressSize);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
+         (double)compressSize / (statbuf.st_size) * 100);
+
+  // Close files.
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Decompress file compressed using LDM_compress.
+ * The input file should have the LDM_HEADER followed by payload.
+ * Returns 0 if succesful, and an error code otherwise. 
+ */
+static int decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* Read the header. */
+  size_t compressSize, decompressSize;
+  LDM_read_header(src, &compressSize, &decompressSize);
+
+#ifdef DEBUG
+  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
+         statbuf.st_size, compressSize, decompressSize);
+#endif
+
+  /* Go to the location corresponding to the last byte. */
+  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
+                                  src + LDM_HEADER_SIZE,
+                                  statbuf.st_size - LDM_HEADER_SIZE);
+#else
+  size_t outSize = LDM_decompress(
+      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
+      dst, decompressSize);
+
+  printf("Ret size out: %zu\n", outSize);
+  #endif
+  ftruncate(fdout, outSize);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Compare two files.
+ * Returns 0 iff they are the same.
+ */
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) break;
+
+    if (0 == result) result = memcmp(b0, b1, r0);
+  }
+  return result;
+}
+
+/* Verify the input file is the same as the decompressed file. */
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+	const int cmp = compare(inpFp, decFp);
+	if(0 == cmp) {
+		printf("verify : OK\n");
+	} else {
+		printf("verify : NG\n");
+	}
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  struct timeval tv1, tv2;
+
+  /* Compress */
+
+  gettimeofday(&tv1, NULL);
+  if (compress(inpFilename, ldmFilename)) {
+      printf("Compress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+         (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* Decompress */
+
+  gettimeofday(&tv1, NULL);
+  if (decompress(ldmFilename, decFilename)) {
+      printf("Decompress error");
+      return 1;
+  }
+  gettimeofday(&tv2, NULL);
+  printf("Total time = %f seconds\n",
+        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+        (double) (tv2.tv_sec - tv1.tv_sec));
+
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+
diff --git a/contrib/long_distance_matching/versions/v0.3/Makefile b/contrib/long_distance_matching/versions/v0.3/Makefile
new file mode 100644
index 00000000..5ffd4eaf
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.3/Makefile
@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+# This Makefile presumes libzstd is installed, using `sudo make install`
+
+CFLAGS  ?= -O3
+DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS)
+
+LDFLAGS += -lzstd
+
+.PHONY: default all clean
+
+default: all
+
+all: main-ldm
+
+
+#main : ldm.c main.c
+#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+main-ldm : util.c ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+clean:
+	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
+	main main-ldm
+	@echo Cleaning completed
+
diff --git a/contrib/long_distance_matching/versions/v0.3/ldm.c b/contrib/long_distance_matching/versions/v0.3/ldm.c
new file mode 100644
index 00000000..1dedf5c3
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.3/ldm.c
@@ -0,0 +1,464 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+
+#include "ldm.h"
+#include "util.h"
+
+#define HASH_EVERY 1
+
+#define LDM_MEMORY_USAGE 16
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
+
+#define LDM_OFFSET_SIZE 4
+
+#define WINDOW_SIZE (1 << 20)
+#define MAX_WINDOW_SIZE 31
+#define HASH_SIZE 4
+#define LDM_HASH_LENGTH 4
+#define MINMATCH 4
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+//#define LDM_ROLLING_HASH
+//#define LDM_DEBUG
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint32_t offset_t;
+typedef uint32_t hash_t;
+typedef signed char schar;
+
+typedef struct LDM_hashEntry {
+  offset_t offset;
+} LDM_hashEntry;
+
+typedef struct LDM_compressStats {
+  U32 numMatches;
+  U32 totalMatchLength;
+  U32 totalLiteralLength;
+  U64 totalOffset;
+} LDM_compressStats;
+
+static void LDM_printCompressStats(const LDM_compressStats *stats) {
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Total number of matches: %u\n", stats->numMatches);
+  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
+                                         (double)stats->numMatches);
+  printf("Average literal length: %.1f\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
+  printf("Average offset length: %.1f\n",
+         ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("=====================\n");
+}
+
+typedef struct LDM_CCtx {
+  size_t isize;             /* Input size */
+  size_t maxOSize;          /* Maximum output size */
+
+  const BYTE *ibase;        /* Base of input */
+  const BYTE *ip;           /* Current input position */
+  const BYTE *iend;         /* End of input */
+
+  // Maximum input position such that hashing at the position does not exceed
+  // end of input.
+  const BYTE *ihashLimit;
+
+  // Maximum input position such that finding a match of at least the minimum
+  // match length does not exceed end of input.
+  const BYTE *imatchLimit;
+
+  const BYTE *obase;        /* Base of output */
+  BYTE *op;                 /* Output */
+
+  const BYTE *anchor;       /* Anchor to start of current (match) block */
+
+  LDM_compressStats stats;            /* Compression statistics */
+
+  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+
+  const BYTE *lastPosHashed;          /* Last position hashed */
+  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
+  const BYTE *nextIp;
+  hash_t nextHash;                    /* Hash corresponding to nextIp */
+
+  unsigned step;
+} LDM_CCtx;
+
+#ifdef LDM_ROLLING_HASH
+/**
+ * Convert a sum computed from LDM_getRollingHash to a hash value in the range
+ * of the hash table.
+ */
+static hash_t LDM_sumToHash(U32 sum) {
+  return sum % (LDM_HASHTABLESIZE >> 2);
+//  return sum & (LDM_HASHTABLESIZE - 1);
+}
+
+static U32 LDM_getRollingHash(const char *data, U32 len) {
+  U32 i;
+  U32 s1, s2;
+  const schar *buf = (const schar *)data;
+
+  s1 = s2 = 0;
+  for (i = 0; i < (len - 4); i += 4) {
+    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
+          (2 * buf[i + 2]) + (buf[i + 3]);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
+  }
+  for(; i < len; i++) {
+    s1 += buf[i];
+    s2 += s1;
+  }
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
+}
+
+typedef struct LDM_sumStruct {
+  U16 s1, s2;
+} LDM_sumStruct;
+
+static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
+  sumStruct->s1 = sum & 0xffff;
+  sumStruct->s2 = sum >> 16;
+}
+
+#else
+static hash_t LDM_hash(U32 sequence) {
+  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
+}
+
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_hash(LDM_read32(p));
+}
+#endif
+
+/*
+static hash_t LDM_hash5(U64 sequence) {
+  static const U64 prime5bytes = 889523592379ULL;
+  static const U64 prime8bytes = 11400714785074694791ULL;
+  const U32 hashLog = LDM_HASHLOG;
+  if (LDM_isLittleEndian())
+    return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
+  else
+    return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
+}
+*/
+
+static void LDM_putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash) {
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+}
+
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  hash_t hash = LDM_hashPosition(cctx->ip);
+  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
+}
+
+static const BYTE *LDM_get_position_on_hash(
+    hash_t h, void *tableBase, const BYTE *srcBase) {
+  const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
+  return hashTable[h].offset + srcBase;
+}
+
+static BYTE LDM_read_byte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
+static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
+                          const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize) {
+  const U32 *ip = (const U32 *)src;
+  *compressSize = *ip++;
+  *decompressSize = *ip;
+}
+
+static void LDM_initializeCCtx(LDM_CCtx *cctx,
+                               const void *src, size_t srcSize,
+                               void *dst, size_t maxDstSize) {
+  cctx->isize = srcSize;
+  cctx->maxOSize = maxDstSize;
+
+  cctx->ibase = (const BYTE *)src;
+  cctx->ip = cctx->ibase;
+  cctx->iend = cctx->ibase + srcSize;
+
+  cctx->ihashLimit = cctx->iend - HASH_SIZE;
+  cctx->imatchLimit = cctx->iend - MINMATCH;
+
+  cctx->obase = (BYTE *)dst;
+  cctx->op = (BYTE *)dst;
+
+  cctx->anchor = cctx->ibase;
+
+  memset(&(cctx->stats), 0, sizeof(cctx->stats));
+  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+
+  cctx->lastPosHashed = NULL;
+  cctx->nextIp = NULL;
+
+  cctx->step = 1;
+}
+
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+  cctx->nextIp = cctx->ip;
+
+  do {
+    hash_t const h = cctx->nextHash;
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
+
+    if (cctx->nextIp > cctx->imatchLimit) {
+      return 1;
+    }
+
+    *match = LDM_get_position_on_hash(h, cctx->hashTable, cctx->ibase);
+
+    cctx->nextHash = LDM_hashPosition(cctx->nextIp);
+    LDM_putHashOfCurrentPositionFromHash(cctx, h);
+  } while (cctx->ip - *match > WINDOW_SIZE ||
+           LDM_read64(*match) != LDM_read64(cctx->ip));
+  return 0;
+}
+
+// TODO: srcSize and maxDstSize is unused
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  LDM_CCtx cctx;
+  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+
+  /* Hash the first position and put it into the hash table. */
+  LDM_putHashOfCurrentPosition(&cctx);
+  cctx.ip++;
+  cctx.nextHash = LDM_hashPosition(cctx.ip);
+
+  // TODO: loop condition is not accurate.
+  while (1) {
+    const BYTE *match;
+
+    /**
+     * Find a match.
+     * If no more matches can be found (i.e. the length of the remaining input
+     * is less than the minimum match length), then stop searching for matches
+     * and encode the final literals.
+     */
+    if (LDM_findBestMatch(&cctx, &match) != 0) {
+      goto _last_literals;
+    }
+
+    cctx.stats.numMatches++;
+
+    /**
+     * Catch up: look back to extend the match backwards from the found match.
+     */
+    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
+           cctx.ip[-1] == match[-1]) {
+      cctx.ip--;
+      match--;
+    }
+
+    /**
+     * Write current block (literals, literal length, match offset, match
+     * length) and update pointers and hashes.
+     */
+    {
+      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
+      unsigned const offset = cctx.ip - match;
+      unsigned const matchLength = LDM_count(
+          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
+      BYTE *token = cctx.op++;
+
+      cctx.stats.totalLiteralLength += literalLength;
+      cctx.stats.totalOffset += offset;
+      cctx.stats.totalMatchLength += matchLength + MINMATCH;
+
+      /* Encode the literal length. */
+      if (literalLength >= RUN_MASK) {
+        int len = (int)literalLength - RUN_MASK;
+        *token = (RUN_MASK << ML_BITS);
+        for (; len >= 255; len -= 255) {
+          *(cctx.op)++ = 255;
+        }
+        *(cctx.op)++ = (BYTE)len;
+      } else {
+        *token = (BYTE)(literalLength << ML_BITS);
+      }
+
+      /* Encode the literals. */
+      memcpy(cctx.op, cctx.anchor, literalLength);
+      cctx.op += literalLength;
+
+      /* Encode the offset. */
+      LDM_write32(cctx.op, offset);
+      cctx.op += LDM_OFFSET_SIZE;
+
+      /* Encode match length */
+      if (matchLength >= ML_MASK) {
+        unsigned matchLengthRemaining = matchLength;
+        *token += ML_MASK;
+        matchLengthRemaining -= ML_MASK;
+        LDM_write32(cctx.op, 0xFFFFFFFF);
+        while (matchLengthRemaining >= 4*0xFF) {
+          cctx.op += 4;
+          LDM_write32(cctx.op, 0xffffffff);
+          matchLengthRemaining -= 4*0xFF;
+        }
+        cctx.op += matchLengthRemaining / 255;
+        *(cctx.op)++ = (BYTE)(matchLengthRemaining % 255);
+      } else {
+        *token += (BYTE)(matchLength);
+      }
+
+      /* Update input pointer, inserting hashes into hash table along the
+       * way.
+       */
+      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
+        LDM_putHashOfCurrentPosition(&cctx);
+        cctx.ip++;
+      }
+    }
+
+    // Set start of next block to current input pointer.
+    cctx.anchor = cctx.ip;
+    LDM_putHashOfCurrentPosition(&cctx);
+    cctx.nextHash = LDM_hashPosition(++cctx.ip);
+  }
+_last_literals:
+  /* Encode the last literals (no more matches). */
+  {
+    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *(cctx.op)++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *(cctx.op)++ = 255;
+      }
+      *(cctx.op)++ = (BYTE)accumulator;
+    } else {
+      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
+    }
+    memcpy(cctx.op, cctx.anchor, lastRun);
+    cctx.op += lastRun;
+  }
+  LDM_printCompressStats(&cctx.stats);
+  return (cctx.op - (const BYTE *)cctx.obase);
+}
+
+typedef struct LDM_DCtx {
+  size_t compressSize;
+  size_t maxDecompressSize;
+
+  const BYTE *ibase;   /* Base of input */
+  const BYTE *ip;      /* Current input position */
+  const BYTE *iend;    /* End of source */
+
+  const BYTE *obase;   /* Base of output */
+  BYTE *op;            /* Current output position */
+  const BYTE *oend;    /* End of output */
+} LDM_DCtx;
+
+static void LDM_initializeDCtx(LDM_DCtx *dctx,
+                               const void *src, size_t compressSize,
+                               void *dst, size_t maxDecompressSize) {
+  dctx->compressSize = compressSize;
+  dctx->maxDecompressSize = maxDecompressSize;
+
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + dctx->compressSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + dctx->maxDecompressSize;
+
+}
+
+size_t LDM_decompress(const void *src, size_t compressSize,
+                      void *dst, size_t maxDecompressSize) {
+  LDM_DCtx dctx;
+  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
+
+  while (dctx.ip < dctx.iend) {
+    BYTE *cpy;
+    const BYTE *match;
+    size_t length, offset;
+
+    /* Get the literal length. */
+    unsigned const token = *(dctx.ip)++;
+    if ((length = (token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+
+    /* Copy literals. */
+    cpy = dctx.op + length;
+    memcpy(dctx.op, dctx.ip, length);
+    dctx.ip += length;
+    dctx.op = cpy;
+
+    //TODO : dynamic offset size
+    offset = LDM_read32(dctx.ip);
+    dctx.ip += LDM_OFFSET_SIZE;
+    match = dctx.op - offset;
+
+    /* Get the match length. */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+
+    /* Copy match. */
+    cpy = dctx.op + length;
+
+    // Inefficient for now
+    while (match < cpy - offset && dctx.op < dctx.oend) {
+      *(dctx.op)++ = *match++;
+    }
+  }
+  return dctx.op - (BYTE *)dst;
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v0.3/ldm.h b/contrib/long_distance_matching/versions/v0.3/ldm.h
new file mode 100644
index 00000000..287d444d
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.3/ldm.h
@@ -0,0 +1,19 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize);
+
+size_t LDM_decompress(const void *src, size_t srcSize,
+                      void *dst, size_t maxDstSize);
+
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.3/main-ldm.c b/contrib/long_distance_matching/versions/v0.3/main-ldm.c
new file mode 100644
index 00000000..724d735d
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.3/main-ldm.c
@@ -0,0 +1,479 @@
+// TODO: file size must fit into a U32
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+// #define BUF_SIZE 16*1024  // Block size
+#define DEBUG
+
+//#define ZSTD
+
+/* Compress file given by fname and output to oname.
+ * Returns 0 if successful, error code otherwise.
+ */
+static int compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+  size_t maxCompressSize, compressSize;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+
+ /* Go to the location corresponding to the last byte. */
+ /* TODO: fallocate? */
+  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+ /* Write a dummy byte at the last location. */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  compressSize = ZSTD_compress(dst, statbuf.st_size,
+                                  src, statbuf.st_size, 1);
+#else
+  compressSize = LDM_HEADER_SIZE +
+      LDM_compress(src, statbuf.st_size,
+                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+
+    // Write compress and decompress size to header
+    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  memcpy(dst, &compressSize, 4);
+  memcpy(dst + 4, &(statbuf.st_size), 4);
+
+#ifdef DEBUG
+  printf("Compressed size: %zu\n", compressSize);
+  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
+#endif
+#endif
+
+  // Truncate file to compressSize.
+  ftruncate(fdout, compressSize);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
+         (double)compressSize / (statbuf.st_size) * 100);
+
+  // Close files.
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Decompress file compressed using LDM_compress.
+ * The input file should have the LDM_HEADER followed by payload.
+ * Returns 0 if succesful, and an error code otherwise.
+ */
+static int decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+  size_t compressSize, decompressSize, outSize;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* Read the header. */
+  LDM_readHeader(src, &compressSize, &decompressSize);
+
+#ifdef DEBUG
+  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
+         (size_t)statbuf.st_size, compressSize, decompressSize);
+#endif
+
+  /* Go to the location corresponding to the last byte. */
+  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+#ifdef ZSTD
+  outSize = ZSTD_decompress(dst, decomrpessed_size,
+                                  src + LDM_HEADER_SIZE,
+                                  statbuf.st_size - LDM_HEADER_SIZE);
+#else
+  outSize = LDM_decompress(
+      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
+      dst, decompressSize);
+
+  printf("Ret size out: %zu\n", outSize);
+  #endif
+  ftruncate(fdout, outSize);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Compare two files.
+ * Returns 0 iff they are the same.
+ */
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) break;
+
+    if (0 == result) result = memcmp(b0, b1, r0);
+  }
+  return result;
+}
+
+/* Verify the input file is the same as the decompressed file. */
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+  {
+    const int cmp = compare(inpFp, decFp);
+    if(0 == cmp) {
+      printf("verify : OK\n");
+    } else {
+      printf("verify : NG\n");
+    }
+  }
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+
+  /* Compress */
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (compress(inpFilename, ldmFilename)) {
+        printf("Compress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total time = %f seconds\n",
+           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+           (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+
+  /* Decompress */
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (decompress(ldmFilename, decFilename)) {
+        printf("Decompress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total time = %f seconds\n",
+          (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+          (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+
diff --git a/contrib/long_distance_matching/versions/v0.3/util.c b/contrib/long_distance_matching/versions/v0.3/util.c
new file mode 100644
index 00000000..9ea4ca1e
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.3/util.c
@@ -0,0 +1,64 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "util.h"
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+unsigned LDM_isLittleEndian(void) {
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+void LDM_write16(void *memPtr, U16 value){
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE *)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+
diff --git a/contrib/long_distance_matching/versions/v0.3/util.h b/contrib/long_distance_matching/versions/v0.3/util.h
new file mode 100644
index 00000000..90726412
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.3/util.h
@@ -0,0 +1,23 @@
+#ifndef LDM_UTIL_H
+#define LDM_UTIL_H
+
+unsigned LDM_isLittleEndian(void);
+
+uint16_t LDM_read16(const void *memPtr);
+
+uint16_t LDM_readLE16(const void *memPtr);
+
+void LDM_write16(void *memPtr, uint16_t value);
+
+void LDM_write32(void *memPtr, uint32_t value);
+
+void LDM_writeLE16(void *memPtr, uint16_t value);
+
+uint32_t LDM_read32(const void *ptr);
+
+uint64_t LDM_read64(const void *ptr);
+
+void LDM_copy8(void *dst, const void *src);
+
+
+#endif /* LDM_UTIL_H */

From 50502519fbd8adeeebed6f0b5232bcfc100bc63a Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 12 Jul 2017 09:47:00 -0700
Subject: [PATCH 035/100] Switch to using rolling hash only

---
 contrib/long_distance_matching/ldm.c          |  40 +-
 .../versions/{v3 => v0.4}/Makefile            |   0
 .../versions/{v3 => v0.4}/ldm.c               | 451 +++++++++++++----
 .../versions/{v3 => v0.4}/ldm.h               |   3 +
 .../versions/{v3 => v0.4}/main-ldm.c          |  15 +-
 .../versions/{v3 => v0.4}/util.c              |   5 +
 .../versions/{v3 => v0.4}/util.h              |   2 +
 .../long_distance_matching/versions/v1/ldm.c  | 394 ---------------
 .../long_distance_matching/versions/v1/ldm.h  |  19 -
 .../versions/v1/main-ldm.c                    | 459 -----------------
 .../versions/v2/Makefile                      |  32 --
 .../long_distance_matching/versions/v2/ldm.c  | 436 ----------------
 .../long_distance_matching/versions/v2/ldm.h  |  19 -
 .../versions/v2/main-ldm.c                    | 474 ------------------
 14 files changed, 404 insertions(+), 1945 deletions(-)
 rename contrib/long_distance_matching/versions/{v3 => v0.4}/Makefile (100%)
 rename contrib/long_distance_matching/versions/{v3 => v0.4}/ldm.c (51%)
 rename contrib/long_distance_matching/versions/{v3 => v0.4}/ldm.h (85%)
 rename contrib/long_distance_matching/versions/{v3 => v0.4}/main-ldm.c (98%)
 rename contrib/long_distance_matching/versions/{v3 => v0.4}/util.c (92%)
 rename contrib/long_distance_matching/versions/{v3 => v0.4}/util.h (91%)
 delete mode 100644 contrib/long_distance_matching/versions/v1/ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v1/ldm.h
 delete mode 100644 contrib/long_distance_matching/versions/v1/main-ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v2/Makefile
 delete mode 100644 contrib/long_distance_matching/versions/v2/ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v2/ldm.h
 delete mode 100644 contrib/long_distance_matching/versions/v2/main-ldm.c

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index ca4f0f2c..79648097 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -20,7 +20,7 @@
 #define WINDOW_SIZE (1 << 23)
 #define MAX_WINDOW_SIZE 31
 #define HASH_SIZE 4
-#define LDM_HASH_LENGTH 100
+#define LDM_HASH_LENGTH 4
 
 // Should be multiple of four
 #define MINMATCH 4
@@ -106,7 +106,8 @@ typedef struct LDM_CCtx {
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
   const BYTE *nextIp;
-  hash_t nextHash;                    /* Hash corresponding to nextIp */
+  const BYTE *nextPosHashed;
+  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
 
   // Members for rolling hash.
   U32 lastSum;
@@ -192,9 +193,9 @@ static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
 */
 
 static void LDM_setNextHash(LDM_CCtx *cctx) {
-  U32 check;
 
 #ifdef RUN_CHECKS
+  U32 check;
   if ((cctx->nextIp - cctx->ibase != 1) &&
       (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
     printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
@@ -204,22 +205,21 @@ static void LDM_setNextHash(LDM_CCtx *cctx) {
   cctx->DEBUG_setNextHash = cctx->nextIp;
 #endif
 
-  cctx->nextSum = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
-  /*
-  check = LDM_updateRollingHash(
+//  cctx->nextSum = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  cctx->nextSum = LDM_updateRollingHash(
       cctx->lastSum, LDM_HASH_LENGTH,
       (schar)((cctx->lastPosHashed)[0]),
       (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
-      */
 
 #ifdef RUN_CHECKS
+  check = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+
   if (check != cctx->nextSum) {
     printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
 //    printf("INFO: %u %u %u\n", LDM_read32(cctx->nextIp),
-  } else {
-//    printf("CHECK: setNextHash passed\n");
   }
 #endif
+  cctx->nextPosHashed = cctx->nextIp;
   cctx->nextHash = LDM_sumToHash(cctx->nextSum);
 
 #ifdef RUN_CHECKS
@@ -254,9 +254,23 @@ static void LDM_putHashOfCurrentPositionFromHash(
   cctx->lastSum = sum;
 }
 
+static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  if (cctx->ip != cctx->nextPosHashed) {
+    printf("CHECK failed: updateLastHashFromNextHash %zu\n", cctx->ip - cctx->ibase);
+  }
+#endif
+  LDM_putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
+}
+
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   U32 sum = LDM_getRollingHash((const char *)cctx->ip, LDM_HASH_LENGTH);
   hash_t hash = LDM_sumToHash(sum);
+#ifdef RUN_CHECKS
+  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
+    printf("CHECK failed: putHashOfCurrentPosition %zu\n", cctx->ip - cctx->ibase);
+  }
+#endif
 //  hash_t hash = LDM_hashPosition(cctx->ip);
   LDM_putHashOfCurrentPositionFromHash(cctx, hash, sum);
 //  printf("Offset %zu\n", cctx->ip - cctx->ibase);
@@ -376,6 +390,7 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
 
   cctx->step = 1;
   cctx->nextIp = cctx->ip + cctx->step;
+  cctx->nextPosHashed = 0;
 
   cctx->DEBUG_setNextHash = 0;
 }
@@ -503,7 +518,8 @@ static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
   while (cctx->ip < cctx->anchor + MINMATCH + matchLength + literalLength) {
 //    printf("Loop\n");
     if (cctx->ip > cctx->lastPosHashed) {
-      LDM_putHashOfCurrentPosition(cctx);
+      LDM_updateLastHashFromNextHash(cctx);
+//      LDM_putHashOfCurrentPosition(cctx);
 #ifdef LDM_ROLLING_HASH
       LDM_setNextHash(cctx);
 #endif
@@ -526,7 +542,6 @@ static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
-  U32 tmp_hash;
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
@@ -579,7 +594,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
     // Set start of next block to current input pointer.
     cctx.anchor = cctx.ip;
-    LDM_putHashOfCurrentPosition(&cctx);
+    LDM_updateLastHashFromNextHash(&cctx);
+//    LDM_putHashOfCurrentPosition(&cctx);
 #ifndef LDM_ROLLING_HASH
     cctx.ip++;
 #endif
diff --git a/contrib/long_distance_matching/versions/v3/Makefile b/contrib/long_distance_matching/versions/v0.4/Makefile
similarity index 100%
rename from contrib/long_distance_matching/versions/v3/Makefile
rename to contrib/long_distance_matching/versions/v0.4/Makefile
diff --git a/contrib/long_distance_matching/versions/v3/ldm.c b/contrib/long_distance_matching/versions/v0.4/ldm.c
similarity index 51%
rename from contrib/long_distance_matching/versions/v3/ldm.c
rename to contrib/long_distance_matching/versions/v0.4/ldm.c
index 1dedf5c3..79648097 100644
--- a/contrib/long_distance_matching/versions/v3/ldm.c
+++ b/contrib/long_distance_matching/versions/v0.4/ldm.c
@@ -9,7 +9,7 @@
 
 #define HASH_EVERY 1
 
-#define LDM_MEMORY_USAGE 16
+#define LDM_MEMORY_USAGE 22
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
@@ -17,10 +17,12 @@
 
 #define LDM_OFFSET_SIZE 4
 
-#define WINDOW_SIZE (1 << 20)
+#define WINDOW_SIZE (1 << 23)
 #define MAX_WINDOW_SIZE 31
 #define HASH_SIZE 4
 #define LDM_HASH_LENGTH 4
+
+// Should be multiple of four
 #define MINMATCH 4
 
 #define ML_BITS 4
@@ -28,7 +30,9 @@
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
-//#define LDM_ROLLING_HASH
+#define LDM_ROLLING_HASH
+#define COMPUTE_STATS
+//#define RUN_CHECKS
 //#define LDM_DEBUG
 
 typedef  uint8_t BYTE;
@@ -41,6 +45,7 @@ typedef uint32_t offset_t;
 typedef uint32_t hash_t;
 typedef signed char schar;
 
+
 typedef struct LDM_hashEntry {
   offset_t offset;
 } LDM_hashEntry;
@@ -50,6 +55,9 @@ typedef struct LDM_compressStats {
   U32 totalMatchLength;
   U32 totalLiteralLength;
   U64 totalOffset;
+
+  U32 numCollisions;
+  U32 numHashInserts;
 } LDM_compressStats;
 
 static void LDM_printCompressStats(const LDM_compressStats *stats) {
@@ -62,6 +70,11 @@ static void LDM_printCompressStats(const LDM_compressStats *stats) {
          ((double)stats->totalLiteralLength) / (double)stats->numMatches);
   printf("Average offset length: %.1f\n",
          ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
+         stats->numCollisions, stats->numHashInserts,
+         stats->numHashInserts == 0 ?
+            1.0 : (100.0 * (double)stats->numCollisions) /
+                  (double)stats->numHashInserts);
   printf("=====================\n");
 }
 
@@ -93,19 +106,46 @@ typedef struct LDM_CCtx {
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
   const BYTE *nextIp;
-  hash_t nextHash;                    /* Hash corresponding to nextIp */
+  const BYTE *nextPosHashed;
+  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
+
+  // Members for rolling hash.
+  U32 lastSum;
+  U32 nextSum;
 
   unsigned step;
+
+  // DEBUG
+  const BYTE *DEBUG_setNextHash;
 } LDM_CCtx;
 
+static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
+  U16 lengthLeft = MINMATCH;
+  const BYTE *curP = p;
+  const BYTE *curMatch = match;
+
+  for (; lengthLeft >= 8; lengthLeft -= 8) {
+    if (LDM_read64(curP) != LDM_read64(curMatch)) {
+      return 0;
+    }
+    curP += 8;
+    curMatch += 8;
+  }
+  if (lengthLeft > 0) {
+    return LDM_read32(curP) == LDM_read32(curMatch);
+  }
+  return 1;
+}
+
+
+
 #ifdef LDM_ROLLING_HASH
 /**
  * Convert a sum computed from LDM_getRollingHash to a hash value in the range
  * of the hash table.
  */
 static hash_t LDM_sumToHash(U32 sum) {
-  return sum % (LDM_HASHTABLESIZE >> 2);
-//  return sum & (LDM_HASHTABLESIZE - 1);
+  return sum & (LDM_HASH_SIZE_U32 - 1);
 }
 
 static U32 LDM_getRollingHash(const char *data, U32 len) {
@@ -126,18 +166,115 @@ static U32 LDM_getRollingHash(const char *data, U32 len) {
   return (s1 & 0xffff) + (s2 << 16);
 }
 
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
-}
-
 typedef struct LDM_sumStruct {
   U16 s1, s2;
 } LDM_sumStruct;
 
+static U32 LDM_updateRollingHash(U32 sum, U32 len,
+                                 schar toRemove, schar toAdd) {
+  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
+  U32 s2 = (sum >> 16) - (toRemove * len) + s1;
+
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+
+/*
+static hash_t LDM_hashPosition(const void * const p) {
+  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
+}
+*/
+
+/*
 static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
   sumStruct->s1 = sum & 0xffff;
   sumStruct->s2 = sum >> 16;
 }
+*/
+
+static void LDM_setNextHash(LDM_CCtx *cctx) {
+
+#ifdef RUN_CHECKS
+  U32 check;
+  if ((cctx->nextIp - cctx->ibase != 1) &&
+      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
+    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
+            cctx->DEBUG_setNextHash - cctx->ibase);
+  }
+
+  cctx->DEBUG_setNextHash = cctx->nextIp;
+#endif
+
+//  cctx->nextSum = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  cctx->nextSum = LDM_updateRollingHash(
+      cctx->lastSum, LDM_HASH_LENGTH,
+      (schar)((cctx->lastPosHashed)[0]),
+      (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
+
+#ifdef RUN_CHECKS
+  check = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+
+  if (check != cctx->nextSum) {
+    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
+//    printf("INFO: %u %u %u\n", LDM_read32(cctx->nextIp),
+  }
+#endif
+  cctx->nextPosHashed = cctx->nextIp;
+  cctx->nextHash = LDM_sumToHash(cctx->nextSum);
+
+#ifdef RUN_CHECKS
+  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
+    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
+            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
+            cctx->ip - cctx->ibase);
+  }
+#endif
+
+}
+
+static void LDM_putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash, U32 sum) {
+  /*
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  */
+#ifdef COMPUTE_STATS
+  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
+    offset_t offset = (cctx->hashTable)[hash].offset;
+    cctx->stats.numHashInserts++;
+    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+      cctx->stats.numCollisions++;
+    }
+  }
+#endif
+  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+  cctx->lastSum = sum;
+}
+
+static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  if (cctx->ip != cctx->nextPosHashed) {
+    printf("CHECK failed: updateLastHashFromNextHash %zu\n", cctx->ip - cctx->ibase);
+  }
+#endif
+  LDM_putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
+}
+
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  U32 sum = LDM_getRollingHash((const char *)cctx->ip, LDM_HASH_LENGTH);
+  hash_t hash = LDM_sumToHash(sum);
+#ifdef RUN_CHECKS
+  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
+    printf("CHECK failed: putHashOfCurrentPosition %zu\n", cctx->ip - cctx->ibase);
+  }
+#endif
+//  hash_t hash = LDM_hashPosition(cctx->ip);
+  LDM_putHashOfCurrentPositionFromHash(cctx, hash, sum);
+//  printf("Offset %zu\n", cctx->ip - cctx->ibase);
+}
 
 #else
 static hash_t LDM_hash(U32 sequence) {
@@ -147,6 +284,39 @@ static hash_t LDM_hash(U32 sequence) {
 static hash_t LDM_hashPosition(const void * const p) {
   return LDM_hash(LDM_read32(p));
 }
+
+static void LDM_putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash) {
+  /*
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
+    return;
+  }
+  */
+#ifdef COMPUTE_STATS
+  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
+    offset_t offset = (cctx->hashTable)[hash].offset;
+    cctx->stats.numHashInserts++;
+    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+      cctx->stats.numCollisions++;
+    }
+  }
+#endif
+
+  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
+#ifdef RUN_CHECKS
+  if (cctx->ip - cctx->lastPosHashed != 1) {
+    printf("putHashError\n");
+  }
+#endif
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+}
+
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  hash_t hash = LDM_hashPosition(cctx->ip);
+  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
+}
+
 #endif
 
 /*
@@ -161,38 +331,19 @@ static hash_t LDM_hash5(U64 sequence) {
 }
 */
 
-static void LDM_putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash) {
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-}
 
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  hash_t hash = LDM_hashPosition(cctx->ip);
-  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
-}
-
-static const BYTE *LDM_get_position_on_hash(
+static const BYTE *LDM_getPositionOnHash(
     hash_t h, void *tableBase, const BYTE *srcBase) {
   const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
   return hashTable[h].offset + srcBase;
 }
 
-static BYTE LDM_read_byte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
 
 static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
                           const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
   while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
+    BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
     if (!diff) {
       pIn++;
       pMatch++;
@@ -220,7 +371,11 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->ip = cctx->ibase;
   cctx->iend = cctx->ibase + srcSize;
 
+#ifdef LDM_ROLLING_HASH
+  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
+#else
   cctx->ihashLimit = cctx->iend - HASH_SIZE;
+#endif
   cctx->imatchLimit = cctx->iend - MINMATCH;
 
   cctx->obase = (BYTE *)dst;
@@ -232,11 +387,47 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
 
   cctx->lastPosHashed = NULL;
-  cctx->nextIp = NULL;
 
   cctx->step = 1;
+  cctx->nextIp = cctx->ip + cctx->step;
+  cctx->nextPosHashed = 0;
+
+  cctx->DEBUG_setNextHash = 0;
 }
 
+#ifdef LDM_ROLLING_HASH
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  do {
+    hash_t h;
+    U32 sum;
+//    printf("Call A\n");
+    LDM_setNextHash(cctx);
+//    printf("End call a\n");
+    h = cctx->nextHash;
+    sum = cctx->nextSum;
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
+
+    if (cctx->ip > cctx->imatchLimit) {
+      return 1;
+    }
+
+    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
+
+//    // Compute cctx->nextSum and cctx->nextHash from cctx->nextIp.
+//    LDM_setNextHash(cctx);
+    LDM_putHashOfCurrentPositionFromHash(cctx, h, sum);
+
+//    printf("%u %u\n", cctx->lastHash, cctx->nextHash);
+  } while (cctx->ip - *match > WINDOW_SIZE ||
+           !LDM_isValidMatch(cctx->ip, *match));
+//           LDM_read64(*match) != LDM_read64(cctx->ip));
+  LDM_setNextHash(cctx);
+  return 0;
+}
+#else
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
   cctx->nextIp = cctx->ip;
 
@@ -245,19 +436,108 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
     cctx->ip = cctx->nextIp;
     cctx->nextIp += cctx->step;
 
-    if (cctx->nextIp > cctx->imatchLimit) {
+    if (cctx->ip > cctx->imatchLimit) {
       return 1;
     }
 
-    *match = LDM_get_position_on_hash(h, cctx->hashTable, cctx->ibase);
+    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
 
     cctx->nextHash = LDM_hashPosition(cctx->nextIp);
     LDM_putHashOfCurrentPositionFromHash(cctx, h);
+
   } while (cctx->ip - *match > WINDOW_SIZE ||
-           LDM_read64(*match) != LDM_read64(cctx->ip));
+           !LDM_isValidMatch(cctx->ip, *match));
   return 0;
 }
 
+#endif
+
+/**
+ * Write current block (literals, literal length, match offset,
+ * match length).
+ *
+ * Update input pointer, inserting hashes into hash table along the
+ * way.
+ */
+static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
+  unsigned const literalLength = (unsigned)(cctx->ip - cctx->anchor);
+  unsigned const offset = cctx->ip - match;
+  unsigned const matchLength = LDM_count(
+      cctx->ip + MINMATCH, match + MINMATCH, cctx->ihashLimit);
+  BYTE *token = cctx->op++;
+
+  cctx->stats.totalLiteralLength += literalLength;
+  cctx->stats.totalOffset += offset;
+  cctx->stats.totalMatchLength += matchLength + MINMATCH;
+
+  /* Encode the literal length. */
+  if (literalLength >= RUN_MASK) {
+    int len = (int)literalLength - RUN_MASK;
+    *token = (RUN_MASK << ML_BITS);
+    for (; len >= 255; len -= 255) {
+      *(cctx->op)++ = 255;
+    }
+    *(cctx->op)++ = (BYTE)len;
+  } else {
+    *token = (BYTE)(literalLength << ML_BITS);
+  }
+
+  /* Encode the literals. */
+  memcpy(cctx->op, cctx->anchor, literalLength);
+  cctx->op += literalLength;
+
+  /* Encode the offset. */
+  LDM_write32(cctx->op, offset);
+  cctx->op += LDM_OFFSET_SIZE;
+
+  /* Encode match length */
+  if (matchLength >= ML_MASK) {
+    unsigned matchLengthRemaining = matchLength;
+    *token += ML_MASK;
+    matchLengthRemaining -= ML_MASK;
+    LDM_write32(cctx->op, 0xFFFFFFFF);
+    while (matchLengthRemaining >= 4*0xFF) {
+      cctx->op += 4;
+      LDM_write32(cctx->op, 0xffffffff);
+      matchLengthRemaining -= 4*0xFF;
+    }
+    cctx->op += matchLengthRemaining / 255;
+    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
+  } else {
+    *token += (BYTE)(matchLength);
+  }
+
+//  LDM_setNextHash(cctx);
+//  cctx->ip = cctx->lastPosHashed + 1;
+//  cctx->nextIp = cctx->ip + cctx->step;
+//  printf("HERE: %zu %zu %zu\n", cctx->ip - cctx->ibase,
+//         cctx->lastPosHashed - cctx->ibase, cctx->nextIp - cctx->ibase);
+
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  while (cctx->ip < cctx->anchor + MINMATCH + matchLength + literalLength) {
+//    printf("Loop\n");
+    if (cctx->ip > cctx->lastPosHashed) {
+      LDM_updateLastHashFromNextHash(cctx);
+//      LDM_putHashOfCurrentPosition(cctx);
+#ifdef LDM_ROLLING_HASH
+      LDM_setNextHash(cctx);
+#endif
+    }
+    /*
+    printf("Call b %zu %zu %zu\n",
+            cctx->lastPosHashed - cctx->ibase,
+            cctx->nextIp - cctx->ibase,
+            cctx->ip - cctx->ibase);
+            */
+//    printf("end call b\n");
+    cctx->ip++;
+    cctx->nextIp++;
+  }
+
+//  printf("There: %zu %zu\n", cctx->ip - cctx->ibase, cctx->lastPosHashed - cctx->ibase);
+}
+
 // TODO: srcSize and maxDstSize is unused
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
@@ -266,12 +546,21 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
+#ifdef LDM_ROLLING_HASH
+//  LDM_setNextHash(&cctx);
+//  tmp_hash = LDM_updateRollingHash(cctx.lastSum, LDM_HASH_LENGTH,
+//                                   cctx.ip[0], cctx.ip[LDM_HASH_LENGTH]);
+//  printf("Update test: %u %u\n", tmp_hash, cctx.nextSum);
+//  cctx.ip++;
+#else
   cctx.ip++;
   cctx.nextHash = LDM_hashPosition(cctx.ip);
+#endif
 
   // TODO: loop condition is not accurate.
   while (1) {
     const BYTE *match;
+//    printf("Start of loop\n");
 
     /**
      * Find a match.
@@ -282,6 +571,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
     if (LDM_findBestMatch(&cctx, &match) != 0) {
       goto _last_literals;
     }
+//    printf("End of match finding\n");
 
     cctx.stats.numMatches++;
 
@@ -290,6 +580,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
      */
     while (cctx.ip > cctx.anchor && match > cctx.ibase &&
            cctx.ip[-1] == match[-1]) {
+//      printf("Catch up\n");
       cctx.ip--;
       match--;
     }
@@ -298,67 +589,25 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * Write current block (literals, literal length, match offset, match
      * length) and update pointers and hashes.
      */
-    {
-      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
-      unsigned const offset = cctx.ip - match;
-      unsigned const matchLength = LDM_count(
-          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
-      BYTE *token = cctx.op++;
-
-      cctx.stats.totalLiteralLength += literalLength;
-      cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + MINMATCH;
-
-      /* Encode the literal length. */
-      if (literalLength >= RUN_MASK) {
-        int len = (int)literalLength - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for (; len >= 255; len -= 255) {
-          *(cctx.op)++ = 255;
-        }
-        *(cctx.op)++ = (BYTE)len;
-      } else {
-        *token = (BYTE)(literalLength << ML_BITS);
-      }
-
-      /* Encode the literals. */
-      memcpy(cctx.op, cctx.anchor, literalLength);
-      cctx.op += literalLength;
-
-      /* Encode the offset. */
-      LDM_write32(cctx.op, offset);
-      cctx.op += LDM_OFFSET_SIZE;
-
-      /* Encode match length */
-      if (matchLength >= ML_MASK) {
-        unsigned matchLengthRemaining = matchLength;
-        *token += ML_MASK;
-        matchLengthRemaining -= ML_MASK;
-        LDM_write32(cctx.op, 0xFFFFFFFF);
-        while (matchLengthRemaining >= 4*0xFF) {
-          cctx.op += 4;
-          LDM_write32(cctx.op, 0xffffffff);
-          matchLengthRemaining -= 4*0xFF;
-        }
-        cctx.op += matchLengthRemaining / 255;
-        *(cctx.op)++ = (BYTE)(matchLengthRemaining % 255);
-      } else {
-        *token += (BYTE)(matchLength);
-      }
-
-      /* Update input pointer, inserting hashes into hash table along the
-       * way.
-       */
-      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
-        LDM_putHashOfCurrentPosition(&cctx);
-        cctx.ip++;
-      }
-    }
+    LDM_outputBlock(&cctx, match);
+//    printf("End of loop\n");
 
     // Set start of next block to current input pointer.
     cctx.anchor = cctx.ip;
+    LDM_updateLastHashFromNextHash(&cctx);
+//    LDM_putHashOfCurrentPosition(&cctx);
+#ifndef LDM_ROLLING_HASH
+    cctx.ip++;
+#endif
+
+    /*
     LDM_putHashOfCurrentPosition(&cctx);
-    cctx.nextHash = LDM_hashPosition(++cctx.ip);
+    printf("Call c\n");
+    LDM_setNextHash(&cctx);
+    printf("End call c\n");
+    cctx.ip++;
+    cctx.nextIp++;
+    */
   }
 _last_literals:
   /* Encode the last literals (no more matches). */
@@ -453,7 +702,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
     /* Copy match. */
     cpy = dctx.op + length;
 
-    // Inefficient for now
+    // Inefficient for now.
     while (match < cpy - offset && dctx.op < dctx.oend) {
       *(dctx.op)++ = *match++;
     }
@@ -461,4 +710,20 @@ size_t LDM_decompress(const void *src, size_t compressSize,
   return dctx.op - (BYTE *)dst;
 }
 
+void LDM_test(const void *src, size_t srcSize,
+              void *dst, size_t maxDstSize) {
+#ifdef LDM_ROLLING_HASH
+  const BYTE *ip = (const BYTE *)src + 1125;
+  U32 sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
+  U32 sum2;
+  ++ip;
+  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
+    sum2 = LDM_updateRollingHash(sum, LDM_HASH_LENGTH,
+                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
+    sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
+    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
+  }
+#endif
+}
+
 
diff --git a/contrib/long_distance_matching/versions/v3/ldm.h b/contrib/long_distance_matching/versions/v0.4/ldm.h
similarity index 85%
rename from contrib/long_distance_matching/versions/v3/ldm.h
rename to contrib/long_distance_matching/versions/v0.4/ldm.h
index 287d444d..a34faac4 100644
--- a/contrib/long_distance_matching/versions/v3/ldm.h
+++ b/contrib/long_distance_matching/versions/v0.4/ldm.h
@@ -16,4 +16,7 @@ size_t LDM_decompress(const void *src, size_t srcSize,
 void LDM_readHeader(const void *src, size_t *compressSize,
                     size_t *decompressSize);
 
+void LDM_test(const void *src, size_t srcSize,
+              void *dst, size_t maxDstSize);
+
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v3/main-ldm.c b/contrib/long_distance_matching/versions/v0.4/main-ldm.c
similarity index 98%
rename from contrib/long_distance_matching/versions/v3/main-ldm.c
rename to contrib/long_distance_matching/versions/v0.4/main-ldm.c
index 724d735d..f8ae5469 100644
--- a/contrib/long_distance_matching/versions/v3/main-ldm.c
+++ b/contrib/long_distance_matching/versions/v0.4/main-ldm.c
@@ -15,6 +15,7 @@
 
 // #define BUF_SIZE 16*1024  // Block size
 #define DEBUG
+//#define TEST
 
 //#define ZSTD
 
@@ -74,6 +75,11 @@ static int compress(const char *fname, const char *oname) {
       return 1;
   }
 
+#ifdef TEST
+  LDM_test(src, statbuf.st_size,
+           dst + LDM_HEADER_SIZE, statbuf.st_size);
+#endif
+
 #ifdef ZSTD
   compressSize = ZSTD_compress(dst, statbuf.st_size,
                                   src, statbuf.st_size, 1);
@@ -144,11 +150,6 @@ static int decompress(const char *fname, const char *oname) {
   /* Read the header. */
   LDM_readHeader(src, &compressSize, &decompressSize);
 
-#ifdef DEBUG
-  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
-         (size_t)statbuf.st_size, compressSize, decompressSize);
-#endif
-
   /* Go to the location corresponding to the last byte. */
   if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
     perror("lseek error");
@@ -256,7 +257,7 @@ int main(int argc, const char *argv[]) {
         return 1;
     }
     gettimeofday(&tv2, NULL);
-    printf("Total time = %f seconds\n",
+    printf("Total compress time = %f seconds\n",
            (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
            (double) (tv2.tv_sec - tv1.tv_sec));
   }
@@ -270,7 +271,7 @@ int main(int argc, const char *argv[]) {
         return 1;
     }
     gettimeofday(&tv2, NULL);
-    printf("Total time = %f seconds\n",
+    printf("Total decompress time = %f seconds\n",
           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
           (double) (tv2.tv_sec - tv1.tv_sec));
   }
diff --git a/contrib/long_distance_matching/versions/v3/util.c b/contrib/long_distance_matching/versions/v0.4/util.c
similarity index 92%
rename from contrib/long_distance_matching/versions/v3/util.c
rename to contrib/long_distance_matching/versions/v0.4/util.c
index 9ea4ca1e..70fcbc2c 100644
--- a/contrib/long_distance_matching/versions/v3/util.c
+++ b/contrib/long_distance_matching/versions/v0.4/util.c
@@ -61,4 +61,9 @@ void LDM_copy8(void *dst, const void *src) {
   memcpy(dst, src, 8);
 }
 
+BYTE LDM_readByte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
 
diff --git a/contrib/long_distance_matching/versions/v3/util.h b/contrib/long_distance_matching/versions/v0.4/util.h
similarity index 91%
rename from contrib/long_distance_matching/versions/v3/util.h
rename to contrib/long_distance_matching/versions/v0.4/util.h
index 90726412..d1c3c999 100644
--- a/contrib/long_distance_matching/versions/v3/util.h
+++ b/contrib/long_distance_matching/versions/v0.4/util.h
@@ -19,5 +19,7 @@ uint64_t LDM_read64(const void *ptr);
 
 void LDM_copy8(void *dst, const void *src);
 
+uint8_t LDM_readByte(const void *ptr);
+
 
 #endif /* LDM_UTIL_H */
diff --git a/contrib/long_distance_matching/versions/v1/ldm.c b/contrib/long_distance_matching/versions/v1/ldm.c
deleted file mode 100644
index 266425f8..00000000
--- a/contrib/long_distance_matching/versions/v1/ldm.c
+++ /dev/null
@@ -1,394 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "ldm.h"
-
-#define LDM_MEMORY_USAGE 14
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
-
-#define WINDOW_SIZE (1 << 20)
-#define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 4
-#define MINMATCH 4
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-//#define LDM_DEBUG
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint64_t tag;
-
-static unsigned LDM_isLittleEndian(void)
-{
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-static U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-static U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-static void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_write16(void *memPtr, U16 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE*)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-static U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-static U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-
-static void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
-  BYTE *d = (BYTE *)dstPtr;
-  const BYTE *s = (const BYTE *)srcPtr;
-  BYTE * const e = (BYTE *)dstEnd;
-
-  do {
-    LDM_copy8(d, s);
-    d += 8;
-    s += 8;
-  } while (d < e);
-
-}
-
-struct hash_entry {
-  U64 offset;
-  tag t;
-};
-
-static U32 LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
-}
-
-static U32 LDM_hash5(U64 sequence) {
-  static const U64 prime5bytes = 889523592379ULL;
-  static const U64 prime8bytes = 11400714785074694791ULL;
-  const U32 hashLog = LDM_HASHLOG;
-  if (LDM_isLittleEndian())
-    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
-  else
-    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
-}
-
-static U32 LDM_hash_position(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-
-static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
-                                     const BYTE *srcBase) {
-  U32 *hashTable = (U32 *) tableBase;
-  hashTable[h] = (U32)(p - srcBase);
-}
-
-static void LDM_put_position(const BYTE *p, void *tableBase,
-                             const BYTE *srcBase) {
-  U32 const h = LDM_hash_position(p);
-  LDM_put_position_on_hash(p, h, tableBase, srcBase);
-}
-
-static const BYTE *LDM_get_position_on_hash(
-    U32 h, void *tableBase, const BYTE *srcBase) {
-  const U32 * const hashTable = (U32*)tableBase;
-  return hashTable[h] + srcBase;
-}
-
-static BYTE LDM_read_byte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
-
-static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
-                          const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (unsigned)(pIn - pStart);
-  }
-  return (unsigned)(pIn - pStart);
-}
-
-void LDM_read_header(void const *source, size_t *compressed_size,
-                     size_t *decompressed_size) {
-  const U32 *ip = (const U32 *)source;
-  *compressed_size = *ip++;
-  *decompressed_size = *ip;
-}
-
-size_t LDM_compress(void const *source, void *dest, size_t source_size,
-                    size_t max_dest_size) {
-  const BYTE * const istart = (const BYTE*)source;
-  const BYTE *ip = istart;
-  const BYTE * const iend = istart + source_size;
-  const BYTE *ilimit = iend - HASH_SIZE;
-  const BYTE * const matchlimit = iend - HASH_SIZE;
-  const BYTE * const mflimit = iend - MINMATCH;
-  BYTE *op = (BYTE*) dest;
-  U32 hashTable[LDM_HASHTABLESIZE_U32];
-  memset(hashTable, 0, sizeof(hashTable));
-
-  const BYTE *anchor = (const BYTE *)source;
-//  struct LDM_cctx cctx;
-  size_t output_size = 0;
-
-  U32 forwardH;
-
-  /* Hash first byte: put into hash table */
-
-  LDM_put_position(ip, hashTable, istart);
-  ip++;
-  forwardH = LDM_hash_position(ip);
-
-  //TODO Loop terminates before ip>=ilimit.
-  while (ip < ilimit) {
-    const BYTE *match;
-    BYTE *token;
-
-    /* Find a match */
-    {
-      const BYTE *forwardIp = ip;
-      unsigned step = 1;
-
-      do {
-        U32 const h = forwardH;
-        ip = forwardIp;
-        forwardIp += step;
-
-        if (forwardIp > mflimit) {
-          goto _last_literals;
-        }
-
-        match = LDM_get_position_on_hash(h, hashTable, istart);
-
-        forwardH = LDM_hash_position(forwardIp);
-        LDM_put_position_on_hash(ip, h, hashTable, istart);
-      } while (ip - match > WINDOW_SIZE ||
-               LDM_read64(match) != LDM_read64(ip));
-    }
-
-    // TODO catchup
-    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
-      ip--;
-      match--;
-    }
-
-    /* Encode literals */
-    {
-      unsigned const litLength = (unsigned)(ip - anchor);
-      token = op++;
-
-#ifdef LDM_DEBUG
-      printf("Cur position: %zu\n", anchor - istart);
-      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
-#endif
-      /*
-      fwrite(match, 4, 1, stdout);
-      printf("\n");
-      */
-
-      if (litLength >= RUN_MASK) {
-        int len = (int)litLength - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for (; len >= 255; len -= 255) {
-          *op++ = 255;
-        }
-        *op++ = (BYTE)len;
-      } else {
-        *token = (BYTE)(litLength << ML_BITS);
-      }
-#ifdef LDM_DEBUG
-      printf("Literals ");
-      fwrite(anchor, litLength, 1, stdout);
-      printf("\n");
-#endif
-      memcpy(op, anchor, litLength);
-      //LDM_wild_copy(op, anchor, op + litLength);
-      op += litLength;
-    }
-_next_match:
-    /* Encode offset */
-    {
-      LDM_write32(op, ip - match);
-      op += 4;
-    }
-
-    /* Encode Match Length */
-    {
-      unsigned matchCode;
-      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
-                            matchlimit);
-#ifdef LDM_DEBUG
-      printf("Match length %zu\n", matchCode + MINMATCH);
-      fwrite(ip, MINMATCH + matchCode, 1, stdout);
-      printf("\n");
-#endif
-      ip += MINMATCH + matchCode;
-      if (matchCode >= ML_MASK) {
-        *token += ML_MASK;
-        matchCode -= ML_MASK;
-        LDM_write32(op, 0xFFFFFFFF);
-        while (matchCode >= 4*0xFF) {
-          op += 4;
-          LDM_write32(op, 0xffffffff);
-          matchCode -= 4*0xFF;
-        }
-        op += matchCode / 255;
-        *op++ = (BYTE)(matchCode % 255);
-      } else {
-        *token += (BYTE)(matchCode);
-      }
-#ifdef LDM_DEBUG
-      printf("\n");
-#endif
-    }
-
-    anchor = ip;
-
-    LDM_put_position(ip, hashTable, istart);
-    forwardH = LDM_hash_position(++ip);
-  }
-_last_literals:
-    /* Encode last literals */
-  {
-    size_t const lastRun = (size_t)(iend - anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *op++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *op++ = 255;
-      }
-      *op++ = (BYTE)accumulator;
-    } else {
-      *op++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(op, anchor, lastRun);
-    op += lastRun;
-  }
-  return (op - (BYTE *)dest);
-}
-
-size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
-                      size_t max_decompressed_size) {
-  const BYTE *ip = (const BYTE *)source;
-  const BYTE * const iend = ip + compressed_size;
-  BYTE *op = (BYTE *)dest;
-  BYTE * const oend = op + max_decompressed_size;
-  BYTE *cpy;
-
-  while (ip < iend) {
-    size_t length;
-    const BYTE *match;
-    size_t offset;
-
-    /* get literal length */
-    unsigned const token = *ip++;
-    if ((length=(token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-#ifdef LDM_DEBUG
-    printf("Literal length: %zu\n", length);
-#endif
-
-    /* copy literals */
-    cpy = op + length;
-#ifdef LDM_DEBUG
-    printf("Literals ");
-    fwrite(ip, length, 1, stdout);
-    printf("\n");
-#endif
-    memcpy(op, ip, length);
-//    LDM_wild_copy(op, ip, cpy);
-    ip += length;
-    op = cpy;
-
-    /* get offset */
-    offset = LDM_read32(ip);
-
-#ifdef LDM_DEBUG
-    printf("Offset: %zu\n", offset);
-#endif
-    ip += 4;
-    match = op - offset;
- //   LDM_write32(op, (U32)offset);
-
-    /* get matchlength */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-    length += MINMATCH;
-#ifdef LDM_DEBUG
-    printf("Match length: %zu\n", length);
-#endif
-    /* copy match */
-    cpy = op + length;
-
-    // Inefficient for now
-
-    while (match < cpy - offset && op < oend) {
-      *op++ = *match++;
-    }
-  }
-//  memcpy(dest, source, compressed_size);
-  return op - (BYTE *)dest;
-}
-
-
diff --git a/contrib/long_distance_matching/versions/v1/ldm.h b/contrib/long_distance_matching/versions/v1/ldm.h
deleted file mode 100644
index f4ca25a3..00000000
--- a/contrib/long_distance_matching/versions/v1/ldm.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef LDM_H
-#define LDM_H
-
-#include <stddef.h>   /* size_t */
-
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
-
-size_t LDM_compress(void const *source, void *dest, size_t source_size,
-                    size_t max_dest_size);
-
-size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
-                      size_t max_decompressed_size);
-
-void LDM_read_header(void const *source, size_t *compressed_size,
-                     size_t *decompressed_size);
-
-#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v1/main-ldm.c b/contrib/long_distance_matching/versions/v1/main-ldm.c
deleted file mode 100644
index 10869cce..00000000
--- a/contrib/long_distance_matching/versions/v1/main-ldm.c
+++ /dev/null
@@ -1,459 +0,0 @@
-// TODO: file size must fit into a U32
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zstd.h>
-
-#include <fcntl.h>
-#include "ldm.h"
-
-// #define BUF_SIZE 16*1024  // Block size
-#define DEBUG
-
-//#define ZSTD
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-#endif
-
-static size_t compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* open the input file */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* open the output file */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* find size of input file */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-  size_t size_in = statbuf.st_size;
-
- /* go to the location corresponding to the last byte */
- if (lseek(fdout, size_in + LDM_HEADER_SIZE - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
- }
-
- /* write a dummy byte at the last location */
- if (write(fdout, "", 1) != 1) {
-     perror("write error");
-     return 1;
- }
-
-  /* mmap the input file */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-  size_t out_size = statbuf.st_size + LDM_HEADER_SIZE;
-
-  /* mmap the output file */
-  if ((dst = mmap(0, out_size, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  #ifdef ZSTD
-    size_t size_out = ZSTD_compress(dst, statbuf.st_size,
-                                    src, statbuf.st_size, 1);
-  #else
-    size_t size_out = LDM_compress(src, dst + LDM_HEADER_SIZE, statbuf.st_size,
-                                   statbuf.st_size);
-    size_out += LDM_HEADER_SIZE;
-
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-    memcpy(dst, &size_out, 4);
-    memcpy(dst + 4, &(statbuf.st_size), 4);
-    printf("Compressed size: %zu\n", size_out);
-    printf("Decompressed size: %zu\n", statbuf.st_size);
-  #endif
-  ftruncate(fdout, size_out);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)size_out, oname,
-         (double)size_out / (statbuf.st_size) * 100);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-static size_t decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* open the input file */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* open the output file */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* find size of input file */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* read header */
-  size_t compressed_size, decompressed_size;
-  LDM_read_header(src, &compressed_size, &decompressed_size);
-
-  printf("Size, compressed_size, decompressed_size: %zu %zu %zu\n",
-         statbuf.st_size, compressed_size, decompressed_size);
-
-  /* go to the location corresponding to the last byte */
-  if (lseek(fdout, decompressed_size - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressed_size, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  /* Copy input file to output file */
-//  memcpy(dst, src, statbuf.st_size);
-
-  #ifdef ZSTD
-    size_t size_out = ZSTD_decompress(dst, decomrpessed_size,
-                                      src + LDM_HEADER_SIZE,
-                                      statbuf.st_size - LDM_HEADER_SIZE);
-  #else
-    size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
-                                     statbuf.st_size - LDM_HEADER_SIZE,
-                                     decompressed_size);
-    printf("Ret size out: %zu\n", size_out);
-  #endif
-  ftruncate(fdout, size_out);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) {
-      break;
-    }
-    if (0 == result) {
-      result = memcmp(b0, b1, r0);
-    }
-  }
-  return result;
-}
-
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-	const int cmp = compare(inpFp, decFp);
-	if(0 == cmp) {
-		printf("verify : OK\n");
-	} else {
-		printf("verify : NG\n");
-	}
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  struct timeval tv1, tv2;
-  /* compress */
-  {
-  gettimeofday(&tv1, NULL);
-  if (compress(inpFilename, ldmFilename)) {
-      printf("Compress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-         (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-
-  /* decompress */
-
-  gettimeofday(&tv1, NULL);
-  if (decompress(ldmFilename, decFilename)) {
-      printf("Decompress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-        (double) (tv2.tv_sec - tv1.tv_sec));
-
-  /* verify */
-  verify(inpFilename, decFilename);
-  return 0;
-}
-
-#if 0
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
-#endif
-
diff --git a/contrib/long_distance_matching/versions/v2/Makefile b/contrib/long_distance_matching/versions/v2/Makefile
deleted file mode 100644
index 4e04fd6a..00000000
--- a/contrib/long_distance_matching/versions/v2/Makefile
+++ /dev/null
@@ -1,32 +0,0 @@
-# ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
-
-# This Makefile presumes libzstd is installed, using `sudo make install`
-
-
-LDFLAGS += -lzstd
-
-.PHONY: default all clean
-
-default: all
-
-all: main-ldm
-
-
-#main : ldm.c main.c
-#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-main-ldm : ldm.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-clean:
-	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main main-ldm
-	@echo Cleaning completed
-
diff --git a/contrib/long_distance_matching/versions/v2/ldm.c b/contrib/long_distance_matching/versions/v2/ldm.c
deleted file mode 100644
index 9081d136..00000000
--- a/contrib/long_distance_matching/versions/v2/ldm.c
+++ /dev/null
@@ -1,436 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "ldm.h"
-
-#define HASH_EVERY 7
-
-#define LDM_MEMORY_USAGE 14
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
-
-#define WINDOW_SIZE (1 << 20)
-#define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 8
-#define MINMATCH 8
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-//#define LDM_DEBUG
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint64_t tag;
-
-static unsigned LDM_isLittleEndian(void) {
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-static U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-static U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-static void LDM_write16(void *memPtr, U16 value){
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE*)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-static U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-static U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-static void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-typedef struct compress_stats {
-  U32 num_matches;
-  U32 total_match_length;
-  U32 total_literal_length;
-  U64 total_offset;
-} compress_stats;
-
-static void LDM_printCompressStats(const compress_stats *stats) {
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->num_matches);
-  printf("Average match length: %.1f\n", ((double)stats->total_match_length) /
-                                         (double)stats->num_matches);
-  printf("Average literal length: %.1f\n",
-         ((double)stats->total_literal_length) / (double)stats->num_matches);
-  printf("Average offset length: %.1f\n",
-         ((double)stats->total_offset) / (double)stats->num_matches);
-  printf("=====================\n");
-}
-
-// TODO: unused.
-struct hash_entry {
-  U64 offset;
-  tag t;
-};
-
-static U32 LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
-}
-
-static U32 LDM_hash5(U64 sequence) {
-  static const U64 prime5bytes = 889523592379ULL;
-  static const U64 prime8bytes = 11400714785074694791ULL;
-  const U32 hashLog = LDM_HASHLOG;
-  if (LDM_isLittleEndian())
-    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
-  else
-    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
-}
-
-static U32 LDM_hash_position(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-
-static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
-                                     const BYTE *srcBase) {
-  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-
-  U32 *hashTable = (U32 *) tableBase;
-  hashTable[h] = (U32)(p - srcBase);
-}
-
-static void LDM_put_position(const BYTE *p, void *tableBase,
-                             const BYTE *srcBase) {
-  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  U32 const h = LDM_hash_position(p);
-  LDM_put_position_on_hash(p, h, tableBase, srcBase);
-}
-
-static const BYTE *LDM_get_position_on_hash(
-    U32 h, void *tableBase, const BYTE *srcBase) {
-  const U32 * const hashTable = (U32*)tableBase;
-  return hashTable[h] + srcBase;
-}
-
-static BYTE LDM_read_byte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
-
-static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
-                          const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (unsigned)(pIn - pStart);
-  }
-  return (unsigned)(pIn - pStart);
-}
-
-void LDM_read_header(const void *src, size_t *compressSize,
-                     size_t *decompressSize) {
-  const U32 *ip = (const U32 *)src;
-  *compressSize = *ip++;
-  *decompressSize = *ip;
-}
-
-// TODO: maxDstSize is unused
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize) {
-  const BYTE * const istart = (const BYTE*)src;
-  const BYTE *ip = istart;
-  const BYTE * const iend = istart + srcSize;
-  const BYTE *ilimit = iend - HASH_SIZE;
-  const BYTE * const matchlimit = iend - HASH_SIZE;
-  const BYTE * const mflimit = iend - MINMATCH;
-  BYTE *op = (BYTE*) dst;
-
-  compress_stats compressStats = { 0 };
-
-  U32 hashTable[LDM_HASHTABLESIZE_U32];
-  memset(hashTable, 0, sizeof(hashTable));
-
-  const BYTE *anchor = (const BYTE *)src;
-//  struct LDM_cctx cctx;
-  size_t output_size = 0;
-
-  U32 forwardH;
-
-  /* Hash first byte: put into hash table */
-
-  LDM_put_position(ip, hashTable, istart);
-  const BYTE *lastHash = ip;
-  ip++;
-  forwardH = LDM_hash_position(ip);
-
-  //TODO Loop terminates before ip>=ilimit.
-  while (ip < ilimit) {
-    const BYTE *match;
-    BYTE *token;
-
-    /* Find a match */
-    {
-      const BYTE *forwardIp = ip;
-      unsigned step = 1;
-
-      do {
-        U32 const h = forwardH;
-        ip = forwardIp;
-        forwardIp += step;
-
-        if (forwardIp > mflimit) {
-          goto _last_literals;
-        }
-
-        match = LDM_get_position_on_hash(h, hashTable, istart);
-
-        forwardH = LDM_hash_position(forwardIp);
-        LDM_put_position_on_hash(ip, h, hashTable, istart);
-        lastHash = ip;
-      } while (ip - match > WINDOW_SIZE ||
-               LDM_read64(match) != LDM_read64(ip));
-    }
-    compressStats.num_matches++;
-
-    /* Catchup: look back to extend match from found match */
-    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
-      ip--;
-      match--;
-    }
-
-    /* Encode literals */
-    {
-      unsigned const litLength = (unsigned)(ip - anchor);
-      token = op++;
-
-      compressStats.total_literal_length += litLength;
-
-#ifdef LDM_DEBUG
-      printf("Cur position: %zu\n", anchor - istart);
-      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
-#endif
-
-      if (litLength >= RUN_MASK) {
-        int len = (int)litLength - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for (; len >= 255; len -= 255) {
-          *op++ = 255;
-        }
-        *op++ = (BYTE)len;
-      } else {
-        *token = (BYTE)(litLength << ML_BITS);
-      }
-#ifdef LDM_DEBUG
-      printf("Literals ");
-      fwrite(anchor, litLength, 1, stdout);
-      printf("\n");
-#endif
-      memcpy(op, anchor, litLength);
-      op += litLength;
-    }
-_next_match:
-    /* Encode offset */
-    {
-      /*
-      LDM_writeLE16(op, ip-match);
-      op += 2;
-      */
-      LDM_write32(op, ip - match);
-      op += 4;
-      compressStats.total_offset += (ip - match);
-    }
-
-    /* Encode Match Length */
-    {
-      unsigned matchCode;
-      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
-                            matchlimit);
-#ifdef LDM_DEBUG
-      printf("Match length %zu\n", matchCode + MINMATCH);
-      fwrite(ip, MINMATCH + matchCode, 1, stdout);
-      printf("\n");
-#endif
-      compressStats.total_match_length += matchCode + MINMATCH;
-      unsigned ctr = 1;
-      ip++;
-      for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
-        LDM_put_position(ip, hashTable, istart);
-      }
-//      ip += MINMATCH + matchCode;
-      if (matchCode >= ML_MASK) {
-        *token += ML_MASK;
-        matchCode -= ML_MASK;
-        LDM_write32(op, 0xFFFFFFFF);
-        while (matchCode >= 4*0xFF) {
-          op += 4;
-          LDM_write32(op, 0xffffffff);
-          matchCode -= 4*0xFF;
-        }
-        op += matchCode / 255;
-        *op++ = (BYTE)(matchCode % 255);
-      } else {
-        *token += (BYTE)(matchCode);
-      }
-#ifdef LDM_DEBUG
-      printf("\n");
-
-#endif
-    }
-
-    anchor = ip;
-
-    LDM_put_position(ip, hashTable, istart);
-    forwardH = LDM_hash_position(++ip);
-    lastHash = ip;
-  }
-_last_literals:
-    /* Encode last literals */
-  {
-    size_t const lastRun = (size_t)(iend - anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *op++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *op++ = 255;
-      }
-      *op++ = (BYTE)accumulator;
-    } else {
-      *op++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(op, anchor, lastRun);
-    op += lastRun;
-  }
-  LDM_printCompressStats(&compressStats);
-  return (op - (BYTE *)dst);
-}
-
-typedef struct LDM_DCtx {
-  const BYTE * const ibase;   /* Pointer to base of input */
-  const BYTE *ip;             /* Pointer to current input position */
-  const BYTE *iend;           /* End of source */
-  BYTE *op;                   /* Pointer to output */
-  const BYTE * const oend;    /* Pointer to end of output */
-
-} LDM_DCtx;
-
-size_t LDM_decompress(const void *src, size_t compressed_size,
-                      void *dst, size_t max_decompressed_size) {
-  const BYTE *ip = (const BYTE *)src;
-  const BYTE * const iend = ip + compressed_size;
-  BYTE *op = (BYTE *)dst;
-  BYTE * const oend = op + max_decompressed_size;
-  BYTE *cpy;
-
-  while (ip < iend) {
-    size_t length;
-    const BYTE *match;
-    size_t offset;
-
-    /* get literal length */
-    unsigned const token = *ip++;
-    if ((length=(token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-#ifdef LDM_DEBUG
-    printf("Literal length: %zu\n", length);
-#endif
-
-    /* copy literals */
-    cpy = op + length;
-#ifdef LDM_DEBUG
-    printf("Literals ");
-    fwrite(ip, length, 1, stdout);
-    printf("\n");
-#endif
-    memcpy(op, ip, length);
-    ip += length;
-    op = cpy;
-
-    /* get offset */
-    /*
-    offset = LDM_readLE16(ip);
-    ip += 2;
-    */
-    offset = LDM_read32(ip);
-    ip += 4;
-#ifdef LDM_DEBUG
-    printf("Offset: %zu\n", offset);
-#endif
-    match = op - offset;
- //   LDM_write32(op, (U32)offset);
-
-    /* get matchlength */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-    length += MINMATCH;
-#ifdef LDM_DEBUG
-    printf("Match length: %zu\n", length);
-#endif
-    /* copy match */
-    cpy = op + length;
-
-    // Inefficient for now
-    while (match < cpy - offset && op < oend) {
-      *op++ = *match++;
-    }
-  }
-  return op - (BYTE *)dst;
-}
-
-
diff --git a/contrib/long_distance_matching/versions/v2/ldm.h b/contrib/long_distance_matching/versions/v2/ldm.h
deleted file mode 100644
index 0ac7b2ec..00000000
--- a/contrib/long_distance_matching/versions/v2/ldm.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef LDM_H
-#define LDM_H
-
-#include <stddef.h>   /* size_t */
-
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
-
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize);
-
-size_t LDM_decompress(const void *src, size_t srcSize,
-                      void *dst, size_t maxDstSize);
-
-void LDM_read_header(const void *src, size_t *compressSize,
-                     size_t *decompressSize);
-
-#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v2/main-ldm.c b/contrib/long_distance_matching/versions/v2/main-ldm.c
deleted file mode 100644
index 0017335b..00000000
--- a/contrib/long_distance_matching/versions/v2/main-ldm.c
+++ /dev/null
@@ -1,474 +0,0 @@
-// TODO: file size must fit into a U32
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zstd.h>
-
-#include <fcntl.h>
-#include "ldm.h"
-
-// #define BUF_SIZE 16*1024  // Block size
-#define DEBUG
-
-//#define ZSTD
-
-/* Compress file given by fname and output to oname.
- * Returns 0 if successful, error code otherwise.
- */
-static int compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
-
- /* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */ 
-  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
- /* Write a dummy byte at the last location. */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef ZSTD
-  size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
-                                  src, statbuf.st_size, 1);
-#else
-  size_t compressSize = LDM_HEADER_SIZE + 
-      LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, statbuf.st_size);
-
-    // Write compress and decompress size to header 
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressSize, 4);
-  memcpy(dst + 4, &(statbuf.st_size), 4);
-
-#ifdef DEBUG
-  printf("Compressed size: %zu\n", compressSize);
-  printf("Decompressed size: %zu\n", statbuf.st_size);
-#endif
-#endif
-
-  // Truncate file to compressSize.
-  ftruncate(fdout, compressSize);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
-         (double)compressSize / (statbuf.st_size) * 100);
-
-  // Close files.
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Decompress file compressed using LDM_compress.
- * The input file should have the LDM_HEADER followed by payload.
- * Returns 0 if succesful, and an error code otherwise. 
- */
-static int decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* Read the header. */
-  size_t compressSize, decompressSize;
-  LDM_read_header(src, &compressSize, &decompressSize);
-
-#ifdef DEBUG
-  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
-         statbuf.st_size, compressSize, decompressSize);
-#endif
-
-  /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef ZSTD
-  size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
-                                  src + LDM_HEADER_SIZE,
-                                  statbuf.st_size - LDM_HEADER_SIZE);
-#else
-  size_t outSize = LDM_decompress(
-      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
-      dst, decompressSize);
-
-  printf("Ret size out: %zu\n", outSize);
-  #endif
-  ftruncate(fdout, outSize);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Compare two files.
- * Returns 0 iff they are the same.
- */
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) break;
-
-    if (0 == result) result = memcmp(b0, b1, r0);
-  }
-  return result;
-}
-
-/* Verify the input file is the same as the decompressed file. */
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-	const int cmp = compare(inpFp, decFp);
-	if(0 == cmp) {
-		printf("verify : OK\n");
-	} else {
-		printf("verify : NG\n");
-	}
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  struct timeval tv1, tv2;
-
-  /* Compress */
-
-  gettimeofday(&tv1, NULL);
-  if (compress(inpFilename, ldmFilename)) {
-      printf("Compress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-         (double) (tv2.tv_sec - tv1.tv_sec));
-
-  /* Decompress */
-
-  gettimeofday(&tv1, NULL);
-  if (decompress(ldmFilename, decFilename)) {
-      printf("Decompress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-        (double) (tv2.tv_sec - tv1.tv_sec));
-
-  /* verify */
-  verify(inpFilename, decFilename);
-  return 0;
-}
-
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
-#endif
-

From e0d416246403c6606bff15eb844b6c12155751de Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 12 Jul 2017 09:50:24 -0700
Subject: [PATCH 036/100] Minor fix for non-rolling hash

---
 contrib/long_distance_matching/ldm.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 79648097..a1fe6174 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -518,10 +518,11 @@ static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
   while (cctx->ip < cctx->anchor + MINMATCH + matchLength + literalLength) {
 //    printf("Loop\n");
     if (cctx->ip > cctx->lastPosHashed) {
-      LDM_updateLastHashFromNextHash(cctx);
-//      LDM_putHashOfCurrentPosition(cctx);
 #ifdef LDM_ROLLING_HASH
+      LDM_updateLastHashFromNextHash(cctx);
       LDM_setNextHash(cctx);
+#else
+      LDM_putHashOfCurrentPosition(cctx);
 #endif
     }
     /*
@@ -594,9 +595,10 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
     // Set start of next block to current input pointer.
     cctx.anchor = cctx.ip;
+#ifdef LDM_ROLLING_HASH
     LDM_updateLastHashFromNextHash(&cctx);
-//    LDM_putHashOfCurrentPosition(&cctx);
-#ifndef LDM_ROLLING_HASH
+#else
+    LDM_putHashOfCurrentPosition(&cctx);
     cctx.ip++;
 #endif
 

From 3a48ffd4fd7a63ea36e2abae141a4fd4b7df847d Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 12 Jul 2017 10:53:19 -0700
Subject: [PATCH 037/100] Fix sumToHash to use hash space more efficiently

---
 contrib/long_distance_matching/ldm.c | 52 +++++++++++++++++++++++-----
 1 file changed, 44 insertions(+), 8 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index a1fe6174..4ed09cff 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -7,9 +7,9 @@
 #include "ldm.h"
 #include "util.h"
 
-#define HASH_EVERY 1
+#define HASH_EVERY 7
 
-#define LDM_MEMORY_USAGE 22
+#define LDM_MEMORY_USAGE 18
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
@@ -132,7 +132,7 @@ static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
     curMatch += 8;
   }
   if (lengthLeft > 0) {
-    return LDM_read32(curP) == LDM_read32(curMatch);
+    return (LDM_read32(curP) == LDM_read32(curMatch));
   }
   return 1;
 }
@@ -144,8 +144,15 @@ static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
  * Convert a sum computed from LDM_getRollingHash to a hash value in the range
  * of the hash table.
  */
+#define LDM_SUM2HASH2(s1,s2) (((s1) + (s2)) & 0xFFFF)
+#define LDM_SUM2HASH(sum) (LDM_SUM2HASH2((sum)&0xFFFF,(sum)>>16))
+
 static hash_t LDM_sumToHash(U32 sum) {
-  return sum & (LDM_HASH_SIZE_U32 - 1);
+//  return sum & (LDM_HASH_SIZE_U32 - 1);
+//  return sum % (LDM_HASHTABLESIZE_U32 );
+
+  return ((sum* 2654435761U) >> ((32)-LDM_HASHLOG));
+//    return LDM_SUM2HASH2(sum&0xFFFF, sum >> 16);
 }
 
 static U32 LDM_getRollingHash(const char *data, U32 len) {
@@ -240,15 +247,32 @@ static void LDM_putHashOfCurrentPositionFromHash(
   }
   */
 #ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
+  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32 ) {
     offset_t offset = (cctx->hashTable)[hash].offset;
     cctx->stats.numHashInserts++;
-    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+    if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+//      printf("%u %u %zu\n", hash, offset, cctx->ip - cctx->ibase);
+//      printf("TST: %u %u\n", LDM_read32(cctx->ip), LDM_read32(offset + cctx->ibase));
       cctx->stats.numCollisions++;
     }
   }
+
 #endif
-  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
+
+  if (((cctx->ip - cctx->ibase) & HASH_EVERY) == HASH_EVERY) {
+#ifdef COMPUTE_STATS
+    /*
+    offset_t offset = (cctx->hashTable)[hash].offset;
+    if (offset == 0) {
+      printf("NEW HASH: %u\n", hash);
+    }
+    */
+#endif
+
+    (cctx->hashTable)[hash] = (LDM_hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
+  }
+
+  // Book-keeping
   cctx->lastPosHashed = cctx->ip;
   cctx->lastHash = hash;
   cctx->lastSum = sum;
@@ -296,7 +320,8 @@ static void LDM_putHashOfCurrentPositionFromHash(
   if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
     offset_t offset = (cctx->hashTable)[hash].offset;
     cctx->stats.numHashInserts++;
-    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+
+    if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
       cctx->stats.numCollisions++;
     }
   }
@@ -629,6 +654,17 @@ _last_literals:
     cctx.op += lastRun;
   }
   LDM_printCompressStats(&cctx.stats);
+
+  {
+    U32 tmp = 0;
+    U32 ctr = 0;
+    for (; tmp < LDM_HASH_SIZE_U32; tmp++) {
+      if ((cctx.hashTable)[tmp].offset == 0) {
+        ctr++;
+      }
+    }
+    printf("HASH: %u %u\n", ctr, LDM_HASH_SIZE_U32);
+  }
   return (cctx.op - (const BYTE *)cctx.obase);
 }
 

From 8ef666c32509e35ebbae8229a2b15ca9d4af493d Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 12 Jul 2017 14:23:34 -0700
Subject: [PATCH 038/100] slightly increased buffer pool, to cover normal "full
 load" scenarios

2 buffers per active worker
+ 1 buffer for input loading
+ 1 buffer for "next input" when submitting current one
+ 1 buffer stuck in queue
---
 NEWS                           | 3 ++-
 lib/compress/zstdmt_compress.c | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 5bf184e8..45710507 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
 v1.3.1
+perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt
 build: fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (#718)
-API exp : breaking change : ZSTD_getframeHeader() 
+API exp : breaking change : ZSTD_getframeHeader() provides more information
 
 v1.3.0
 cli : new : `--list` command, by Paul Cruz
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 3fdfe9e1..677e96f0 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -93,7 +93,7 @@ typedef struct ZSTDMT_bufferPool_s {
 
 static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_customMem cMem)
 {
-    unsigned const maxNbBuffers = 2*nbThreads + 2;
+    unsigned const maxNbBuffers = 2*nbThreads + 3;
     ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
         sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
     if (bufPool==NULL) return NULL;

From 8ff8cdb15bae9c2d5db78477958ff90172cb83b9 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 12 Jul 2017 15:11:06 -0700
Subject: [PATCH 039/100] [ldm] Clean up code

---
 contrib/long_distance_matching/ldm.c          | 515 ++++++--------
 contrib/long_distance_matching/ldm.h          |  33 +
 contrib/long_distance_matching/main-ldm.c     |  20 +-
 .../versions/v0.5/Makefile                    |  40 ++
 .../versions/v0.5/ldm.c                       | 659 ++++++++++++++++++
 .../versions/v0.5/ldm.h                       |  26 +
 .../versions/v0.5/main-ldm.c                  | 468 +++++++++++++
 .../versions/v0.5/util.c                      |  69 ++
 .../versions/v0.5/util.h                      |  25 +
 9 files changed, 1528 insertions(+), 327 deletions(-)
 create mode 100644 contrib/long_distance_matching/versions/v0.5/Makefile
 create mode 100644 contrib/long_distance_matching/versions/v0.5/ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.5/ldm.h
 create mode 100644 contrib/long_distance_matching/versions/v0.5/main-ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.5/util.c
 create mode 100644 contrib/long_distance_matching/versions/v0.5/util.h

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 4ed09cff..b17a0f15 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -3,34 +3,30 @@
 #include <stdint.h>
 #include <stdio.h>
 
-
 #include "ldm.h"
 #include "util.h"
 
-#define HASH_EVERY 7
+// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
+#define HASH_ONLY_EVERY 0
 
-#define LDM_MEMORY_USAGE 18
+#define LDM_MEMORY_USAGE 20
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
 
 #define LDM_OFFSET_SIZE 4
 
-#define WINDOW_SIZE (1 << 23)
-#define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 4
-#define LDM_HASH_LENGTH 4
+#define WINDOW_SIZE (1 << 20)
 
-// Should be multiple of four
-#define MINMATCH 4
+//These should be multiples of four.
+#define LDM_HASH_LENGTH 100
+#define MINMATCH 100
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
-#define LDM_ROLLING_HASH
 #define COMPUTE_STATS
 //#define RUN_CHECKS
 //#define LDM_DEBUG
@@ -45,10 +41,9 @@ typedef uint32_t offset_t;
 typedef uint32_t hash_t;
 typedef signed char schar;
 
-
-typedef struct LDM_hashEntry {
+typedef struct hashEntry {
   offset_t offset;
-} LDM_hashEntry;
+} hashEntry;
 
 typedef struct LDM_compressStats {
   U32 numMatches;
@@ -60,24 +55,6 @@ typedef struct LDM_compressStats {
   U32 numHashInserts;
 } LDM_compressStats;
 
-static void LDM_printCompressStats(const LDM_compressStats *stats) {
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->numMatches);
-  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
-                                         (double)stats->numMatches);
-  printf("Average literal length: %.1f\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
-  printf("Average offset length: %.1f\n",
-         ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
-         stats->numCollisions, stats->numHashInserts,
-         stats->numHashInserts == 0 ?
-            1.0 : (100.0 * (double)stats->numCollisions) /
-                  (double)stats->numHashInserts);
-  printf("=====================\n");
-}
-
 typedef struct LDM_CCtx {
   size_t isize;             /* Input size */
   size_t maxOSize;          /* Maximum output size */
@@ -101,25 +78,79 @@ typedef struct LDM_CCtx {
 
   LDM_compressStats stats;            /* Compression statistics */
 
-  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+  hashEntry hashTable[LDM_HASHTABLESIZE_U32];
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  const BYTE *nextIp;
+  U32 lastSum;
+
+  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
   const BYTE *nextPosHashed;
   hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
-
-  // Members for rolling hash.
-  U32 lastSum;
   U32 nextSum;
 
-  unsigned step;
+  unsigned step;                      // ip step, should be 1.
 
   // DEBUG
   const BYTE *DEBUG_setNextHash;
 } LDM_CCtx;
 
+/**
+ * Outputs compression statistics.
+ */
+static void printCompressStats(const LDM_CCtx *cctx) {
+  const LDM_compressStats *stats = &(cctx->stats);
+#ifdef COMPUTE_STATS
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Total number of matches: %u\n", stats->numMatches);
+  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
+                                         (double)stats->numMatches);
+  printf("Average literal length: %.1f\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
+  printf("Average offset length: %.1f\n",
+         ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
+         stats->numCollisions, stats->numHashInserts,
+         stats->numHashInserts == 0 ?
+            1.0 : (100.0 * (double)stats->numCollisions) /
+                  (double)stats->numHashInserts);
+
+  // Output occupancy of hash table.
+  {
+    U32 i = 0;
+    U32 ctr = 0;
+    for (; i < LDM_HASHTABLESIZE_U32; i++) {
+      if ((cctx->hashTable)[i].offset == 0) {
+        ctr++;
+      }
+    }
+    printf("Hash table size, empty slots, %% empty: %u %u %.3f\n",
+           LDM_HASHTABLESIZE_U32, ctr,
+           100.0 * (double)(ctr) / (double)LDM_HASHTABLESIZE_U32);
+  }
+
+  printf("=====================\n");
+#endif
+}
+
+/**
+ * Checks whether the MINMATCH bytes from p are the same as the MINMATCH
+ * bytes from match.
+ *
+ * This assumes MINMATCH is a multiple of four.
+ *
+ * Return 1 if valid, 0 otherwise.
+ */
 static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
+  /*
+  if (memcmp(p, match, MINMATCH) == 0) {
+    return 1;
+  }
+  return 0;
+  */
+
+  //TODO: This seems to be faster for some reason?
   U16 lengthLeft = MINMATCH;
   const BYTE *curP = p;
   const BYTE *curMatch = match;
@@ -137,25 +168,22 @@ static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
   return 1;
 }
 
-
-
-#ifdef LDM_ROLLING_HASH
 /**
- * Convert a sum computed from LDM_getRollingHash to a hash value in the range
+ * Convert a sum computed from getChecksum to a hash value in the range
  * of the hash table.
  */
-#define LDM_SUM2HASH2(s1,s2) (((s1) + (s2)) & 0xFFFF)
-#define LDM_SUM2HASH(sum) (LDM_SUM2HASH2((sum)&0xFFFF,(sum)>>16))
-
-static hash_t LDM_sumToHash(U32 sum) {
-//  return sum & (LDM_HASH_SIZE_U32 - 1);
-//  return sum % (LDM_HASHTABLESIZE_U32 );
-
-  return ((sum* 2654435761U) >> ((32)-LDM_HASHLOG));
-//    return LDM_SUM2HASH2(sum&0xFFFF, sum >> 16);
+static hash_t checksumToHash(U32 sum) {
+  return ((sum * 2654435761U) >> ((32)-LDM_HASHLOG));
 }
 
-static U32 LDM_getRollingHash(const char *data, U32 len) {
+/**
+ * Computes a checksum based on rsync's checksum.
+ *
+ * a(k,l) = \sum_{i = k}^l x_i (mod M)
+ * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
+ * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
+  */
+static U32 getChecksum(const char *data, U32 len) {
   U32 i;
   U32 s1, s2;
   const schar *buf = (const schar *)data;
@@ -173,34 +201,31 @@ static U32 LDM_getRollingHash(const char *data, U32 len) {
   return (s1 & 0xffff) + (s2 << 16);
 }
 
-typedef struct LDM_sumStruct {
-  U16 s1, s2;
-} LDM_sumStruct;
-
-static U32 LDM_updateRollingHash(U32 sum, U32 len,
-                                 schar toRemove, schar toAdd) {
+/**
+ * Update a checksum computed from getChecksum(data, len).
+ *
+ * The checksum can be updated along its ends as follows:
+ * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
+ * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
+ *
+ * Thus toRemove should correspond to data[0].
+ */
+static U32 updateChecksum(U32 sum, U32 len,
+                          schar toRemove, schar toAdd) {
   U32 s1 = (sum & 0xffff) - toRemove + toAdd;
   U32 s2 = (sum >> 16) - (toRemove * len) + s1;
 
   return (s1 & 0xffff) + (s2 << 16);
 }
 
-
-/*
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
-}
-*/
-
-/*
-static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
-  sumStruct->s1 = sum & 0xffff;
-  sumStruct->s2 = sum >> 16;
-}
-*/
-
-static void LDM_setNextHash(LDM_CCtx *cctx) {
-
+/**
+ * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
+ * based on cctx->lastSum and cctx->lastPosHashed.
+ *
+ * This uses a rolling hash and requires that the last position hashed
+ * corresponds to cctx->nextIp - step.
+ */
+static void setNextHash(LDM_CCtx *cctx) {
 #ifdef RUN_CHECKS
   U32 check;
   if ((cctx->nextIp - cctx->ibase != 1) &&
@@ -212,160 +237,100 @@ static void LDM_setNextHash(LDM_CCtx *cctx) {
   cctx->DEBUG_setNextHash = cctx->nextIp;
 #endif
 
-//  cctx->nextSum = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
-  cctx->nextSum = LDM_updateRollingHash(
+//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  cctx->nextSum = updateChecksum(
       cctx->lastSum, LDM_HASH_LENGTH,
       (schar)((cctx->lastPosHashed)[0]),
       (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
+  cctx->nextPosHashed = cctx->nextIp;
+  cctx->nextHash = checksumToHash(cctx->nextSum);
 
 #ifdef RUN_CHECKS
-  check = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  check = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
 
   if (check != cctx->nextSum) {
     printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
-//    printf("INFO: %u %u %u\n", LDM_read32(cctx->nextIp),
   }
-#endif
-  cctx->nextPosHashed = cctx->nextIp;
-  cctx->nextHash = LDM_sumToHash(cctx->nextSum);
 
-#ifdef RUN_CHECKS
   if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
     printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
             cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
             cctx->ip - cctx->ibase);
   }
 #endif
-
 }
 
-static void LDM_putHashOfCurrentPositionFromHash(
+static void putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash, U32 sum) {
-  /*
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  */
 #ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32 ) {
+  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
     offset_t offset = (cctx->hashTable)[hash].offset;
     cctx->stats.numHashInserts++;
     if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
-//      printf("%u %u %zu\n", hash, offset, cctx->ip - cctx->ibase);
-//      printf("TST: %u %u\n", LDM_read32(cctx->ip), LDM_read32(offset + cctx->ibase));
       cctx->stats.numCollisions++;
     }
   }
-
 #endif
 
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) == HASH_EVERY) {
-#ifdef COMPUTE_STATS
-    /*
-    offset_t offset = (cctx->hashTable)[hash].offset;
-    if (offset == 0) {
-      printf("NEW HASH: %u\n", hash);
-    }
-    */
-#endif
-
-    (cctx->hashTable)[hash] = (LDM_hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
+  // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
+  // Note: this works only when cctx->step is 1.
+  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+    (cctx->hashTable)[hash] = (hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
   }
 
-  // Book-keeping
   cctx->lastPosHashed = cctx->ip;
   cctx->lastHash = hash;
   cctx->lastSum = sum;
 }
 
+/**
+ * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
+ * fields from the "next" fields.
+ *
+ * This requires that cctx->ip == cctx->nextPosHashed.
+ */
 static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
 #ifdef RUN_CHECKS
   if (cctx->ip != cctx->nextPosHashed) {
-    printf("CHECK failed: updateLastHashFromNextHash %zu\n", cctx->ip - cctx->ibase);
+    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
+           cctx->ip - cctx->ibase);
   }
 #endif
-  LDM_putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
+  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
 }
 
+/**
+ * Insert hash of the current position into the hash table.
+ */
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = LDM_getRollingHash((const char *)cctx->ip, LDM_HASH_LENGTH);
-  hash_t hash = LDM_sumToHash(sum);
+  U32 sum = getChecksum((const char *)cctx->ip, LDM_HASH_LENGTH);
+  hash_t hash = checksumToHash(sum);
+
 #ifdef RUN_CHECKS
   if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
-    printf("CHECK failed: putHashOfCurrentPosition %zu\n", cctx->ip - cctx->ibase);
-  }
-#endif
-//  hash_t hash = LDM_hashPosition(cctx->ip);
-  LDM_putHashOfCurrentPositionFromHash(cctx, hash, sum);
-//  printf("Offset %zu\n", cctx->ip - cctx->ibase);
-}
-
-#else
-static hash_t LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
-}
-
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-
-static void LDM_putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash) {
-  /*
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  */
-#ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = (cctx->hashTable)[hash].offset;
-    cctx->stats.numHashInserts++;
-
-    if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
-      cctx->stats.numCollisions++;
-    }
+    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
+           cctx->ip - cctx->ibase);
   }
 #endif
 
-  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
-#ifdef RUN_CHECKS
-  if (cctx->ip - cctx->lastPosHashed != 1) {
-    printf("putHashError\n");
-  }
-#endif
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
+  putHashOfCurrentPositionFromHash(cctx, hash, sum);
 }
 
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  hash_t hash = LDM_hashPosition(cctx->ip);
-  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
+/**
+ * Returns the position of the entry at hashTable[hash].
+ */
+static const BYTE *getPositionOnHash(LDM_CCtx *cctx, hash_t hash) {
+  return cctx->hashTable[hash].offset + cctx->ibase;
 }
 
-#endif
-
-/*
-static hash_t LDM_hash5(U64 sequence) {
-  static const U64 prime5bytes = 889523592379ULL;
-  static const U64 prime8bytes = 11400714785074694791ULL;
-  const U32 hashLog = LDM_HASHLOG;
-  if (LDM_isLittleEndian())
-    return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
-  else
-    return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
-}
-*/
-
-
-static const BYTE *LDM_getPositionOnHash(
-    hash_t h, void *tableBase, const BYTE *srcBase) {
-  const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
-  return hashTable[h].offset + srcBase;
-}
-
-
-static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
-                          const BYTE *pInLimit) {
+/**
+ *  Counts the number of bytes that match from pIn and pMatch,
+ *  up to pInLimit.
+ *
+ *  TODO: make more efficient.
+ */
+static unsigned countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                                 const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
   while (pIn < pInLimit - 1) {
     BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
@@ -386,9 +351,12 @@ void LDM_readHeader(const void *src, size_t *compressSize,
   *decompressSize = *ip;
 }
 
-static void LDM_initializeCCtx(LDM_CCtx *cctx,
-                               const void *src, size_t srcSize,
-                               void *dst, size_t maxDstSize) {
+/**
+ * Initialize a compression context.
+ */
+static void initializeCCtx(LDM_CCtx *cctx,
+                           const void *src, size_t srcSize,
+                           void *dst, size_t maxDstSize) {
   cctx->isize = srcSize;
   cctx->maxOSize = maxDstSize;
 
@@ -396,11 +364,7 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->ip = cctx->ibase;
   cctx->iend = cctx->ibase + srcSize;
 
-#ifdef LDM_ROLLING_HASH
   cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
-#else
-  cctx->ihashLimit = cctx->iend - HASH_SIZE;
-#endif
   cctx->imatchLimit = cctx->iend - MINMATCH;
 
   cctx->obase = (BYTE *)dst;
@@ -413,23 +377,27 @@ static void LDM_initializeCCtx(LDM_CCtx *cctx,
 
   cctx->lastPosHashed = NULL;
 
-  cctx->step = 1;
+  cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
   cctx->nextIp = cctx->ip + cctx->step;
   cctx->nextPosHashed = 0;
 
   cctx->DEBUG_setNextHash = 0;
 }
 
-#ifdef LDM_ROLLING_HASH
+/**
+ * Finds the "best" match.
+ *
+ * Returns 0 if successful and 1 otherwise (i.e. no match can be found
+ * in the remaining input that is long enough).
+ *
+ */
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
   cctx->nextIp = cctx->ip + cctx->step;
 
   do {
     hash_t h;
     U32 sum;
-//    printf("Call A\n");
-    LDM_setNextHash(cctx);
-//    printf("End call a\n");
+    setNextHash(cctx);
     h = cctx->nextHash;
     sum = cctx->nextSum;
     cctx->ip = cctx->nextIp;
@@ -439,62 +407,27 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
       return 1;
     }
 
-    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
-
-//    // Compute cctx->nextSum and cctx->nextHash from cctx->nextIp.
-//    LDM_setNextHash(cctx);
-    LDM_putHashOfCurrentPositionFromHash(cctx, h, sum);
-
-//    printf("%u %u\n", cctx->lastHash, cctx->nextHash);
-  } while (cctx->ip - *match > WINDOW_SIZE ||
-           !LDM_isValidMatch(cctx->ip, *match));
-//           LDM_read64(*match) != LDM_read64(cctx->ip));
-  LDM_setNextHash(cctx);
-  return 0;
-}
-#else
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
-  cctx->nextIp = cctx->ip;
-
-  do {
-    hash_t const h = cctx->nextHash;
-    cctx->ip = cctx->nextIp;
-    cctx->nextIp += cctx->step;
-
-    if (cctx->ip > cctx->imatchLimit) {
-      return 1;
-    }
-
-    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
-
-    cctx->nextHash = LDM_hashPosition(cctx->nextIp);
-    LDM_putHashOfCurrentPositionFromHash(cctx, h);
+    *match = getPositionOnHash(cctx, h);
+    putHashOfCurrentPositionFromHash(cctx, h, sum);
 
   } while (cctx->ip - *match > WINDOW_SIZE ||
            !LDM_isValidMatch(cctx->ip, *match));
+  setNextHash(cctx);
   return 0;
 }
 
-#endif
-
 /**
  * Write current block (literals, literal length, match offset,
  * match length).
  *
- * Update input pointer, inserting hashes into hash table along the
- * way.
+ * Update input pointer, inserting hashes into hash table along the way.
  */
-static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
-  unsigned const literalLength = (unsigned)(cctx->ip - cctx->anchor);
-  unsigned const offset = cctx->ip - match;
-  unsigned const matchLength = LDM_count(
-      cctx->ip + MINMATCH, match + MINMATCH, cctx->ihashLimit);
+static void outputBlock(LDM_CCtx *cctx,
+                        unsigned const literalLength,
+                        unsigned const offset,
+                        unsigned const matchLength) {
   BYTE *token = cctx->op++;
 
-  cctx->stats.totalLiteralLength += literalLength;
-  cctx->stats.totalOffset += offset;
-  cctx->stats.totalMatchLength += matchLength + MINMATCH;
-
   /* Encode the literal length. */
   if (literalLength >= RUN_MASK) {
     int len = (int)literalLength - RUN_MASK;
@@ -515,7 +448,7 @@ static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
   LDM_write32(cctx->op, offset);
   cctx->op += LDM_OFFSET_SIZE;
 
-  /* Encode match length */
+  /* Encode the match length. */
   if (matchLength >= ML_MASK) {
     unsigned matchLengthRemaining = matchLength;
     *token += ML_MASK;
@@ -531,62 +464,21 @@ static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
   } else {
     *token += (BYTE)(matchLength);
   }
-
-//  LDM_setNextHash(cctx);
-//  cctx->ip = cctx->lastPosHashed + 1;
-//  cctx->nextIp = cctx->ip + cctx->step;
-//  printf("HERE: %zu %zu %zu\n", cctx->ip - cctx->ibase,
-//         cctx->lastPosHashed - cctx->ibase, cctx->nextIp - cctx->ibase);
-
-  cctx->nextIp = cctx->ip + cctx->step;
-
-  while (cctx->ip < cctx->anchor + MINMATCH + matchLength + literalLength) {
-//    printf("Loop\n");
-    if (cctx->ip > cctx->lastPosHashed) {
-#ifdef LDM_ROLLING_HASH
-      LDM_updateLastHashFromNextHash(cctx);
-      LDM_setNextHash(cctx);
-#else
-      LDM_putHashOfCurrentPosition(cctx);
-#endif
-    }
-    /*
-    printf("Call b %zu %zu %zu\n",
-            cctx->lastPosHashed - cctx->ibase,
-            cctx->nextIp - cctx->ibase,
-            cctx->ip - cctx->ibase);
-            */
-//    printf("end call b\n");
-    cctx->ip++;
-    cctx->nextIp++;
-  }
-
-//  printf("There: %zu %zu\n", cctx->ip - cctx->ibase, cctx->lastPosHashed - cctx->ibase);
 }
 
 // TODO: srcSize and maxDstSize is unused
+// This is based upon lz4.
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
-  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+  initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
-#ifdef LDM_ROLLING_HASH
-//  LDM_setNextHash(&cctx);
-//  tmp_hash = LDM_updateRollingHash(cctx.lastSum, LDM_HASH_LENGTH,
-//                                   cctx.ip[0], cctx.ip[LDM_HASH_LENGTH]);
-//  printf("Update test: %u %u\n", tmp_hash, cctx.nextSum);
-//  cctx.ip++;
-#else
-  cctx.ip++;
-  cctx.nextHash = LDM_hashPosition(cctx.ip);
-#endif
 
   // TODO: loop condition is not accurate.
   while (1) {
     const BYTE *match;
-//    printf("Start of loop\n");
 
     /**
      * Find a match.
@@ -597,16 +489,15 @@ size_t LDM_compress(const void *src, size_t srcSize,
     if (LDM_findBestMatch(&cctx, &match) != 0) {
       goto _last_literals;
     }
-//    printf("End of match finding\n");
-
+#ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
+#endif
 
     /**
      * Catch up: look back to extend the match backwards from the found match.
      */
     while (cctx.ip > cctx.anchor && match > cctx.ibase &&
            cctx.ip[-1] == match[-1]) {
-//      printf("Catch up\n");
       cctx.ip--;
       match--;
     }
@@ -615,26 +506,35 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * Write current block (literals, literal length, match offset, match
      * length) and update pointers and hashes.
      */
-    LDM_outputBlock(&cctx, match);
-//    printf("End of loop\n");
+    {
+      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
+      unsigned const offset = cctx.ip - match;
+      unsigned const matchLength = countMatchLength(
+          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
+
+#ifdef COMPUTE_STATS
+      cctx.stats.totalLiteralLength += literalLength;
+      cctx.stats.totalOffset += offset;
+      cctx.stats.totalMatchLength += matchLength + MINMATCH;
+#endif
+      outputBlock(&cctx, literalLength, offset, matchLength);
+
+      // Move ip to end of block, inserting hashes at each position.
+      cctx.nextIp = cctx.ip + cctx.step;
+      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
+        if (cctx.ip > cctx.lastPosHashed) {
+          // TODO: Simplify.
+          LDM_updateLastHashFromNextHash(&cctx);
+          setNextHash(&cctx);
+        }
+        cctx.ip++;
+        cctx.nextIp++;
+      }
+    }
 
     // Set start of next block to current input pointer.
     cctx.anchor = cctx.ip;
-#ifdef LDM_ROLLING_HASH
     LDM_updateLastHashFromNextHash(&cctx);
-#else
-    LDM_putHashOfCurrentPosition(&cctx);
-    cctx.ip++;
-#endif
-
-    /*
-    LDM_putHashOfCurrentPosition(&cctx);
-    printf("Call c\n");
-    LDM_setNextHash(&cctx);
-    printf("End call c\n");
-    cctx.ip++;
-    cctx.nextIp++;
-    */
   }
 _last_literals:
   /* Encode the last literals (no more matches). */
@@ -653,18 +553,11 @@ _last_literals:
     memcpy(cctx.op, cctx.anchor, lastRun);
     cctx.op += lastRun;
   }
-  LDM_printCompressStats(&cctx.stats);
 
-  {
-    U32 tmp = 0;
-    U32 ctr = 0;
-    for (; tmp < LDM_HASH_SIZE_U32; tmp++) {
-      if ((cctx.hashTable)[tmp].offset == 0) {
-        ctr++;
-      }
-    }
-    printf("HASH: %u %u\n", ctr, LDM_HASH_SIZE_U32);
-  }
+#ifdef COMPUTE_STATS
+  printCompressStats(&cctx);
+#endif
+
   return (cctx.op - (const BYTE *)cctx.obase);
 }
 
@@ -715,7 +608,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
       } while (s == 255);
     }
 
-    /* Copy literals. */
+    /* Copy the literals. */
     cpy = dctx.op + length;
     memcpy(dctx.op, dctx.ip, length);
     dctx.ip += length;
@@ -748,20 +641,20 @@ size_t LDM_decompress(const void *src, size_t compressSize,
   return dctx.op - (BYTE *)dst;
 }
 
+/*
 void LDM_test(const void *src, size_t srcSize,
               void *dst, size_t maxDstSize) {
-#ifdef LDM_ROLLING_HASH
   const BYTE *ip = (const BYTE *)src + 1125;
-  U32 sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
+  U32 sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
   U32 sum2;
   ++ip;
   for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
-    sum2 = LDM_updateRollingHash(sum, LDM_HASH_LENGTH,
+    sum2 = updateChecksum(sum, LDM_HASH_LENGTH,
                                  ip[-1], ip[LDM_HASH_LENGTH - 1]);
-    sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
+    sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
     printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
   }
-#endif
 }
+*/
 
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index a34faac4..d7f977d9 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -7,12 +7,45 @@
 #define LDM_DECOMPRESS_SIZE 4
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
 
+/**
+ *  Compresses src into dst.
+ *
+ *  NB: This currently ignores maxDstSize and assumes enough space is available.
+ *
+ *  Block format (see lz4 documentation for more information):
+ *  github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
+ *
+ *  A block is composed of sequences. Each sequence begins with a token, which
+ *  is a one-byte value separated into two 4-bit fields.
+ *
+ *  The first field uses the four high bits of the token and encodes the literal
+ *  length. If the field value is 0, there is no literal. If it is 15,
+ *  additional bytes are added (each ranging from 0 to 255) to the previous
+ *  value to produce a total length.
+ *
+ *  Following the token and optional length bytes are the literals.
+ *
+ *  Next are the 4 bytes representing the offset of the match (2 in lz4),
+ *  representing the position to copy the literals.
+ *
+ *  The lower four bits of the token encode the match length. With additional
+ *  bytes added similarly to the additional literal length bytes after the offset.
+ *
+ *  The last sequence is incomplete and stops right after the lieterals.
+ *
+ */
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize);
 
 size_t LDM_decompress(const void *src, size_t srcSize,
                       void *dst, size_t maxDstSize);
 
+/**
+ * Reads the header from src and writes the compressed size and
+ * decompressed size into compressSize and decompressSize respectively.
+ *
+ * NB: LDM_compress and LDM_decompress currently do not add/read headers.
+ */
 void LDM_readHeader(const void *src, size_t *compressSize,
                     size_t *decompressSize);
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index f8ae5469..fbfd789b 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -13,12 +13,9 @@
 #include <fcntl.h>
 #include "ldm.h"
 
-// #define BUF_SIZE 16*1024  // Block size
 #define DEBUG
 //#define TEST
 
-//#define ZSTD
-
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
  */
@@ -75,28 +72,25 @@ static int compress(const char *fname, const char *oname) {
       return 1;
   }
 
+/*
 #ifdef TEST
   LDM_test(src, statbuf.st_size,
            dst + LDM_HEADER_SIZE, statbuf.st_size);
 #endif
+*/
 
-#ifdef ZSTD
-  compressSize = ZSTD_compress(dst, statbuf.st_size,
-                                  src, statbuf.st_size, 1);
-#else
   compressSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
                    dst + LDM_HEADER_SIZE, statbuf.st_size);
 
-    // Write compress and decompress size to header
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  // Write compress and decompress size to header
+  // TODO: should depend on LDM_DECOMPRESS_SIZE write32
   memcpy(dst, &compressSize, 4);
   memcpy(dst + 4, &(statbuf.st_size), 4);
 
 #ifdef DEBUG
   printf("Compressed size: %zu\n", compressSize);
   printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
-#endif
 #endif
 
   // Truncate file to compressSize.
@@ -169,17 +163,11 @@ static int decompress(const char *fname, const char *oname) {
       return 1;
   }
 
-#ifdef ZSTD
-  outSize = ZSTD_decompress(dst, decomrpessed_size,
-                                  src + LDM_HEADER_SIZE,
-                                  statbuf.st_size - LDM_HEADER_SIZE);
-#else
   outSize = LDM_decompress(
       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
       dst, decompressSize);
 
   printf("Ret size out: %zu\n", outSize);
-  #endif
   ftruncate(fdout, outSize);
 
   close(fdin);
diff --git a/contrib/long_distance_matching/versions/v0.5/Makefile b/contrib/long_distance_matching/versions/v0.5/Makefile
new file mode 100644
index 00000000..5ffd4eaf
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.5/Makefile
@@ -0,0 +1,40 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+# This Makefile presumes libzstd is installed, using `sudo make install`
+
+CFLAGS  ?= -O3
+DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS)
+
+LDFLAGS += -lzstd
+
+.PHONY: default all clean
+
+default: all
+
+all: main-ldm
+
+
+#main : ldm.c main.c
+#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+main-ldm : util.c ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+clean:
+	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
+	main main-ldm
+	@echo Cleaning completed
+
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.c b/contrib/long_distance_matching/versions/v0.5/ldm.c
new file mode 100644
index 00000000..325c5040
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.5/ldm.c
@@ -0,0 +1,659 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "ldm.h"
+#include "util.h"
+
+// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
+#define HASH_ONLY_EVERY 0
+
+#define LDM_MEMORY_USAGE 20
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+
+#define LDM_OFFSET_SIZE 4
+
+#define WINDOW_SIZE (1 << 20)
+
+//These should be multiples of four.
+#define LDM_HASH_LENGTH 100
+#define MINMATCH 100
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+#define COMPUTE_STATS
+//#define RUN_CHECKS
+//#define LDM_DEBUG
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+typedef uint32_t offset_t;
+typedef uint32_t hash_t;
+typedef signed char schar;
+
+typedef struct hashEntry {
+  offset_t offset;
+} hashEntry;
+
+typedef struct LDM_compressStats {
+  U32 numMatches;
+  U32 totalMatchLength;
+  U32 totalLiteralLength;
+  U64 totalOffset;
+
+  U32 numCollisions;
+  U32 numHashInserts;
+} LDM_compressStats;
+
+typedef struct LDM_CCtx {
+  size_t isize;             /* Input size */
+  size_t maxOSize;          /* Maximum output size */
+
+  const BYTE *ibase;        /* Base of input */
+  const BYTE *ip;           /* Current input position */
+  const BYTE *iend;         /* End of input */
+
+  // Maximum input position such that hashing at the position does not exceed
+  // end of input.
+  const BYTE *ihashLimit;
+
+  // Maximum input position such that finding a match of at least the minimum
+  // match length does not exceed end of input.
+  const BYTE *imatchLimit;
+
+  const BYTE *obase;        /* Base of output */
+  BYTE *op;                 /* Output */
+
+  const BYTE *anchor;       /* Anchor to start of current (match) block */
+
+  LDM_compressStats stats;            /* Compression statistics */
+
+  hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+
+  const BYTE *lastPosHashed;          /* Last position hashed */
+  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
+  U32 lastSum;
+
+  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
+  const BYTE *nextPosHashed;
+  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
+  U32 nextSum;
+
+  unsigned step;                      // ip step, should be 1.
+
+  // DEBUG
+  const BYTE *DEBUG_setNextHash;
+} LDM_CCtx;
+
+/**
+ * Outputs compression statistics.
+ */
+static void printCompressStats(const LDM_CCtx *cctx) {
+  const LDM_compressStats *stats = &(cctx->stats);
+#ifdef COMPUTE_STATS
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Total number of matches: %u\n", stats->numMatches);
+  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
+                                         (double)stats->numMatches);
+  printf("Average literal length: %.1f\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
+  printf("Average offset length: %.1f\n",
+         ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
+         stats->numCollisions, stats->numHashInserts,
+         stats->numHashInserts == 0 ?
+            1.0 : (100.0 * (double)stats->numCollisions) /
+                  (double)stats->numHashInserts);
+
+  // Output occupancy of hash table.
+  {
+    U32 i = 0;
+    U32 ctr = 0;
+    for (; i < LDM_HASHTABLESIZE_U32; i++) {
+      if ((cctx->hashTable)[i].offset == 0) {
+        ctr++;
+      }
+    }
+    printf("Hash table size, empty slots, %% empty: %u %u %.3f\n",
+           LDM_HASHTABLESIZE_U32, ctr,
+           100.0 * (double)(ctr) / (double)LDM_HASHTABLESIZE_U32);
+  }
+
+  printf("=====================\n");
+#endif
+}
+
+/**
+ * Checks whether the MINMATCH bytes from p are the same as the MINMATCH
+ * bytes from match.
+ *
+ * This assumes MINMATCH is a multiple of four.
+ *
+ * Return 1 if valid, 0 otherwise.
+ */
+static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
+  /*
+  if (memcmp(p, match, MINMATCH) == 0) {
+    return 1;
+  }
+  return 0;
+  */
+
+  //TODO: This seems to be faster for some reason?
+  U16 lengthLeft = MINMATCH;
+  const BYTE *curP = p;
+  const BYTE *curMatch = match;
+
+  for (; lengthLeft >= 8; lengthLeft -= 8) {
+    if (LDM_read64(curP) != LDM_read64(curMatch)) {
+      return 0;
+    }
+    curP += 8;
+    curMatch += 8;
+  }
+  if (lengthLeft > 0) {
+    return (LDM_read32(curP) == LDM_read32(curMatch));
+  }
+  return 1;
+}
+
+/**
+ * Convert a sum computed from getChecksum to a hash value in the range
+ * of the hash table.
+ */
+static hash_t checksumToHash(U32 sum) {
+  return ((sum * 2654435761U) >> ((32)-LDM_HASHLOG));
+}
+
+/**
+ * Computes a checksum based on rsync's checksum.
+ *
+ * a(k,l) = \sum_{i = k}^l x_i (mod M)
+ * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
+ * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
+  */
+static U32 getChecksum(const char *data, U32 len) {
+  U32 i;
+  U32 s1, s2;
+  const schar *buf = (const schar *)data;
+
+  s1 = s2 = 0;
+  for (i = 0; i < (len - 4); i += 4) {
+    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
+          (2 * buf[i + 2]) + (buf[i + 3]);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
+  }
+  for(; i < len; i++) {
+    s1 += buf[i];
+    s2 += s1;
+  }
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+/**
+ * Update a checksum computed from getChecksum(data, len).
+ *
+ * The checksum can be updated along its ends as follows:
+ * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
+ * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
+ *
+ * Thus toRemove should correspond to data[0].
+ */
+static U32 updateChecksum(U32 sum, U32 len,
+                          schar toRemove, schar toAdd) {
+  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
+  U32 s2 = (sum >> 16) - (toRemove * len) + s1;
+
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+/**
+ * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
+ * based on cctx->lastSum and cctx->lastPosHashed.
+ *
+ * This uses a rolling hash and requires that the last position hashed
+ * corresponds to cctx->nextIp - step.
+ */
+static void setNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  U32 check;
+  if ((cctx->nextIp - cctx->ibase != 1) &&
+      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
+    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
+            cctx->DEBUG_setNextHash - cctx->ibase);
+  }
+
+  cctx->DEBUG_setNextHash = cctx->nextIp;
+#endif
+
+//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  cctx->nextSum = updateChecksum(
+      cctx->lastSum, LDM_HASH_LENGTH,
+      (schar)((cctx->lastPosHashed)[0]),
+      (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
+  cctx->nextPosHashed = cctx->nextIp;
+  cctx->nextHash = checksumToHash(cctx->nextSum);
+
+#ifdef RUN_CHECKS
+  check = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+
+  if (check != cctx->nextSum) {
+    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
+  }
+
+  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
+    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
+            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
+            cctx->ip - cctx->ibase);
+  }
+#endif
+}
+
+static void putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash, U32 sum) {
+#ifdef COMPUTE_STATS
+  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
+    offset_t offset = (cctx->hashTable)[hash].offset;
+    cctx->stats.numHashInserts++;
+    if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
+      cctx->stats.numCollisions++;
+    }
+  }
+#endif
+
+  // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
+  // Note: this works only when cctx->step is 1.
+  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+    (cctx->hashTable)[hash] = (hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
+  }
+
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+  cctx->lastSum = sum;
+}
+
+/**
+ * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
+ * fields from the "next" fields.
+ *
+ * This requires that cctx->ip == cctx->nextPosHashed.
+ */
+static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  if (cctx->ip != cctx->nextPosHashed) {
+    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
+           cctx->ip - cctx->ibase);
+  }
+#endif
+  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
+}
+
+/**
+ * Insert hash of the current position into the hash table.
+ */
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  U32 sum = getChecksum((const char *)cctx->ip, LDM_HASH_LENGTH);
+  hash_t hash = checksumToHash(sum);
+
+#ifdef RUN_CHECKS
+  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
+    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
+           cctx->ip - cctx->ibase);
+  }
+#endif
+
+  putHashOfCurrentPositionFromHash(cctx, hash, sum);
+}
+
+/**
+ * Returns the position of the entry at hashTable[hash].
+ */
+static const BYTE *getPositionOnHash(LDM_CCtx *cctx, hash_t hash) {
+  return cctx->hashTable[hash].offset + cctx->ibase;
+}
+
+/**
+ *  Counts the number of bytes that match from pIn and pMatch,
+ *  up to pInLimit.
+ *
+ *  TODO: make more efficient.
+ */
+static unsigned countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                                 const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (unsigned)(pIn - pStart);
+  }
+  return (unsigned)(pIn - pStart);
+}
+
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize) {
+  const U32 *ip = (const U32 *)src;
+  *compressSize = *ip++;
+  *decompressSize = *ip;
+}
+
+/**
+ * Initialize a compression context.
+ */
+static void initializeCCtx(LDM_CCtx *cctx,
+                           const void *src, size_t srcSize,
+                           void *dst, size_t maxDstSize) {
+  cctx->isize = srcSize;
+  cctx->maxOSize = maxDstSize;
+
+  cctx->ibase = (const BYTE *)src;
+  cctx->ip = cctx->ibase;
+  cctx->iend = cctx->ibase + srcSize;
+
+  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
+  cctx->imatchLimit = cctx->iend - MINMATCH;
+
+  cctx->obase = (BYTE *)dst;
+  cctx->op = (BYTE *)dst;
+
+  cctx->anchor = cctx->ibase;
+
+  memset(&(cctx->stats), 0, sizeof(cctx->stats));
+  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+
+  cctx->lastPosHashed = NULL;
+
+  cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
+  cctx->nextIp = cctx->ip + cctx->step;
+  cctx->nextPosHashed = 0;
+
+  cctx->DEBUG_setNextHash = 0;
+}
+
+/**
+ * Finds the "best" match.
+ *
+ * Returns 0 if successful and 1 otherwise (i.e. no match can be found
+ * in the remaining input that is long enough).
+ *
+ */
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  do {
+    hash_t h;
+    U32 sum;
+    setNextHash(cctx);
+    h = cctx->nextHash;
+    sum = cctx->nextSum;
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
+
+    if (cctx->ip > cctx->imatchLimit) {
+      return 1;
+    }
+
+    *match = getPositionOnHash(cctx, h);
+    putHashOfCurrentPositionFromHash(cctx, h, sum);
+
+  } while (cctx->ip - *match > WINDOW_SIZE ||
+           !LDM_isValidMatch(cctx->ip, *match));
+  setNextHash(cctx);
+  return 0;
+}
+
+/**
+ * Write current block (literals, literal length, match offset,
+ * match length).
+ *
+ * Update input pointer, inserting hashes into hash table along the way.
+ */
+static void outputBlock(LDM_CCtx *cctx,
+                        unsigned const literalLength,
+                        unsigned const offset,
+                        unsigned const matchLength) {
+  BYTE *token = cctx->op++;
+
+  /* Encode the literal length. */
+  if (literalLength >= RUN_MASK) {
+    int len = (int)literalLength - RUN_MASK;
+    *token = (RUN_MASK << ML_BITS);
+    for (; len >= 255; len -= 255) {
+      *(cctx->op)++ = 255;
+    }
+    *(cctx->op)++ = (BYTE)len;
+  } else {
+    *token = (BYTE)(literalLength << ML_BITS);
+  }
+
+  /* Encode the literals. */
+  memcpy(cctx->op, cctx->anchor, literalLength);
+  cctx->op += literalLength;
+
+  /* Encode the offset. */
+  LDM_write32(cctx->op, offset);
+  cctx->op += LDM_OFFSET_SIZE;
+
+  /* Encode the match length. */
+  if (matchLength >= ML_MASK) {
+    unsigned matchLengthRemaining = matchLength;
+    *token += ML_MASK;
+    matchLengthRemaining -= ML_MASK;
+    LDM_write32(cctx->op, 0xFFFFFFFF);
+    while (matchLengthRemaining >= 4*0xFF) {
+      cctx->op += 4;
+      LDM_write32(cctx->op, 0xffffffff);
+      matchLengthRemaining -= 4*0xFF;
+    }
+    cctx->op += matchLengthRemaining / 255;
+    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
+  } else {
+    *token += (BYTE)(matchLength);
+  }
+}
+
+// TODO: srcSize and maxDstSize is unused
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  LDM_CCtx cctx;
+  initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+
+  /* Hash the first position and put it into the hash table. */
+  LDM_putHashOfCurrentPosition(&cctx);
+
+  // TODO: loop condition is not accurate.
+  while (1) {
+    const BYTE *match;
+
+    /**
+     * Find a match.
+     * If no more matches can be found (i.e. the length of the remaining input
+     * is less than the minimum match length), then stop searching for matches
+     * and encode the final literals.
+     */
+    if (LDM_findBestMatch(&cctx, &match) != 0) {
+      goto _last_literals;
+    }
+#ifdef COMPUTE_STATS
+    cctx.stats.numMatches++;
+#endif
+
+    /**
+     * Catch up: look back to extend the match backwards from the found match.
+     */
+    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
+           cctx.ip[-1] == match[-1]) {
+      cctx.ip--;
+      match--;
+    }
+
+    /**
+     * Write current block (literals, literal length, match offset, match
+     * length) and update pointers and hashes.
+     */
+    {
+      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
+      unsigned const offset = cctx.ip - match;
+      unsigned const matchLength = countMatchLength(
+          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
+
+#ifdef COMPUTE_STATS
+      cctx.stats.totalLiteralLength += literalLength;
+      cctx.stats.totalOffset += offset;
+      cctx.stats.totalMatchLength += matchLength + MINMATCH;
+#endif
+      outputBlock(&cctx, literalLength, offset, matchLength);
+
+      // Move ip to end of block, inserting hashes at each position.
+      cctx.nextIp = cctx.ip + cctx.step;
+      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
+        if (cctx.ip > cctx.lastPosHashed) {
+          // TODO: Simplify.
+          LDM_updateLastHashFromNextHash(&cctx);
+          setNextHash(&cctx);
+        }
+        cctx.ip++;
+        cctx.nextIp++;
+      }
+    }
+
+    // Set start of next block to current input pointer.
+    cctx.anchor = cctx.ip;
+    LDM_updateLastHashFromNextHash(&cctx);
+  }
+_last_literals:
+  /* Encode the last literals (no more matches). */
+  {
+    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
+    if (lastRun >= RUN_MASK) {
+      size_t accumulator = lastRun - RUN_MASK;
+      *(cctx.op)++ = RUN_MASK << ML_BITS;
+      for(; accumulator >= 255; accumulator -= 255) {
+        *(cctx.op)++ = 255;
+      }
+      *(cctx.op)++ = (BYTE)accumulator;
+    } else {
+      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
+    }
+    memcpy(cctx.op, cctx.anchor, lastRun);
+    cctx.op += lastRun;
+  }
+
+#ifdef COMPUTE_STATS
+  printCompressStats(&cctx);
+#endif
+
+  return (cctx.op - (const BYTE *)cctx.obase);
+}
+
+typedef struct LDM_DCtx {
+  size_t compressSize;
+  size_t maxDecompressSize;
+
+  const BYTE *ibase;   /* Base of input */
+  const BYTE *ip;      /* Current input position */
+  const BYTE *iend;    /* End of source */
+
+  const BYTE *obase;   /* Base of output */
+  BYTE *op;            /* Current output position */
+  const BYTE *oend;    /* End of output */
+} LDM_DCtx;
+
+static void LDM_initializeDCtx(LDM_DCtx *dctx,
+                               const void *src, size_t compressSize,
+                               void *dst, size_t maxDecompressSize) {
+  dctx->compressSize = compressSize;
+  dctx->maxDecompressSize = maxDecompressSize;
+
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + dctx->compressSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + dctx->maxDecompressSize;
+
+}
+
+size_t LDM_decompress(const void *src, size_t compressSize,
+                      void *dst, size_t maxDecompressSize) {
+  LDM_DCtx dctx;
+  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
+
+  while (dctx.ip < dctx.iend) {
+    BYTE *cpy;
+    const BYTE *match;
+    size_t length, offset;
+
+    /* Get the literal length. */
+    unsigned const token = *(dctx.ip)++;
+    if ((length = (token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+
+    /* Copy the literals. */
+    cpy = dctx.op + length;
+    memcpy(dctx.op, dctx.ip, length);
+    dctx.ip += length;
+    dctx.op = cpy;
+
+    //TODO : dynamic offset size
+    offset = LDM_read32(dctx.ip);
+    dctx.ip += LDM_OFFSET_SIZE;
+    match = dctx.op - offset;
+
+    /* Get the match length. */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+    length += MINMATCH;
+
+    /* Copy match. */
+    cpy = dctx.op + length;
+
+    // Inefficient for now.
+    while (match < cpy - offset && dctx.op < dctx.oend) {
+      *(dctx.op)++ = *match++;
+    }
+  }
+  return dctx.op - (BYTE *)dst;
+}
+
+/*
+void LDM_test(const void *src, size_t srcSize,
+              void *dst, size_t maxDstSize) {
+  const BYTE *ip = (const BYTE *)src + 1125;
+  U32 sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
+  U32 sum2;
+  ++ip;
+  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
+    sum2 = updateChecksum(sum, LDM_HASH_LENGTH,
+                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
+    sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
+    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
+  }
+}
+*/
+
+
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.h b/contrib/long_distance_matching/versions/v0.5/ldm.h
new file mode 100644
index 00000000..1bd19745
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.5/ldm.h
@@ -0,0 +1,26 @@
+#ifndef LDM_H
+#define LDM_H
+
+#include <stddef.h>   /* size_t */
+
+#define LDM_COMPRESS_SIZE 4
+#define LDM_DECOMPRESS_SIZE 4
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize);
+
+size_t LDM_decompress(const void *src, size_t srcSize,
+                      void *dst, size_t maxDstSize);
+
+/**
+ * Reads the header from src and writes the compressed size and
+ * decompressed size into compressSize and decompressSize respectively.
+ */
+void LDM_readHeader(const void *src, size_t *compressSize,
+                    size_t *decompressSize);
+
+void LDM_test(const void *src, size_t srcSize,
+              void *dst, size_t maxDstSize);
+
+#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.5/main-ldm.c b/contrib/long_distance_matching/versions/v0.5/main-ldm.c
new file mode 100644
index 00000000..fbfd789b
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.5/main-ldm.c
@@ -0,0 +1,468 @@
+// TODO: file size must fit into a U32
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <zstd.h>
+
+#include <fcntl.h>
+#include "ldm.h"
+
+#define DEBUG
+//#define TEST
+
+/* Compress file given by fname and output to oname.
+ * Returns 0 if successful, error code otherwise.
+ */
+static int compress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+  size_t maxCompressSize, compressSize;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+
+ /* Go to the location corresponding to the last byte. */
+ /* TODO: fallocate? */
+  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+ /* Write a dummy byte at the last location. */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+/*
+#ifdef TEST
+  LDM_test(src, statbuf.st_size,
+           dst + LDM_HEADER_SIZE, statbuf.st_size);
+#endif
+*/
+
+  compressSize = LDM_HEADER_SIZE +
+      LDM_compress(src, statbuf.st_size,
+                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+
+  // Write compress and decompress size to header
+  // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  memcpy(dst, &compressSize, 4);
+  memcpy(dst + 4, &(statbuf.st_size), 4);
+
+#ifdef DEBUG
+  printf("Compressed size: %zu\n", compressSize);
+  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
+#endif
+
+  // Truncate file to compressSize.
+  ftruncate(fdout, compressSize);
+
+  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
+         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
+         (double)compressSize / (statbuf.st_size) * 100);
+
+  // Close files.
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Decompress file compressed using LDM_compress.
+ * The input file should have the LDM_HEADER followed by payload.
+ * Returns 0 if succesful, and an error code otherwise.
+ */
+static int decompress(const char *fname, const char *oname) {
+  int fdin, fdout;
+  struct stat statbuf;
+  char *src, *dst;
+  size_t compressSize, decompressSize, outSize;
+
+  /* Open the input file. */
+  if ((fdin = open(fname, O_RDONLY)) < 0) {
+    perror("Error in file opening");
+    return 1;
+  }
+
+  /* Open the output file. */
+  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
+    perror("Can't create output file");
+    return 1;
+  }
+
+  /* Find the size of the input file. */
+  if (fstat (fdin, &statbuf) < 0) {
+    perror("Fstat error");
+    return 1;
+  }
+
+  /* mmap the input file. */
+  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
+          == (caddr_t) - 1) {
+      perror("mmap error for input");
+      return 1;
+  }
+
+  /* Read the header. */
+  LDM_readHeader(src, &compressSize, &decompressSize);
+
+  /* Go to the location corresponding to the last byte. */
+  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+    perror("lseek error");
+    return 1;
+  }
+
+  /* write a dummy byte at the last location */
+  if (write(fdout, "", 1) != 1) {
+    perror("write error");
+    return 1;
+  }
+
+  /* mmap the output file */
+  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
+      perror("mmap error for output");
+      return 1;
+  }
+
+  outSize = LDM_decompress(
+      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
+      dst, decompressSize);
+
+  printf("Ret size out: %zu\n", outSize);
+  ftruncate(fdout, outSize);
+
+  close(fdin);
+  close(fdout);
+  return 0;
+}
+
+/* Compare two files.
+ * Returns 0 iff they are the same.
+ */
+static int compare(FILE *fp0, FILE *fp1) {
+  int result = 0;
+  while (result == 0) {
+    char b0[1024];
+    char b1[1024];
+    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+
+    result = (int)r0 - (int)r1;
+
+    if (0 == r0 || 0 == r1) break;
+
+    if (0 == result) result = memcmp(b0, b1, r0);
+  }
+  return result;
+}
+
+/* Verify the input file is the same as the decompressed file. */
+static void verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp = fopen(inpFilename, "rb");
+  FILE *decFp = fopen(decFilename, "rb");
+
+  printf("verify : %s <-> %s\n", inpFilename, decFilename);
+  {
+    const int cmp = compare(inpFp, decFp);
+    if(0 == cmp) {
+      printf("verify : OK\n");
+    } else {
+      printf("verify : NG\n");
+    }
+  }
+
+	fclose(decFp);
+	fclose(inpFp);
+}
+
+int main(int argc, const char *argv[]) {
+  const char * const exeName = argv[0];
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Wrong arguments\n");
+    printf("Usage:\n");
+    printf("%s FILE\n", exeName);
+    return 1;
+  }
+
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+ 	printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+
+  /* Compress */
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (compress(inpFilename, ldmFilename)) {
+        printf("Compress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total compress time = %f seconds\n",
+           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+           (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+
+  /* Decompress */
+  {
+    struct timeval tv1, tv2;
+    gettimeofday(&tv1, NULL);
+    if (decompress(ldmFilename, decFilename)) {
+        printf("Decompress error");
+        return 1;
+    }
+    gettimeofday(&tv2, NULL);
+    printf("Total decompress time = %f seconds\n",
+          (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+          (double) (tv2.tv_sec - tv1.tv_sec));
+  }
+  /* verify */
+  verify(inpFilename, decFilename);
+  return 0;
+}
+
+
+#if 0
+static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
+                            size_t *size_out) {
+  char *src, *buf = NULL;
+  size_t r = 1;
+  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
+
+  src = malloc(BUF_SIZE);
+  if (!src) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+  size = BUF_SIZE + LDM_HEADER_SIZE;
+  buf = malloc(size);
+  if (!buf) {
+    printf("Not enough memory\n");
+    goto cleanup;
+  }
+
+
+  for (;;) {
+    k = fread(src, 1, BUF_SIZE, in);
+    if (k == 0)
+      break;
+    count_in += k;
+
+    n = LDM_compress(src, buf, k, BUF_SIZE);
+
+    // n = k;
+    // offset += n;
+    offset = k;
+    count_out += k;
+
+//    k = fwrite(src, 1, offset, out);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+      if (ferror(out))
+        printf("Write failed\n");
+      else
+        printf("Short write\n");
+      goto cleanup;
+    }
+
+  }
+  *size_in = count_in;
+  *size_out = count_out;
+  r = 0;
+ cleanup:
+  free(src);
+  free(buf);
+  return r;
+}
+
+static size_t decompress_file(FILE *in, FILE *out) {
+  void *src = malloc(BUF_SIZE);
+  void *dst = NULL;
+  size_t dst_capacity = BUF_SIZE;
+  size_t ret = 1;
+  size_t bytes_written = 0;
+
+  if (!src) {
+    perror("decompress_file(src)");
+    goto cleanup;
+  }
+
+  while (ret != 0) {
+    /* Load more input */
+    size_t src_size = fread(src, 1, BUF_SIZE, in);
+    void *src_ptr = src;
+    void *src_end = src_ptr + src_size;
+    if (src_size == 0 || ferror(in)) {
+      printf("(TODO): Decompress: not enough input or error reading file\n");
+      //TODO
+      ret = 0;
+      goto cleanup;
+    }
+
+    /* Allocate destination buffer if it hasn't been allocated already */
+    if (!dst) {
+      dst = malloc(dst_capacity);
+      if (!dst) {
+        perror("decompress_file(dst)");
+        goto cleanup;
+      }
+    }
+
+    // TODO
+
+    /* Decompress:
+     * Continue while there is more input to read.
+     */
+    while (src_ptr != src_end && ret != 0) {
+      // size_t dst_size = src_size;
+      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
+      size_t written = fwrite(dst, 1, dst_size, out);
+//      printf("Writing %zu bytes\n", dst_size);
+      bytes_written += dst_size;
+      if (written != dst_size) {
+        printf("Decompress: Failed to write to file\n");
+        goto cleanup;
+      }
+      src_ptr += src_size;
+      src_size = src_end - src_ptr;
+    }
+
+    /* Update input */
+
+  }
+
+  printf("Wrote %zu bytes\n", bytes_written);
+
+ cleanup:
+  free(src);
+  free(dst);
+
+  return ret;
+}
+
+int main2(int argc, char *argv[]) {
+  char inpFilename[256] = { 0 };
+  char ldmFilename[256] = { 0 };
+  char decFilename[256] = { 0 };
+
+  if (argc < 2) {
+    printf("Please specify input filename\n");
+    return 0;
+  }
+  snprintf(inpFilename, 256, "%s", argv[1]);
+  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
+  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+	printf("ldm = [%s]\n", ldmFilename);
+	printf("dec = [%s]\n", decFilename);
+
+  /* compress */
+  {
+    FILE *inpFp = fopen(inpFilename, "rb");
+    FILE *outFp = fopen(ldmFilename, "wb");
+    size_t sizeIn = 0;
+    size_t sizeOut = 0;
+    size_t ret;
+		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
+		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+		if (ret) {
+			printf("compress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("%s: %zu → %zu bytes, %.1f%%\n",
+			inpFilename, sizeIn, sizeOut,
+			(double)sizeOut / sizeIn * 100);
+		printf("compress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+  }
+
+	/* decompress */
+	{
+    FILE *inpFp = fopen(ldmFilename, "rb");
+		FILE *outFp = fopen(decFilename, "wb");
+		size_t ret;
+
+		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
+		ret = decompress_file(inpFp, outFp);
+		if (ret) {
+			printf("decompress : failed with code %zu\n", ret);
+			return ret;
+		}
+		printf("decompress : done\n");
+
+		fclose(outFp);
+		fclose(inpFp);
+	}
+
+	/* verify */
+	{
+    FILE *inpFp = fopen(inpFilename, "rb");
+		FILE *decFp = fopen(decFilename, "rb");
+
+		printf("verify : %s <-> %s\n", inpFilename, decFilename);
+		const int cmp = compare(inpFp, decFp);
+		if(0 == cmp) {
+			printf("verify : OK\n");
+		} else {
+			printf("verify : NG\n");
+		}
+
+		fclose(decFp);
+		fclose(inpFp);
+	}
+  return 0;
+}
+#endif
+
diff --git a/contrib/long_distance_matching/versions/v0.5/util.c b/contrib/long_distance_matching/versions/v0.5/util.c
new file mode 100644
index 00000000..70fcbc2c
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.5/util.c
@@ -0,0 +1,69 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "util.h"
+
+typedef  uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef  int32_t S32;
+typedef uint64_t U64;
+
+unsigned LDM_isLittleEndian(void) {
+    const union { U32 u; BYTE c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+U16 LDM_read16(const void *memPtr) {
+  U16 val;
+  memcpy(&val, memPtr, sizeof(val));
+  return val;
+}
+
+U16 LDM_readLE16(const void *memPtr) {
+  if (LDM_isLittleEndian()) {
+    return LDM_read16(memPtr);
+  } else {
+    const BYTE *p = (const BYTE *)memPtr;
+    return (U16)((U16)p[0] + (p[1] << 8));
+  }
+}
+
+void LDM_write16(void *memPtr, U16 value){
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_write32(void *memPtr, U32 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_writeLE16(void *memPtr, U16 value) {
+  if (LDM_isLittleEndian()) {
+    LDM_write16(memPtr, value);
+  } else {
+    BYTE* p = (BYTE *)memPtr;
+    p[0] = (BYTE) value;
+    p[1] = (BYTE)(value>>8);
+  }
+}
+
+U32 LDM_read32(const void *ptr) {
+  return *(const U32 *)ptr;
+}
+
+U64 LDM_read64(const void *ptr) {
+  return *(const U64 *)ptr;
+}
+
+void LDM_copy8(void *dst, const void *src) {
+  memcpy(dst, src, 8);
+}
+
+BYTE LDM_readByte(const void *memPtr) {
+  BYTE val;
+  memcpy(&val, memPtr, 1);
+  return val;
+}
+
diff --git a/contrib/long_distance_matching/versions/v0.5/util.h b/contrib/long_distance_matching/versions/v0.5/util.h
new file mode 100644
index 00000000..d1c3c999
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.5/util.h
@@ -0,0 +1,25 @@
+#ifndef LDM_UTIL_H
+#define LDM_UTIL_H
+
+unsigned LDM_isLittleEndian(void);
+
+uint16_t LDM_read16(const void *memPtr);
+
+uint16_t LDM_readLE16(const void *memPtr);
+
+void LDM_write16(void *memPtr, uint16_t value);
+
+void LDM_write32(void *memPtr, uint32_t value);
+
+void LDM_writeLE16(void *memPtr, uint16_t value);
+
+uint32_t LDM_read32(const void *ptr);
+
+uint64_t LDM_read64(const void *ptr);
+
+void LDM_copy8(void *dst, const void *src);
+
+uint8_t LDM_readByte(const void *ptr);
+
+
+#endif /* LDM_UTIL_H */

From 8de82b6eb045d66dc8f0c4d54a7bbe7f144a05cc Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 12 Jul 2017 16:31:31 -0700
Subject: [PATCH 040/100] [ldm] Clean up versions

---
 contrib/long_distance_matching/ldm.c          |   5 +-
 contrib/long_distance_matching/util.c         |   1 -
 .../versions/v0.1/ldm.c                       | 394 ----------
 .../versions/v0.1/ldm.h                       |  19 -
 .../versions/v0.1/main-ldm.c                  | 459 -----------
 .../versions/v0.2/Makefile                    |  32 -
 .../versions/v0.2/ldm.c                       | 436 -----------
 .../versions/v0.2/ldm.h                       |  19 -
 .../versions/v0.2/main-ldm.c                  | 474 ------------
 .../versions/v0.3/Makefile                    |  10 -
 .../versions/v0.3/README                      |   3 +
 .../versions/v0.4/Makefile                    |  40 -
 .../versions/v0.4/ldm.c                       | 729 ------------------
 .../versions/v0.4/ldm.h                       |  22 -
 .../versions/v0.4/main-ldm.c                  | 480 ------------
 .../versions/v0.4/util.c                      |  69 --
 .../versions/v0.4/util.h                      |  25 -
 .../versions/v0.5/Makefile                    |  15 +-
 .../versions/v0.5/README                      |   5 +
 .../versions/v0.5/ldm.c                       |   8 +-
 20 files changed, 15 insertions(+), 3230 deletions(-)
 delete mode 100644 contrib/long_distance_matching/versions/v0.1/ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.1/ldm.h
 delete mode 100644 contrib/long_distance_matching/versions/v0.1/main-ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.2/Makefile
 delete mode 100644 contrib/long_distance_matching/versions/v0.2/ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.2/ldm.h
 delete mode 100644 contrib/long_distance_matching/versions/v0.2/main-ldm.c
 create mode 100644 contrib/long_distance_matching/versions/v0.3/README
 delete mode 100644 contrib/long_distance_matching/versions/v0.4/Makefile
 delete mode 100644 contrib/long_distance_matching/versions/v0.4/ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.4/ldm.h
 delete mode 100644 contrib/long_distance_matching/versions/v0.4/main-ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.4/util.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.4/util.h
 create mode 100644 contrib/long_distance_matching/versions/v0.5/README

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index b17a0f15..87645b76 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -19,8 +19,8 @@
 #define WINDOW_SIZE (1 << 20)
 
 //These should be multiples of four.
-#define LDM_HASH_LENGTH 100
-#define MINMATCH 100
+#define LDM_HASH_LENGTH 4
+#define MINMATCH 4
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
@@ -585,7 +585,6 @@ static void LDM_initializeDCtx(LDM_DCtx *dctx,
   dctx->iend = dctx->ip + dctx->compressSize;
   dctx->op = dst;
   dctx->oend = dctx->op + dctx->maxDecompressSize;
-
 }
 
 size_t LDM_decompress(const void *src, size_t compressSize,
diff --git a/contrib/long_distance_matching/util.c b/contrib/long_distance_matching/util.c
index 70fcbc2c..47ac8a12 100644
--- a/contrib/long_distance_matching/util.c
+++ b/contrib/long_distance_matching/util.c
@@ -66,4 +66,3 @@ BYTE LDM_readByte(const void *memPtr) {
   memcpy(&val, memPtr, 1);
   return val;
 }
-
diff --git a/contrib/long_distance_matching/versions/v0.1/ldm.c b/contrib/long_distance_matching/versions/v0.1/ldm.c
deleted file mode 100644
index 266425f8..00000000
--- a/contrib/long_distance_matching/versions/v0.1/ldm.c
+++ /dev/null
@@ -1,394 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "ldm.h"
-
-#define LDM_MEMORY_USAGE 14
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
-
-#define WINDOW_SIZE (1 << 20)
-#define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 4
-#define MINMATCH 4
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-//#define LDM_DEBUG
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint64_t tag;
-
-static unsigned LDM_isLittleEndian(void)
-{
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-static U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-static U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-static void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_write16(void *memPtr, U16 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE*)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-static U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-static U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-
-static void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
-  BYTE *d = (BYTE *)dstPtr;
-  const BYTE *s = (const BYTE *)srcPtr;
-  BYTE * const e = (BYTE *)dstEnd;
-
-  do {
-    LDM_copy8(d, s);
-    d += 8;
-    s += 8;
-  } while (d < e);
-
-}
-
-struct hash_entry {
-  U64 offset;
-  tag t;
-};
-
-static U32 LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
-}
-
-static U32 LDM_hash5(U64 sequence) {
-  static const U64 prime5bytes = 889523592379ULL;
-  static const U64 prime8bytes = 11400714785074694791ULL;
-  const U32 hashLog = LDM_HASHLOG;
-  if (LDM_isLittleEndian())
-    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
-  else
-    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
-}
-
-static U32 LDM_hash_position(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-
-static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
-                                     const BYTE *srcBase) {
-  U32 *hashTable = (U32 *) tableBase;
-  hashTable[h] = (U32)(p - srcBase);
-}
-
-static void LDM_put_position(const BYTE *p, void *tableBase,
-                             const BYTE *srcBase) {
-  U32 const h = LDM_hash_position(p);
-  LDM_put_position_on_hash(p, h, tableBase, srcBase);
-}
-
-static const BYTE *LDM_get_position_on_hash(
-    U32 h, void *tableBase, const BYTE *srcBase) {
-  const U32 * const hashTable = (U32*)tableBase;
-  return hashTable[h] + srcBase;
-}
-
-static BYTE LDM_read_byte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
-
-static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
-                          const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (unsigned)(pIn - pStart);
-  }
-  return (unsigned)(pIn - pStart);
-}
-
-void LDM_read_header(void const *source, size_t *compressed_size,
-                     size_t *decompressed_size) {
-  const U32 *ip = (const U32 *)source;
-  *compressed_size = *ip++;
-  *decompressed_size = *ip;
-}
-
-size_t LDM_compress(void const *source, void *dest, size_t source_size,
-                    size_t max_dest_size) {
-  const BYTE * const istart = (const BYTE*)source;
-  const BYTE *ip = istart;
-  const BYTE * const iend = istart + source_size;
-  const BYTE *ilimit = iend - HASH_SIZE;
-  const BYTE * const matchlimit = iend - HASH_SIZE;
-  const BYTE * const mflimit = iend - MINMATCH;
-  BYTE *op = (BYTE*) dest;
-  U32 hashTable[LDM_HASHTABLESIZE_U32];
-  memset(hashTable, 0, sizeof(hashTable));
-
-  const BYTE *anchor = (const BYTE *)source;
-//  struct LDM_cctx cctx;
-  size_t output_size = 0;
-
-  U32 forwardH;
-
-  /* Hash first byte: put into hash table */
-
-  LDM_put_position(ip, hashTable, istart);
-  ip++;
-  forwardH = LDM_hash_position(ip);
-
-  //TODO Loop terminates before ip>=ilimit.
-  while (ip < ilimit) {
-    const BYTE *match;
-    BYTE *token;
-
-    /* Find a match */
-    {
-      const BYTE *forwardIp = ip;
-      unsigned step = 1;
-
-      do {
-        U32 const h = forwardH;
-        ip = forwardIp;
-        forwardIp += step;
-
-        if (forwardIp > mflimit) {
-          goto _last_literals;
-        }
-
-        match = LDM_get_position_on_hash(h, hashTable, istart);
-
-        forwardH = LDM_hash_position(forwardIp);
-        LDM_put_position_on_hash(ip, h, hashTable, istart);
-      } while (ip - match > WINDOW_SIZE ||
-               LDM_read64(match) != LDM_read64(ip));
-    }
-
-    // TODO catchup
-    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
-      ip--;
-      match--;
-    }
-
-    /* Encode literals */
-    {
-      unsigned const litLength = (unsigned)(ip - anchor);
-      token = op++;
-
-#ifdef LDM_DEBUG
-      printf("Cur position: %zu\n", anchor - istart);
-      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
-#endif
-      /*
-      fwrite(match, 4, 1, stdout);
-      printf("\n");
-      */
-
-      if (litLength >= RUN_MASK) {
-        int len = (int)litLength - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for (; len >= 255; len -= 255) {
-          *op++ = 255;
-        }
-        *op++ = (BYTE)len;
-      } else {
-        *token = (BYTE)(litLength << ML_BITS);
-      }
-#ifdef LDM_DEBUG
-      printf("Literals ");
-      fwrite(anchor, litLength, 1, stdout);
-      printf("\n");
-#endif
-      memcpy(op, anchor, litLength);
-      //LDM_wild_copy(op, anchor, op + litLength);
-      op += litLength;
-    }
-_next_match:
-    /* Encode offset */
-    {
-      LDM_write32(op, ip - match);
-      op += 4;
-    }
-
-    /* Encode Match Length */
-    {
-      unsigned matchCode;
-      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
-                            matchlimit);
-#ifdef LDM_DEBUG
-      printf("Match length %zu\n", matchCode + MINMATCH);
-      fwrite(ip, MINMATCH + matchCode, 1, stdout);
-      printf("\n");
-#endif
-      ip += MINMATCH + matchCode;
-      if (matchCode >= ML_MASK) {
-        *token += ML_MASK;
-        matchCode -= ML_MASK;
-        LDM_write32(op, 0xFFFFFFFF);
-        while (matchCode >= 4*0xFF) {
-          op += 4;
-          LDM_write32(op, 0xffffffff);
-          matchCode -= 4*0xFF;
-        }
-        op += matchCode / 255;
-        *op++ = (BYTE)(matchCode % 255);
-      } else {
-        *token += (BYTE)(matchCode);
-      }
-#ifdef LDM_DEBUG
-      printf("\n");
-#endif
-    }
-
-    anchor = ip;
-
-    LDM_put_position(ip, hashTable, istart);
-    forwardH = LDM_hash_position(++ip);
-  }
-_last_literals:
-    /* Encode last literals */
-  {
-    size_t const lastRun = (size_t)(iend - anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *op++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *op++ = 255;
-      }
-      *op++ = (BYTE)accumulator;
-    } else {
-      *op++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(op, anchor, lastRun);
-    op += lastRun;
-  }
-  return (op - (BYTE *)dest);
-}
-
-size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
-                      size_t max_decompressed_size) {
-  const BYTE *ip = (const BYTE *)source;
-  const BYTE * const iend = ip + compressed_size;
-  BYTE *op = (BYTE *)dest;
-  BYTE * const oend = op + max_decompressed_size;
-  BYTE *cpy;
-
-  while (ip < iend) {
-    size_t length;
-    const BYTE *match;
-    size_t offset;
-
-    /* get literal length */
-    unsigned const token = *ip++;
-    if ((length=(token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-#ifdef LDM_DEBUG
-    printf("Literal length: %zu\n", length);
-#endif
-
-    /* copy literals */
-    cpy = op + length;
-#ifdef LDM_DEBUG
-    printf("Literals ");
-    fwrite(ip, length, 1, stdout);
-    printf("\n");
-#endif
-    memcpy(op, ip, length);
-//    LDM_wild_copy(op, ip, cpy);
-    ip += length;
-    op = cpy;
-
-    /* get offset */
-    offset = LDM_read32(ip);
-
-#ifdef LDM_DEBUG
-    printf("Offset: %zu\n", offset);
-#endif
-    ip += 4;
-    match = op - offset;
- //   LDM_write32(op, (U32)offset);
-
-    /* get matchlength */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-    length += MINMATCH;
-#ifdef LDM_DEBUG
-    printf("Match length: %zu\n", length);
-#endif
-    /* copy match */
-    cpy = op + length;
-
-    // Inefficient for now
-
-    while (match < cpy - offset && op < oend) {
-      *op++ = *match++;
-    }
-  }
-//  memcpy(dest, source, compressed_size);
-  return op - (BYTE *)dest;
-}
-
-
diff --git a/contrib/long_distance_matching/versions/v0.1/ldm.h b/contrib/long_distance_matching/versions/v0.1/ldm.h
deleted file mode 100644
index f4ca25a3..00000000
--- a/contrib/long_distance_matching/versions/v0.1/ldm.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef LDM_H
-#define LDM_H
-
-#include <stddef.h>   /* size_t */
-
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
-
-size_t LDM_compress(void const *source, void *dest, size_t source_size,
-                    size_t max_dest_size);
-
-size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
-                      size_t max_decompressed_size);
-
-void LDM_read_header(void const *source, size_t *compressed_size,
-                     size_t *decompressed_size);
-
-#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.1/main-ldm.c b/contrib/long_distance_matching/versions/v0.1/main-ldm.c
deleted file mode 100644
index 10869cce..00000000
--- a/contrib/long_distance_matching/versions/v0.1/main-ldm.c
+++ /dev/null
@@ -1,459 +0,0 @@
-// TODO: file size must fit into a U32
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zstd.h>
-
-#include <fcntl.h>
-#include "ldm.h"
-
-// #define BUF_SIZE 16*1024  // Block size
-#define DEBUG
-
-//#define ZSTD
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-#endif
-
-static size_t compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* open the input file */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* open the output file */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* find size of input file */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-  size_t size_in = statbuf.st_size;
-
- /* go to the location corresponding to the last byte */
- if (lseek(fdout, size_in + LDM_HEADER_SIZE - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
- }
-
- /* write a dummy byte at the last location */
- if (write(fdout, "", 1) != 1) {
-     perror("write error");
-     return 1;
- }
-
-  /* mmap the input file */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-  size_t out_size = statbuf.st_size + LDM_HEADER_SIZE;
-
-  /* mmap the output file */
-  if ((dst = mmap(0, out_size, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  #ifdef ZSTD
-    size_t size_out = ZSTD_compress(dst, statbuf.st_size,
-                                    src, statbuf.st_size, 1);
-  #else
-    size_t size_out = LDM_compress(src, dst + LDM_HEADER_SIZE, statbuf.st_size,
-                                   statbuf.st_size);
-    size_out += LDM_HEADER_SIZE;
-
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-    memcpy(dst, &size_out, 4);
-    memcpy(dst + 4, &(statbuf.st_size), 4);
-    printf("Compressed size: %zu\n", size_out);
-    printf("Decompressed size: %zu\n", statbuf.st_size);
-  #endif
-  ftruncate(fdout, size_out);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)size_out, oname,
-         (double)size_out / (statbuf.st_size) * 100);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-static size_t decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* open the input file */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* open the output file */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* find size of input file */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* read header */
-  size_t compressed_size, decompressed_size;
-  LDM_read_header(src, &compressed_size, &decompressed_size);
-
-  printf("Size, compressed_size, decompressed_size: %zu %zu %zu\n",
-         statbuf.st_size, compressed_size, decompressed_size);
-
-  /* go to the location corresponding to the last byte */
-  if (lseek(fdout, decompressed_size - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressed_size, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  /* Copy input file to output file */
-//  memcpy(dst, src, statbuf.st_size);
-
-  #ifdef ZSTD
-    size_t size_out = ZSTD_decompress(dst, decomrpessed_size,
-                                      src + LDM_HEADER_SIZE,
-                                      statbuf.st_size - LDM_HEADER_SIZE);
-  #else
-    size_t size_out = LDM_decompress(src + LDM_HEADER_SIZE, dst,
-                                     statbuf.st_size - LDM_HEADER_SIZE,
-                                     decompressed_size);
-    printf("Ret size out: %zu\n", size_out);
-  #endif
-  ftruncate(fdout, size_out);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) {
-      break;
-    }
-    if (0 == result) {
-      result = memcmp(b0, b1, r0);
-    }
-  }
-  return result;
-}
-
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-	const int cmp = compare(inpFp, decFp);
-	if(0 == cmp) {
-		printf("verify : OK\n");
-	} else {
-		printf("verify : NG\n");
-	}
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  struct timeval tv1, tv2;
-  /* compress */
-  {
-  gettimeofday(&tv1, NULL);
-  if (compress(inpFilename, ldmFilename)) {
-      printf("Compress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-         (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-
-  /* decompress */
-
-  gettimeofday(&tv1, NULL);
-  if (decompress(ldmFilename, decFilename)) {
-      printf("Decompress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-        (double) (tv2.tv_sec - tv1.tv_sec));
-
-  /* verify */
-  verify(inpFilename, decFilename);
-  return 0;
-}
-
-#if 0
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
-#endif
-
diff --git a/contrib/long_distance_matching/versions/v0.2/Makefile b/contrib/long_distance_matching/versions/v0.2/Makefile
deleted file mode 100644
index 4e04fd6a..00000000
--- a/contrib/long_distance_matching/versions/v0.2/Makefile
+++ /dev/null
@@ -1,32 +0,0 @@
-# ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
-
-# This Makefile presumes libzstd is installed, using `sudo make install`
-
-
-LDFLAGS += -lzstd
-
-.PHONY: default all clean
-
-default: all
-
-all: main-ldm
-
-
-#main : ldm.c main.c
-#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-main-ldm : ldm.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-clean:
-	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main main-ldm
-	@echo Cleaning completed
-
diff --git a/contrib/long_distance_matching/versions/v0.2/ldm.c b/contrib/long_distance_matching/versions/v0.2/ldm.c
deleted file mode 100644
index 9081d136..00000000
--- a/contrib/long_distance_matching/versions/v0.2/ldm.c
+++ /dev/null
@@ -1,436 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "ldm.h"
-
-#define HASH_EVERY 7
-
-#define LDM_MEMORY_USAGE 14
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
-
-#define WINDOW_SIZE (1 << 20)
-#define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 8
-#define MINMATCH 8
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-//#define LDM_DEBUG
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint64_t tag;
-
-static unsigned LDM_isLittleEndian(void) {
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-static U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-static U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-static void LDM_write16(void *memPtr, U16 value){
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-static void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE*)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-static U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-static U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-static void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-typedef struct compress_stats {
-  U32 num_matches;
-  U32 total_match_length;
-  U32 total_literal_length;
-  U64 total_offset;
-} compress_stats;
-
-static void LDM_printCompressStats(const compress_stats *stats) {
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->num_matches);
-  printf("Average match length: %.1f\n", ((double)stats->total_match_length) /
-                                         (double)stats->num_matches);
-  printf("Average literal length: %.1f\n",
-         ((double)stats->total_literal_length) / (double)stats->num_matches);
-  printf("Average offset length: %.1f\n",
-         ((double)stats->total_offset) / (double)stats->num_matches);
-  printf("=====================\n");
-}
-
-// TODO: unused.
-struct hash_entry {
-  U64 offset;
-  tag t;
-};
-
-static U32 LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
-}
-
-static U32 LDM_hash5(U64 sequence) {
-  static const U64 prime5bytes = 889523592379ULL;
-  static const U64 prime8bytes = 11400714785074694791ULL;
-  const U32 hashLog = LDM_HASHLOG;
-  if (LDM_isLittleEndian())
-    return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
-  else
-    return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
-}
-
-static U32 LDM_hash_position(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-
-static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
-                                     const BYTE *srcBase) {
-  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-
-  U32 *hashTable = (U32 *) tableBase;
-  hashTable[h] = (U32)(p - srcBase);
-}
-
-static void LDM_put_position(const BYTE *p, void *tableBase,
-                             const BYTE *srcBase) {
-  if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  U32 const h = LDM_hash_position(p);
-  LDM_put_position_on_hash(p, h, tableBase, srcBase);
-}
-
-static const BYTE *LDM_get_position_on_hash(
-    U32 h, void *tableBase, const BYTE *srcBase) {
-  const U32 * const hashTable = (U32*)tableBase;
-  return hashTable[h] + srcBase;
-}
-
-static BYTE LDM_read_byte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
-
-static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
-                          const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (unsigned)(pIn - pStart);
-  }
-  return (unsigned)(pIn - pStart);
-}
-
-void LDM_read_header(const void *src, size_t *compressSize,
-                     size_t *decompressSize) {
-  const U32 *ip = (const U32 *)src;
-  *compressSize = *ip++;
-  *decompressSize = *ip;
-}
-
-// TODO: maxDstSize is unused
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize) {
-  const BYTE * const istart = (const BYTE*)src;
-  const BYTE *ip = istart;
-  const BYTE * const iend = istart + srcSize;
-  const BYTE *ilimit = iend - HASH_SIZE;
-  const BYTE * const matchlimit = iend - HASH_SIZE;
-  const BYTE * const mflimit = iend - MINMATCH;
-  BYTE *op = (BYTE*) dst;
-
-  compress_stats compressStats = { 0 };
-
-  U32 hashTable[LDM_HASHTABLESIZE_U32];
-  memset(hashTable, 0, sizeof(hashTable));
-
-  const BYTE *anchor = (const BYTE *)src;
-//  struct LDM_cctx cctx;
-  size_t output_size = 0;
-
-  U32 forwardH;
-
-  /* Hash first byte: put into hash table */
-
-  LDM_put_position(ip, hashTable, istart);
-  const BYTE *lastHash = ip;
-  ip++;
-  forwardH = LDM_hash_position(ip);
-
-  //TODO Loop terminates before ip>=ilimit.
-  while (ip < ilimit) {
-    const BYTE *match;
-    BYTE *token;
-
-    /* Find a match */
-    {
-      const BYTE *forwardIp = ip;
-      unsigned step = 1;
-
-      do {
-        U32 const h = forwardH;
-        ip = forwardIp;
-        forwardIp += step;
-
-        if (forwardIp > mflimit) {
-          goto _last_literals;
-        }
-
-        match = LDM_get_position_on_hash(h, hashTable, istart);
-
-        forwardH = LDM_hash_position(forwardIp);
-        LDM_put_position_on_hash(ip, h, hashTable, istart);
-        lastHash = ip;
-      } while (ip - match > WINDOW_SIZE ||
-               LDM_read64(match) != LDM_read64(ip));
-    }
-    compressStats.num_matches++;
-
-    /* Catchup: look back to extend match from found match */
-    while (ip > anchor && match > istart && ip[-1] == match[-1]) {
-      ip--;
-      match--;
-    }
-
-    /* Encode literals */
-    {
-      unsigned const litLength = (unsigned)(ip - anchor);
-      token = op++;
-
-      compressStats.total_literal_length += litLength;
-
-#ifdef LDM_DEBUG
-      printf("Cur position: %zu\n", anchor - istart);
-      printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
-#endif
-
-      if (litLength >= RUN_MASK) {
-        int len = (int)litLength - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for (; len >= 255; len -= 255) {
-          *op++ = 255;
-        }
-        *op++ = (BYTE)len;
-      } else {
-        *token = (BYTE)(litLength << ML_BITS);
-      }
-#ifdef LDM_DEBUG
-      printf("Literals ");
-      fwrite(anchor, litLength, 1, stdout);
-      printf("\n");
-#endif
-      memcpy(op, anchor, litLength);
-      op += litLength;
-    }
-_next_match:
-    /* Encode offset */
-    {
-      /*
-      LDM_writeLE16(op, ip-match);
-      op += 2;
-      */
-      LDM_write32(op, ip - match);
-      op += 4;
-      compressStats.total_offset += (ip - match);
-    }
-
-    /* Encode Match Length */
-    {
-      unsigned matchCode;
-      matchCode = LDM_count(ip + MINMATCH, match + MINMATCH,
-                            matchlimit);
-#ifdef LDM_DEBUG
-      printf("Match length %zu\n", matchCode + MINMATCH);
-      fwrite(ip, MINMATCH + matchCode, 1, stdout);
-      printf("\n");
-#endif
-      compressStats.total_match_length += matchCode + MINMATCH;
-      unsigned ctr = 1;
-      ip++;
-      for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
-        LDM_put_position(ip, hashTable, istart);
-      }
-//      ip += MINMATCH + matchCode;
-      if (matchCode >= ML_MASK) {
-        *token += ML_MASK;
-        matchCode -= ML_MASK;
-        LDM_write32(op, 0xFFFFFFFF);
-        while (matchCode >= 4*0xFF) {
-          op += 4;
-          LDM_write32(op, 0xffffffff);
-          matchCode -= 4*0xFF;
-        }
-        op += matchCode / 255;
-        *op++ = (BYTE)(matchCode % 255);
-      } else {
-        *token += (BYTE)(matchCode);
-      }
-#ifdef LDM_DEBUG
-      printf("\n");
-
-#endif
-    }
-
-    anchor = ip;
-
-    LDM_put_position(ip, hashTable, istart);
-    forwardH = LDM_hash_position(++ip);
-    lastHash = ip;
-  }
-_last_literals:
-    /* Encode last literals */
-  {
-    size_t const lastRun = (size_t)(iend - anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *op++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *op++ = 255;
-      }
-      *op++ = (BYTE)accumulator;
-    } else {
-      *op++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(op, anchor, lastRun);
-    op += lastRun;
-  }
-  LDM_printCompressStats(&compressStats);
-  return (op - (BYTE *)dst);
-}
-
-typedef struct LDM_DCtx {
-  const BYTE * const ibase;   /* Pointer to base of input */
-  const BYTE *ip;             /* Pointer to current input position */
-  const BYTE *iend;           /* End of source */
-  BYTE *op;                   /* Pointer to output */
-  const BYTE * const oend;    /* Pointer to end of output */
-
-} LDM_DCtx;
-
-size_t LDM_decompress(const void *src, size_t compressed_size,
-                      void *dst, size_t max_decompressed_size) {
-  const BYTE *ip = (const BYTE *)src;
-  const BYTE * const iend = ip + compressed_size;
-  BYTE *op = (BYTE *)dst;
-  BYTE * const oend = op + max_decompressed_size;
-  BYTE *cpy;
-
-  while (ip < iend) {
-    size_t length;
-    const BYTE *match;
-    size_t offset;
-
-    /* get literal length */
-    unsigned const token = *ip++;
-    if ((length=(token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-#ifdef LDM_DEBUG
-    printf("Literal length: %zu\n", length);
-#endif
-
-    /* copy literals */
-    cpy = op + length;
-#ifdef LDM_DEBUG
-    printf("Literals ");
-    fwrite(ip, length, 1, stdout);
-    printf("\n");
-#endif
-    memcpy(op, ip, length);
-    ip += length;
-    op = cpy;
-
-    /* get offset */
-    /*
-    offset = LDM_readLE16(ip);
-    ip += 2;
-    */
-    offset = LDM_read32(ip);
-    ip += 4;
-#ifdef LDM_DEBUG
-    printf("Offset: %zu\n", offset);
-#endif
-    match = op - offset;
- //   LDM_write32(op, (U32)offset);
-
-    /* get matchlength */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *ip++;
-        length += s;
-      } while (s == 255);
-    }
-    length += MINMATCH;
-#ifdef LDM_DEBUG
-    printf("Match length: %zu\n", length);
-#endif
-    /* copy match */
-    cpy = op + length;
-
-    // Inefficient for now
-    while (match < cpy - offset && op < oend) {
-      *op++ = *match++;
-    }
-  }
-  return op - (BYTE *)dst;
-}
-
-
diff --git a/contrib/long_distance_matching/versions/v0.2/ldm.h b/contrib/long_distance_matching/versions/v0.2/ldm.h
deleted file mode 100644
index 0ac7b2ec..00000000
--- a/contrib/long_distance_matching/versions/v0.2/ldm.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef LDM_H
-#define LDM_H
-
-#include <stddef.h>   /* size_t */
-
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
-
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize);
-
-size_t LDM_decompress(const void *src, size_t srcSize,
-                      void *dst, size_t maxDstSize);
-
-void LDM_read_header(const void *src, size_t *compressSize,
-                     size_t *decompressSize);
-
-#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.2/main-ldm.c b/contrib/long_distance_matching/versions/v0.2/main-ldm.c
deleted file mode 100644
index 0017335b..00000000
--- a/contrib/long_distance_matching/versions/v0.2/main-ldm.c
+++ /dev/null
@@ -1,474 +0,0 @@
-// TODO: file size must fit into a U32
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zstd.h>
-
-#include <fcntl.h>
-#include "ldm.h"
-
-// #define BUF_SIZE 16*1024  // Block size
-#define DEBUG
-
-//#define ZSTD
-
-/* Compress file given by fname and output to oname.
- * Returns 0 if successful, error code otherwise.
- */
-static int compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
-
- /* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */ 
-  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
- /* Write a dummy byte at the last location. */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef ZSTD
-  size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
-                                  src, statbuf.st_size, 1);
-#else
-  size_t compressSize = LDM_HEADER_SIZE + 
-      LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, statbuf.st_size);
-
-    // Write compress and decompress size to header 
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressSize, 4);
-  memcpy(dst + 4, &(statbuf.st_size), 4);
-
-#ifdef DEBUG
-  printf("Compressed size: %zu\n", compressSize);
-  printf("Decompressed size: %zu\n", statbuf.st_size);
-#endif
-#endif
-
-  // Truncate file to compressSize.
-  ftruncate(fdout, compressSize);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
-         (double)compressSize / (statbuf.st_size) * 100);
-
-  // Close files.
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Decompress file compressed using LDM_compress.
- * The input file should have the LDM_HEADER followed by payload.
- * Returns 0 if succesful, and an error code otherwise. 
- */
-static int decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* Read the header. */
-  size_t compressSize, decompressSize;
-  LDM_read_header(src, &compressSize, &decompressSize);
-
-#ifdef DEBUG
-  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
-         statbuf.st_size, compressSize, decompressSize);
-#endif
-
-  /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef ZSTD
-  size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
-                                  src + LDM_HEADER_SIZE,
-                                  statbuf.st_size - LDM_HEADER_SIZE);
-#else
-  size_t outSize = LDM_decompress(
-      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
-      dst, decompressSize);
-
-  printf("Ret size out: %zu\n", outSize);
-  #endif
-  ftruncate(fdout, outSize);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Compare two files.
- * Returns 0 iff they are the same.
- */
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) break;
-
-    if (0 == result) result = memcmp(b0, b1, r0);
-  }
-  return result;
-}
-
-/* Verify the input file is the same as the decompressed file. */
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-	const int cmp = compare(inpFp, decFp);
-	if(0 == cmp) {
-		printf("verify : OK\n");
-	} else {
-		printf("verify : NG\n");
-	}
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  struct timeval tv1, tv2;
-
-  /* Compress */
-
-  gettimeofday(&tv1, NULL);
-  if (compress(inpFilename, ldmFilename)) {
-      printf("Compress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-         (double) (tv2.tv_sec - tv1.tv_sec));
-
-  /* Decompress */
-
-  gettimeofday(&tv1, NULL);
-  if (decompress(ldmFilename, decFilename)) {
-      printf("Decompress error");
-      return 1;
-  }
-  gettimeofday(&tv2, NULL);
-  printf("Total time = %f seconds\n",
-        (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-        (double) (tv2.tv_sec - tv1.tv_sec));
-
-  /* verify */
-  verify(inpFilename, decFilename);
-  return 0;
-}
-
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
-#endif
-
diff --git a/contrib/long_distance_matching/versions/v0.3/Makefile b/contrib/long_distance_matching/versions/v0.3/Makefile
index 5ffd4eaf..e5153970 100644
--- a/contrib/long_distance_matching/versions/v0.3/Makefile
+++ b/contrib/long_distance_matching/versions/v0.3/Makefile
@@ -1,12 +1,3 @@
-# ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
-
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
 CFLAGS  ?= -O3
@@ -26,7 +17,6 @@ default: all
 
 all: main-ldm
 
-
 #main : ldm.c main.c
 #	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
diff --git a/contrib/long_distance_matching/versions/v0.3/README b/contrib/long_distance_matching/versions/v0.3/README
new file mode 100644
index 00000000..8699562e
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.3/README
@@ -0,0 +1,3 @@
+This version uses simple lz4-style compression:
+- A 4-byte hash is inserted into the hash table for every position.
+- Hash table replacement policy: direct overwrite.
diff --git a/contrib/long_distance_matching/versions/v0.4/Makefile b/contrib/long_distance_matching/versions/v0.4/Makefile
deleted file mode 100644
index 5ffd4eaf..00000000
--- a/contrib/long_distance_matching/versions/v0.4/Makefile
+++ /dev/null
@@ -1,40 +0,0 @@
-# ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
-
-# This Makefile presumes libzstd is installed, using `sudo make install`
-
-CFLAGS  ?= -O3
-DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls
-CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CFLAGS)
-
-LDFLAGS += -lzstd
-
-.PHONY: default all clean
-
-default: all
-
-all: main-ldm
-
-
-#main : ldm.c main.c
-#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-main-ldm : util.c ldm.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-clean:
-	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main main-ldm
-	@echo Cleaning completed
-
diff --git a/contrib/long_distance_matching/versions/v0.4/ldm.c b/contrib/long_distance_matching/versions/v0.4/ldm.c
deleted file mode 100644
index 79648097..00000000
--- a/contrib/long_distance_matching/versions/v0.4/ldm.c
+++ /dev/null
@@ -1,729 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-
-#include "ldm.h"
-#include "util.h"
-
-#define HASH_EVERY 1
-
-#define LDM_MEMORY_USAGE 22
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
-
-#define LDM_OFFSET_SIZE 4
-
-#define WINDOW_SIZE (1 << 23)
-#define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 4
-#define LDM_HASH_LENGTH 4
-
-// Should be multiple of four
-#define MINMATCH 4
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-#define LDM_ROLLING_HASH
-#define COMPUTE_STATS
-//#define RUN_CHECKS
-//#define LDM_DEBUG
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint32_t offset_t;
-typedef uint32_t hash_t;
-typedef signed char schar;
-
-
-typedef struct LDM_hashEntry {
-  offset_t offset;
-} LDM_hashEntry;
-
-typedef struct LDM_compressStats {
-  U32 numMatches;
-  U32 totalMatchLength;
-  U32 totalLiteralLength;
-  U64 totalOffset;
-
-  U32 numCollisions;
-  U32 numHashInserts;
-} LDM_compressStats;
-
-static void LDM_printCompressStats(const LDM_compressStats *stats) {
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->numMatches);
-  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
-                                         (double)stats->numMatches);
-  printf("Average literal length: %.1f\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
-  printf("Average offset length: %.1f\n",
-         ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
-         stats->numCollisions, stats->numHashInserts,
-         stats->numHashInserts == 0 ?
-            1.0 : (100.0 * (double)stats->numCollisions) /
-                  (double)stats->numHashInserts);
-  printf("=====================\n");
-}
-
-typedef struct LDM_CCtx {
-  size_t isize;             /* Input size */
-  size_t maxOSize;          /* Maximum output size */
-
-  const BYTE *ibase;        /* Base of input */
-  const BYTE *ip;           /* Current input position */
-  const BYTE *iend;         /* End of input */
-
-  // Maximum input position such that hashing at the position does not exceed
-  // end of input.
-  const BYTE *ihashLimit;
-
-  // Maximum input position such that finding a match of at least the minimum
-  // match length does not exceed end of input.
-  const BYTE *imatchLimit;
-
-  const BYTE *obase;        /* Base of output */
-  BYTE *op;                 /* Output */
-
-  const BYTE *anchor;       /* Anchor to start of current (match) block */
-
-  LDM_compressStats stats;            /* Compression statistics */
-
-  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
-
-  const BYTE *lastPosHashed;          /* Last position hashed */
-  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  const BYTE *nextIp;
-  const BYTE *nextPosHashed;
-  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
-
-  // Members for rolling hash.
-  U32 lastSum;
-  U32 nextSum;
-
-  unsigned step;
-
-  // DEBUG
-  const BYTE *DEBUG_setNextHash;
-} LDM_CCtx;
-
-static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
-  U16 lengthLeft = MINMATCH;
-  const BYTE *curP = p;
-  const BYTE *curMatch = match;
-
-  for (; lengthLeft >= 8; lengthLeft -= 8) {
-    if (LDM_read64(curP) != LDM_read64(curMatch)) {
-      return 0;
-    }
-    curP += 8;
-    curMatch += 8;
-  }
-  if (lengthLeft > 0) {
-    return LDM_read32(curP) == LDM_read32(curMatch);
-  }
-  return 1;
-}
-
-
-
-#ifdef LDM_ROLLING_HASH
-/**
- * Convert a sum computed from LDM_getRollingHash to a hash value in the range
- * of the hash table.
- */
-static hash_t LDM_sumToHash(U32 sum) {
-  return sum & (LDM_HASH_SIZE_U32 - 1);
-}
-
-static U32 LDM_getRollingHash(const char *data, U32 len) {
-  U32 i;
-  U32 s1, s2;
-  const schar *buf = (const schar *)data;
-
-  s1 = s2 = 0;
-  for (i = 0; i < (len - 4); i += 4) {
-    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
-  }
-  for(; i < len; i++) {
-    s1 += buf[i];
-    s2 += s1;
-  }
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-typedef struct LDM_sumStruct {
-  U16 s1, s2;
-} LDM_sumStruct;
-
-static U32 LDM_updateRollingHash(U32 sum, U32 len,
-                                 schar toRemove, schar toAdd) {
-  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
-  U32 s2 = (sum >> 16) - (toRemove * len) + s1;
-
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-
-/*
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
-}
-*/
-
-/*
-static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
-  sumStruct->s1 = sum & 0xffff;
-  sumStruct->s2 = sum >> 16;
-}
-*/
-
-static void LDM_setNextHash(LDM_CCtx *cctx) {
-
-#ifdef RUN_CHECKS
-  U32 check;
-  if ((cctx->nextIp - cctx->ibase != 1) &&
-      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
-    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
-            cctx->DEBUG_setNextHash - cctx->ibase);
-  }
-
-  cctx->DEBUG_setNextHash = cctx->nextIp;
-#endif
-
-//  cctx->nextSum = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
-  cctx->nextSum = LDM_updateRollingHash(
-      cctx->lastSum, LDM_HASH_LENGTH,
-      (schar)((cctx->lastPosHashed)[0]),
-      (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
-
-#ifdef RUN_CHECKS
-  check = LDM_getRollingHash((const char *)cctx->nextIp, LDM_HASH_LENGTH);
-
-  if (check != cctx->nextSum) {
-    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
-//    printf("INFO: %u %u %u\n", LDM_read32(cctx->nextIp),
-  }
-#endif
-  cctx->nextPosHashed = cctx->nextIp;
-  cctx->nextHash = LDM_sumToHash(cctx->nextSum);
-
-#ifdef RUN_CHECKS
-  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
-    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
-            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
-            cctx->ip - cctx->ibase);
-  }
-#endif
-
-}
-
-static void LDM_putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash, U32 sum) {
-  /*
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  */
-#ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = (cctx->hashTable)[hash].offset;
-    cctx->stats.numHashInserts++;
-    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
-      cctx->stats.numCollisions++;
-    }
-  }
-#endif
-  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-  cctx->lastSum = sum;
-}
-
-static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  if (cctx->ip != cctx->nextPosHashed) {
-    printf("CHECK failed: updateLastHashFromNextHash %zu\n", cctx->ip - cctx->ibase);
-  }
-#endif
-  LDM_putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
-}
-
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = LDM_getRollingHash((const char *)cctx->ip, LDM_HASH_LENGTH);
-  hash_t hash = LDM_sumToHash(sum);
-#ifdef RUN_CHECKS
-  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
-    printf("CHECK failed: putHashOfCurrentPosition %zu\n", cctx->ip - cctx->ibase);
-  }
-#endif
-//  hash_t hash = LDM_hashPosition(cctx->ip);
-  LDM_putHashOfCurrentPositionFromHash(cctx, hash, sum);
-//  printf("Offset %zu\n", cctx->ip - cctx->ibase);
-}
-
-#else
-static hash_t LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
-}
-
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-
-static void LDM_putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash) {
-  /*
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  */
-#ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = (cctx->hashTable)[hash].offset;
-    cctx->stats.numHashInserts++;
-    if (offset == 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
-      cctx->stats.numCollisions++;
-    }
-  }
-#endif
-
-  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
-#ifdef RUN_CHECKS
-  if (cctx->ip - cctx->lastPosHashed != 1) {
-    printf("putHashError\n");
-  }
-#endif
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-}
-
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  hash_t hash = LDM_hashPosition(cctx->ip);
-  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
-}
-
-#endif
-
-/*
-static hash_t LDM_hash5(U64 sequence) {
-  static const U64 prime5bytes = 889523592379ULL;
-  static const U64 prime8bytes = 11400714785074694791ULL;
-  const U32 hashLog = LDM_HASHLOG;
-  if (LDM_isLittleEndian())
-    return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
-  else
-    return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
-}
-*/
-
-
-static const BYTE *LDM_getPositionOnHash(
-    hash_t h, void *tableBase, const BYTE *srcBase) {
-  const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
-  return hashTable[h].offset + srcBase;
-}
-
-
-static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
-                          const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (unsigned)(pIn - pStart);
-  }
-  return (unsigned)(pIn - pStart);
-}
-
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize) {
-  const U32 *ip = (const U32 *)src;
-  *compressSize = *ip++;
-  *decompressSize = *ip;
-}
-
-static void LDM_initializeCCtx(LDM_CCtx *cctx,
-                               const void *src, size_t srcSize,
-                               void *dst, size_t maxDstSize) {
-  cctx->isize = srcSize;
-  cctx->maxOSize = maxDstSize;
-
-  cctx->ibase = (const BYTE *)src;
-  cctx->ip = cctx->ibase;
-  cctx->iend = cctx->ibase + srcSize;
-
-#ifdef LDM_ROLLING_HASH
-  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
-#else
-  cctx->ihashLimit = cctx->iend - HASH_SIZE;
-#endif
-  cctx->imatchLimit = cctx->iend - MINMATCH;
-
-  cctx->obase = (BYTE *)dst;
-  cctx->op = (BYTE *)dst;
-
-  cctx->anchor = cctx->ibase;
-
-  memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
-
-  cctx->lastPosHashed = NULL;
-
-  cctx->step = 1;
-  cctx->nextIp = cctx->ip + cctx->step;
-  cctx->nextPosHashed = 0;
-
-  cctx->DEBUG_setNextHash = 0;
-}
-
-#ifdef LDM_ROLLING_HASH
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
-  cctx->nextIp = cctx->ip + cctx->step;
-
-  do {
-    hash_t h;
-    U32 sum;
-//    printf("Call A\n");
-    LDM_setNextHash(cctx);
-//    printf("End call a\n");
-    h = cctx->nextHash;
-    sum = cctx->nextSum;
-    cctx->ip = cctx->nextIp;
-    cctx->nextIp += cctx->step;
-
-    if (cctx->ip > cctx->imatchLimit) {
-      return 1;
-    }
-
-    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
-
-//    // Compute cctx->nextSum and cctx->nextHash from cctx->nextIp.
-//    LDM_setNextHash(cctx);
-    LDM_putHashOfCurrentPositionFromHash(cctx, h, sum);
-
-//    printf("%u %u\n", cctx->lastHash, cctx->nextHash);
-  } while (cctx->ip - *match > WINDOW_SIZE ||
-           !LDM_isValidMatch(cctx->ip, *match));
-//           LDM_read64(*match) != LDM_read64(cctx->ip));
-  LDM_setNextHash(cctx);
-  return 0;
-}
-#else
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
-  cctx->nextIp = cctx->ip;
-
-  do {
-    hash_t const h = cctx->nextHash;
-    cctx->ip = cctx->nextIp;
-    cctx->nextIp += cctx->step;
-
-    if (cctx->ip > cctx->imatchLimit) {
-      return 1;
-    }
-
-    *match = LDM_getPositionOnHash(h, cctx->hashTable, cctx->ibase);
-
-    cctx->nextHash = LDM_hashPosition(cctx->nextIp);
-    LDM_putHashOfCurrentPositionFromHash(cctx, h);
-
-  } while (cctx->ip - *match > WINDOW_SIZE ||
-           !LDM_isValidMatch(cctx->ip, *match));
-  return 0;
-}
-
-#endif
-
-/**
- * Write current block (literals, literal length, match offset,
- * match length).
- *
- * Update input pointer, inserting hashes into hash table along the
- * way.
- */
-static void LDM_outputBlock(LDM_CCtx *cctx, const BYTE *match) {
-  unsigned const literalLength = (unsigned)(cctx->ip - cctx->anchor);
-  unsigned const offset = cctx->ip - match;
-  unsigned const matchLength = LDM_count(
-      cctx->ip + MINMATCH, match + MINMATCH, cctx->ihashLimit);
-  BYTE *token = cctx->op++;
-
-  cctx->stats.totalLiteralLength += literalLength;
-  cctx->stats.totalOffset += offset;
-  cctx->stats.totalMatchLength += matchLength + MINMATCH;
-
-  /* Encode the literal length. */
-  if (literalLength >= RUN_MASK) {
-    int len = (int)literalLength - RUN_MASK;
-    *token = (RUN_MASK << ML_BITS);
-    for (; len >= 255; len -= 255) {
-      *(cctx->op)++ = 255;
-    }
-    *(cctx->op)++ = (BYTE)len;
-  } else {
-    *token = (BYTE)(literalLength << ML_BITS);
-  }
-
-  /* Encode the literals. */
-  memcpy(cctx->op, cctx->anchor, literalLength);
-  cctx->op += literalLength;
-
-  /* Encode the offset. */
-  LDM_write32(cctx->op, offset);
-  cctx->op += LDM_OFFSET_SIZE;
-
-  /* Encode match length */
-  if (matchLength >= ML_MASK) {
-    unsigned matchLengthRemaining = matchLength;
-    *token += ML_MASK;
-    matchLengthRemaining -= ML_MASK;
-    LDM_write32(cctx->op, 0xFFFFFFFF);
-    while (matchLengthRemaining >= 4*0xFF) {
-      cctx->op += 4;
-      LDM_write32(cctx->op, 0xffffffff);
-      matchLengthRemaining -= 4*0xFF;
-    }
-    cctx->op += matchLengthRemaining / 255;
-    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
-  } else {
-    *token += (BYTE)(matchLength);
-  }
-
-//  LDM_setNextHash(cctx);
-//  cctx->ip = cctx->lastPosHashed + 1;
-//  cctx->nextIp = cctx->ip + cctx->step;
-//  printf("HERE: %zu %zu %zu\n", cctx->ip - cctx->ibase,
-//         cctx->lastPosHashed - cctx->ibase, cctx->nextIp - cctx->ibase);
-
-  cctx->nextIp = cctx->ip + cctx->step;
-
-  while (cctx->ip < cctx->anchor + MINMATCH + matchLength + literalLength) {
-//    printf("Loop\n");
-    if (cctx->ip > cctx->lastPosHashed) {
-      LDM_updateLastHashFromNextHash(cctx);
-//      LDM_putHashOfCurrentPosition(cctx);
-#ifdef LDM_ROLLING_HASH
-      LDM_setNextHash(cctx);
-#endif
-    }
-    /*
-    printf("Call b %zu %zu %zu\n",
-            cctx->lastPosHashed - cctx->ibase,
-            cctx->nextIp - cctx->ibase,
-            cctx->ip - cctx->ibase);
-            */
-//    printf("end call b\n");
-    cctx->ip++;
-    cctx->nextIp++;
-  }
-
-//  printf("There: %zu %zu\n", cctx->ip - cctx->ibase, cctx->lastPosHashed - cctx->ibase);
-}
-
-// TODO: srcSize and maxDstSize is unused
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize) {
-  LDM_CCtx cctx;
-  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
-
-  /* Hash the first position and put it into the hash table. */
-  LDM_putHashOfCurrentPosition(&cctx);
-#ifdef LDM_ROLLING_HASH
-//  LDM_setNextHash(&cctx);
-//  tmp_hash = LDM_updateRollingHash(cctx.lastSum, LDM_HASH_LENGTH,
-//                                   cctx.ip[0], cctx.ip[LDM_HASH_LENGTH]);
-//  printf("Update test: %u %u\n", tmp_hash, cctx.nextSum);
-//  cctx.ip++;
-#else
-  cctx.ip++;
-  cctx.nextHash = LDM_hashPosition(cctx.ip);
-#endif
-
-  // TODO: loop condition is not accurate.
-  while (1) {
-    const BYTE *match;
-//    printf("Start of loop\n");
-
-    /**
-     * Find a match.
-     * If no more matches can be found (i.e. the length of the remaining input
-     * is less than the minimum match length), then stop searching for matches
-     * and encode the final literals.
-     */
-    if (LDM_findBestMatch(&cctx, &match) != 0) {
-      goto _last_literals;
-    }
-//    printf("End of match finding\n");
-
-    cctx.stats.numMatches++;
-
-    /**
-     * Catch up: look back to extend the match backwards from the found match.
-     */
-    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
-           cctx.ip[-1] == match[-1]) {
-//      printf("Catch up\n");
-      cctx.ip--;
-      match--;
-    }
-
-    /**
-     * Write current block (literals, literal length, match offset, match
-     * length) and update pointers and hashes.
-     */
-    LDM_outputBlock(&cctx, match);
-//    printf("End of loop\n");
-
-    // Set start of next block to current input pointer.
-    cctx.anchor = cctx.ip;
-    LDM_updateLastHashFromNextHash(&cctx);
-//    LDM_putHashOfCurrentPosition(&cctx);
-#ifndef LDM_ROLLING_HASH
-    cctx.ip++;
-#endif
-
-    /*
-    LDM_putHashOfCurrentPosition(&cctx);
-    printf("Call c\n");
-    LDM_setNextHash(&cctx);
-    printf("End call c\n");
-    cctx.ip++;
-    cctx.nextIp++;
-    */
-  }
-_last_literals:
-  /* Encode the last literals (no more matches). */
-  {
-    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *(cctx.op)++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *(cctx.op)++ = 255;
-      }
-      *(cctx.op)++ = (BYTE)accumulator;
-    } else {
-      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(cctx.op, cctx.anchor, lastRun);
-    cctx.op += lastRun;
-  }
-  LDM_printCompressStats(&cctx.stats);
-  return (cctx.op - (const BYTE *)cctx.obase);
-}
-
-typedef struct LDM_DCtx {
-  size_t compressSize;
-  size_t maxDecompressSize;
-
-  const BYTE *ibase;   /* Base of input */
-  const BYTE *ip;      /* Current input position */
-  const BYTE *iend;    /* End of source */
-
-  const BYTE *obase;   /* Base of output */
-  BYTE *op;            /* Current output position */
-  const BYTE *oend;    /* End of output */
-} LDM_DCtx;
-
-static void LDM_initializeDCtx(LDM_DCtx *dctx,
-                               const void *src, size_t compressSize,
-                               void *dst, size_t maxDecompressSize) {
-  dctx->compressSize = compressSize;
-  dctx->maxDecompressSize = maxDecompressSize;
-
-  dctx->ibase = src;
-  dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressSize;
-  dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressSize;
-
-}
-
-size_t LDM_decompress(const void *src, size_t compressSize,
-                      void *dst, size_t maxDecompressSize) {
-  LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
-
-  while (dctx.ip < dctx.iend) {
-    BYTE *cpy;
-    const BYTE *match;
-    size_t length, offset;
-
-    /* Get the literal length. */
-    unsigned const token = *(dctx.ip)++;
-    if ((length = (token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-
-    /* Copy literals. */
-    cpy = dctx.op + length;
-    memcpy(dctx.op, dctx.ip, length);
-    dctx.ip += length;
-    dctx.op = cpy;
-
-    //TODO : dynamic offset size
-    offset = LDM_read32(dctx.ip);
-    dctx.ip += LDM_OFFSET_SIZE;
-    match = dctx.op - offset;
-
-    /* Get the match length. */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-    length += MINMATCH;
-
-    /* Copy match. */
-    cpy = dctx.op + length;
-
-    // Inefficient for now.
-    while (match < cpy - offset && dctx.op < dctx.oend) {
-      *(dctx.op)++ = *match++;
-    }
-  }
-  return dctx.op - (BYTE *)dst;
-}
-
-void LDM_test(const void *src, size_t srcSize,
-              void *dst, size_t maxDstSize) {
-#ifdef LDM_ROLLING_HASH
-  const BYTE *ip = (const BYTE *)src + 1125;
-  U32 sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
-  U32 sum2;
-  ++ip;
-  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
-    sum2 = LDM_updateRollingHash(sum, LDM_HASH_LENGTH,
-                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
-    sum = LDM_getRollingHash((const char *)ip, LDM_HASH_LENGTH);
-    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
-  }
-#endif
-}
-
-
diff --git a/contrib/long_distance_matching/versions/v0.4/ldm.h b/contrib/long_distance_matching/versions/v0.4/ldm.h
deleted file mode 100644
index a34faac4..00000000
--- a/contrib/long_distance_matching/versions/v0.4/ldm.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef LDM_H
-#define LDM_H
-
-#include <stddef.h>   /* size_t */
-
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
-
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize);
-
-size_t LDM_decompress(const void *src, size_t srcSize,
-                      void *dst, size_t maxDstSize);
-
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize);
-
-void LDM_test(const void *src, size_t srcSize,
-              void *dst, size_t maxDstSize);
-
-#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.4/main-ldm.c b/contrib/long_distance_matching/versions/v0.4/main-ldm.c
deleted file mode 100644
index f8ae5469..00000000
--- a/contrib/long_distance_matching/versions/v0.4/main-ldm.c
+++ /dev/null
@@ -1,480 +0,0 @@
-// TODO: file size must fit into a U32
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zstd.h>
-
-#include <fcntl.h>
-#include "ldm.h"
-
-// #define BUF_SIZE 16*1024  // Block size
-#define DEBUG
-//#define TEST
-
-//#define ZSTD
-
-/* Compress file given by fname and output to oname.
- * Returns 0 if successful, error code otherwise.
- */
-static int compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-  size_t maxCompressSize, compressSize;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
-
- /* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */
-  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
- /* Write a dummy byte at the last location. */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef TEST
-  LDM_test(src, statbuf.st_size,
-           dst + LDM_HEADER_SIZE, statbuf.st_size);
-#endif
-
-#ifdef ZSTD
-  compressSize = ZSTD_compress(dst, statbuf.st_size,
-                                  src, statbuf.st_size, 1);
-#else
-  compressSize = LDM_HEADER_SIZE +
-      LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, statbuf.st_size);
-
-    // Write compress and decompress size to header
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressSize, 4);
-  memcpy(dst + 4, &(statbuf.st_size), 4);
-
-#ifdef DEBUG
-  printf("Compressed size: %zu\n", compressSize);
-  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
-#endif
-#endif
-
-  // Truncate file to compressSize.
-  ftruncate(fdout, compressSize);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
-         (double)compressSize / (statbuf.st_size) * 100);
-
-  // Close files.
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Decompress file compressed using LDM_compress.
- * The input file should have the LDM_HEADER followed by payload.
- * Returns 0 if succesful, and an error code otherwise.
- */
-static int decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-  size_t compressSize, decompressSize, outSize;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* Read the header. */
-  LDM_readHeader(src, &compressSize, &decompressSize);
-
-  /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef ZSTD
-  outSize = ZSTD_decompress(dst, decomrpessed_size,
-                                  src + LDM_HEADER_SIZE,
-                                  statbuf.st_size - LDM_HEADER_SIZE);
-#else
-  outSize = LDM_decompress(
-      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
-      dst, decompressSize);
-
-  printf("Ret size out: %zu\n", outSize);
-  #endif
-  ftruncate(fdout, outSize);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Compare two files.
- * Returns 0 iff they are the same.
- */
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) break;
-
-    if (0 == result) result = memcmp(b0, b1, r0);
-  }
-  return result;
-}
-
-/* Verify the input file is the same as the decompressed file. */
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-  {
-    const int cmp = compare(inpFp, decFp);
-    if(0 == cmp) {
-      printf("verify : OK\n");
-    } else {
-      printf("verify : NG\n");
-    }
-  }
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-
-  /* Compress */
-  {
-    struct timeval tv1, tv2;
-    gettimeofday(&tv1, NULL);
-    if (compress(inpFilename, ldmFilename)) {
-        printf("Compress error");
-        return 1;
-    }
-    gettimeofday(&tv2, NULL);
-    printf("Total compress time = %f seconds\n",
-           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-           (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-
-  /* Decompress */
-  {
-    struct timeval tv1, tv2;
-    gettimeofday(&tv1, NULL);
-    if (decompress(ldmFilename, decFilename)) {
-        printf("Decompress error");
-        return 1;
-    }
-    gettimeofday(&tv2, NULL);
-    printf("Total decompress time = %f seconds\n",
-          (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-          (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-  /* verify */
-  verify(inpFilename, decFilename);
-  return 0;
-}
-
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
-#endif
-
diff --git a/contrib/long_distance_matching/versions/v0.4/util.c b/contrib/long_distance_matching/versions/v0.4/util.c
deleted file mode 100644
index 70fcbc2c..00000000
--- a/contrib/long_distance_matching/versions/v0.4/util.c
+++ /dev/null
@@ -1,69 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "util.h"
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-unsigned LDM_isLittleEndian(void) {
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-void LDM_write16(void *memPtr, U16 value){
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE *)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-BYTE LDM_readByte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
-
diff --git a/contrib/long_distance_matching/versions/v0.4/util.h b/contrib/long_distance_matching/versions/v0.4/util.h
deleted file mode 100644
index d1c3c999..00000000
--- a/contrib/long_distance_matching/versions/v0.4/util.h
+++ /dev/null
@@ -1,25 +0,0 @@
-#ifndef LDM_UTIL_H
-#define LDM_UTIL_H
-
-unsigned LDM_isLittleEndian(void);
-
-uint16_t LDM_read16(const void *memPtr);
-
-uint16_t LDM_readLE16(const void *memPtr);
-
-void LDM_write16(void *memPtr, uint16_t value);
-
-void LDM_write32(void *memPtr, uint32_t value);
-
-void LDM_writeLE16(void *memPtr, uint16_t value);
-
-uint32_t LDM_read32(const void *ptr);
-
-uint64_t LDM_read64(const void *ptr);
-
-void LDM_copy8(void *dst, const void *src);
-
-uint8_t LDM_readByte(const void *ptr);
-
-
-#endif /* LDM_UTIL_H */
diff --git a/contrib/long_distance_matching/versions/v0.5/Makefile b/contrib/long_distance_matching/versions/v0.5/Makefile
index 5ffd4eaf..fa4abce6 100644
--- a/contrib/long_distance_matching/versions/v0.5/Makefile
+++ b/contrib/long_distance_matching/versions/v0.5/Makefile
@@ -1,12 +1,3 @@
-# ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
-
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
 CFLAGS  ?= -O3
@@ -26,15 +17,11 @@ default: all
 
 all: main-ldm
 
-
-#main : ldm.c main.c
-#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
 main-ldm : util.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main main-ldm
+	main-ldm
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/versions/v0.5/README b/contrib/long_distance_matching/versions/v0.5/README
new file mode 100644
index 00000000..7901ae76
--- /dev/null
+++ b/contrib/long_distance_matching/versions/v0.5/README
@@ -0,0 +1,5 @@
+This version uses simple lz4-style compression with a rolling hash.
+- A rolling checksum based on rsync's Adler-32 style checksum is used.
+- The checksum is hashed using lz4's hash function.
+- Hash table replacement policy: direct overwrite.
+- The length of input to the hash function can be set with LDM_HASH_LENGTH.
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.c b/contrib/long_distance_matching/versions/v0.5/ldm.c
index 325c5040..5fa20c06 100644
--- a/contrib/long_distance_matching/versions/v0.5/ldm.c
+++ b/contrib/long_distance_matching/versions/v0.5/ldm.c
@@ -19,8 +19,8 @@
 #define WINDOW_SIZE (1 << 20)
 
 //These should be multiples of four.
-#define LDM_HASH_LENGTH 100
-#define MINMATCH 100
+#define LDM_HASH_LENGTH 4
+#define MINMATCH 4
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
@@ -95,12 +95,12 @@ typedef struct LDM_CCtx {
   const BYTE *DEBUG_setNextHash;
 } LDM_CCtx;
 
+#ifdef COMPUTE_STATS
 /**
  * Outputs compression statistics.
  */
 static void printCompressStats(const LDM_CCtx *cctx) {
   const LDM_compressStats *stats = &(cctx->stats);
-#ifdef COMPUTE_STATS
   printf("=====================\n");
   printf("Compression statistics\n");
   printf("Total number of matches: %u\n", stats->numMatches);
@@ -131,8 +131,8 @@ static void printCompressStats(const LDM_CCtx *cctx) {
   }
 
   printf("=====================\n");
-#endif
 }
+#endif
 
 /**
  * Checks whether the MINMATCH bytes from p are the same as the MINMATCH

From 92bed4a7e0051c06dc6ceca058b5559615fcc2a8 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 12 Jul 2017 18:47:26 -0700
Subject: [PATCH 041/100] [ldm] Add CHAR_OFFSET in hash function and extend
 header size

---
 contrib/long_distance_matching/ldm.c      |  18 +-
 contrib/long_distance_matching/ldm.h      |   4 +-
 contrib/long_distance_matching/main-ldm.c | 203 +---------------------
 contrib/long_distance_matching/util.c     |   5 +
 contrib/long_distance_matching/util.h     |   2 +
 5 files changed, 22 insertions(+), 210 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 87645b76..e64d2865 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -28,6 +28,7 @@
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
 #define COMPUTE_STATS
+#define CHECKSUM_CHAR_OFFSET 2
 //#define RUN_CHECKS
 //#define LDM_DEBUG
 
@@ -191,11 +192,14 @@ static U32 getChecksum(const char *data, U32 len) {
   s1 = s2 = 0;
   for (i = 0; i < (len - 4); i += 4) {
     s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
+          (2 * buf[i + 2]) + (buf[i + 3]) +
+          (10 * CHECKSUM_CHAR_OFFSET);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
+          + (4 * CHECKSUM_CHAR_OFFSET);
+
   }
   for(; i < len; i++) {
-    s1 += buf[i];
+    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
     s2 += s1;
   }
   return (s1 & 0xffff) + (s2 << 16);
@@ -213,7 +217,7 @@ static U32 getChecksum(const char *data, U32 len) {
 static U32 updateChecksum(U32 sum, U32 len,
                           schar toRemove, schar toAdd) {
   U32 s1 = (sum & 0xffff) - toRemove + toAdd;
-  U32 s2 = (sum >> 16) - (toRemove * len) + s1;
+  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
 
   return (s1 & 0xffff) + (s2 << 16);
 }
@@ -344,9 +348,9 @@ static unsigned countMatchLength(const BYTE *pIn, const BYTE *pMatch,
   return (unsigned)(pIn - pStart);
 }
 
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize) {
-  const U32 *ip = (const U32 *)src;
+void LDM_readHeader(const void *src, U64 *compressSize,
+                    U64 *decompressSize) {
+  const U64 *ip = (const U64 *)src;
   *compressSize = *ip++;
   *decompressSize = *ip;
 }
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index d7f977d9..f04b6e95 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -3,8 +3,8 @@
 
 #include <stddef.h>   /* size_t */
 
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
+#define LDM_COMPRESS_SIZE 8
+#define LDM_DECOMPRESS_SIZE 8
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
 
 /**
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index fbfd789b..8354b795 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -85,8 +85,8 @@ static int compress(const char *fname, const char *oname) {
 
   // Write compress and decompress size to header
   // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressSize, 4);
-  memcpy(dst + 4, &(statbuf.st_size), 4);
+  memcpy(dst, &compressSize, 8);
+  memcpy(dst + 8, &(statbuf.st_size), 8);
 
 #ifdef DEBUG
   printf("Compressed size: %zu\n", compressSize);
@@ -267,202 +267,3 @@ int main(int argc, const char *argv[]) {
   verify(inpFilename, decFilename);
   return 0;
 }
-
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
-#endif
-
diff --git a/contrib/long_distance_matching/util.c b/contrib/long_distance_matching/util.c
index 47ac8a12..62749215 100644
--- a/contrib/long_distance_matching/util.c
+++ b/contrib/long_distance_matching/util.c
@@ -53,6 +53,11 @@ U32 LDM_read32(const void *ptr) {
   return *(const U32 *)ptr;
 }
 
+//TODO: endianness?
+void LDM_write64(void *memPtr, U64 value) {
+  memcpy(memPtr, &value, sizeof(value));
+}
+
 U64 LDM_read64(const void *ptr) {
   return *(const U64 *)ptr;
 }
diff --git a/contrib/long_distance_matching/util.h b/contrib/long_distance_matching/util.h
index d1c3c999..dbf55cbc 100644
--- a/contrib/long_distance_matching/util.h
+++ b/contrib/long_distance_matching/util.h
@@ -21,5 +21,7 @@ void LDM_copy8(void *dst, const void *src);
 
 uint8_t LDM_readByte(const void *ptr);
 
+void LDM_write64(void *memPtr, uint64_t value);
+
 
 #endif /* LDM_UTIL_H */

From de0414b7365246ffb7848db72ed08ff6c07d8a91 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 12 Jul 2017 19:08:24 -0700
Subject: [PATCH 042/100] [libzstd] Pull CTables into sub-structure

---
 lib/common/fse.h                 |  11 +--
 lib/common/huf.h                 |  11 ++-
 lib/common/zstd_internal.h       |  14 ++++
 lib/compress/zstd_compress.c     | 130 ++++++++++---------------------
 lib/decompress/zstd_decompress.c |   8 +-
 5 files changed, 70 insertions(+), 104 deletions(-)

diff --git a/lib/common/fse.h b/lib/common/fse.h
index 6d5d41de..54ac98b1 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -31,13 +31,14 @@
    You can contact the author at :
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 ****************************************************************** */
-#ifndef FSE_H
-#define FSE_H
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef FSE_H
+#define FSE_H
+
 
 /*-*****************************************
 *  Dependencies
@@ -297,8 +298,10 @@ FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<
 If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
 */
 
+#endif  /* FSE_H */
 
-#ifdef FSE_STATIC_LINKING_ONLY
+#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
+#define FSE_H_FSE_STATIC_LINKING_ONLY
 
 /* *** Dependency *** */
 #include "bitstream.h"
@@ -694,5 +697,3 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
 #if defined (__cplusplus)
 }
 #endif
-
-#endif  /* FSE_H */
diff --git a/lib/common/huf.h b/lib/common/huf.h
index dabd3599..2b3015a8 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -31,13 +31,13 @@
    You can contact the author at :
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 ****************************************************************** */
-#ifndef HUF_H_298734234
-#define HUF_H_298734234
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef HUF_H_298734234
+#define HUF_H_298734234
 
 /* *** Dependencies *** */
 #include <stddef.h>    /* size_t */
@@ -124,6 +124,7 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
 #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
 #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
 
+#endif   /* HUF_H_298734234 */
 
 /* ******************************************************************
  *  WARNING !!
@@ -132,7 +133,8 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
  *  because they are not guaranteed to remain stable in the future.
  *  Only consider them in association with static linking.
  *******************************************************************/
-#ifdef HUF_STATIC_LINKING_ONLY
+#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
+#define HUF_H_HUF_STATIC_LINKING_ONLY
 
 /* *** Dependencies *** */
 #include "mem.h"   /* U32 */
@@ -295,9 +297,6 @@ size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* c
 
 #endif /* HUF_STATIC_LINKING_ONLY */
 
-
 #if defined (__cplusplus)
 }
 #endif
-
-#endif   /* HUF_H_298734234 */
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f49f6a13..42e5e7b5 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -50,6 +50,10 @@
 #include "error_private.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
+#define HUF_STATIC_LINKING_ONLY
+#include "huf.h"
 #ifndef XXH_STATIC_LINKING_ONLY
 #  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
 #endif
@@ -266,6 +270,16 @@ typedef struct {
     const BYTE* cachedLiterals;
 } seqStore_t;
 
+typedef struct {
+    HUF_repeat hufCTable_repeatMode;
+    U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
+    U32 fseCTables_ready;
+    FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+    FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+    FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+    U32 workspace[HUF_WORKSPACE_SIZE_U32];
+} ZSTD_entropyCTables_t;
+
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9300357f..64da67d5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -36,13 +36,6 @@ static const U32 g_searchStrength = 8;   /* control skip over incompressible dat
 #define HASH_READ_SIZE 8
 typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
 
-/* entropy tables always have same size */
-static size_t const hufCTable_size = HUF_CTABLE_SIZE(255);
-static size_t const litlengthCTable_size = FSE_CTABLE_SIZE(LLFSELog, MaxLL);
-static size_t const offcodeCTable_size = FSE_CTABLE_SIZE(OffFSELog, MaxOff);
-static size_t const matchlengthCTable_size = FSE_CTABLE_SIZE(MLFSELog, MaxML);
-static size_t const entropyScratchSpace_size = HUF_WORKSPACE_SIZE;
-
 
 /*-*************************************
 *  Helper functions
@@ -108,13 +101,7 @@ struct ZSTD_CCtx_s {
     U32* hashTable;
     U32* hashTable3;
     U32* chainTable;
-    HUF_repeat hufCTable_repeatMode;
-    HUF_CElt* hufCTable;
-    U32 fseCTables_ready;
-    FSE_CTable* offcodeCTable;
-    FSE_CTable* matchlengthCTable;
-    FSE_CTable* litlengthCTable;
-    unsigned* entropyScratchSpace;
+    ZSTD_entropyCTables_t* entropy;
 
     /* streaming */
     char*  inBuff;
@@ -174,19 +161,9 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
     cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx);
 
     /* entropy space (never moves) */
-    /* note : this code should be shared with resetCCtx, rather than copy/pasted */
-    {   void* ptr = cctx->workSpace;
-        cctx->hufCTable = (HUF_CElt*)ptr;
-        ptr = (char*)cctx->hufCTable + hufCTable_size;
-        cctx->offcodeCTable = (FSE_CTable*) ptr;
-        ptr = (char*)ptr + offcodeCTable_size;
-        cctx->matchlengthCTable = (FSE_CTable*) ptr;
-        ptr = (char*)ptr + matchlengthCTable_size;
-        cctx->litlengthCTable = (FSE_CTable*) ptr;
-        ptr = (char*)ptr + litlengthCTable_size;
-        assert(((size_t)ptr & 3) == 0);   /* ensure correct alignment */
-        cctx->entropyScratchSpace = (unsigned*) ptr;
-    }
+    if (cctx->workSpaceSize < sizeof(ZSTD_entropyCTables_t)) return NULL;
+    assert(((size_t)cctx->workSpace & 7) == 0);   /* ensure correct alignment */
+    cctx->entropy = (ZSTD_entropyCTables_t*)cctx->workSpace;
 
     return cctx;
 }
@@ -551,9 +528,7 @@ size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams)
     size_t const hSize = ((size_t)1) << cParams.hashLog;
     U32    const hashLog3 = (cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
     size_t const h3Size = ((size_t)1) << hashLog3;
-    size_t const entropySpace = hufCTable_size + litlengthCTable_size
-                              + offcodeCTable_size + matchlengthCTable_size
-                              + entropyScratchSpace_size;
+    size_t const entropySpace = sizeof(ZSTD_entropyCTables_t);
     size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
 
     size_t const optBudget = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
@@ -641,8 +616,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     if (crp == ZSTDcrp_continue) {
         if (ZSTD_equivalentParams(params.cParams, zc->appliedParams.cParams)) {
             DEBUGLOG(5, "ZSTD_equivalentParams()==1");
-            zc->fseCTables_ready = 0;
-            zc->hufCTable_repeatMode = HUF_repeat_none;
+            zc->entropy->fseCTables_ready = 0;
+            zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
             return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
     }   }
 
@@ -662,9 +637,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         void* ptr;
 
         /* Check if workSpace is large enough, alloc a new one if needed */
-        {   size_t const entropySpace = hufCTable_size + litlengthCTable_size
-                                  + offcodeCTable_size + matchlengthCTable_size
-                                  + entropyScratchSpace_size;
+        {   size_t const entropySpace = sizeof(ZSTD_entropyCTables_t);
             size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32)
                                   + (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t));
             size_t const optSpace = ( (params.cParams.strategy == ZSTD_btopt)
@@ -689,16 +662,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                 ptr = zc->workSpace;
 
                 /* entropy space */
-                zc->hufCTable = (HUF_CElt*)ptr;
-                ptr = (char*)zc->hufCTable + hufCTable_size;  /* note : HUF_CElt* is incomplete type, size is estimated via macro */
-                zc->offcodeCTable = (FSE_CTable*) ptr;
-                ptr = (char*)ptr + offcodeCTable_size;
-                zc->matchlengthCTable = (FSE_CTable*) ptr;
-                ptr = (char*)ptr + matchlengthCTable_size;
-                zc->litlengthCTable = (FSE_CTable*) ptr;
-                ptr = (char*)ptr + litlengthCTable_size;
-                assert(((size_t)ptr & 3) == 0);   /* ensure correct alignment */
-                zc->entropyScratchSpace = (unsigned*) ptr;
+                assert(((size_t)zc->workSpace & 3) == 0);   /* ensure correct alignment */
+                assert(zc->workSpaceSize >= sizeof(ZSTD_entropyCTables_t));
+                zc->entropy = (ZSTD_entropyCTables_t*)zc->workSpace;
         }   }
 
         /* init params */
@@ -715,8 +681,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->stage = ZSTDcs_init;
         zc->dictID = 0;
         zc->loadedDictEnd = 0;
-        zc->fseCTables_ready = 0;
-        zc->hufCTable_repeatMode = HUF_repeat_none;
+        zc->entropy->fseCTables_ready = 0;
+        zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
         zc->nextToUpdate = 1;
         zc->nextSrc = NULL;
         zc->base = NULL;
@@ -727,13 +693,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->hashLog3 = hashLog3;
         zc->seqStore.litLengthSum = 0;
 
-        /* ensure entropy tables are close together at the beginning */
-        assert((void*)zc->hufCTable == zc->workSpace);
-        assert((char*)zc->offcodeCTable == (char*)zc->hufCTable + hufCTable_size);
-        assert((char*)zc->matchlengthCTable == (char*)zc->offcodeCTable + offcodeCTable_size);
-        assert((char*)zc->litlengthCTable == (char*)zc->matchlengthCTable + matchlengthCTable_size);
-        assert((char*)zc->entropyScratchSpace == (char*)zc->litlengthCTable + litlengthCTable_size);
-        ptr = (char*)zc->entropyScratchSpace + entropyScratchSpace_size;
+        ptr = zc->entropy + 1;
 
         /* opt parser space */
         if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) {
@@ -830,16 +790,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
     dstCCtx->dictID       = srcCCtx->dictID;
 
     /* copy entropy tables */
-    dstCCtx->fseCTables_ready = srcCCtx->fseCTables_ready;
-    if (srcCCtx->fseCTables_ready) {
-        memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, litlengthCTable_size);
-        memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, matchlengthCTable_size);
-        memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, offcodeCTable_size);
-    }
-    dstCCtx->hufCTable_repeatMode = srcCCtx->hufCTable_repeatMode;
-    if (srcCCtx->hufCTable_repeatMode) {
-        memcpy(dstCCtx->hufCTable, srcCCtx->hufCTable, hufCTable_size);
-    }
+    memcpy(dstCCtx->entropy, srcCCtx->entropy, sizeof(ZSTD_entropyCTables_t));
 
     return 0;
 }
@@ -970,28 +921,28 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
 
     /* small ? don't even attempt compression (speed opt) */
 #   define LITERAL_NOENTROPY 63
-    {   size_t const minLitSize = zc->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
+    {   size_t const minLitSize = zc->entropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
         if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }
 
     if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
-    {   HUF_repeat repeat = zc->hufCTable_repeatMode;
+    {   HUF_repeat repeat = zc->entropy->hufCTable_repeatMode;
         int const preferRepeat = zc->appliedParams.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
         cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
-                                      zc->entropyScratchSpace, entropyScratchSpace_size, zc->hufCTable, &repeat, preferRepeat)
+                                      zc->entropy->workspace, sizeof(zc->entropy->workspace), (HUF_CElt*)zc->entropy->hufCTable, &repeat, preferRepeat)
                                 : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
-                                      zc->entropyScratchSpace, entropyScratchSpace_size, zc->hufCTable, &repeat, preferRepeat);
+                                      zc->entropy->workspace, sizeof(zc->entropy->workspace), (HUF_CElt*)zc->entropy->hufCTable, &repeat, preferRepeat);
         if (repeat != HUF_repeat_none) { hType = set_repeat; }    /* reused the existing table */
-        else { zc->hufCTable_repeatMode = HUF_repeat_check; }       /* now have a table to reuse */
+        else { zc->entropy->hufCTable_repeatMode = HUF_repeat_check; }       /* now have a table to reuse */
     }
 
     if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) {
-        zc->hufCTable_repeatMode = HUF_repeat_none;
+        zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }
     if (cLitSize==1) {
-        zc->hufCTable_repeatMode = HUF_repeat_none;
+        zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
         return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
     }
 
@@ -1070,9 +1021,9 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
     const seqStore_t* seqStorePtr = &(zc->seqStore);
     U32 count[MaxSeq+1];
     S16 norm[MaxSeq+1];
-    FSE_CTable* CTable_LitLength = zc->litlengthCTable;
-    FSE_CTable* CTable_OffsetBits = zc->offcodeCTable;
-    FSE_CTable* CTable_MatchLength = zc->matchlengthCTable;
+    FSE_CTable* CTable_LitLength = zc->entropy->litlengthCTable;
+    FSE_CTable* CTable_OffsetBits = zc->entropy->offcodeCTable;
+    FSE_CTable* CTable_MatchLength = zc->entropy->matchlengthCTable;
     U32 LLtype, Offtype, MLtype;   /* compressed, raw or rle */
     const seqDef* const sequences = seqStorePtr->sequencesStart;
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
@@ -1111,12 +1062,12 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
 
     /* CTable for Literal Lengths */
     {   U32 max = MaxLL;
-        size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->entropyScratchSpace);
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->entropy->workspace);
         if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
             *op++ = llCodeTable[0];
             FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
             LLtype = set_rle;
-        } else if ((zc->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+        } else if ((zc->entropy->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
             LLtype = set_repeat;
         } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) {
             FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
@@ -1135,12 +1086,12 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
 
     /* CTable for Offsets */
     {   U32 max = MaxOff;
-        size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->entropyScratchSpace);
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->entropy->workspace);
         if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
             *op++ = ofCodeTable[0];
             FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
             Offtype = set_rle;
-        } else if ((zc->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+        } else if ((zc->entropy->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
             Offtype = set_repeat;
         } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) {
             FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
@@ -1159,12 +1110,12 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
 
     /* CTable for MatchLengths */
     {   U32 max = MaxML;
-        size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->entropyScratchSpace);
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->entropy->workspace);
         if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
             *op++ = *mlCodeTable;
             FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
             MLtype = set_rle;
-        } else if ((zc->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+        } else if ((zc->entropy->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
             MLtype = set_repeat;
         } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) {
             FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
@@ -1182,7 +1133,7 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
     }   }
 
     *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
-    zc->fseCTables_ready = 0;
+    zc->entropy->fseCTables_ready = 0;
 
     /* Encoding Sequences */
     {   BIT_CStream_t blockStream;
@@ -1261,7 +1212,7 @@ _check_compressibility:
     {   size_t const minGain = ZSTD_minGain(srcSize);
         size_t const maxCSize = srcSize - minGain;
         if ((size_t)(op-ostart) >= maxCSize) {
-            zc->hufCTable_repeatMode = HUF_repeat_none;
+            zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
             return 0;
     }   }
 
@@ -3106,13 +3057,14 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
     const BYTE* const dictEnd = dictPtr + dictSize;
     short offcodeNCount[MaxOff+1];
     unsigned offcodeMaxValue = MaxOff;
-    BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)];
+
+    ZSTD_STATIC_ASSERT(sizeof(cctx->entropy->workspace) >= (1<<MAX(MLFSELog,LLFSELog)));
 
     dictPtr += 4;   /* skip magic number */
     cctx->dictID = cctx->appliedParams.fParams.noDictIDFlag ? 0 :  MEM_readLE32(dictPtr);
     dictPtr += 4;
 
-    {   size_t const hufHeaderSize = HUF_readCTable(cctx->hufCTable, 255, dictPtr, dictEnd-dictPtr);
+    {   size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)cctx->entropy->hufCTable, 255, dictPtr, dictEnd-dictPtr);
         if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
         dictPtr += hufHeaderSize;
     }
@@ -3122,7 +3074,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
         /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
-        CHECK_E( FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)),
+        CHECK_E( FSE_buildCTable_wksp(cctx->entropy->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)),
                  dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
@@ -3134,7 +3086,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
         if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
         /* Every match length code must have non-zero probability */
         CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
-        CHECK_E( FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)),
+        CHECK_E( FSE_buildCTable_wksp(cctx->entropy->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)),
                  dictionary_corrupted);
         dictPtr += matchlengthHeaderSize;
     }
@@ -3146,7 +3098,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
         if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
         /* Every literal length code must have non-zero probability */
         CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
-        CHECK_E( FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)),
+        CHECK_E( FSE_buildCTable_wksp(cctx->entropy->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)),
                  dictionary_corrupted);
         dictPtr += litlengthHeaderSize;
     }
@@ -3172,8 +3124,8 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
                 if (cctx->rep[u] > dictContentSize) return ERROR(dictionary_corrupted);
         }   }
 
-        cctx->fseCTables_ready = 1;
-        cctx->hufCTable_repeatMode = HUF_repeat_valid;
+        cctx->entropy->fseCTables_ready = 1;
+        cctx->entropy->hufCTable_repeatMode = HUF_repeat_valid;
         return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize);
     }
 }
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index a145dbf8..4e96504e 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -95,7 +95,7 @@ typedef struct {
     HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
     U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
     U32 rep[ZSTD_REP_NUM];
-} ZSTD_entropyTables_t;
+} ZSTD_entropyDTables_t;
 
 struct ZSTD_DCtx_s
 {
@@ -103,7 +103,7 @@ struct ZSTD_DCtx_s
     const FSE_DTable* MLTptr;
     const FSE_DTable* OFTptr;
     const HUF_DTable* HUFptr;
-    ZSTD_entropyTables_t entropy;
+    ZSTD_entropyDTables_t entropy;
     const void* previousDstEnd;   /* detect continuity */
     const void* base;             /* start of current segment */
     const void* vBase;            /* virtual start of previous segment if it was just before current one */
@@ -1842,7 +1842,7 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
 /* ZSTD_loadEntropy() :
  * dict : must point at beginning of a valid zstd dictionary
  * @return : size of entropy tables read */
-static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const dict, size_t const dictSize)
+static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize)
 {
     const BYTE* dictPtr = (const BYTE*)dict;
     const BYTE* const dictEnd = dictPtr + dictSize;
@@ -1933,7 +1933,7 @@ struct ZSTD_DDict_s {
     void* dictBuffer;
     const void* dictContent;
     size_t dictSize;
-    ZSTD_entropyTables_t entropy;
+    ZSTD_entropyDTables_t entropy;
     U32 dictID;
     U32 entropyPresent;
     ZSTD_customMem cMem;

From 4e77f7761da1b4b35222ac75bd10b1c8c8dcea0d Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 13 Jul 2017 02:09:07 -0700
Subject: [PATCH 043/100] clarified comment on ZSTD_p_contentSizeFlag

---
 lib/zstd.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 291c6df2..a2a756df 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -955,7 +955,9 @@ typedef enum {
                               * Special: value 0 means "do not change strategy". */
 
     /* frame parameters */
-    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
+    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1)
+                              * note that content size must be known at the beginning,
+                              * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 

From 132e6efd760b97403f1e84c6c4e4ee6569fad605 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 13 Jul 2017 02:22:58 -0700
Subject: [PATCH 044/100] switched ZSTDMT_compress_advanced() last argument to
 overlapLog

overlapRLog (== 9 - overlapLog) was a bit "strange"
as all other public entry points use overlapLog
---
 doc/zstd_manual.html           |  4 +++-
 lib/compress/zstdmt_compress.c | 22 ++++++++++++----------
 lib/compress/zstdmt_compress.h |  2 +-
 3 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 1058503f..c166e725 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -813,7 +813,9 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
                               * Special: value 0 means "do not change strategy". */
 
     </b>/* frame parameters */<b>
-    ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1) */<b>
+    ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1)<b>
+                              * note that content size must be known at the beginning,
+                              * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_p_checksumFlag,     </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
     ZSTD_p_dictIDFlag,       </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 677e96f0..ed4b9117 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -10,6 +10,7 @@
 
 /* ======   Tuning parameters   ====== */
 #define ZSTDMT_NBTHREADS_MAX 128
+#define ZSTDMT_OVERLAPLOG_DEFAULT 6
 
 
 /* ======   Compiler specifics   ====== */
@@ -383,7 +384,7 @@ struct ZSTDMT_CCtx_s {
     unsigned nextJobID;
     unsigned frameEnded;
     unsigned allJobsCompleted;
-    unsigned overlapRLog;
+    unsigned overlapLog;
     unsigned long long frameContentSize;
     size_t sectionSize;
     ZSTD_customMem cMem;
@@ -417,7 +418,7 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
     mtctx->nbThreads = nbThreads;
     mtctx->allJobsCompleted = 1;
     mtctx->sectionSize = 0;
-    mtctx->overlapRLog = 3;
+    mtctx->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT;
     mtctx->factory = POOL_create(nbThreads, 1);
     mtctx->jobs = ZSTDMT_allocJobsTable(&nbJobs, cMem);
     mtctx->jobIDMask = nbJobs - 1;
@@ -491,7 +492,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
         return 0;
     case ZSTDMT_p_overlapSectionLog :
         DEBUGLOG(5, "ZSTDMT_p_overlapSectionLog : %u", value);
-        mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
+        mtctx->overlapLog = (value >= 9) ? 9 : value;
         return 0;
     default :
         return ERROR(compressionParameter_unsupported);
@@ -520,8 +521,9 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
                          const void* src, size_t srcSize,
                          const ZSTD_CDict* cdict,
                                ZSTD_parameters const params,
-                               unsigned overlapRLog)
+                               unsigned overlapLog)
 {
+    unsigned const overlapRLog = (overlapLog>9) ? 0 : 9-overlapLog;
     size_t const overlapSize = (overlapRLog>=9) ? 0 : (size_t)1 << (params.cParams.windowLog - overlapRLog);
     unsigned nbChunks = computeNbChunks(srcSize, params.cParams.windowLog, mtctx->nbThreads);
     size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
@@ -538,7 +540,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
         if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, params.fParams);
         return ZSTD_compress_advanced(cctx, dst, dstCapacity, src, srcSize, NULL, 0, params);
     }
-    assert(avgChunkSize >= 256 KB);  /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), which is useful to avoid allocating extra buffers */
+    assert(avgChunkSize >= 256 KB);  /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), which is required for compressWithinDst */
     ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgChunkSize) );
     XXH64_reset(&xxh64, 0);
 
@@ -642,10 +644,10 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
                      const void* src, size_t srcSize,
                            int compressionLevel)
 {
-    U32 const overlapRLog = (compressionLevel >= ZSTD_maxCLevel()) ? 0 : 3;
+    U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT;
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
     params.fParams.contentSizeFlag = 1;
-    return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapRLog);
+    return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog);
 }
 
 
@@ -710,8 +712,8 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
         zcs->cdict = cdict;
     }
 
-    zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog);
-    DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog);
+    zcs->targetDictSize = (zcs->overlapLog==0) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - (9 - zcs->overlapLog));
+    DEBUGLOG(4, "overlapLog : %u ", zcs->overlapLog);
     DEBUGLOG(4, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
     zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
     zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
@@ -918,7 +920,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
         size_t const cSize = ZSTDMT_compress_advanced(mtctx,
                 (char*)output->dst + output->pos, output->size - output->pos,
                 (const char*)input->src + input->pos, input->size - input->pos,
-                mtctx->cdict, mtctx->params, mtctx->overlapRLog);
+                mtctx->cdict, mtctx->params, mtctx->overlapLog);
         if (ZSTD_isError(cSize)) return cSize;
         input->pos = input->size;
         output->pos += cSize;
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 7584007f..843a240a 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -68,7 +68,7 @@ ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
                                      const void* src, size_t srcSize,
                                      const ZSTD_CDict* cdict,
                                            ZSTD_parameters const params,
-                                           unsigned overlapRLog);            /* overlapRLog = 9 - overlapLog */
+                                           unsigned overlapLog);
 
 ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
                                         const void* dict, size_t dictSize,   /* dict can be released after init, a local copy is preserved within zcs */

From 3a60efd3a92ce2e3b222e69ff0965a223d26978e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 13 Jul 2017 10:10:13 -0700
Subject: [PATCH 045/100] policy change : ZSTDMT automatically caps nbThreads
 to ZSTDMT_NBTHREADS_MAX (#760)

Previously, ZSTDMT would refuse to create the compressor.
Also : increased ZSTDMT_NBTHREADS_MAX to 256,
updated doc,
and added relevant test
---
 lib/compress/zstdmt_compress.c | 5 +++--
 programs/zstd.1                | 4 ++--
 programs/zstd.1.md             | 1 +
 tests/playTests.sh             | 3 ++-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index ed4b9117..3be850c6 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -9,7 +9,7 @@
 
 
 /* ======   Tuning parameters   ====== */
-#define ZSTDMT_NBTHREADS_MAX 128
+#define ZSTDMT_NBTHREADS_MAX 256
 #define ZSTDMT_OVERLAPLOG_DEFAULT 6
 
 
@@ -407,7 +407,8 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
     U32 nbJobs = nbThreads + 2;
     DEBUGLOG(3, "ZSTDMT_createCCtx_advanced");
 
-    if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
+    if (nbThreads < 1) return NULL;
+    nbThreads = MIN(nbThreads , ZSTDMT_NBTHREADS_MAX);
     if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL))
         /* invalid custom allocator */
         return NULL;
diff --git a/programs/zstd.1 b/programs/zstd.1
index 5df45db2..2b80659c 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "June 2017" "zstd 1.3.0" "User Commands"
+.TH "ZSTD" "1" "July 2017" "zstd 1.3.1" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -105,7 +105,7 @@ unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note
 .
 .TP
 \fB\-T#\fR, \fB\-\-threads=#\fR
-Compress using \fB#\fR threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
+Compress using \fB#\fR threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==256\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
 .
 .TP
 \fB\-D file\fR
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 24e25a2f..ba51c843 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -108,6 +108,7 @@ the last one takes effect.
 * `-T#`, `--threads=#`:
     Compress using `#` threads (default: 1).
     If `#` is 0, attempt to detect and use the number of physical CPU cores.
+    In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==256.
     This modifier does nothing if `zstd` is compiled without multithread support.
 * `-D file`:
     use `file` as Dictionary to compress or decompress FILE(s)
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 88a1c2ab..dd0f2dbf 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -643,7 +643,8 @@ then
     $ECHO "\n**** zstdmt long round-trip tests **** "
     roundTripTest -g99000000 -P99 "20 -T2"
     roundTripTest -g6000000000 -P99 "1 -T2"
-    fileRoundTripTest -g4193M -P98 " -T0"
+    roundTripTest -g1500000000 -P97 "1 -T999"
+    fileRoundTripTest -g4195M -P98 " -T0"
 else
     $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
 fi

From 68c4560701cca29837dfa5a57a730a56be0199fb Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 13 Jul 2017 10:38:19 -0700
Subject: [PATCH 046/100] [ldm] Add TODO and comment for segfaulting in
 compress function

---
 contrib/long_distance_matching/ldm.c      |  7 +++++--
 contrib/long_distance_matching/main-ldm.c | 10 ++++++----
 contrib/long_distance_matching/util.c     |  5 -----
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index e64d2865..7a545073 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -28,7 +28,7 @@
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
 #define COMPUTE_STATS
-#define CHECKSUM_CHAR_OFFSET 2
+#define CHECKSUM_CHAR_OFFSET 0
 //#define RUN_CHECKS
 //#define LDM_DEBUG
 
@@ -470,7 +470,10 @@ static void outputBlock(LDM_CCtx *cctx,
   }
 }
 
-// TODO: srcSize and maxDstSize is unused
+// TODO: maxDstSize is unused. This function may seg fault when writing
+// beyond the size of dst, as it does not check maxDstSize. Writing to
+// a buffer and performing checks is a possible solution.
+//
 // This is based upon lz4.
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 8354b795..9e7d4526 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -1,5 +1,3 @@
-// TODO: file size must fit into a U32
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -12,12 +10,17 @@
 
 #include <fcntl.h>
 #include "ldm.h"
+#include "zstd.h"
 
 #define DEBUG
 //#define TEST
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
+ *
+ * TODO: This currently seg faults if the compressed size is > the decompress
+ * size due to the mmapping and output file size allocated to be the input size.
+ * The compress function should check before writing or buffer writes.
  */
 static int compress(const char *fname, const char *oname) {
   int fdin, fdout;
@@ -78,10 +81,9 @@ static int compress(const char *fname, const char *oname) {
            dst + LDM_HEADER_SIZE, statbuf.st_size);
 #endif
 */
-
   compressSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+                   dst + LDM_HEADER_SIZE, maxCompressSize);
 
   // Write compress and decompress size to header
   // TODO: should depend on LDM_DECOMPRESS_SIZE write32
diff --git a/contrib/long_distance_matching/util.c b/contrib/long_distance_matching/util.c
index 62749215..47ac8a12 100644
--- a/contrib/long_distance_matching/util.c
+++ b/contrib/long_distance_matching/util.c
@@ -53,11 +53,6 @@ U32 LDM_read32(const void *ptr) {
   return *(const U32 *)ptr;
 }
 
-//TODO: endianness?
-void LDM_write64(void *memPtr, U64 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
 U64 LDM_read64(const void *ptr) {
   return *(const U64 *)ptr;
 }

From 50421d9474710b44618d843fb000b08b3a96df65 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 13 Jul 2017 11:45:00 -0700
Subject: [PATCH 047/100] [ldm] Remove old main files

---
 contrib/long_distance_matching/main.c | 240 --------------------------
 contrib/long_distance_matching/main.h |   7 -
 2 files changed, 247 deletions(-)
 delete mode 100644 contrib/long_distance_matching/main.c
 delete mode 100644 contrib/long_distance_matching/main.h

diff --git a/contrib/long_distance_matching/main.c b/contrib/long_distance_matching/main.c
deleted file mode 100644
index 67144166..00000000
--- a/contrib/long_distance_matching/main.c
+++ /dev/null
@@ -1,240 +0,0 @@
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-
-#include "ldm.h"
-
-#define BUF_SIZE 16*1024  // Block size
-#define LDM_HEADER_SIZE 8
-
-/*
-static size_t compress_file_mmap(FILE *in, FILE *out, size_t *size_in,
-                                 size_t *size_out) {
-  char *src, *dst;
-  struct stat statbuf;
-
-  if (fstat(in, &statbuf) < 0) {
-    printf("fstat error\n");
-    return 1;
-  }
-
-
-  return 0;
-}
-*/
-
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = n;
-    count_out += n;
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) {
-      break;
-    }
-    if (0 == result) {
-      result = memcmp(b0, b1, r0);
-    }
-  }
-  return result;
-}
-
-int main(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-
-  return 0;
-}
-
-
diff --git a/contrib/long_distance_matching/main.h b/contrib/long_distance_matching/main.h
deleted file mode 100644
index a0b03012..00000000
--- a/contrib/long_distance_matching/main.h
+++ /dev/null
@@ -1,7 +0,0 @@
-#ifndef _MAIN_H
-#define _MAIN_H
-
-void compress_file(FILE *in, FILE *out, int argc, char *argv[]);
-void decompress_file(FILE *in, FILE *out, int argc, char *argv[]);
-
-#endif  /* _MAIN_H */

From 830ef4152a8b7568916ecb600a0aac1a5212c08f Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 13 Jul 2017 12:45:39 -0700
Subject: [PATCH 048/100] [libzstd] Increase granularity of FSECTable repeat
 mode

---
 lib/common/fse.h             |  5 +++++
 lib/common/zstd_internal.h   |  6 ++++--
 lib/compress/zstd_compress.c | 31 ++++++++++++++++++++++++-------
 3 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/lib/common/fse.h b/lib/common/fse.h
index 54ac98b1..1c44f837 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -384,6 +384,11 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
 size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
 /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
 
+typedef enum {
+   FSE_repeat_none,  /**< Cannot use the previous table */
+   FSE_repeat_check, /**< Can use the previous table but it must be checked */
+   FSE_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+ } FSE_repeat;
 
 /* *****************************************
 *  FSE symbol compression API
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 42e5e7b5..f3779e84 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -271,13 +271,15 @@ typedef struct {
 } seqStore_t;
 
 typedef struct {
-    HUF_repeat hufCTable_repeatMode;
     U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
-    U32 fseCTables_ready;
     FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
     FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
     FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
     U32 workspace[HUF_WORKSPACE_SIZE_U32];
+    HUF_repeat hufCTable_repeatMode;
+    FSE_repeat offcode_repeatMode;
+    FSE_repeat matchlength_repeatMode;
+    FSE_repeat litlength_repeatMode;
 } ZSTD_entropyCTables_t;
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b362a191..dfcb0266 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -616,8 +616,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     if (crp == ZSTDcrp_continue) {
         if (ZSTD_equivalentParams(params.cParams, zc->appliedParams.cParams)) {
             DEBUGLOG(5, "ZSTD_equivalentParams()==1");
-            zc->entropy->fseCTables_ready = 0;
             zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
+            zc->entropy->offcode_repeatMode = FSE_repeat_none;
+            zc->entropy->matchlength_repeatMode = FSE_repeat_none;
+            zc->entropy->litlength_repeatMode = FSE_repeat_none;
             return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
     }   }
 
@@ -681,8 +683,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->stage = ZSTDcs_init;
         zc->dictID = 0;
         zc->loadedDictEnd = 0;
-        zc->entropy->fseCTables_ready = 0;
         zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
+        zc->entropy->offcode_repeatMode = FSE_repeat_none;
+        zc->entropy->matchlength_repeatMode = FSE_repeat_none;
+        zc->entropy->litlength_repeatMode = FSE_repeat_none;
         zc->nextToUpdate = 1;
         zc->nextSrc = NULL;
         zc->base = NULL;
@@ -1067,11 +1071,13 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
             *op++ = llCodeTable[0];
             FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
             LLtype = set_rle;
-        } else if ((zc->entropy->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+            zc->entropy->litlength_repeatMode = FSE_repeat_check;
+        } else if ((zc->entropy->litlength_repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
             LLtype = set_repeat;
         } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) {
             FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
             LLtype = set_basic;
+            zc->entropy->litlength_repeatMode = FSE_repeat_valid;
         } else {
             size_t nbSeq_1 = nbSeq;
             const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
@@ -1082,6 +1088,7 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
               op += NCountSize; }
             FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
             LLtype = set_compressed;
+            zc->entropy->litlength_repeatMode = FSE_repeat_check;
     }   }
 
     /* CTable for Offsets */
@@ -1091,11 +1098,13 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
             *op++ = ofCodeTable[0];
             FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
             Offtype = set_rle;
-        } else if ((zc->entropy->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+            zc->entropy->offcode_repeatMode = FSE_repeat_check;
+        } else if ((zc->entropy->offcode_repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
             Offtype = set_repeat;
         } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) {
             FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
             Offtype = set_basic;
+            zc->entropy->offcode_repeatMode = FSE_repeat_valid;
         } else {
             size_t nbSeq_1 = nbSeq;
             const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
@@ -1106,6 +1115,7 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
               op += NCountSize; }
             FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
             Offtype = set_compressed;
+            zc->entropy->offcode_repeatMode = FSE_repeat_check;
     }   }
 
     /* CTable for MatchLengths */
@@ -1115,11 +1125,13 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
             *op++ = *mlCodeTable;
             FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
             MLtype = set_rle;
-        } else if ((zc->entropy->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+            zc->entropy->matchlength_repeatMode = FSE_repeat_check;
+        } else if ((zc->entropy->matchlength_repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
             MLtype = set_repeat;
         } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) {
             FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
             MLtype = set_basic;
+            zc->entropy->matchlength_repeatMode = FSE_repeat_valid;
         } else {
             size_t nbSeq_1 = nbSeq;
             const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
@@ -1130,10 +1142,10 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
               op += NCountSize; }
             FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
             MLtype = set_compressed;
+            zc->entropy->matchlength_repeatMode = FSE_repeat_check;
     }   }
 
     *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
-    zc->entropy->fseCTables_ready = 0;
 
     /* Encoding Sequences */
     {   BIT_CStream_t blockStream;
@@ -1213,6 +1225,9 @@ _check_compressibility:
         size_t const maxCSize = srcSize - minGain;
         if ((size_t)(op-ostart) >= maxCSize) {
             zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
+            zc->entropy->offcode_repeatMode = FSE_repeat_none;
+            zc->entropy->matchlength_repeatMode = FSE_repeat_none;
+            zc->entropy->litlength_repeatMode = FSE_repeat_none;
             return 0;
     }   }
 
@@ -3124,8 +3139,10 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
                 if (cctx->rep[u] > dictContentSize) return ERROR(dictionary_corrupted);
         }   }
 
-        cctx->entropy->fseCTables_ready = 1;
         cctx->entropy->hufCTable_repeatMode = HUF_repeat_valid;
+        cctx->entropy->offcode_repeatMode = FSE_repeat_valid;
+        cctx->entropy->matchlength_repeatMode = FSE_repeat_valid;
+        cctx->entropy->litlength_repeatMode = FSE_repeat_valid;
         return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize);
     }
 }

From 9306feb8fabdbaa1e4c4ccee90eb3cf8848556e6 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 13 Jul 2017 13:44:48 -0700
Subject: [PATCH 049/100] [ldm] Switch to using lib/common/mem.h and move
 typedefs to ldm.h

Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

Blame Revision:
---
 contrib/long_distance_matching/Makefile   |  7 +-
 contrib/long_distance_matching/ldm.c      | 82 +++++++++++------------
 contrib/long_distance_matching/ldm.h      | 15 +++--
 contrib/long_distance_matching/main-ldm.c | 13 ++--
 contrib/long_distance_matching/util.c     | 68 -------------------
 contrib/long_distance_matching/util.h     | 27 --------
 6 files changed, 57 insertions(+), 155 deletions(-)
 delete mode 100644 contrib/long_distance_matching/util.c
 delete mode 100644 contrib/long_distance_matching/util.h

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 5ffd4eaf..8ba16d03 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -9,6 +9,7 @@
 
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
+CPPFLAGS+= -I../../lib/common
 CFLAGS  ?= -O3
 DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@@ -26,11 +27,7 @@ default: all
 
 all: main-ldm
 
-
-#main : ldm.c main.c
-#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-main-ldm : util.c ldm.c main-ldm.c
+main-ldm : ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 7a545073..00099fbe 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -4,23 +4,22 @@
 #include <stdio.h>
 
 #include "ldm.h"
-#include "util.h"
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
 #define HASH_ONLY_EVERY 0
 
-#define LDM_MEMORY_USAGE 20
+#define LDM_MEMORY_USAGE 22
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 
 #define LDM_OFFSET_SIZE 4
 
-#define WINDOW_SIZE (1 << 20)
+#define WINDOW_SIZE (1 << 29)
 
 //These should be multiples of four.
-#define LDM_HASH_LENGTH 4
-#define MINMATCH 4
+#define LDM_HASH_LENGTH 8
+#define MINMATCH 8
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
@@ -32,21 +31,11 @@
 //#define RUN_CHECKS
 //#define LDM_DEBUG
 
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint32_t offset_t;
-typedef uint32_t hash_t;
-typedef signed char schar;
-
-typedef struct hashEntry {
+struct LDM_hashEntry {
   offset_t offset;
-} hashEntry;
+};
 
-typedef struct LDM_compressStats {
+struct LDM_compressStats {
   U32 numMatches;
   U32 totalMatchLength;
   U32 totalLiteralLength;
@@ -54,9 +43,9 @@ typedef struct LDM_compressStats {
 
   U32 numCollisions;
   U32 numHashInserts;
-} LDM_compressStats;
+};
 
-typedef struct LDM_CCtx {
+struct LDM_CCtx {
   size_t isize;             /* Input size */
   size_t maxOSize;          /* Maximum output size */
 
@@ -79,7 +68,7 @@ typedef struct LDM_CCtx {
 
   LDM_compressStats stats;            /* Compression statistics */
 
-  hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
@@ -94,7 +83,7 @@ typedef struct LDM_CCtx {
 
   // DEBUG
   const BYTE *DEBUG_setNextHash;
-} LDM_CCtx;
+};
 
 /**
  * Outputs compression statistics.
@@ -157,14 +146,14 @@ static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
   const BYTE *curMatch = match;
 
   for (; lengthLeft >= 8; lengthLeft -= 8) {
-    if (LDM_read64(curP) != LDM_read64(curMatch)) {
+    if (MEM_read64(curP) != MEM_read64(curMatch)) {
       return 0;
     }
     curP += 8;
     curMatch += 8;
   }
   if (lengthLeft > 0) {
-    return (LDM_read32(curP) == LDM_read32(curMatch));
+    return (MEM_read32(curP) == MEM_read32(curMatch));
   }
   return 1;
 }
@@ -184,10 +173,9 @@ static hash_t checksumToHash(U32 sum) {
  * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
  * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
   */
-static U32 getChecksum(const char *data, U32 len) {
+static U32 getChecksum(const BYTE *buf, U32 len) {
   U32 i;
   U32 s1, s2;
-  const schar *buf = (const schar *)data;
 
   s1 = s2 = 0;
   for (i = 0; i < (len - 4); i += 4) {
@@ -215,7 +203,7 @@ static U32 getChecksum(const char *data, U32 len) {
  * Thus toRemove should correspond to data[0].
  */
 static U32 updateChecksum(U32 sum, U32 len,
-                          schar toRemove, schar toAdd) {
+                          BYTE toRemove, BYTE toAdd) {
   U32 s1 = (sum & 0xffff) - toRemove + toAdd;
   U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
 
@@ -244,13 +232,13 @@ static void setNextHash(LDM_CCtx *cctx) {
 //  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
   cctx->nextSum = updateChecksum(
       cctx->lastSum, LDM_HASH_LENGTH,
-      (schar)((cctx->lastPosHashed)[0]),
-      (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
+      (cctx->lastPosHashed)[0],
+      (cctx->lastPosHashed)[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
   cctx->nextHash = checksumToHash(cctx->nextSum);
 
 #ifdef RUN_CHECKS
-  check = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
 
   if (check != cctx->nextSum) {
     printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
@@ -279,7 +267,8 @@ static void putHashOfCurrentPositionFromHash(
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    (cctx->hashTable)[hash] = (hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
+    (cctx->hashTable)[hash] =
+        (LDM_hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
   }
 
   cctx->lastPosHashed = cctx->ip;
@@ -307,7 +296,7 @@ static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
  * Insert hash of the current position into the hash table.
  */
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = getChecksum((const char *)cctx->ip, LDM_HASH_LENGTH);
+  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
   hash_t hash = checksumToHash(sum);
 
 #ifdef RUN_CHECKS
@@ -337,7 +326,7 @@ static unsigned countMatchLength(const BYTE *pIn, const BYTE *pMatch,
                                  const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
   while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
+    BYTE const diff = (*pMatch) ^ *(pIn);
     if (!diff) {
       pIn++;
       pMatch++;
@@ -427,9 +416,9 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
  * Update input pointer, inserting hashes into hash table along the way.
  */
 static void outputBlock(LDM_CCtx *cctx,
-                        unsigned const literalLength,
-                        unsigned const offset,
-                        unsigned const matchLength) {
+                        const unsigned literalLength,
+                        const unsigned offset,
+                        const unsigned matchLength) {
   BYTE *token = cctx->op++;
 
   /* Encode the literal length. */
@@ -449,7 +438,7 @@ static void outputBlock(LDM_CCtx *cctx,
   cctx->op += literalLength;
 
   /* Encode the offset. */
-  LDM_write32(cctx->op, offset);
+  MEM_write32(cctx->op, offset);
   cctx->op += LDM_OFFSET_SIZE;
 
   /* Encode the match length. */
@@ -457,10 +446,10 @@ static void outputBlock(LDM_CCtx *cctx,
     unsigned matchLengthRemaining = matchLength;
     *token += ML_MASK;
     matchLengthRemaining -= ML_MASK;
-    LDM_write32(cctx->op, 0xFFFFFFFF);
+    MEM_write32(cctx->op, 0xFFFFFFFF);
     while (matchLengthRemaining >= 4*0xFF) {
       cctx->op += 4;
-      LDM_write32(cctx->op, 0xffffffff);
+      MEM_write32(cctx->op, 0xffffffff);
       matchLengthRemaining -= 4*0xFF;
     }
     cctx->op += matchLengthRemaining / 255;
@@ -514,9 +503,9 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * length) and update pointers and hashes.
      */
     {
-      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
-      unsigned const offset = cctx.ip - match;
-      unsigned const matchLength = countMatchLength(
+      const unsigned literalLength = (unsigned)(cctx.ip - cctx.anchor);
+      const unsigned offset = cctx.ip - match;
+      const unsigned matchLength = countMatchLength(
           cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
 
 #ifdef COMPUTE_STATS
@@ -605,7 +594,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
     size_t length, offset;
 
     /* Get the literal length. */
-    unsigned const token = *(dctx.ip)++;
+    const unsigned token = *(dctx.ip)++;
     if ((length = (token >> ML_BITS)) == RUN_MASK) {
       unsigned s;
       do {
@@ -621,7 +610,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
     dctx.op = cpy;
 
     //TODO : dynamic offset size
-    offset = LDM_read32(dctx.ip);
+    offset = MEM_read32(dctx.ip);
     dctx.ip += LDM_OFFSET_SIZE;
     match = dctx.op - offset;
 
@@ -647,6 +636,11 @@ size_t LDM_decompress(const void *src, size_t compressSize,
   return dctx.op - (BYTE *)dst;
 }
 
+// TODO: implement and test hash function
+void LDM_test(void) {
+
+}
+
 /*
 void LDM_test(const void *src, size_t srcSize,
               void *dst, size_t maxDstSize) {
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index f04b6e95..fd8c2ab8 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -3,10 +3,18 @@
 
 #include <stddef.h>   /* size_t */
 
+#include "mem.h"    // from /lib/common/mem.h
+
 #define LDM_COMPRESS_SIZE 8
 #define LDM_DECOMPRESS_SIZE 8
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
 
+typedef U32 offset_t;
+typedef U32 hash_t;
+typedef struct LDM_hashEntry LDM_hashEntry;
+typedef struct LDM_compressStats LDM_compressStats;
+typedef struct LDM_CCtx LDM_CCtx;
+
 /**
  *  Compresses src into dst.
  *
@@ -46,10 +54,9 @@ size_t LDM_decompress(const void *src, size_t srcSize,
  *
  * NB: LDM_compress and LDM_decompress currently do not add/read headers.
  */
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize);
+void LDM_readHeader(const void *src, U64 *compressSize,
+                    U64 *decompressSize);
 
-void LDM_test(const void *src, size_t srcSize,
-              void *dst, size_t maxDstSize);
+void LDM_test(void);
 
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 9e7d4526..2017cf4e 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -75,12 +75,6 @@ static int compress(const char *fname, const char *oname) {
       return 1;
   }
 
-/*
-#ifdef TEST
-  LDM_test(src, statbuf.st_size,
-           dst + LDM_HEADER_SIZE, statbuf.st_size);
-#endif
-*/
   compressSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
                    dst + LDM_HEADER_SIZE, maxCompressSize);
@@ -116,7 +110,8 @@ static int decompress(const char *fname, const char *oname) {
   int fdin, fdout;
   struct stat statbuf;
   char *src, *dst;
-  size_t compressSize, decompressSize, outSize;
+  U64 compressSize, decompressSize;
+  size_t outSize;
 
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
@@ -267,5 +262,9 @@ int main(int argc, const char *argv[]) {
   }
   /* verify */
   verify(inpFilename, decFilename);
+
+#ifdef TEST
+  LDM_test();
+#endif
   return 0;
 }
diff --git a/contrib/long_distance_matching/util.c b/contrib/long_distance_matching/util.c
deleted file mode 100644
index 47ac8a12..00000000
--- a/contrib/long_distance_matching/util.c
+++ /dev/null
@@ -1,68 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "util.h"
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-unsigned LDM_isLittleEndian(void) {
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-void LDM_write16(void *memPtr, U16 value){
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE *)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-BYTE LDM_readByte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
diff --git a/contrib/long_distance_matching/util.h b/contrib/long_distance_matching/util.h
deleted file mode 100644
index dbf55cbc..00000000
--- a/contrib/long_distance_matching/util.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef LDM_UTIL_H
-#define LDM_UTIL_H
-
-unsigned LDM_isLittleEndian(void);
-
-uint16_t LDM_read16(const void *memPtr);
-
-uint16_t LDM_readLE16(const void *memPtr);
-
-void LDM_write16(void *memPtr, uint16_t value);
-
-void LDM_write32(void *memPtr, uint32_t value);
-
-void LDM_writeLE16(void *memPtr, uint16_t value);
-
-uint32_t LDM_read32(const void *ptr);
-
-uint64_t LDM_read64(const void *ptr);
-
-void LDM_copy8(void *dst, const void *src);
-
-uint8_t LDM_readByte(const void *ptr);
-
-void LDM_write64(void *memPtr, uint64_t value);
-
-
-#endif /* LDM_UTIL_H */

From 2b3c7e4199842a7ae2038e31adb36be69bb0725d Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 13 Jul 2017 14:39:35 -0700
Subject: [PATCH 050/100] [ldm] Make some functions shared

---
 contrib/long_distance_matching/ldm.c      | 178 +++++++++-------------
 contrib/long_distance_matching/ldm.h      |  71 ++++++++-
 contrib/long_distance_matching/main-ldm.c |  34 ++---
 3 files changed, 160 insertions(+), 123 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 00099fbe..a5057aec 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -19,7 +19,6 @@
 
 //These should be multiples of four.
 #define LDM_HASH_LENGTH 8
-#define MINMATCH 8
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
@@ -85,12 +84,9 @@ struct LDM_CCtx {
   const BYTE *DEBUG_setNextHash;
 };
 
-/**
- * Outputs compression statistics.
- */
-static void printCompressStats(const LDM_CCtx *cctx) {
-  const LDM_compressStats *stats = &(cctx->stats);
-#ifdef COMPUTE_STATS
+void LDM_printCompressStats(const LDM_compressStats *stats,
+                            const LDM_hashEntry *hashTable,
+                            U32 hashTableSize) {
   printf("=====================\n");
   printf("Compression statistics\n");
   printf("Total number of matches: %u\n", stats->numMatches);
@@ -110,50 +106,41 @@ static void printCompressStats(const LDM_CCtx *cctx) {
   {
     U32 i = 0;
     U32 ctr = 0;
-    for (; i < LDM_HASHTABLESIZE_U32; i++) {
-      if ((cctx->hashTable)[i].offset == 0) {
+    for (; i < hashTableSize; i++) {
+      if (hashTable[i].offset == 0) {
         ctr++;
       }
     }
     printf("Hash table size, empty slots, %% empty: %u %u %.3f\n",
-           LDM_HASHTABLESIZE_U32, ctr,
-           100.0 * (double)(ctr) / (double)LDM_HASHTABLESIZE_U32);
+           hashTableSize, ctr,
+           100.0 * (double)(ctr) / (double)hashTableSize);
   }
 
   printf("=====================\n");
-#endif
 }
 
-/**
- * Checks whether the MINMATCH bytes from p are the same as the MINMATCH
- * bytes from match.
- *
- * This assumes MINMATCH is a multiple of four.
- *
- * Return 1 if valid, 0 otherwise.
- */
-static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
+int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
   /*
-  if (memcmp(p, match, MINMATCH) == 0) {
+  if (memcmp(pIn, pMatch, LDM_MIN_MATCH_LENGTH) == 0) {
     return 1;
   }
   return 0;
   */
 
   //TODO: This seems to be faster for some reason?
-  U16 lengthLeft = MINMATCH;
-  const BYTE *curP = p;
-  const BYTE *curMatch = match;
+  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
+  const BYTE *curIn = pIn;
+  const BYTE *curMatch = pMatch;
 
   for (; lengthLeft >= 8; lengthLeft -= 8) {
-    if (MEM_read64(curP) != MEM_read64(curMatch)) {
+    if (MEM_read64(curIn) != MEM_read64(curMatch)) {
       return 0;
     }
-    curP += 8;
+    curIn += 8;
     curMatch += 8;
   }
   if (lengthLeft > 0) {
-    return (MEM_read32(curP) == MEM_read32(curMatch));
+    return (MEM_read32(curIn) == MEM_read32(curMatch));
   }
   return 1;
 }
@@ -316,14 +303,8 @@ static const BYTE *getPositionOnHash(LDM_CCtx *cctx, hash_t hash) {
   return cctx->hashTable[hash].offset + cctx->ibase;
 }
 
-/**
- *  Counts the number of bytes that match from pIn and pMatch,
- *  up to pInLimit.
- *
- *  TODO: make more efficient.
- */
-static unsigned countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                                 const BYTE *pInLimit) {
+U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
   while (pIn < pInLimit - 1) {
     BYTE const diff = (*pMatch) ^ *(pIn);
@@ -332,24 +313,23 @@ static unsigned countMatchLength(const BYTE *pIn, const BYTE *pMatch,
       pMatch++;
       continue;
     }
-    return (unsigned)(pIn - pStart);
+    return (U32)(pIn - pStart);
   }
-  return (unsigned)(pIn - pStart);
+  return (U32)(pIn - pStart);
 }
 
-void LDM_readHeader(const void *src, U64 *compressSize,
-                    U64 *decompressSize) {
-  const U64 *ip = (const U64 *)src;
-  *compressSize = *ip++;
-  *decompressSize = *ip;
+void LDM_readHeader(const void *src, U64 *compressedSize,
+                    U64 *decompressedSize) {
+  const BYTE *ip = (const BYTE *)src;
+  *compressedSize = MEM_readLE64(ip);
+  ip += sizeof(U64);
+  *decompressedSize = MEM_readLE64(ip);
+  // ip += sizeof(U64);
 }
 
-/**
- * Initialize a compression context.
- */
-static void initializeCCtx(LDM_CCtx *cctx,
-                           const void *src, size_t srcSize,
-                           void *dst, size_t maxDstSize) {
+void LDM_initializeCCtx(LDM_CCtx *cctx,
+                        const void *src, size_t srcSize,
+                        void *dst, size_t maxDstSize) {
   cctx->isize = srcSize;
   cctx->maxOSize = maxDstSize;
 
@@ -358,7 +338,7 @@ static void initializeCCtx(LDM_CCtx *cctx,
   cctx->iend = cctx->ibase + srcSize;
 
   cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
-  cctx->imatchLimit = cctx->iend - MINMATCH;
+  cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
 
   cctx->obase = (BYTE *)dst;
   cctx->op = (BYTE *)dst;
@@ -409,33 +389,33 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
   return 0;
 }
 
-/**
- * Write current block (literals, literal length, match offset,
- * match length).
- *
- * Update input pointer, inserting hashes into hash table along the way.
- */
-static void outputBlock(LDM_CCtx *cctx,
-                        const unsigned literalLength,
-                        const unsigned offset,
-                        const unsigned matchLength) {
-  BYTE *token = cctx->op++;
-
+void LDM_encodeLiteralLengthAndLiterals(
+    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
   /* Encode the literal length. */
   if (literalLength >= RUN_MASK) {
     int len = (int)literalLength - RUN_MASK;
-    *token = (RUN_MASK << ML_BITS);
+    *pToken = (RUN_MASK << ML_BITS);
     for (; len >= 255; len -= 255) {
       *(cctx->op)++ = 255;
     }
     *(cctx->op)++ = (BYTE)len;
   } else {
-    *token = (BYTE)(literalLength << ML_BITS);
+    *pToken = (BYTE)(literalLength << ML_BITS);
   }
 
   /* Encode the literals. */
   memcpy(cctx->op, cctx->anchor, literalLength);
   cctx->op += literalLength;
+}
+
+void LDM_outputBlock(LDM_CCtx *cctx,
+                     const U32 literalLength,
+                     const U32 offset,
+                     const U32 matchLength) {
+  BYTE *pToken = cctx->op++;
+
+  /* Encode the literal length and literals. */
+  LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
 
   /* Encode the offset. */
   MEM_write32(cctx->op, offset);
@@ -444,7 +424,7 @@ static void outputBlock(LDM_CCtx *cctx,
   /* Encode the match length. */
   if (matchLength >= ML_MASK) {
     unsigned matchLengthRemaining = matchLength;
-    *token += ML_MASK;
+    *pToken += ML_MASK;
     matchLengthRemaining -= ML_MASK;
     MEM_write32(cctx->op, 0xFFFFFFFF);
     while (matchLengthRemaining >= 4*0xFF) {
@@ -455,7 +435,7 @@ static void outputBlock(LDM_CCtx *cctx,
     cctx->op += matchLengthRemaining / 255;
     *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
   } else {
-    *token += (BYTE)(matchLength);
+    *pToken += (BYTE)(matchLength);
   }
 }
 
@@ -467,7 +447,7 @@ static void outputBlock(LDM_CCtx *cctx,
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
-  initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
@@ -503,21 +483,23 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * length) and update pointers and hashes.
      */
     {
-      const unsigned literalLength = (unsigned)(cctx.ip - cctx.anchor);
-      const unsigned offset = cctx.ip - match;
-      const unsigned matchLength = countMatchLength(
-          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
+      const U32 literalLength = cctx.ip - cctx.anchor;
+      const U32 offset = cctx.ip - match;
+      const U32 matchLength = LDM_countMatchLength(
+          cctx.ip + LDM_MIN_MATCH_LENGTH, match + LDM_MIN_MATCH_LENGTH,
+          cctx.ihashLimit);
 
 #ifdef COMPUTE_STATS
       cctx.stats.totalLiteralLength += literalLength;
       cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + MINMATCH;
+      cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
 #endif
-      outputBlock(&cctx, literalLength, offset, matchLength);
+      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
       // Move ip to end of block, inserting hashes at each position.
       cctx.nextIp = cctx.ip + cctx.step;
-      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
+      while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
+                       matchLength + literalLength) {
         if (cctx.ip > cctx.lastPosHashed) {
           // TODO: Simplify.
           LDM_updateLastHashFromNextHash(&cctx);
@@ -535,31 +517,21 @@ size_t LDM_compress(const void *src, size_t srcSize,
 _last_literals:
   /* Encode the last literals (no more matches). */
   {
-    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *(cctx.op)++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *(cctx.op)++ = 255;
-      }
-      *(cctx.op)++ = (BYTE)accumulator;
-    } else {
-      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(cctx.op, cctx.anchor, lastRun);
-    cctx.op += lastRun;
+    const size_t lastRun = (size_t)(cctx.iend - cctx.anchor);
+    BYTE *pToken = cctx.op++;
+    LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
   }
 
 #ifdef COMPUTE_STATS
-  printCompressStats(&cctx);
+  LDM_printCompressStats(&cctx.stats, cctx.hashTable, LDM_HASHTABLESIZE_U32);
 #endif
 
   return (cctx.op - (const BYTE *)cctx.obase);
 }
 
-typedef struct LDM_DCtx {
-  size_t compressSize;
-  size_t maxDecompressSize;
+struct LDM_DCtx {
+  size_t compressedSize;
+  size_t maxDecompressedSize;
 
   const BYTE *ibase;   /* Base of input */
   const BYTE *ip;      /* Current input position */
@@ -568,25 +540,25 @@ typedef struct LDM_DCtx {
   const BYTE *obase;   /* Base of output */
   BYTE *op;            /* Current output position */
   const BYTE *oend;    /* End of output */
-} LDM_DCtx;
+};
 
-static void LDM_initializeDCtx(LDM_DCtx *dctx,
-                               const void *src, size_t compressSize,
-                               void *dst, size_t maxDecompressSize) {
-  dctx->compressSize = compressSize;
-  dctx->maxDecompressSize = maxDecompressSize;
+void LDM_initializeDCtx(LDM_DCtx *dctx,
+                        const void *src, size_t compressedSize,
+                        void *dst, size_t maxDecompressedSize) {
+  dctx->compressedSize = compressedSize;
+  dctx->maxDecompressedSize = maxDecompressedSize;
 
   dctx->ibase = src;
   dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressSize;
+  dctx->iend = dctx->ip + dctx->compressedSize;
   dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressSize;
+  dctx->oend = dctx->op + dctx->maxDecompressedSize;
 }
 
-size_t LDM_decompress(const void *src, size_t compressSize,
-                      void *dst, size_t maxDecompressSize) {
+size_t LDM_decompress(const void *src, size_t compressedSize,
+                      void *dst, size_t maxDecompressedSize) {
   LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
+  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
 
   while (dctx.ip < dctx.iend) {
     BYTE *cpy;
@@ -623,7 +595,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
         length += s;
       } while (s == 255);
     }
-    length += MINMATCH;
+    length += LDM_MIN_MATCH_LENGTH;
 
     /* Copy match. */
     cpy = dctx.op + length;
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index fd8c2ab8..19d475dc 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -9,11 +9,15 @@
 #define LDM_DECOMPRESS_SIZE 8
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
 
+// This should be a multiple of four.
+#define LDM_MIN_MATCH_LENGTH 8
+
 typedef U32 offset_t;
 typedef U32 hash_t;
 typedef struct LDM_hashEntry LDM_hashEntry;
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
+typedef struct LDM_DCtx LDM_DCtx;
 
 /**
  *  Compresses src into dst.
@@ -45,17 +49,78 @@ typedef struct LDM_CCtx LDM_CCtx;
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize);
 
+/**
+ * Initialize the compression context.
+ */
+void LDM_initializeCCtx(LDM_CCtx *cctx,
+                        const void *src, size_t srcSize,
+                        void *dst, size_t maxDstSize);
+/**
+ * Outputs compression statistics to stdout.
+ */
+void LDM_printCompressStats(const LDM_compressStats *stats,
+                            const LDM_hashEntry *hashTable,
+                            U32 hashTableSize);
+/**
+ * Checks whether the LDM_MIN_MATCH_LENGTH bytes from p are the same as the
+ * LDM_MIN_MATCH_LENGTH bytes from match.
+ *
+ * This assumes LDM_MIN_MATCH_LENGTH is a multiple of four.
+ *
+ * Return 1 if valid, 0 otherwise.
+ */
+int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch);
+
+/**
+ *  Counts the number of bytes that match from pIn and pMatch,
+ *  up to pInLimit.
+ */
+U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *pInLimit);
+
+/**
+ * Encode the literal length followed by the literals.
+ *
+ * The literal length is written to the upper four bits of pToken, with
+ * additional bytes written to the output as needed (see lz4).
+ *
+ * This is followed by literalLength bytes corresponding to the literals.
+ */
+void LDM_encodeLiteralLengthAndLiterals(
+    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength);
+
+/**
+ * Write current block (literals, literal length, match offset,
+ * match length).
+ */
+void LDM_outputBlock(LDM_CCtx *cctx,
+                     const U32 literalLength,
+                     const U32 offset,
+                     const U32 matchLength);
+
+/**
+ * Decompresses src into dst.
+ *
+ * Note: assumes src does not have a header.
+ */
 size_t LDM_decompress(const void *src, size_t srcSize,
                       void *dst, size_t maxDstSize);
 
+/**
+ * Initialize the decompression context.
+ */
+void LDM_initializeDCtx(LDM_DCtx *dctx,
+                        const void *src, size_t compressedSize,
+                        void *dst, size_t maxDecompressedSize);
+
 /**
  * Reads the header from src and writes the compressed size and
- * decompressed size into compressSize and decompressSize respectively.
+ * decompressed size into compressedSize and decompressedSize respectively.
  *
  * NB: LDM_compress and LDM_decompress currently do not add/read headers.
  */
-void LDM_readHeader(const void *src, U64 *compressSize,
-                    U64 *decompressSize);
+void LDM_readHeader(const void *src, U64 *compressedSize,
+                    U64 *decompressedSize);
 
 void LDM_test(void);
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 2017cf4e..40afef8c 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -26,7 +26,7 @@ static int compress(const char *fname, const char *oname) {
   int fdin, fdout;
   struct stat statbuf;
   char *src, *dst;
-  size_t maxCompressSize, compressSize;
+  size_t maxCompressedSize, compressedSize;
 
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
@@ -46,11 +46,11 @@ static int compress(const char *fname, const char *oname) {
     return 1;
   }
 
-  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+  maxCompressedSize = statbuf.st_size + LDM_HEADER_SIZE;
 
  /* Go to the location corresponding to the last byte. */
  /* TODO: fallocate? */
-  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+  if (lseek(fdout, maxCompressedSize - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
   }
@@ -69,32 +69,32 @@ static int compress(const char *fname, const char *oname) {
   }
 
   /* mmap the output file */
-  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+  if ((dst = mmap(0, maxCompressedSize, PROT_READ | PROT_WRITE,
                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
       perror("mmap error for output");
       return 1;
   }
 
-  compressSize = LDM_HEADER_SIZE +
+  compressedSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, maxCompressSize);
+                   dst + LDM_HEADER_SIZE, maxCompressedSize);
 
   // Write compress and decompress size to header
   // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressSize, 8);
+  memcpy(dst, &compressedSize, 8);
   memcpy(dst + 8, &(statbuf.st_size), 8);
 
 #ifdef DEBUG
-  printf("Compressed size: %zu\n", compressSize);
+  printf("Compressed size: %zu\n", compressedSize);
   printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
 #endif
 
-  // Truncate file to compressSize.
-  ftruncate(fdout, compressSize);
+  // Truncate file to compressedSize.
+  ftruncate(fdout, compressedSize);
 
   printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
-         (double)compressSize / (statbuf.st_size) * 100);
+         (unsigned)statbuf.st_size, (unsigned)compressedSize, oname,
+         (double)compressedSize / (statbuf.st_size) * 100);
 
   // Close files.
   close(fdin);
@@ -110,7 +110,7 @@ static int decompress(const char *fname, const char *oname) {
   int fdin, fdout;
   struct stat statbuf;
   char *src, *dst;
-  U64 compressSize, decompressSize;
+  U64 compressedSize, decompressedSize;
   size_t outSize;
 
   /* Open the input file. */
@@ -139,10 +139,10 @@ static int decompress(const char *fname, const char *oname) {
   }
 
   /* Read the header. */
-  LDM_readHeader(src, &compressSize, &decompressSize);
+  LDM_readHeader(src, &compressedSize, &decompressedSize);
 
   /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+  if (lseek(fdout, decompressedSize - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
   }
@@ -154,7 +154,7 @@ static int decompress(const char *fname, const char *oname) {
   }
 
   /* mmap the output file */
-  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+  if ((dst = mmap(0, decompressedSize, PROT_READ | PROT_WRITE,
                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
       perror("mmap error for output");
       return 1;
@@ -162,7 +162,7 @@ static int decompress(const char *fname, const char *oname) {
 
   outSize = LDM_decompress(
       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
-      dst, decompressSize);
+      dst, decompressedSize);
 
   printf("Ret size out: %zu\n", outSize);
   ftruncate(fdout, outSize);

From 361c06df75d154b793cfb02b7bd279e9a6846cbd Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 13 Jul 2017 15:29:41 -0700
Subject: [PATCH 051/100] Add min/max offset to stats

---
 contrib/long_distance_matching/ldm.c          |  97 +++---
 contrib/long_distance_matching/ldm.h          |  24 +-
 .../versions/v0.5/Makefile                    |  14 +-
 .../versions/v0.5/ldm.c                       | 328 ++++++++----------
 .../versions/v0.5/ldm.h                       | 135 ++++++-
 .../versions/v0.5/main-ldm.c                  | 254 ++------------
 .../versions/v0.5/util.c                      |  69 ----
 .../versions/v0.5/util.h                      |  25 --
 8 files changed, 386 insertions(+), 560 deletions(-)
 delete mode 100644 contrib/long_distance_matching/versions/v0.5/util.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.5/util.h

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index a5057aec..b8e8c63b 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -1,25 +1,14 @@
-#include <stdlib.h>
-#include <string.h>
+#include <limits.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 #include "ldm.h"
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
 #define HASH_ONLY_EVERY 0
 
-#define LDM_MEMORY_USAGE 22
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-
-#define LDM_OFFSET_SIZE 4
-
-#define WINDOW_SIZE (1 << 29)
-
-//These should be multiples of four.
-#define LDM_HASH_LENGTH 8
-
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
 #define RUN_BITS (8-ML_BITS)
@@ -34,19 +23,24 @@ struct LDM_hashEntry {
   offset_t offset;
 };
 
+// TODO: Add offset histogram by powers of two
+// TODO: Scanning speed
+// TODO: Memory usage
 struct LDM_compressStats {
   U32 numMatches;
-  U32 totalMatchLength;
-  U32 totalLiteralLength;
+  U64 totalMatchLength;
+  U64 totalLiteralLength;
   U64 totalOffset;
 
+  U32 minOffset, maxOffset;
+
   U32 numCollisions;
   U32 numHashInserts;
 };
 
 struct LDM_CCtx {
-  size_t isize;             /* Input size */
-  size_t maxOSize;          /* Maximum output size */
+  U64 isize;             /* Input size */
+  U64 maxOSize;          /* Maximum output size */
 
   const BYTE *ibase;        /* Base of input */
   const BYTE *ip;           /* Current input position */
@@ -84,39 +78,40 @@ struct LDM_CCtx {
   const BYTE *DEBUG_setNextHash;
 };
 
-void LDM_printCompressStats(const LDM_compressStats *stats,
-                            const LDM_hashEntry *hashTable,
-                            U32 hashTableSize) {
+void LDM_outputHashtableOccupancy(
+    const LDM_hashEntry *hashTable, U32 hashTableSize) {
+  U32 i = 0;
+  U32 ctr = 0;
+  for (; i < hashTableSize; i++) {
+    if (hashTable[i].offset == 0) {
+      ctr++;
+    }
+  }
+  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
+         hashTableSize, ctr,
+         100.0 * (double)(ctr) / (double)hashTableSize);
+}
+
+void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("=====================\n");
   printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->numMatches);
-  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
+  //TODO: compute percentage matched?
+  printf("num matches, total match length: %u, %llu\n",
+          stats->numMatches,
+          stats->totalMatchLength);
+  printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
                                          (double)stats->numMatches);
-  printf("Average literal length: %.1f\n",
+  printf("avg literal length: %.1f\n",
          ((double)stats->totalLiteralLength) / (double)stats->numMatches);
-  printf("Average offset length: %.1f\n",
+  printf("avg offset length: %.1f\n",
          ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
+  printf("min offset, max offset: %u %u\n",
+         stats->minOffset, stats->maxOffset);
+  printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
          stats->numCollisions, stats->numHashInserts,
          stats->numHashInserts == 0 ?
             1.0 : (100.0 * (double)stats->numCollisions) /
                   (double)stats->numHashInserts);
-
-  // Output occupancy of hash table.
-  {
-    U32 i = 0;
-    U32 ctr = 0;
-    for (; i < hashTableSize; i++) {
-      if (hashTable[i].offset == 0) {
-        ctr++;
-      }
-    }
-    printf("Hash table size, empty slots, %% empty: %u %u %.3f\n",
-           hashTableSize, ctr,
-           100.0 * (double)(ctr) / (double)hashTableSize);
-  }
-
-  printf("=====================\n");
 }
 
 int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
@@ -219,8 +214,8 @@ static void setNextHash(LDM_CCtx *cctx) {
 //  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
   cctx->nextSum = updateChecksum(
       cctx->lastSum, LDM_HASH_LENGTH,
-      (cctx->lastPosHashed)[0],
-      (cctx->lastPosHashed)[LDM_HASH_LENGTH]);
+      cctx->lastPosHashed[0],
+      cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
   cctx->nextHash = checksumToHash(cctx->nextSum);
 
@@ -243,7 +238,7 @@ static void putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash, U32 sum) {
 #ifdef COMPUTE_STATS
   if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = (cctx->hashTable)[hash].offset;
+    offset_t offset = cctx->hashTable[hash].offset;
     cctx->stats.numHashInserts++;
     if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
       cctx->stats.numCollisions++;
@@ -254,8 +249,8 @@ static void putHashOfCurrentPositionFromHash(
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    (cctx->hashTable)[hash] =
-        (LDM_hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
+    cctx->hashTable[hash] = entry;
   }
 
   cctx->lastPosHashed = cctx->ip;
@@ -347,6 +342,7 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
   memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+  cctx->stats.minOffset = UINT_MAX;
 
   cctx->lastPosHashed = NULL;
 
@@ -493,6 +489,10 @@ size_t LDM_compress(const void *src, size_t srcSize,
       cctx.stats.totalLiteralLength += literalLength;
       cctx.stats.totalOffset += offset;
       cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
+      cctx.stats.minOffset =
+          offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
+      cctx.stats.maxOffset =
+          offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
 #endif
       LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
@@ -523,7 +523,8 @@ _last_literals:
   }
 
 #ifdef COMPUTE_STATS
-  LDM_printCompressStats(&cctx.stats, cctx.hashTable, LDM_HASHTABLESIZE_U32);
+  LDM_printCompressStats(&cctx.stats);
+  LDM_outputHashtableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
 #endif
 
   return (cctx.op - (const BYTE *)cctx.obase);
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 19d475dc..5da3c3b9 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -8,9 +8,19 @@
 #define LDM_COMPRESS_SIZE 8
 #define LDM_DECOMPRESS_SIZE 8
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+#define LDM_OFFSET_SIZE 4
 
-// This should be a multiple of four.
+// Defines the size of the hash table.
+#define LDM_MEMORY_USAGE 22
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+
+#define WINDOW_SIZE (1 << 25)
+
+//These should be multiples of four.
 #define LDM_MIN_MATCH_LENGTH 8
+#define LDM_HASH_LENGTH 8
 
 typedef U32 offset_t;
 typedef U32 hash_t;
@@ -55,12 +65,18 @@ size_t LDM_compress(const void *src, size_t srcSize,
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize);
+
+/**
+ * Prints the percentage of the hash table occupied (where occupied is defined
+ * as the entry being non-zero).
+ */
+void LDM_outputHashtableOccupancy(const LDM_hashEntry *hashTable,
+                                  U32 hashTableSize);
+
 /**
  * Outputs compression statistics to stdout.
  */
-void LDM_printCompressStats(const LDM_compressStats *stats,
-                            const LDM_hashEntry *hashTable,
-                            U32 hashTableSize);
+void LDM_printCompressStats(const LDM_compressStats *stats);
 /**
  * Checks whether the LDM_MIN_MATCH_LENGTH bytes from p are the same as the
  * LDM_MIN_MATCH_LENGTH bytes from match.
diff --git a/contrib/long_distance_matching/versions/v0.5/Makefile b/contrib/long_distance_matching/versions/v0.5/Makefile
index fa4abce6..dee686bc 100644
--- a/contrib/long_distance_matching/versions/v0.5/Makefile
+++ b/contrib/long_distance_matching/versions/v0.5/Makefile
@@ -1,5 +1,15 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
+CPPFLAGS+= -I../../../../lib/common
 CFLAGS  ?= -O3
 DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@@ -17,11 +27,11 @@ default: all
 
 all: main-ldm
 
-main-ldm : util.c ldm.c main-ldm.c
+main-ldm : ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-ldm
+	main main-ldm
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.c b/contrib/long_distance_matching/versions/v0.5/ldm.c
index 5fa20c06..b8e8c63b 100644
--- a/contrib/long_distance_matching/versions/v0.5/ldm.c
+++ b/contrib/long_distance_matching/versions/v0.5/ldm.c
@@ -1,63 +1,46 @@
-#include <stdlib.h>
-#include <string.h>
+#include <limits.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 #include "ldm.h"
-#include "util.h"
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
 #define HASH_ONLY_EVERY 0
 
-#define LDM_MEMORY_USAGE 20
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-
-#define LDM_OFFSET_SIZE 4
-
-#define WINDOW_SIZE (1 << 20)
-
-//These should be multiples of four.
-#define LDM_HASH_LENGTH 4
-#define MINMATCH 4
-
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
 #define COMPUTE_STATS
+#define CHECKSUM_CHAR_OFFSET 0
 //#define RUN_CHECKS
 //#define LDM_DEBUG
 
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint32_t offset_t;
-typedef uint32_t hash_t;
-typedef signed char schar;
-
-typedef struct hashEntry {
+struct LDM_hashEntry {
   offset_t offset;
-} hashEntry;
+};
 
-typedef struct LDM_compressStats {
+// TODO: Add offset histogram by powers of two
+// TODO: Scanning speed
+// TODO: Memory usage
+struct LDM_compressStats {
   U32 numMatches;
-  U32 totalMatchLength;
-  U32 totalLiteralLength;
+  U64 totalMatchLength;
+  U64 totalLiteralLength;
   U64 totalOffset;
 
+  U32 minOffset, maxOffset;
+
   U32 numCollisions;
   U32 numHashInserts;
-} LDM_compressStats;
+};
 
-typedef struct LDM_CCtx {
-  size_t isize;             /* Input size */
-  size_t maxOSize;          /* Maximum output size */
+struct LDM_CCtx {
+  U64 isize;             /* Input size */
+  U64 maxOSize;          /* Maximum output size */
 
   const BYTE *ibase;        /* Base of input */
   const BYTE *ip;           /* Current input position */
@@ -78,7 +61,7 @@ typedef struct LDM_CCtx {
 
   LDM_compressStats stats;            /* Compression statistics */
 
-  hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
@@ -93,77 +76,66 @@ typedef struct LDM_CCtx {
 
   // DEBUG
   const BYTE *DEBUG_setNextHash;
-} LDM_CCtx;
+};
 
-#ifdef COMPUTE_STATS
-/**
- * Outputs compression statistics.
- */
-static void printCompressStats(const LDM_CCtx *cctx) {
-  const LDM_compressStats *stats = &(cctx->stats);
+void LDM_outputHashtableOccupancy(
+    const LDM_hashEntry *hashTable, U32 hashTableSize) {
+  U32 i = 0;
+  U32 ctr = 0;
+  for (; i < hashTableSize; i++) {
+    if (hashTable[i].offset == 0) {
+      ctr++;
+    }
+  }
+  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
+         hashTableSize, ctr,
+         100.0 * (double)(ctr) / (double)hashTableSize);
+}
+
+void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("=====================\n");
   printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->numMatches);
-  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
+  //TODO: compute percentage matched?
+  printf("num matches, total match length: %u, %llu\n",
+          stats->numMatches,
+          stats->totalMatchLength);
+  printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
                                          (double)stats->numMatches);
-  printf("Average literal length: %.1f\n",
+  printf("avg literal length: %.1f\n",
          ((double)stats->totalLiteralLength) / (double)stats->numMatches);
-  printf("Average offset length: %.1f\n",
+  printf("avg offset length: %.1f\n",
          ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("Num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
+  printf("min offset, max offset: %u %u\n",
+         stats->minOffset, stats->maxOffset);
+  printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
          stats->numCollisions, stats->numHashInserts,
          stats->numHashInserts == 0 ?
             1.0 : (100.0 * (double)stats->numCollisions) /
                   (double)stats->numHashInserts);
-
-  // Output occupancy of hash table.
-  {
-    U32 i = 0;
-    U32 ctr = 0;
-    for (; i < LDM_HASHTABLESIZE_U32; i++) {
-      if ((cctx->hashTable)[i].offset == 0) {
-        ctr++;
-      }
-    }
-    printf("Hash table size, empty slots, %% empty: %u %u %.3f\n",
-           LDM_HASHTABLESIZE_U32, ctr,
-           100.0 * (double)(ctr) / (double)LDM_HASHTABLESIZE_U32);
-  }
-
-  printf("=====================\n");
 }
-#endif
 
-/**
- * Checks whether the MINMATCH bytes from p are the same as the MINMATCH
- * bytes from match.
- *
- * This assumes MINMATCH is a multiple of four.
- *
- * Return 1 if valid, 0 otherwise.
- */
-static int LDM_isValidMatch(const BYTE *p, const BYTE *match) {
+int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
   /*
-  if (memcmp(p, match, MINMATCH) == 0) {
+  if (memcmp(pIn, pMatch, LDM_MIN_MATCH_LENGTH) == 0) {
     return 1;
   }
   return 0;
   */
 
   //TODO: This seems to be faster for some reason?
-  U16 lengthLeft = MINMATCH;
-  const BYTE *curP = p;
-  const BYTE *curMatch = match;
+  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
+  const BYTE *curIn = pIn;
+  const BYTE *curMatch = pMatch;
 
   for (; lengthLeft >= 8; lengthLeft -= 8) {
-    if (LDM_read64(curP) != LDM_read64(curMatch)) {
+    if (MEM_read64(curIn) != MEM_read64(curMatch)) {
       return 0;
     }
-    curP += 8;
+    curIn += 8;
     curMatch += 8;
   }
   if (lengthLeft > 0) {
-    return (LDM_read32(curP) == LDM_read32(curMatch));
+    return (MEM_read32(curIn) == MEM_read32(curMatch));
   }
   return 1;
 }
@@ -183,19 +155,21 @@ static hash_t checksumToHash(U32 sum) {
  * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
  * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
   */
-static U32 getChecksum(const char *data, U32 len) {
+static U32 getChecksum(const BYTE *buf, U32 len) {
   U32 i;
   U32 s1, s2;
-  const schar *buf = (const schar *)data;
 
   s1 = s2 = 0;
   for (i = 0; i < (len - 4); i += 4) {
     s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
+          (2 * buf[i + 2]) + (buf[i + 3]) +
+          (10 * CHECKSUM_CHAR_OFFSET);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
+          + (4 * CHECKSUM_CHAR_OFFSET);
+
   }
   for(; i < len; i++) {
-    s1 += buf[i];
+    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
     s2 += s1;
   }
   return (s1 & 0xffff) + (s2 << 16);
@@ -211,9 +185,9 @@ static U32 getChecksum(const char *data, U32 len) {
  * Thus toRemove should correspond to data[0].
  */
 static U32 updateChecksum(U32 sum, U32 len,
-                          schar toRemove, schar toAdd) {
+                          BYTE toRemove, BYTE toAdd) {
   U32 s1 = (sum & 0xffff) - toRemove + toAdd;
-  U32 s2 = (sum >> 16) - (toRemove * len) + s1;
+  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
 
   return (s1 & 0xffff) + (s2 << 16);
 }
@@ -240,13 +214,13 @@ static void setNextHash(LDM_CCtx *cctx) {
 //  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
   cctx->nextSum = updateChecksum(
       cctx->lastSum, LDM_HASH_LENGTH,
-      (schar)((cctx->lastPosHashed)[0]),
-      (schar)((cctx->lastPosHashed)[LDM_HASH_LENGTH]));
+      cctx->lastPosHashed[0],
+      cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
   cctx->nextHash = checksumToHash(cctx->nextSum);
 
 #ifdef RUN_CHECKS
-  check = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
 
   if (check != cctx->nextSum) {
     printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
@@ -264,7 +238,7 @@ static void putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash, U32 sum) {
 #ifdef COMPUTE_STATS
   if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = (cctx->hashTable)[hash].offset;
+    offset_t offset = cctx->hashTable[hash].offset;
     cctx->stats.numHashInserts++;
     if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
       cctx->stats.numCollisions++;
@@ -275,7 +249,8 @@ static void putHashOfCurrentPositionFromHash(
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    (cctx->hashTable)[hash] = (hashEntry){ (offset_t)(cctx->ip - cctx->ibase) };
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
+    cctx->hashTable[hash] = entry;
   }
 
   cctx->lastPosHashed = cctx->ip;
@@ -303,7 +278,7 @@ static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
  * Insert hash of the current position into the hash table.
  */
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = getChecksum((const char *)cctx->ip, LDM_HASH_LENGTH);
+  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
   hash_t hash = checksumToHash(sum);
 
 #ifdef RUN_CHECKS
@@ -323,40 +298,33 @@ static const BYTE *getPositionOnHash(LDM_CCtx *cctx, hash_t hash) {
   return cctx->hashTable[hash].offset + cctx->ibase;
 }
 
-/**
- *  Counts the number of bytes that match from pIn and pMatch,
- *  up to pInLimit.
- *
- *  TODO: make more efficient.
- */
-static unsigned countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                                 const BYTE *pInLimit) {
+U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
   while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_readByte(pMatch) ^ LDM_readByte(pIn);
+    BYTE const diff = (*pMatch) ^ *(pIn);
     if (!diff) {
       pIn++;
       pMatch++;
       continue;
     }
-    return (unsigned)(pIn - pStart);
+    return (U32)(pIn - pStart);
   }
-  return (unsigned)(pIn - pStart);
+  return (U32)(pIn - pStart);
 }
 
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize) {
-  const U32 *ip = (const U32 *)src;
-  *compressSize = *ip++;
-  *decompressSize = *ip;
+void LDM_readHeader(const void *src, U64 *compressedSize,
+                    U64 *decompressedSize) {
+  const BYTE *ip = (const BYTE *)src;
+  *compressedSize = MEM_readLE64(ip);
+  ip += sizeof(U64);
+  *decompressedSize = MEM_readLE64(ip);
+  // ip += sizeof(U64);
 }
 
-/**
- * Initialize a compression context.
- */
-static void initializeCCtx(LDM_CCtx *cctx,
-                           const void *src, size_t srcSize,
-                           void *dst, size_t maxDstSize) {
+void LDM_initializeCCtx(LDM_CCtx *cctx,
+                        const void *src, size_t srcSize,
+                        void *dst, size_t maxDstSize) {
   cctx->isize = srcSize;
   cctx->maxOSize = maxDstSize;
 
@@ -365,7 +333,7 @@ static void initializeCCtx(LDM_CCtx *cctx,
   cctx->iend = cctx->ibase + srcSize;
 
   cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
-  cctx->imatchLimit = cctx->iend - MINMATCH;
+  cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
 
   cctx->obase = (BYTE *)dst;
   cctx->op = (BYTE *)dst;
@@ -374,6 +342,7 @@ static void initializeCCtx(LDM_CCtx *cctx,
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
   memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+  cctx->stats.minOffset = UINT_MAX;
 
   cctx->lastPosHashed = NULL;
 
@@ -416,61 +385,65 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
   return 0;
 }
 
-/**
- * Write current block (literals, literal length, match offset,
- * match length).
- *
- * Update input pointer, inserting hashes into hash table along the way.
- */
-static void outputBlock(LDM_CCtx *cctx,
-                        unsigned const literalLength,
-                        unsigned const offset,
-                        unsigned const matchLength) {
-  BYTE *token = cctx->op++;
-
+void LDM_encodeLiteralLengthAndLiterals(
+    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
   /* Encode the literal length. */
   if (literalLength >= RUN_MASK) {
     int len = (int)literalLength - RUN_MASK;
-    *token = (RUN_MASK << ML_BITS);
+    *pToken = (RUN_MASK << ML_BITS);
     for (; len >= 255; len -= 255) {
       *(cctx->op)++ = 255;
     }
     *(cctx->op)++ = (BYTE)len;
   } else {
-    *token = (BYTE)(literalLength << ML_BITS);
+    *pToken = (BYTE)(literalLength << ML_BITS);
   }
 
   /* Encode the literals. */
   memcpy(cctx->op, cctx->anchor, literalLength);
   cctx->op += literalLength;
+}
+
+void LDM_outputBlock(LDM_CCtx *cctx,
+                     const U32 literalLength,
+                     const U32 offset,
+                     const U32 matchLength) {
+  BYTE *pToken = cctx->op++;
+
+  /* Encode the literal length and literals. */
+  LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
 
   /* Encode the offset. */
-  LDM_write32(cctx->op, offset);
+  MEM_write32(cctx->op, offset);
   cctx->op += LDM_OFFSET_SIZE;
 
   /* Encode the match length. */
   if (matchLength >= ML_MASK) {
     unsigned matchLengthRemaining = matchLength;
-    *token += ML_MASK;
+    *pToken += ML_MASK;
     matchLengthRemaining -= ML_MASK;
-    LDM_write32(cctx->op, 0xFFFFFFFF);
+    MEM_write32(cctx->op, 0xFFFFFFFF);
     while (matchLengthRemaining >= 4*0xFF) {
       cctx->op += 4;
-      LDM_write32(cctx->op, 0xffffffff);
+      MEM_write32(cctx->op, 0xffffffff);
       matchLengthRemaining -= 4*0xFF;
     }
     cctx->op += matchLengthRemaining / 255;
     *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
   } else {
-    *token += (BYTE)(matchLength);
+    *pToken += (BYTE)(matchLength);
   }
 }
 
-// TODO: srcSize and maxDstSize is unused
+// TODO: maxDstSize is unused. This function may seg fault when writing
+// beyond the size of dst, as it does not check maxDstSize. Writing to
+// a buffer and performing checks is a possible solution.
+//
+// This is based upon lz4.
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
-  initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
@@ -506,21 +479,27 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * length) and update pointers and hashes.
      */
     {
-      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
-      unsigned const offset = cctx.ip - match;
-      unsigned const matchLength = countMatchLength(
-          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
+      const U32 literalLength = cctx.ip - cctx.anchor;
+      const U32 offset = cctx.ip - match;
+      const U32 matchLength = LDM_countMatchLength(
+          cctx.ip + LDM_MIN_MATCH_LENGTH, match + LDM_MIN_MATCH_LENGTH,
+          cctx.ihashLimit);
 
 #ifdef COMPUTE_STATS
       cctx.stats.totalLiteralLength += literalLength;
       cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + MINMATCH;
+      cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
+      cctx.stats.minOffset =
+          offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
+      cctx.stats.maxOffset =
+          offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
 #endif
-      outputBlock(&cctx, literalLength, offset, matchLength);
+      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
       // Move ip to end of block, inserting hashes at each position.
       cctx.nextIp = cctx.ip + cctx.step;
-      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
+      while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
+                       matchLength + literalLength) {
         if (cctx.ip > cctx.lastPosHashed) {
           // TODO: Simplify.
           LDM_updateLastHashFromNextHash(&cctx);
@@ -538,31 +517,22 @@ size_t LDM_compress(const void *src, size_t srcSize,
 _last_literals:
   /* Encode the last literals (no more matches). */
   {
-    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *(cctx.op)++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *(cctx.op)++ = 255;
-      }
-      *(cctx.op)++ = (BYTE)accumulator;
-    } else {
-      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(cctx.op, cctx.anchor, lastRun);
-    cctx.op += lastRun;
+    const size_t lastRun = (size_t)(cctx.iend - cctx.anchor);
+    BYTE *pToken = cctx.op++;
+    LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
   }
 
 #ifdef COMPUTE_STATS
-  printCompressStats(&cctx);
+  LDM_printCompressStats(&cctx.stats);
+  LDM_outputHashtableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
 #endif
 
   return (cctx.op - (const BYTE *)cctx.obase);
 }
 
-typedef struct LDM_DCtx {
-  size_t compressSize;
-  size_t maxDecompressSize;
+struct LDM_DCtx {
+  size_t compressedSize;
+  size_t maxDecompressedSize;
 
   const BYTE *ibase;   /* Base of input */
   const BYTE *ip;      /* Current input position */
@@ -571,26 +541,25 @@ typedef struct LDM_DCtx {
   const BYTE *obase;   /* Base of output */
   BYTE *op;            /* Current output position */
   const BYTE *oend;    /* End of output */
-} LDM_DCtx;
+};
 
-static void LDM_initializeDCtx(LDM_DCtx *dctx,
-                               const void *src, size_t compressSize,
-                               void *dst, size_t maxDecompressSize) {
-  dctx->compressSize = compressSize;
-  dctx->maxDecompressSize = maxDecompressSize;
+void LDM_initializeDCtx(LDM_DCtx *dctx,
+                        const void *src, size_t compressedSize,
+                        void *dst, size_t maxDecompressedSize) {
+  dctx->compressedSize = compressedSize;
+  dctx->maxDecompressedSize = maxDecompressedSize;
 
   dctx->ibase = src;
   dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressSize;
+  dctx->iend = dctx->ip + dctx->compressedSize;
   dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressSize;
-
+  dctx->oend = dctx->op + dctx->maxDecompressedSize;
 }
 
-size_t LDM_decompress(const void *src, size_t compressSize,
-                      void *dst, size_t maxDecompressSize) {
+size_t LDM_decompress(const void *src, size_t compressedSize,
+                      void *dst, size_t maxDecompressedSize) {
   LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
+  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
 
   while (dctx.ip < dctx.iend) {
     BYTE *cpy;
@@ -598,7 +567,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
     size_t length, offset;
 
     /* Get the literal length. */
-    unsigned const token = *(dctx.ip)++;
+    const unsigned token = *(dctx.ip)++;
     if ((length = (token >> ML_BITS)) == RUN_MASK) {
       unsigned s;
       do {
@@ -614,7 +583,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
     dctx.op = cpy;
 
     //TODO : dynamic offset size
-    offset = LDM_read32(dctx.ip);
+    offset = MEM_read32(dctx.ip);
     dctx.ip += LDM_OFFSET_SIZE;
     match = dctx.op - offset;
 
@@ -627,7 +596,7 @@ size_t LDM_decompress(const void *src, size_t compressSize,
         length += s;
       } while (s == 255);
     }
-    length += MINMATCH;
+    length += LDM_MIN_MATCH_LENGTH;
 
     /* Copy match. */
     cpy = dctx.op + length;
@@ -640,6 +609,11 @@ size_t LDM_decompress(const void *src, size_t compressSize,
   return dctx.op - (BYTE *)dst;
 }
 
+// TODO: implement and test hash function
+void LDM_test(void) {
+
+}
+
 /*
 void LDM_test(const void *src, size_t srcSize,
               void *dst, size_t maxDstSize) {
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.h b/contrib/long_distance_matching/versions/v0.5/ldm.h
index 1bd19745..5da3c3b9 100644
--- a/contrib/long_distance_matching/versions/v0.5/ldm.h
+++ b/contrib/long_distance_matching/versions/v0.5/ldm.h
@@ -3,24 +3,141 @@
 
 #include <stddef.h>   /* size_t */
 
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+#include "mem.h"    // from /lib/common/mem.h
 
+#define LDM_COMPRESS_SIZE 8
+#define LDM_DECOMPRESS_SIZE 8
+#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+#define LDM_OFFSET_SIZE 4
+
+// Defines the size of the hash table.
+#define LDM_MEMORY_USAGE 22
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+
+#define WINDOW_SIZE (1 << 25)
+
+//These should be multiples of four.
+#define LDM_MIN_MATCH_LENGTH 8
+#define LDM_HASH_LENGTH 8
+
+typedef U32 offset_t;
+typedef U32 hash_t;
+typedef struct LDM_hashEntry LDM_hashEntry;
+typedef struct LDM_compressStats LDM_compressStats;
+typedef struct LDM_CCtx LDM_CCtx;
+typedef struct LDM_DCtx LDM_DCtx;
+
+/**
+ *  Compresses src into dst.
+ *
+ *  NB: This currently ignores maxDstSize and assumes enough space is available.
+ *
+ *  Block format (see lz4 documentation for more information):
+ *  github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
+ *
+ *  A block is composed of sequences. Each sequence begins with a token, which
+ *  is a one-byte value separated into two 4-bit fields.
+ *
+ *  The first field uses the four high bits of the token and encodes the literal
+ *  length. If the field value is 0, there is no literal. If it is 15,
+ *  additional bytes are added (each ranging from 0 to 255) to the previous
+ *  value to produce a total length.
+ *
+ *  Following the token and optional length bytes are the literals.
+ *
+ *  Next are the 4 bytes representing the offset of the match (2 in lz4),
+ *  representing the position to copy the literals.
+ *
+ *  The lower four bits of the token encode the match length. With additional
+ *  bytes added similarly to the additional literal length bytes after the offset.
+ *
+ *  The last sequence is incomplete and stops right after the lieterals.
+ *
+ */
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize);
 
+/**
+ * Initialize the compression context.
+ */
+void LDM_initializeCCtx(LDM_CCtx *cctx,
+                        const void *src, size_t srcSize,
+                        void *dst, size_t maxDstSize);
+
+/**
+ * Prints the percentage of the hash table occupied (where occupied is defined
+ * as the entry being non-zero).
+ */
+void LDM_outputHashtableOccupancy(const LDM_hashEntry *hashTable,
+                                  U32 hashTableSize);
+
+/**
+ * Outputs compression statistics to stdout.
+ */
+void LDM_printCompressStats(const LDM_compressStats *stats);
+/**
+ * Checks whether the LDM_MIN_MATCH_LENGTH bytes from p are the same as the
+ * LDM_MIN_MATCH_LENGTH bytes from match.
+ *
+ * This assumes LDM_MIN_MATCH_LENGTH is a multiple of four.
+ *
+ * Return 1 if valid, 0 otherwise.
+ */
+int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch);
+
+/**
+ *  Counts the number of bytes that match from pIn and pMatch,
+ *  up to pInLimit.
+ */
+U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *pInLimit);
+
+/**
+ * Encode the literal length followed by the literals.
+ *
+ * The literal length is written to the upper four bits of pToken, with
+ * additional bytes written to the output as needed (see lz4).
+ *
+ * This is followed by literalLength bytes corresponding to the literals.
+ */
+void LDM_encodeLiteralLengthAndLiterals(
+    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength);
+
+/**
+ * Write current block (literals, literal length, match offset,
+ * match length).
+ */
+void LDM_outputBlock(LDM_CCtx *cctx,
+                     const U32 literalLength,
+                     const U32 offset,
+                     const U32 matchLength);
+
+/**
+ * Decompresses src into dst.
+ *
+ * Note: assumes src does not have a header.
+ */
 size_t LDM_decompress(const void *src, size_t srcSize,
                       void *dst, size_t maxDstSize);
 
 /**
- * Reads the header from src and writes the compressed size and
- * decompressed size into compressSize and decompressSize respectively.
+ * Initialize the decompression context.
  */
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize);
+void LDM_initializeDCtx(LDM_DCtx *dctx,
+                        const void *src, size_t compressedSize,
+                        void *dst, size_t maxDecompressedSize);
 
-void LDM_test(const void *src, size_t srcSize,
-              void *dst, size_t maxDstSize);
+/**
+ * Reads the header from src and writes the compressed size and
+ * decompressed size into compressedSize and decompressedSize respectively.
+ *
+ * NB: LDM_compress and LDM_decompress currently do not add/read headers.
+ */
+void LDM_readHeader(const void *src, U64 *compressedSize,
+                    U64 *decompressedSize);
+
+void LDM_test(void);
 
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.5/main-ldm.c b/contrib/long_distance_matching/versions/v0.5/main-ldm.c
index fbfd789b..40afef8c 100644
--- a/contrib/long_distance_matching/versions/v0.5/main-ldm.c
+++ b/contrib/long_distance_matching/versions/v0.5/main-ldm.c
@@ -1,5 +1,3 @@
-// TODO: file size must fit into a U32
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -12,18 +10,23 @@
 
 #include <fcntl.h>
 #include "ldm.h"
+#include "zstd.h"
 
 #define DEBUG
 //#define TEST
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
+ *
+ * TODO: This currently seg faults if the compressed size is > the decompress
+ * size due to the mmapping and output file size allocated to be the input size.
+ * The compress function should check before writing or buffer writes.
  */
 static int compress(const char *fname, const char *oname) {
   int fdin, fdout;
   struct stat statbuf;
   char *src, *dst;
-  size_t maxCompressSize, compressSize;
+  size_t maxCompressedSize, compressedSize;
 
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
@@ -43,11 +46,11 @@ static int compress(const char *fname, const char *oname) {
     return 1;
   }
 
-  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+  maxCompressedSize = statbuf.st_size + LDM_HEADER_SIZE;
 
  /* Go to the location corresponding to the last byte. */
  /* TODO: fallocate? */
-  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
+  if (lseek(fdout, maxCompressedSize - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
   }
@@ -66,39 +69,32 @@ static int compress(const char *fname, const char *oname) {
   }
 
   /* mmap the output file */
-  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
+  if ((dst = mmap(0, maxCompressedSize, PROT_READ | PROT_WRITE,
                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
       perror("mmap error for output");
       return 1;
   }
 
-/*
-#ifdef TEST
-  LDM_test(src, statbuf.st_size,
-           dst + LDM_HEADER_SIZE, statbuf.st_size);
-#endif
-*/
-
-  compressSize = LDM_HEADER_SIZE +
+  compressedSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, statbuf.st_size);
+                   dst + LDM_HEADER_SIZE, maxCompressedSize);
 
   // Write compress and decompress size to header
   // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressSize, 4);
-  memcpy(dst + 4, &(statbuf.st_size), 4);
+  memcpy(dst, &compressedSize, 8);
+  memcpy(dst + 8, &(statbuf.st_size), 8);
 
 #ifdef DEBUG
-  printf("Compressed size: %zu\n", compressSize);
+  printf("Compressed size: %zu\n", compressedSize);
   printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
 #endif
 
-  // Truncate file to compressSize.
-  ftruncate(fdout, compressSize);
+  // Truncate file to compressedSize.
+  ftruncate(fdout, compressedSize);
 
   printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
-         (double)compressSize / (statbuf.st_size) * 100);
+         (unsigned)statbuf.st_size, (unsigned)compressedSize, oname,
+         (double)compressedSize / (statbuf.st_size) * 100);
 
   // Close files.
   close(fdin);
@@ -114,7 +110,8 @@ static int decompress(const char *fname, const char *oname) {
   int fdin, fdout;
   struct stat statbuf;
   char *src, *dst;
-  size_t compressSize, decompressSize, outSize;
+  U64 compressedSize, decompressedSize;
+  size_t outSize;
 
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
@@ -142,10 +139,10 @@ static int decompress(const char *fname, const char *oname) {
   }
 
   /* Read the header. */
-  LDM_readHeader(src, &compressSize, &decompressSize);
+  LDM_readHeader(src, &compressedSize, &decompressedSize);
 
   /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
+  if (lseek(fdout, decompressedSize - 1, SEEK_SET) == -1) {
     perror("lseek error");
     return 1;
   }
@@ -157,7 +154,7 @@ static int decompress(const char *fname, const char *oname) {
   }
 
   /* mmap the output file */
-  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
+  if ((dst = mmap(0, decompressedSize, PROT_READ | PROT_WRITE,
                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
       perror("mmap error for output");
       return 1;
@@ -165,7 +162,7 @@ static int decompress(const char *fname, const char *oname) {
 
   outSize = LDM_decompress(
       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
-      dst, decompressSize);
+      dst, decompressedSize);
 
   printf("Ret size out: %zu\n", outSize);
   ftruncate(fdout, outSize);
@@ -265,204 +262,9 @@ int main(int argc, const char *argv[]) {
   }
   /* verify */
   verify(inpFilename, decFilename);
-  return 0;
-}
 
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
+#ifdef TEST
+  LDM_test();
 #endif
-
+  return 0;
+}
diff --git a/contrib/long_distance_matching/versions/v0.5/util.c b/contrib/long_distance_matching/versions/v0.5/util.c
deleted file mode 100644
index 70fcbc2c..00000000
--- a/contrib/long_distance_matching/versions/v0.5/util.c
+++ /dev/null
@@ -1,69 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "util.h"
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-unsigned LDM_isLittleEndian(void) {
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-void LDM_write16(void *memPtr, U16 value){
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE *)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-BYTE LDM_readByte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
-
diff --git a/contrib/long_distance_matching/versions/v0.5/util.h b/contrib/long_distance_matching/versions/v0.5/util.h
deleted file mode 100644
index d1c3c999..00000000
--- a/contrib/long_distance_matching/versions/v0.5/util.h
+++ /dev/null
@@ -1,25 +0,0 @@
-#ifndef LDM_UTIL_H
-#define LDM_UTIL_H
-
-unsigned LDM_isLittleEndian(void);
-
-uint16_t LDM_read16(const void *memPtr);
-
-uint16_t LDM_readLE16(const void *memPtr);
-
-void LDM_write16(void *memPtr, uint16_t value);
-
-void LDM_write32(void *memPtr, uint32_t value);
-
-void LDM_writeLE16(void *memPtr, uint16_t value);
-
-uint32_t LDM_read32(const void *ptr);
-
-uint64_t LDM_read64(const void *ptr);
-
-void LDM_copy8(void *dst, const void *src);
-
-uint8_t LDM_readByte(const void *ptr);
-
-
-#endif /* LDM_UTIL_H */

From 6733c0777ccf277cd5b41b0f0dac78a9cdb24c3e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 13 Jul 2017 15:34:44 -0700
Subject: [PATCH 052/100] updated NEWS regarding #760

---
 NEWS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS b/NEWS
index 45710507..68539d75 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
 v1.3.1
 perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt
+perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
 build: fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (#718)
 API exp : breaking change : ZSTD_getframeHeader() provides more information
 

From 175a6c602928cd1277d961271b7d782d496520eb Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 13 Jul 2017 16:16:31 -0700
Subject: [PATCH 053/100] [ldm] Minor refactoring

---
 contrib/long_distance_matching/ldm.c | 30 ++++++++++++----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index b8e8c63b..437feb1c 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -443,24 +443,19 @@ void LDM_outputBlock(LDM_CCtx *cctx,
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
+  const BYTE *match;
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
 
-  // TODO: loop condition is not accurate.
-  while (1) {
-    const BYTE *match;
-
-    /**
-     * Find a match.
-     * If no more matches can be found (i.e. the length of the remaining input
-     * is less than the minimum match length), then stop searching for matches
-     * and encode the final literals.
-     */
-    if (LDM_findBestMatch(&cctx, &match) != 0) {
-      goto _last_literals;
-    }
+  /**
+   * Find a match.
+   * If no more matches can be found (i.e. the length of the remaining input
+   * is less than the minimum match length), then stop searching for matches
+   * and encode the final literals.
+   */
+  while (LDM_findBestMatch(&cctx, &match) == 0) {
 #ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
 #endif
@@ -485,6 +480,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
           cctx.ip + LDM_MIN_MATCH_LENGTH, match + LDM_MIN_MATCH_LENGTH,
           cctx.ihashLimit);
 
+      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
+
 #ifdef COMPUTE_STATS
       cctx.stats.totalLiteralLength += literalLength;
       cctx.stats.totalOffset += offset;
@@ -494,7 +491,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
       cctx.stats.maxOffset =
           offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
 #endif
-      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
       // Move ip to end of block, inserting hashes at each position.
       cctx.nextIp = cctx.ip + cctx.step;
@@ -514,10 +510,10 @@ size_t LDM_compress(const void *src, size_t srcSize,
     cctx.anchor = cctx.ip;
     LDM_updateLastHashFromNextHash(&cctx);
   }
-_last_literals:
+
   /* Encode the last literals (no more matches). */
   {
-    const size_t lastRun = (size_t)(cctx.iend - cctx.anchor);
+    const size_t lastRun = cctx.iend - cctx.anchor;
     BYTE *pToken = cctx.op++;
     LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
   }
@@ -527,7 +523,7 @@ _last_literals:
   LDM_outputHashtableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
 #endif
 
-  return (cctx.op - (const BYTE *)cctx.obase);
+  return cctx.op - cctx.obase;
 }
 
 struct LDM_DCtx {

From 2bd6440be0b36304722b20a6fcc7052c197fd33d Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 13 Jul 2017 17:12:16 -0700
Subject: [PATCH 054/100] pinned down error code enum values

Note : all error codes are changed by this new version,
but it's expected to be the last change for existing codes.

Codes are now grouped by category, and receive a manually attributed value.
The objective is to guarantee that
error code values will not change in the future
when introducing new codes.
Intentionnal empty spaces and ranges are defined
in order to keep room for potential new codes.
---
 NEWS                             |  1 +
 lib/common/error_private.c       | 10 +++---
 lib/common/zstd_errors.h         | 54 +++++++++++++++-----------------
 lib/compress/zstd_compress.c     | 23 +++++++-------
 lib/compress/zstdmt_compress.c   |  2 +-
 lib/decompress/zstd_decompress.c |  2 +-
 lib/dictBuilder/zdict.c          |  2 +-
 lib/legacy/zstd_v04.c            |  2 +-
 lib/legacy/zstd_v05.c            |  2 +-
 lib/legacy/zstd_v06.c            |  2 +-
 10 files changed, 49 insertions(+), 51 deletions(-)

diff --git a/NEWS b/NEWS
index 68539d75..b3c2613f 100644
--- a/NEWS
+++ b/NEWS
@@ -3,6 +3,7 @@ perf: substantially decreased memory usage in Multi-threading mode, thanks to re
 perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
 build: fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (#718)
 API exp : breaking change : ZSTD_getframeHeader() provides more information
+API exp : breaking change : pinned down values of error codes
 
 v1.3.0
 cli : new : `--list` command, by Paul Cruz
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 2d752cd2..c3a38621 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -20,19 +20,17 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(GENERIC):  return "Error (generic)";
     case PREFIX(prefix_unknown): return "Unknown frame descriptor";
     case PREFIX(version_unsupported): return "Version not supported";
-    case PREFIX(parameter_unknown): return "Unknown parameter type";
     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
-    case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
-    case PREFIX(compressionParameter_unsupported): return "Compression parameter is not supported";
-    case PREFIX(compressionParameter_outOfBound): return "Compression parameter is out of bound";
+    case PREFIX(corruption_detected): return "Corrupted block detected";
+    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(parameter_unsupported): return "Unsupported parameter";
+    case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
     case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
     case PREFIX(srcSize_wrong): return "Src size is incorrect";
-    case PREFIX(corruption_detected): return "Corrupted block detected";
-    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
     case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
     case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
     case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h
index 19f1597a..fbc13d19 100644
--- a/lib/common/zstd_errors.h
+++ b/lib/common/zstd_errors.h
@@ -37,43 +37,41 @@ extern "C" {
 /*-****************************************
  *  error codes list
  *  note : this API is still considered unstable
- *         it should not be used with a dynamic library
+ *         and shall not be used with a dynamic library.
  *         only static linking is allowed
  ******************************************/
 typedef enum {
-  ZSTD_error_no_error,
-  ZSTD_error_GENERIC,
-  ZSTD_error_prefix_unknown,
-  ZSTD_error_version_unsupported,
-  ZSTD_error_parameter_unknown,
-  ZSTD_error_frameParameter_unsupported,
-  ZSTD_error_frameParameter_unsupportedBy32bits,
-  ZSTD_error_frameParameter_windowTooLarge,
-  ZSTD_error_compressionParameter_unsupported,
-  ZSTD_error_compressionParameter_outOfBound,
-  ZSTD_error_init_missing,
-  ZSTD_error_memory_allocation,
-  ZSTD_error_stage_wrong,
-  ZSTD_error_dstSize_tooSmall,
-  ZSTD_error_srcSize_wrong,
-  ZSTD_error_corruption_detected,
-  ZSTD_error_checksum_wrong,
-  ZSTD_error_tableLog_tooLarge,
-  ZSTD_error_maxSymbolValue_tooLarge,
-  ZSTD_error_maxSymbolValue_tooSmall,
-  ZSTD_error_dictionary_corrupted,
-  ZSTD_error_dictionary_wrong,
-  ZSTD_error_dictionaryCreation_failed,
-  ZSTD_error_frameIndex_tooLarge,
-  ZSTD_error_seekableIO,
-  ZSTD_error_maxCode
+  ZSTD_error_no_error = 0,
+  ZSTD_error_GENERIC  = 1,
+  ZSTD_error_prefix_unknown                = 10,
+  ZSTD_error_version_unsupported           = 12,
+  ZSTD_error_frameParameter_unsupported    = 14,
+  ZSTD_error_frameParameter_windowTooLarge = 16,
+  ZSTD_error_corruption_detected = 20,
+  ZSTD_error_checksum_wrong      = 22,
+  ZSTD_error_dictionary_corrupted      = 30,
+  ZSTD_error_dictionary_wrong          = 32,
+  ZSTD_error_dictionaryCreation_failed = 34,
+  ZSTD_error_parameter_unsupported   = 40,
+  ZSTD_error_parameter_outOfBound    = 42,
+  ZSTD_error_tableLog_tooLarge       = 44,
+  ZSTD_error_maxSymbolValue_tooLarge = 46,
+  ZSTD_error_maxSymbolValue_tooSmall = 48,
+  ZSTD_error_stage_wrong       = 60,
+  ZSTD_error_init_missing      = 62,
+  ZSTD_error_memory_allocation = 64,
+  ZSTD_error_dstSize_tooSmall = 70,
+  ZSTD_error_srcSize_wrong    = 72,
+  ZSTD_error_frameIndex_tooLarge = 100,
+  ZSTD_error_seekableIO          = 102,
+  ZSTD_error_maxCode = 120  /* never EVER use this value directly, it may change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;
 
 /*! ZSTD_getErrorCode() :
     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
     which can be used to compare with enum list published above */
 ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);   /*< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
 
 
 #if defined (__cplusplus)
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b362a191..4ccb7ce6 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -214,7 +214,7 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
     ZSTD_STATIC_ASSERT(ZSTD_dm_auto==0);
     ZSTD_STATIC_ASSERT(ZSTD_dm_rawContent==1);
     case ZSTD_p_forceRawDict : cctx->dictMode = (ZSTD_dictMode_e)(value>0); return 0;
-    default: return ERROR(parameter_unknown);
+    default: return ERROR(parameter_unsupported);
     }
 }
 
@@ -228,9 +228,9 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
     cctx->compressionLevel = ZSTD_CLEVEL_CUSTOM;
 }
 
-#define CLAMPCHECK(val,min,max) {                       \
-    if (((val)<(min)) | ((val)>(max))) {                \
-        return ERROR(compressionParameter_outOfBound);  \
+#define CLAMPCHECK(val,min,max) {            \
+    if (((val)<(min)) | ((val)>(max))) {     \
+        return ERROR(parameter_outOfBound);  \
 }   }
 
 size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
@@ -326,7 +326,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         /* restrict dictionary mode, to "rawContent" or "fullDict" only */
         ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
         if (value > (unsigned)ZSTD_dm_fullDict)
-            return ERROR(compressionParameter_outOfBound);
+            return ERROR(parameter_outOfBound);
         cctx->dictMode = (ZSTD_dictMode_e)value;
         return 0;
 
@@ -347,11 +347,11 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         if (value==0) return 0;
         DEBUGLOG(5, " setting nbThreads : %u", value);
 #ifndef ZSTD_MULTITHREAD
-        if (value > 1) return ERROR(compressionParameter_unsupported);
+        if (value > 1) return ERROR(parameter_unsupported);
 #endif
         if ((value>1) && (cctx->nbThreads != value)) {
             if (cctx->staticSize)  /* MT not compatible with static alloc */
-                return ERROR(compressionParameter_unsupported);
+                return ERROR(parameter_unsupported);
             ZSTDMT_freeCCtx(cctx->mtctx);
             cctx->nbThreads = 1;
             cctx->mtctx = ZSTDMT_createCCtx_advanced(value, cctx->customMem);
@@ -361,17 +361,17 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         return 0;
 
     case ZSTD_p_jobSize:
-        if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
+        if (cctx->nbThreads <= 1) return ERROR(parameter_unsupported);
         assert(cctx->mtctx != NULL);
         return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_sectionSize, value);
 
     case ZSTD_p_overlapSizeLog:
         DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->nbThreads);
-        if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
+        if (cctx->nbThreads <= 1) return ERROR(parameter_unsupported);
         assert(cctx->mtctx != NULL);
         return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
 
-    default: return ERROR(parameter_unknown);
+    default: return ERROR(parameter_unsupported);
     }
 }
 
@@ -451,7 +451,8 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
     CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
     CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
     CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
-    if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) return ERROR(compressionParameter_unsupported);
+    if ((U32)(cParams.strategy) > (U32)ZSTD_btultra)
+        return ERROR(parameter_unsupported);
     return 0;
 }
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 3be850c6..ccc20c78 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -496,7 +496,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
         mtctx->overlapLog = (value >= 9) ? 9 : value;
         return 0;
     default :
-        return ERROR(compressionParameter_unsupported);
+        return ERROR(parameter_unsupported);
     }
 }
 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 4e96504e..92e80c1a 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2227,7 +2227,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
 {
     switch(paramType)
     {
-        default : return ERROR(parameter_unknown);
+        default : return ERROR(parameter_unsupported);
         case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
     }
     return 0;
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 742586ea..113d205f 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -695,7 +695,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
         DISPLAYLEVEL(1, "Not enough memory \n");
         goto _cleanup;
     }
-    if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; }   /* too large dictionary */
+    if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionaryCreation_failed); goto _cleanup; }   /* too large dictionary */
     for (u=0; u<256; u++) countLit[u] = 1;   /* any character must be described */
     for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1;
     for (u=0; u<=MaxML; u++) matchLengthCount[u] = 1;
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 8b8e23cb..2ba75a87 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -2776,7 +2776,7 @@ static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_
     size_t result;
     if (srcSize != zc->headerSize) return ERROR(srcSize_wrong);
     result = ZSTD_getFrameParams(&(zc->params), src, srcSize);
-    if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
+    if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupported);
     return result;
 }
 
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index e929618a..bd3e8c11 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -2888,7 +2888,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
     if (srcSize != zc->headerSize)
         return ERROR(srcSize_wrong);
     result = ZSTDv05_getFrameParams(&(zc->params), src, srcSize);
-    if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
+    if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupported);
     return result;
 }
 
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 26f0929d..3fb47c2b 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3084,7 +3084,7 @@ size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src,
 static size_t ZSTDv06_decodeFrameHeader(ZSTDv06_DCtx* zc, const void* src, size_t srcSize)
 {
     size_t const result = ZSTDv06_getFrameParams(&(zc->fParams), src, srcSize);
-    if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
+    if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupported);
     return result;
 }
 

From 3b0cff3c33724c9dd538a2c1a3f326a879a438f8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 13 Jul 2017 18:58:30 -0700
Subject: [PATCH 055/100] fixed clang's -Wdocumentation

---
 lib/common/zstd_errors.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h
index fbc13d19..a645a9e8 100644
--- a/lib/common/zstd_errors.h
+++ b/lib/common/zstd_errors.h
@@ -71,7 +71,7 @@ typedef enum {
     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
     which can be used to compare with enum list published above */
 ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);   /*< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);   /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
 
 
 #if defined (__cplusplus)

From 4db7f12ef3a3466684a786de4f61b8ad8ef6950e Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 14 Jul 2017 10:52:03 -0700
Subject: [PATCH 056/100] Add offset histogram

---
 contrib/long_distance_matching/Makefile   |  2 +-
 contrib/long_distance_matching/ldm.c      | 41 +++++++++++++++++++++--
 contrib/long_distance_matching/ldm.h      |  6 ++--
 contrib/long_distance_matching/main-ldm.c |  2 +-
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 8ba16d03..cff78644 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -27,7 +27,7 @@ default: all
 
 all: main-ldm
 
-main-ldm : ldm.c main-ldm.c
+main-ldm : ldm.h ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 437feb1c..186fa08e 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -15,7 +15,7 @@
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
 #define COMPUTE_STATS
-#define CHECKSUM_CHAR_OFFSET 0
+#define CHECKSUM_CHAR_OFFSET 10
 //#define RUN_CHECKS
 //#define LDM_DEBUG
 
@@ -23,6 +23,16 @@ struct LDM_hashEntry {
   offset_t offset;
 };
 
+typedef struct LDM_hashTable {
+  U32 numEntries;
+  U32 minimumTagMask;  // TODO: what if tag == offset?
+
+  // Maximum number of elements in the table.
+  U32 limit;
+
+  LDM_hashEntry *entries;
+} LDM_hashTable;
+
 // TODO: Add offset histogram by powers of two
 // TODO: Scanning speed
 // TODO: Memory usage
@@ -36,6 +46,8 @@ struct LDM_compressStats {
 
   U32 numCollisions;
   U32 numHashInserts;
+
+  U32 offsetHistogram[32];
 };
 
 struct LDM_CCtx {
@@ -92,7 +104,19 @@ void LDM_outputHashtableOccupancy(
          100.0 * (double)(ctr) / (double)hashTableSize);
 }
 
+// TODO: This can be done more efficienctly but is not that important as it
+// is only used for computing stats.
+//
+static int intLog2(U32 x) {
+  int ret = 0;
+  while (x >>= 1) {
+    ret++;
+  }
+  return ret;
+}
+
 void LDM_printCompressStats(const LDM_compressStats *stats) {
+  int i = 0;
   printf("=====================\n");
   printf("Compression statistics\n");
   //TODO: compute percentage matched?
@@ -107,11 +131,22 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
          ((double)stats->totalOffset) / (double)stats->numMatches);
   printf("min offset, max offset: %u %u\n",
          stats->minOffset, stats->maxOffset);
+
+  printf("\n");
+  printf("offset histogram\n");
+  for (; i <= intLog2(stats->maxOffset); i++) {
+    printf("2^%*d: %10u\n", 2, i, stats->offsetHistogram[i]);
+  }
+  printf("\n");
+
+
   printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
          stats->numCollisions, stats->numHashInserts,
          stats->numHashInserts == 0 ?
             1.0 : (100.0 * (double)stats->numCollisions) /
                   (double)stats->numHashInserts);
+  printf("=====================\n");
+
 }
 
 int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
@@ -145,7 +180,7 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
  * of the hash table.
  */
 static hash_t checksumToHash(U32 sum) {
-  return ((sum * 2654435761U) >> ((32)-LDM_HASHLOG));
+  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
 }
 
 /**
@@ -490,6 +525,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
           offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
       cctx.stats.maxOffset =
           offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
+      cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
 #endif
 
       // Move ip to end of block, inserting hashes at each position.
@@ -607,7 +643,6 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
 
 // TODO: implement and test hash function
 void LDM_test(void) {
-
 }
 
 /*
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 5da3c3b9..0e54faa7 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -11,7 +11,7 @@
 #define LDM_OFFSET_SIZE 4
 
 // Defines the size of the hash table.
-#define LDM_MEMORY_USAGE 22
+#define LDM_MEMORY_USAGE 20
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
@@ -19,8 +19,8 @@
 #define WINDOW_SIZE (1 << 25)
 
 //These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 8
-#define LDM_HASH_LENGTH 8
+#define LDM_MIN_MATCH_LENGTH 4
+#define LDM_HASH_LENGTH 4
 
 typedef U32 offset_t;
 typedef U32 hash_t;
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 40afef8c..ea6375ba 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -13,7 +13,7 @@
 #include "zstd.h"
 
 #define DEBUG
-//#define TEST
+#define TEST
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.

From 55f960e8db9e3df9322ae9c973c699f671f51c90 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 14 Jul 2017 11:00:20 -0700
Subject: [PATCH 057/100] Add percentages to offset histogram

---
 contrib/long_distance_matching/ldm.c | 5 ++++-
 contrib/long_distance_matching/ldm.h | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 186fa08e..d935a2bd 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -135,7 +135,10 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("\n");
   printf("offset histogram\n");
   for (; i <= intLog2(stats->maxOffset); i++) {
-    printf("2^%*d: %10u\n", 2, i, stats->offsetHistogram[i]);
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           stats->offsetHistogram[i],
+           100.0 * (double) stats->offsetHistogram[i] /
+                   (double)stats->numMatches);
   }
   printf("\n");
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 0e54faa7..3c8c04ec 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -19,8 +19,8 @@
 #define WINDOW_SIZE (1 << 25)
 
 //These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 4
-#define LDM_HASH_LENGTH 4
+#define LDM_MIN_MATCH_LENGTH 1024
+#define LDM_HASH_LENGTH 1024
 
 typedef U32 offset_t;
 typedef U32 hash_t;

From 2d8e6c6608bc0de2d24a40ad6fe7a2fb8e8377bf Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 14 Jul 2017 12:31:01 -0700
Subject: [PATCH 058/100] Add more statistics

---
 contrib/long_distance_matching/ldm.c | 61 ++++++++++++++++++++++------
 contrib/long_distance_matching/ldm.h | 19 ++++++---
 2 files changed, 62 insertions(+), 18 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index d935a2bd..c9c6a709 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -37,6 +37,7 @@ typedef struct LDM_hashTable {
 // TODO: Scanning speed
 // TODO: Memory usage
 struct LDM_compressStats {
+  U32 windowSizeLog, hashTableSizeLog;
   U32 numMatches;
   U64 totalMatchLength;
   U64 totalLiteralLength;
@@ -73,7 +74,9 @@ struct LDM_CCtx {
 
   LDM_compressStats stats;            /* Compression statistics */
 
-  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+  LDM_hashEntry *hashTable;
+
+//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
@@ -90,7 +93,7 @@ struct LDM_CCtx {
   const BYTE *DEBUG_setNextHash;
 };
 
-void LDM_outputHashtableOccupancy(
+void LDM_outputHashTableOccupancy(
     const LDM_hashEntry *hashTable, U32 hashTableSize) {
   U32 i = 0;
   U32 ctr = 0;
@@ -104,9 +107,8 @@ void LDM_outputHashtableOccupancy(
          100.0 * (double)(ctr) / (double)hashTableSize);
 }
 
-// TODO: This can be done more efficienctly but is not that important as it
-// is only used for computing stats.
-//
+// TODO: This can be done more efficiently (but it is not that important as it
+// is only used for computing stats).
 static int intLog2(U32 x) {
   int ret = 0;
   while (x >>= 1) {
@@ -115,30 +117,57 @@ static int intLog2(U32 x) {
   return ret;
 }
 
+// TODO: Maybe we would eventually prefer to have linear rather than
+// exponential buckets.
+void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx) {
+  int i = 0;
+  int buckets[32] = { 0 };
+
+  printf("\n");
+  printf("Hash table histogram\n");
+  for (; i < LDM_HASHTABLESIZE_U32; i++) {
+    int offset = (cctx->ip - cctx->ibase) - cctx->hashTable[i].offset;
+    buckets[intLog2(offset)]++;
+  }
+
+  i = 0;
+  for (; i < 32; i++) {
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           buckets[i],
+           100.0 * (double) buckets[i] /
+                   (double) LDM_HASHTABLESIZE_U32);
+  }
+  printf("\n");
+}
+
 void LDM_printCompressStats(const LDM_compressStats *stats) {
   int i = 0;
   printf("=====================\n");
   printf("Compression statistics\n");
   //TODO: compute percentage matched?
+  printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
+          stats->windowSizeLog, stats->hashTableSizeLog);
   printf("num matches, total match length: %u, %llu\n",
           stats->numMatches,
           stats->totalMatchLength);
   printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
                                          (double)stats->numMatches);
-  printf("avg literal length: %.1f\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
+  printf("avg literal length, total literalLength: %.1f, %llu\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches,
+         stats->totalLiteralLength);
   printf("avg offset length: %.1f\n",
          ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("min offset, max offset: %u %u\n",
+  printf("min offset, max offset: %u, %u\n",
          stats->minOffset, stats->maxOffset);
 
   printf("\n");
-  printf("offset histogram\n");
+  printf("offset histogram: offset, num matches, %% of matches\n");
+
   for (; i <= intLog2(stats->maxOffset); i++) {
     printf("2^%*d: %10u    %6.3f%%\n", 2, i,
            stats->offsetHistogram[i],
            100.0 * (double) stats->offsetHistogram[i] /
-                   (double)stats->numMatches);
+                   (double) stats->numMatches);
   }
   printf("\n");
 
@@ -379,8 +408,12 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+  cctx->hashTable = calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
+//  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
   cctx->stats.minOffset = UINT_MAX;
+  cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
+  cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
+
 
   cctx->lastPosHashed = NULL;
 
@@ -417,7 +450,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
     *match = getPositionOnHash(cctx, h);
     putHashOfCurrentPositionFromHash(cctx, h, sum);
 
-  } while (cctx->ip - *match > WINDOW_SIZE ||
+  } while (cctx->ip - *match > LDM_WINDOW_SIZE ||
            !LDM_isValidMatch(cctx->ip, *match));
   setNextHash(cctx);
   return 0;
@@ -550,6 +583,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
     LDM_updateLastHashFromNextHash(&cctx);
   }
 
+  // LDM_outputHashTableOffsetHistogram(&cctx);
+
   /* Encode the last literals (no more matches). */
   {
     const size_t lastRun = cctx.iend - cctx.anchor;
@@ -559,7 +594,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
 #ifdef COMPUTE_STATS
   LDM_printCompressStats(&cctx.stats);
-  LDM_outputHashtableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
+  LDM_outputHashTableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
 #endif
 
   return cctx.op - cctx.obase;
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 3c8c04ec..87444359 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -11,16 +11,17 @@
 #define LDM_OFFSET_SIZE 4
 
 // Defines the size of the hash table.
-#define LDM_MEMORY_USAGE 20
+#define LDM_MEMORY_USAGE 16
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 
-#define WINDOW_SIZE (1 << 25)
+#define LDM_WINDOW_SIZE_LOG 25
+#define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 1024
-#define LDM_HASH_LENGTH 1024
+#define LDM_MIN_MATCH_LENGTH 4
+#define LDM_HASH_LENGTH 4
 
 typedef U32 offset_t;
 typedef U32 hash_t;
@@ -70,9 +71,17 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
  * Prints the percentage of the hash table occupied (where occupied is defined
  * as the entry being non-zero).
  */
-void LDM_outputHashtableOccupancy(const LDM_hashEntry *hashTable,
+void LDM_outputHashTableOccupancy(const LDM_hashEntry *hashTable,
                                   U32 hashTableSize);
 
+/**
+ * Prints the distribution of offsets in the hash table.
+ *
+ * The offsets are defined as the distance of the hash table entry from the
+ * current input position of the cctx.
+ */
+void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx);
+
 /**
  * Outputs compression statistics to stdout.
  */

From 6e443b4960091fb7b885b83372b1edf79f873564 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 14 Jul 2017 14:27:55 -0700
Subject: [PATCH 059/100] Move hash table access for own functions

---
 contrib/long_distance_matching/ldm.c          |  77 ++++++----
 contrib/long_distance_matching/ldm.h          |  11 +-
 .../versions/v0.5/Makefile                    |   4 +-
 .../versions/v0.5/ldm.c                       | 133 ++++++++++++++----
 .../versions/v0.5/ldm.h                       |  26 +++-
 .../versions/v0.5/main-ldm.c                  |   2 +-
 6 files changed, 189 insertions(+), 64 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index c9c6a709..08cb856c 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -23,17 +23,23 @@ struct LDM_hashEntry {
   offset_t offset;
 };
 
-typedef struct LDM_hashTable {
-  U32 numEntries;
-  U32 minimumTagMask;  // TODO: what if tag == offset?
-
-  // Maximum number of elements in the table.
-  U32 limit;
-
+// TODO: move to its own file.
+struct LDM_hashTable {
+  U32 size;
   LDM_hashEntry *entries;
-} LDM_hashTable;
+};
+
+LDM_hashEntry *HASH_getHash(
+    const LDM_hashTable *table, const hash_t hash) {
+  return &(table->entries[hash]);
+}
+
+void HASH_insert(LDM_hashTable *table,
+                 const hash_t hash, const LDM_hashEntry entry) {
+  *HASH_getHash(table, hash) = entry;
+}
+
 
-// TODO: Add offset histogram by powers of two
 // TODO: Scanning speed
 // TODO: Memory usage
 struct LDM_compressStats {
@@ -74,7 +80,9 @@ struct LDM_CCtx {
 
   LDM_compressStats stats;            /* Compression statistics */
 
-  LDM_hashEntry *hashTable;
+  LDM_hashTable hashTable;
+
+//  LDM_hashEntry *hashTable;
 
 //  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
 
@@ -93,18 +101,19 @@ struct LDM_CCtx {
   const BYTE *DEBUG_setNextHash;
 };
 
-void LDM_outputHashTableOccupancy(
-    const LDM_hashEntry *hashTable, U32 hashTableSize) {
+
+
+void LDM_outputHashTableOccupancy(const LDM_hashTable *hashTable) {
   U32 i = 0;
   U32 ctr = 0;
-  for (; i < hashTableSize; i++) {
-    if (hashTable[i].offset == 0) {
+  for (; i < hashTable->size; i++) {
+    if (HASH_getHash(hashTable, i)->offset == 0) {
       ctr++;
     }
   }
   printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         hashTableSize, ctr,
-         100.0 * (double)(ctr) / (double)hashTableSize);
+         hashTable->size, ctr,
+         100.0 * (double)(ctr) / (double)hashTable->size);
 }
 
 // TODO: This can be done more efficiently (but it is not that important as it
@@ -120,13 +129,14 @@ static int intLog2(U32 x) {
 // TODO: Maybe we would eventually prefer to have linear rather than
 // exponential buckets.
 void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx) {
-  int i = 0;
+  U32 i = 0;
   int buckets[32] = { 0 };
 
   printf("\n");
   printf("Hash table histogram\n");
-  for (; i < LDM_HASHTABLESIZE_U32; i++) {
-    int offset = (cctx->ip - cctx->ibase) - cctx->hashTable[i].offset;
+  for (; i < cctx->hashTable.size; i++) {
+    int offset = (cctx->ip - cctx->ibase) -
+                 HASH_getHash(&cctx->hashTable, i)->offset;
     buckets[intLog2(offset)]++;
   }
 
@@ -135,7 +145,7 @@ void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx) {
     printf("2^%*d: %10u    %6.3f%%\n", 2, i,
            buckets[i],
            100.0 * (double) buckets[i] /
-                   (double) LDM_HASHTABLESIZE_U32);
+                   (double) cctx->hashTable.size);
   }
   printf("\n");
 }
@@ -305,7 +315,7 @@ static void putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash, U32 sum) {
 #ifdef COMPUTE_STATS
   if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = cctx->hashTable[hash].offset;
+    offset_t offset = HASH_getHash(&cctx->hashTable, hash)->offset;
     cctx->stats.numHashInserts++;
     if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
       cctx->stats.numCollisions++;
@@ -317,7 +327,7 @@ static void putHashOfCurrentPositionFromHash(
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
-    cctx->hashTable[hash] = entry;
+    HASH_insert(&cctx->hashTable, hash, entry);
   }
 
   cctx->lastPosHashed = cctx->ip;
@@ -362,7 +372,7 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
  * Returns the position of the entry at hashTable[hash].
  */
 static const BYTE *getPositionOnHash(LDM_CCtx *cctx, hash_t hash) {
-  return cctx->hashTable[hash].offset + cctx->ibase;
+  return HASH_getHash(&cctx->hashTable, hash)->offset + cctx->ibase;
 }
 
 U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
@@ -389,6 +399,11 @@ void LDM_readHeader(const void *src, U64 *compressedSize,
   // ip += sizeof(U64);
 }
 
+static void LDM_initializeHashTable(LDM_hashTable *table) {
+  table->size = LDM_HASHTABLESIZE_U32;
+  table->entries = calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
+}
+
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
@@ -408,7 +423,9 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  cctx->hashTable = calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
+
+  LDM_initializeHashTable(&cctx->hashTable);
+//    calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
 //  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
   cctx->stats.minOffset = UINT_MAX;
   cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
@@ -424,6 +441,10 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->DEBUG_setNextHash = 0;
 }
 
+void LDM_destroyCCtx(LDM_CCtx *cctx) {
+  free((cctx->hashTable).entries);
+}
+
 /**
  * Finds the "best" match.
  *
@@ -594,10 +615,14 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
 #ifdef COMPUTE_STATS
   LDM_printCompressStats(&cctx.stats);
-  LDM_outputHashTableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
+  LDM_outputHashTableOccupancy(&cctx.hashTable);
 #endif
 
-  return cctx.op - cctx.obase;
+  {
+    const size_t ret = cctx.op - cctx.obase;
+    LDM_destroyCCtx(&cctx);
+    return ret;
+  }
 }
 
 struct LDM_DCtx {
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 87444359..8c3aa4e6 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -26,6 +26,7 @@
 typedef U32 offset_t;
 typedef U32 hash_t;
 typedef struct LDM_hashEntry LDM_hashEntry;
+typedef struct LDM_hashTable LDM_hashTable;
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
 typedef struct LDM_DCtx LDM_DCtx;
@@ -62,17 +63,23 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
 /**
  * Initialize the compression context.
+ *
+ * Allocates memory for the hash table.
  */
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize);
 
+/**
+ * Frees up memory allocating in initializeCCtx
+ */
+void LDM_destroyCCtx(LDM_CCtx *cctx);
+
 /**
  * Prints the percentage of the hash table occupied (where occupied is defined
  * as the entry being non-zero).
  */
-void LDM_outputHashTableOccupancy(const LDM_hashEntry *hashTable,
-                                  U32 hashTableSize);
+void LDM_outputHashTableOccupancy(const LDM_hashTable *hashTable);
 
 /**
  * Prints the distribution of offsets in the hash table.
diff --git a/contrib/long_distance_matching/versions/v0.5/Makefile b/contrib/long_distance_matching/versions/v0.5/Makefile
index dee686bc..cff78644 100644
--- a/contrib/long_distance_matching/versions/v0.5/Makefile
+++ b/contrib/long_distance_matching/versions/v0.5/Makefile
@@ -9,7 +9,7 @@
 
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
-CPPFLAGS+= -I../../../../lib/common
+CPPFLAGS+= -I../../lib/common
 CFLAGS  ?= -O3
 DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@@ -27,7 +27,7 @@ default: all
 
 all: main-ldm
 
-main-ldm : ldm.c main-ldm.c
+main-ldm : ldm.h ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.c b/contrib/long_distance_matching/versions/v0.5/ldm.c
index b8e8c63b..06c97bc4 100644
--- a/contrib/long_distance_matching/versions/v0.5/ldm.c
+++ b/contrib/long_distance_matching/versions/v0.5/ldm.c
@@ -15,7 +15,7 @@
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
 #define COMPUTE_STATS
-#define CHECKSUM_CHAR_OFFSET 0
+#define CHECKSUM_CHAR_OFFSET 10
 //#define RUN_CHECKS
 //#define LDM_DEBUG
 
@@ -23,10 +23,21 @@ struct LDM_hashEntry {
   offset_t offset;
 };
 
+typedef struct LDM_hashTable {
+  U32 numEntries;
+  U32 minimumTagMask;  // TODO: what if tag == offset?
+
+  // Maximum number of elements in the table.
+  U32 limit;
+
+  LDM_hashEntry *entries;
+} LDM_hashTable;
+
 // TODO: Add offset histogram by powers of two
 // TODO: Scanning speed
 // TODO: Memory usage
 struct LDM_compressStats {
+  U32 windowSizeLog, hashTableSizeLog;
   U32 numMatches;
   U64 totalMatchLength;
   U64 totalLiteralLength;
@@ -36,6 +47,8 @@ struct LDM_compressStats {
 
   U32 numCollisions;
   U32 numHashInserts;
+
+  U32 offsetHistogram[32];
 };
 
 struct LDM_CCtx {
@@ -61,7 +74,9 @@ struct LDM_CCtx {
 
   LDM_compressStats stats;            /* Compression statistics */
 
-  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+  LDM_hashEntry *hashTable;
+
+//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
@@ -78,7 +93,7 @@ struct LDM_CCtx {
   const BYTE *DEBUG_setNextHash;
 };
 
-void LDM_outputHashtableOccupancy(
+void LDM_outputHashTableOccupancy(
     const LDM_hashEntry *hashTable, U32 hashTableSize) {
   U32 i = 0;
   U32 ctr = 0;
@@ -92,26 +107,78 @@ void LDM_outputHashtableOccupancy(
          100.0 * (double)(ctr) / (double)hashTableSize);
 }
 
+// TODO: This can be done more efficiently (but it is not that important as it
+// is only used for computing stats).
+static int intLog2(U32 x) {
+  int ret = 0;
+  while (x >>= 1) {
+    ret++;
+  }
+  return ret;
+}
+
+// TODO: Maybe we would eventually prefer to have linear rather than
+// exponential buckets.
+void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx) {
+  int i = 0;
+  int buckets[32] = { 0 };
+
+  printf("\n");
+  printf("Hash table histogram\n");
+  for (; i < LDM_HASHTABLESIZE_U32; i++) {
+    int offset = (cctx->ip - cctx->ibase) - cctx->hashTable[i].offset;
+    buckets[intLog2(offset)]++;
+  }
+
+  i = 0;
+  for (; i < 32; i++) {
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           buckets[i],
+           100.0 * (double) buckets[i] /
+                   (double) LDM_HASHTABLESIZE_U32);
+  }
+  printf("\n");
+}
+
 void LDM_printCompressStats(const LDM_compressStats *stats) {
+  int i = 0;
   printf("=====================\n");
   printf("Compression statistics\n");
   //TODO: compute percentage matched?
+  printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
+          stats->windowSizeLog, stats->hashTableSizeLog);
   printf("num matches, total match length: %u, %llu\n",
           stats->numMatches,
           stats->totalMatchLength);
   printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
                                          (double)stats->numMatches);
-  printf("avg literal length: %.1f\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
+  printf("avg literal length, total literalLength: %.1f, %llu\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches,
+         stats->totalLiteralLength);
   printf("avg offset length: %.1f\n",
          ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("min offset, max offset: %u %u\n",
+  printf("min offset, max offset: %u, %u\n",
          stats->minOffset, stats->maxOffset);
+
+  printf("\n");
+  printf("offset histogram: offset, num matches, %% of matches\n");
+
+  for (; i <= intLog2(stats->maxOffset); i++) {
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           stats->offsetHistogram[i],
+           100.0 * (double) stats->offsetHistogram[i] /
+                   (double) stats->numMatches);
+  }
+  printf("\n");
+
+
   printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
          stats->numCollisions, stats->numHashInserts,
          stats->numHashInserts == 0 ?
             1.0 : (100.0 * (double)stats->numCollisions) /
                   (double)stats->numHashInserts);
+  printf("=====================\n");
+
 }
 
 int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
@@ -145,7 +212,7 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
  * of the hash table.
  */
 static hash_t checksumToHash(U32 sum) {
-  return ((sum * 2654435761U) >> ((32)-LDM_HASHLOG));
+  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
 }
 
 /**
@@ -341,8 +408,12 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
+  cctx->hashTable = calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
+//  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
   cctx->stats.minOffset = UINT_MAX;
+  cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
+  cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
+
 
   cctx->lastPosHashed = NULL;
 
@@ -353,6 +424,10 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->DEBUG_setNextHash = 0;
 }
 
+void LDM_destroyCCtx(LDM_CCtx *cctx) {
+  free(cctx->hashTable);
+}
+
 /**
  * Finds the "best" match.
  *
@@ -379,7 +454,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
     *match = getPositionOnHash(cctx, h);
     putHashOfCurrentPositionFromHash(cctx, h, sum);
 
-  } while (cctx->ip - *match > WINDOW_SIZE ||
+  } while (cctx->ip - *match > LDM_WINDOW_SIZE ||
            !LDM_isValidMatch(cctx->ip, *match));
   setNextHash(cctx);
   return 0;
@@ -443,24 +518,19 @@ void LDM_outputBlock(LDM_CCtx *cctx,
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
+  const BYTE *match;
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
 
-  // TODO: loop condition is not accurate.
-  while (1) {
-    const BYTE *match;
-
-    /**
-     * Find a match.
-     * If no more matches can be found (i.e. the length of the remaining input
-     * is less than the minimum match length), then stop searching for matches
-     * and encode the final literals.
-     */
-    if (LDM_findBestMatch(&cctx, &match) != 0) {
-      goto _last_literals;
-    }
+  /**
+   * Find a match.
+   * If no more matches can be found (i.e. the length of the remaining input
+   * is less than the minimum match length), then stop searching for matches
+   * and encode the final literals.
+   */
+  while (LDM_findBestMatch(&cctx, &match) == 0) {
 #ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
 #endif
@@ -485,6 +555,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
           cctx.ip + LDM_MIN_MATCH_LENGTH, match + LDM_MIN_MATCH_LENGTH,
           cctx.ihashLimit);
 
+      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
+
 #ifdef COMPUTE_STATS
       cctx.stats.totalLiteralLength += literalLength;
       cctx.stats.totalOffset += offset;
@@ -493,8 +565,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
           offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
       cctx.stats.maxOffset =
           offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
+      cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
 #endif
-      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
       // Move ip to end of block, inserting hashes at each position.
       cctx.nextIp = cctx.ip + cctx.step;
@@ -514,20 +586,26 @@ size_t LDM_compress(const void *src, size_t srcSize,
     cctx.anchor = cctx.ip;
     LDM_updateLastHashFromNextHash(&cctx);
   }
-_last_literals:
+
+  // LDM_outputHashTableOffsetHistogram(&cctx);
+
   /* Encode the last literals (no more matches). */
   {
-    const size_t lastRun = (size_t)(cctx.iend - cctx.anchor);
+    const size_t lastRun = cctx.iend - cctx.anchor;
     BYTE *pToken = cctx.op++;
     LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
   }
 
 #ifdef COMPUTE_STATS
   LDM_printCompressStats(&cctx.stats);
-  LDM_outputHashtableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
+  LDM_outputHashTableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
 #endif
 
-  return (cctx.op - (const BYTE *)cctx.obase);
+  {
+    const size_t ret = cctx.op - cctx.obase;
+    LDM_destroyCCtx(&cctx);
+    return ret;
+  }
 }
 
 struct LDM_DCtx {
@@ -611,7 +689,6 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
 
 // TODO: implement and test hash function
 void LDM_test(void) {
-
 }
 
 /*
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.h b/contrib/long_distance_matching/versions/v0.5/ldm.h
index 5da3c3b9..70cda8b8 100644
--- a/contrib/long_distance_matching/versions/v0.5/ldm.h
+++ b/contrib/long_distance_matching/versions/v0.5/ldm.h
@@ -11,16 +11,17 @@
 #define LDM_OFFSET_SIZE 4
 
 // Defines the size of the hash table.
-#define LDM_MEMORY_USAGE 22
+#define LDM_MEMORY_USAGE 16
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 
-#define WINDOW_SIZE (1 << 25)
+#define LDM_WINDOW_SIZE_LOG 25
+#define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 8
-#define LDM_HASH_LENGTH 8
+#define LDM_MIN_MATCH_LENGTH 4
+#define LDM_HASH_LENGTH 4
 
 typedef U32 offset_t;
 typedef U32 hash_t;
@@ -61,18 +62,33 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
 /**
  * Initialize the compression context.
+ *
+ * Allocates memory for the hash table.
  */
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize);
 
+/**
+ * Frees up memory allocating in initializeCCtx
+ */
+void LDM_destroyCCtx(LDM_CCtx *cctx);
+
 /**
  * Prints the percentage of the hash table occupied (where occupied is defined
  * as the entry being non-zero).
  */
-void LDM_outputHashtableOccupancy(const LDM_hashEntry *hashTable,
+void LDM_outputHashTableOccupancy(const LDM_hashEntry *hashTable,
                                   U32 hashTableSize);
 
+/**
+ * Prints the distribution of offsets in the hash table.
+ *
+ * The offsets are defined as the distance of the hash table entry from the
+ * current input position of the cctx.
+ */
+void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx);
+
 /**
  * Outputs compression statistics to stdout.
  */
diff --git a/contrib/long_distance_matching/versions/v0.5/main-ldm.c b/contrib/long_distance_matching/versions/v0.5/main-ldm.c
index 40afef8c..ea6375ba 100644
--- a/contrib/long_distance_matching/versions/v0.5/main-ldm.c
+++ b/contrib/long_distance_matching/versions/v0.5/main-ldm.c
@@ -13,7 +13,7 @@
 #include "zstd.h"
 
 #define DEBUG
-//#define TEST
+#define TEST
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.

From ca300ce6e0004a447327d19a2ed879923e8c8baa Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 14 Jul 2017 17:17:00 -0700
Subject: [PATCH 060/100] Decouple hash table from compression function

---
 contrib/long_distance_matching/Makefile       |  2 +-
 contrib/long_distance_matching/basic_table.c  | 56 +++++++++++
 contrib/long_distance_matching/ldm.c          | 93 +++++++------------
 contrib/long_distance_matching/ldm.h          | 13 ---
 .../long_distance_matching/ldm_hashtable.h    | 36 +++++++
 5 files changed, 127 insertions(+), 73 deletions(-)
 create mode 100644 contrib/long_distance_matching/basic_table.c
 create mode 100644 contrib/long_distance_matching/ldm_hashtable.h

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index cff78644..0d4dea06 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -27,7 +27,7 @@ default: all
 
 all: main-ldm
 
-main-ldm : ldm.h ldm.c main-ldm.c
+main-ldm : basic_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
new file mode 100644
index 00000000..007086fe
--- /dev/null
+++ b/contrib/long_distance_matching/basic_table.c
@@ -0,0 +1,56 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "ldm_hashtable.h"
+
+struct LDM_hashTable {
+  U32 size;
+  LDM_hashEntry *entries;
+};
+
+LDM_hashTable *HASH_createTable(U32 size) {
+  LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
+  table->size = size;
+  table->entries = calloc(size, sizeof(LDM_hashEntry));
+  return table;
+}
+
+void HASH_initializeTable(LDM_hashTable *table, U32 size) {
+  table->size = size;
+  table->entries = calloc(size, sizeof(LDM_hashEntry));
+}
+
+
+LDM_hashEntry *HASH_getEntryFromHash(
+    const LDM_hashTable *table, const hash_t hash) {
+  return &(table->entries[hash]);
+}
+
+void HASH_insert(LDM_hashTable *table,
+                 const hash_t hash, const LDM_hashEntry entry) {
+  *HASH_getEntryFromHash(table, hash) = entry;
+}
+
+U32 HASH_getSize(const LDM_hashTable *table) {
+  return table->size;
+}
+
+void HASH_destroyTable(LDM_hashTable *table) {
+  free(table->entries);
+  free(table);
+}
+
+void HASH_outputTableOccupancy(const LDM_hashTable *hashTable) {
+  U32 i = 0;
+  U32 ctr = 0;
+  for (; i < HASH_getSize(hashTable); i++) {
+    if (HASH_getEntryFromHash(hashTable, i)->offset == 0) {
+      ctr++;
+    }
+  }
+  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
+         HASH_getSize(hashTable), ctr,
+         100.0 * (double)(ctr) / (double)HASH_getSize(hashTable));
+}
+
+
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 08cb856c..32da40f8 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -4,11 +4,13 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "ldm.h"
-
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
 #define HASH_ONLY_EVERY 0
 
+#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
 #define RUN_BITS (8-ML_BITS)
@@ -18,27 +20,10 @@
 #define CHECKSUM_CHAR_OFFSET 10
 //#define RUN_CHECKS
 //#define LDM_DEBUG
+//
+#include "ldm.h"
 
-struct LDM_hashEntry {
-  offset_t offset;
-};
-
-// TODO: move to its own file.
-struct LDM_hashTable {
-  U32 size;
-  LDM_hashEntry *entries;
-};
-
-LDM_hashEntry *HASH_getHash(
-    const LDM_hashTable *table, const hash_t hash) {
-  return &(table->entries[hash]);
-}
-
-void HASH_insert(LDM_hashTable *table,
-                 const hash_t hash, const LDM_hashEntry entry) {
-  *HASH_getHash(table, hash) = entry;
-}
-
+#include "ldm_hashtable.h"
 
 // TODO: Scanning speed
 // TODO: Memory usage
@@ -54,6 +39,8 @@ struct LDM_compressStats {
   U32 numCollisions;
   U32 numHashInserts;
 
+//  U64 numInvalidHashes, numValidHashes; // tmp
+
   U32 offsetHistogram[32];
 };
 
@@ -80,9 +67,7 @@ struct LDM_CCtx {
 
   LDM_compressStats stats;            /* Compression statistics */
 
-  LDM_hashTable hashTable;
-
-//  LDM_hashEntry *hashTable;
+  LDM_hashTable *hashTable;
 
 //  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
 
@@ -101,21 +86,6 @@ struct LDM_CCtx {
   const BYTE *DEBUG_setNextHash;
 };
 
-
-
-void LDM_outputHashTableOccupancy(const LDM_hashTable *hashTable) {
-  U32 i = 0;
-  U32 ctr = 0;
-  for (; i < hashTable->size; i++) {
-    if (HASH_getHash(hashTable, i)->offset == 0) {
-      ctr++;
-    }
-  }
-  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         hashTable->size, ctr,
-         100.0 * (double)(ctr) / (double)hashTable->size);
-}
-
 // TODO: This can be done more efficiently (but it is not that important as it
 // is only used for computing stats).
 static int intLog2(U32 x) {
@@ -128,15 +98,15 @@ static int intLog2(U32 x) {
 
 // TODO: Maybe we would eventually prefer to have linear rather than
 // exponential buckets.
-void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx) {
+void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
   U32 i = 0;
   int buckets[32] = { 0 };
 
   printf("\n");
   printf("Hash table histogram\n");
-  for (; i < cctx->hashTable.size; i++) {
+  for (; i < HASH_getSize(cctx->hashTable); i++) {
     int offset = (cctx->ip - cctx->ibase) -
-                 HASH_getHash(&cctx->hashTable, i)->offset;
+                 HASH_getEntryFromHash(cctx->hashTable, i)->offset;
     buckets[intLog2(offset)]++;
   }
 
@@ -145,7 +115,7 @@ void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx) {
     printf("2^%*d: %10u    %6.3f%%\n", 2, i,
            buckets[i],
            100.0 * (double) buckets[i] /
-                   (double) cctx->hashTable.size);
+                   (double) HASH_getSize(cctx->hashTable));
   }
   printf("\n");
 }
@@ -181,7 +151,10 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   }
   printf("\n");
 
-
+  /*
+  printf("Num invalid hashes, num valid hashes, %llu %llu\n",
+         stats->numInvalidHashes, stats->numValidHashes);
+  */
   printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
          stats->numCollisions, stats->numHashInserts,
          stats->numHashInserts == 0 ?
@@ -315,7 +288,7 @@ static void putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash, U32 sum) {
 #ifdef COMPUTE_STATS
   if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = HASH_getHash(&cctx->hashTable, hash)->offset;
+    U32 offset = HASH_getEntryFromHash(cctx->hashTable, hash)->offset;
     cctx->stats.numHashInserts++;
     if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
       cctx->stats.numCollisions++;
@@ -327,7 +300,7 @@ static void putHashOfCurrentPositionFromHash(
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
-    HASH_insert(&cctx->hashTable, hash, entry);
+    HASH_insert(cctx->hashTable, hash, entry);
   }
 
   cctx->lastPosHashed = cctx->ip;
@@ -371,9 +344,11 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
 /**
  * Returns the position of the entry at hashTable[hash].
  */
-static const BYTE *getPositionOnHash(LDM_CCtx *cctx, hash_t hash) {
-  return HASH_getHash(&cctx->hashTable, hash)->offset + cctx->ibase;
+/*
+static const BYTE *getPositionOnHash(const LDM_CCtx *cctx, const hash_t hash) {
+  return HASH_getEntryFromHash(cctx->hashTable, hash)->offset + cctx->ibase;
 }
+*/
 
 U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
                          const BYTE *pInLimit) {
@@ -399,11 +374,6 @@ void LDM_readHeader(const void *src, U64 *compressedSize,
   // ip += sizeof(U64);
 }
 
-static void LDM_initializeHashTable(LDM_hashTable *table) {
-  table->size = LDM_HASHTABLESIZE_U32;
-  table->entries = calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
-}
-
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
@@ -423,8 +393,10 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
+  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U32);
+
+  //HASH_initializeTable(cctx->hashTable, LDM_HASHTABLESIZE_U32);
 
-  LDM_initializeHashTable(&cctx->hashTable);
 //    calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
 //  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
   cctx->stats.minOffset = UINT_MAX;
@@ -442,7 +414,7 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
 }
 
 void LDM_destroyCCtx(LDM_CCtx *cctx) {
-  free((cctx->hashTable).entries);
+  HASH_destroyTable(cctx->hashTable);
 }
 
 /**
@@ -458,6 +430,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
   do {
     hash_t h;
     U32 sum;
+    LDM_hashEntry *entry;
     setNextHash(cctx);
     h = cctx->nextHash;
     sum = cctx->nextSum;
@@ -468,7 +441,9 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
       return 1;
     }
 
-    *match = getPositionOnHash(cctx, h);
+    entry = HASH_getEntryFromHash(cctx->hashTable, h);
+    *match = entry->offset + cctx->ibase;
+
     putHashOfCurrentPositionFromHash(cctx, h, sum);
 
   } while (cctx->ip - *match > LDM_WINDOW_SIZE ||
@@ -604,7 +579,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
     LDM_updateLastHashFromNextHash(&cctx);
   }
 
-  // LDM_outputHashTableOffsetHistogram(&cctx);
+  // HASH_outputTableOffsetHistogram(&cctx);
 
   /* Encode the last literals (no more matches). */
   {
@@ -615,7 +590,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
 #ifdef COMPUTE_STATS
   LDM_printCompressStats(&cctx.stats);
-  LDM_outputHashTableOccupancy(&cctx.hashTable);
+  HASH_outputTableOccupancy(cctx.hashTable);
 #endif
 
   {
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 8c3aa4e6..18b64e37 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -12,9 +12,6 @@
 
 // Defines the size of the hash table.
 #define LDM_MEMORY_USAGE 16
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 
 #define LDM_WINDOW_SIZE_LOG 25
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
@@ -23,10 +20,6 @@
 #define LDM_MIN_MATCH_LENGTH 4
 #define LDM_HASH_LENGTH 4
 
-typedef U32 offset_t;
-typedef U32 hash_t;
-typedef struct LDM_hashEntry LDM_hashEntry;
-typedef struct LDM_hashTable LDM_hashTable;
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
 typedef struct LDM_DCtx LDM_DCtx;
@@ -75,12 +68,6 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
  */
 void LDM_destroyCCtx(LDM_CCtx *cctx);
 
-/**
- * Prints the percentage of the hash table occupied (where occupied is defined
- * as the entry being non-zero).
- */
-void LDM_outputHashTableOccupancy(const LDM_hashTable *hashTable);
-
 /**
  * Prints the distribution of offsets in the hash table.
  *
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
new file mode 100644
index 00000000..690c47a1
--- /dev/null
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -0,0 +1,36 @@
+#ifndef LDM_HASHTABLE_H
+#define LDM_HASHTABLE_H
+
+#include "mem.h"
+
+typedef U32 hash_t;
+
+typedef struct LDM_hashEntry {
+  U32 offset;
+} LDM_hashEntry;
+
+typedef struct LDM_hashTable LDM_hashTable;
+
+// TODO: rename functions
+// TODO: comments
+
+LDM_hashTable *HASH_createTable(U32 size);
+
+LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
+                                     const hash_t hash);
+
+void HASH_insert(LDM_hashTable *table, const hash_t hash,
+                        const LDM_hashEntry entry);
+
+U32 HASH_getSize(const LDM_hashTable *table);
+
+void HASH_destroyTable(LDM_hashTable *table);
+
+/**
+ * Prints the percentage of the hash table occupied (where occupied is defined
+ * as the entry being non-zero).
+ */
+void HASH_outputTableOccupancy(const LDM_hashTable *hashTable);
+
+
+#endif /* LDM_HASHTABLE_H */

From 634f01242053966e4e30ac383d6edb914179376f Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 14 Jul 2017 19:11:58 -0700
Subject: [PATCH 061/100] [libzstd] Refactor ZSTD_compressSequences()

---
 lib/compress/zstd_compress.c | 310 +++++++++++++++++++----------------
 1 file changed, 166 insertions(+), 144 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index dfcb0266..3c792a85 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1017,6 +1017,144 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
         mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
 }
 
+MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType(FSE_repeat* repeatMode,
+        size_t const mostFrequent, size_t nbSeq, U32 defaultNormLog)
+{
+#define MIN_SEQ_FOR_DYNAMIC_FSE   64
+#define MAX_SEQ_FOR_STATIC_FSE  1000
+
+    if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
+        *repeatMode = FSE_repeat_check;
+        return set_rle;
+    }
+    if ((*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+        return set_repeat;
+    }
+    if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1)))) {
+        *repeatMode = FSE_repeat_valid;
+        return set_basic;
+    }
+    *repeatMode = FSE_repeat_check;
+    return set_compressed;
+}
+
+MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
+        FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type,
+        U32* count, U32 max,
+        BYTE const* codeTable, size_t nbSeq,
+        S16 const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+        void* workspace, size_t workspaceSize)
+{
+    BYTE* op = (BYTE*)dst;
+    BYTE const* const oend = op + dstCapacity;
+
+    switch (type) {
+    case set_rle:
+        *op = codeTable[0];
+        CHECK_F(FSE_buildCTable_rle(CTable, (BYTE)max));
+        return 1;
+    case set_repeat:
+        return 0;
+    case set_basic:
+        CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize));
+        return 0;
+    case set_compressed: {
+        S16 norm[MaxSeq + 1];
+        size_t nbSeq_1 = nbSeq;
+        const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
+        if (count[codeTable[nbSeq-1]] > 1) {
+            count[codeTable[nbSeq-1]]--;
+            nbSeq_1--;
+        }
+        CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
+        {   size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog);   /* overflow protected */
+            if (FSE_isError(NCountSize)) return NCountSize;
+            CHECK_F(FSE_buildCTable_wksp(CTable, norm, max, tableLog, workspace, workspaceSize));
+            return NCountSize;
+        }
+    }
+    default: return assert(0), ERROR(GENERIC);
+    }
+}
+
+MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity,
+    FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
+    FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
+    FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
+    seqDef const* sequences, size_t nbSeq, int longOffsets)
+{
+    BIT_CStream_t blockStream;
+    FSE_CState_t  stateMatchLength;
+    FSE_CState_t  stateOffsetBits;
+    FSE_CState_t  stateLitLength;
+
+    CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */
+
+    /* first symbols */
+    FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
+    FSE_initCState2(&stateOffsetBits,  CTable_OffsetBits,  ofCodeTable[nbSeq-1]);
+    FSE_initCState2(&stateLitLength,   CTable_LitLength,   llCodeTable[nbSeq-1]);
+    BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
+    if (MEM_32bits()) BIT_flushBits(&blockStream);
+    BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
+    if (MEM_32bits()) BIT_flushBits(&blockStream);
+    if (longOffsets) {
+        U32 const ofBits = ofCodeTable[nbSeq-1];
+        int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+        if (extraBits) {
+            BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
+            BIT_flushBits(&blockStream);
+        }
+        BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
+                    ofBits - extraBits);
+    } else {
+        BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+    }
+    BIT_flushBits(&blockStream);
+
+    {   size_t n;
+        for (n=nbSeq-2 ; n<nbSeq ; n--) {      /* intentional underflow */
+            BYTE const llCode = llCodeTable[n];
+            BYTE const ofCode = ofCodeTable[n];
+            BYTE const mlCode = mlCodeTable[n];
+            U32  const llBits = LL_bits[llCode];
+            U32  const ofBits = ofCode;                                     /* 32b*/  /* 64b*/
+            U32  const mlBits = ML_bits[mlCode];
+                                                                            /* (7)*/  /* (7)*/
+            FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode);       /* 15 */  /* 15 */
+            FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode);      /* 24 */  /* 24 */
+            if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
+            FSE_encodeSymbol(&blockStream, &stateLitLength, llCode);        /* 16 */  /* 33 */
+            if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog)))
+                BIT_flushBits(&blockStream);                                /* (7)*/
+            BIT_addBits(&blockStream, sequences[n].litLength, llBits);
+            if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
+            BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
+            if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
+            if (longOffsets) {
+                int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+                if (extraBits) {
+                    BIT_addBits(&blockStream, sequences[n].offset, extraBits);
+                    BIT_flushBits(&blockStream);                            /* (7)*/
+                }
+                BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
+                            ofBits - extraBits);                            /* 31 */
+            } else {
+                BIT_addBits(&blockStream, sequences[n].offset, ofBits);     /* 31 */
+            }
+            BIT_flushBits(&blockStream);                                    /* (7)*/
+    }   }
+
+    FSE_flushCState(&blockStream, &stateMatchLength);
+    FSE_flushCState(&blockStream, &stateOffsetBits);
+    FSE_flushCState(&blockStream, &stateLitLength);
+
+    {   size_t const streamSize = BIT_closeCStream(&blockStream);
+        if (streamSize==0) return ERROR(dstSize_tooSmall);   /* not enough space */
+        return streamSize;
+    }
+}
+
 MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
                               size_t srcSize)
@@ -1024,7 +1162,6 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
     const int longOffsets = zc->appliedParams.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
     const seqStore_t* seqStorePtr = &(zc->seqStore);
     U32 count[MaxSeq+1];
-    S16 norm[MaxSeq+1];
     FSE_CTable* CTable_LitLength = zc->entropy->litlengthCTable;
     FSE_CTable* CTable_OffsetBits = zc->entropy->offcodeCTable;
     FSE_CTable* CTable_MatchLength = zc->entropy->matchlengthCTable;
@@ -1038,7 +1175,8 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
     BYTE* op = ostart;
     size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
     BYTE* seqHead;
-    BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)];
+
+    ZSTD_STATIC_ASSERT(sizeof(zc->entropy->workspace) >= (1<<MAX(MLFSELog,LLFSELog)));
 
     /* Compress literals */
     {   const BYTE* const literals = seqStorePtr->litStart;
@@ -1058,166 +1196,50 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
     /* seqHead : flags for FSE encoding type */
     seqHead = op++;
 
-#define MIN_SEQ_FOR_DYNAMIC_FSE   64
-#define MAX_SEQ_FOR_STATIC_FSE  1000
-
     /* convert length/distances into codes */
     ZSTD_seqToCodes(seqStorePtr);
-
     /* CTable for Literal Lengths */
     {   U32 max = MaxLL;
         size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->entropy->workspace);
-        if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
-            *op++ = llCodeTable[0];
-            FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
-            LLtype = set_rle;
-            zc->entropy->litlength_repeatMode = FSE_repeat_check;
-        } else if ((zc->entropy->litlength_repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
-            LLtype = set_repeat;
-        } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) {
-            FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
-            LLtype = set_basic;
-            zc->entropy->litlength_repeatMode = FSE_repeat_valid;
-        } else {
-            size_t nbSeq_1 = nbSeq;
-            const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
-            if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; }
-            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
-            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-              if (FSE_isError(NCountSize)) return NCountSize;
-              op += NCountSize; }
-            FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
-            LLtype = set_compressed;
-            zc->entropy->litlength_repeatMode = FSE_repeat_check;
+        LLtype = ZSTD_selectEncodingType(&zc->entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog);
+        {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
+                    count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
+                    zc->entropy->workspace, sizeof(zc->entropy->workspace));
+            if (ZSTD_isError(countSize)) return countSize;
+            op += countSize;
     }   }
-
     /* CTable for Offsets */
     {   U32 max = MaxOff;
         size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->entropy->workspace);
-        if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
-            *op++ = ofCodeTable[0];
-            FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
-            Offtype = set_rle;
-            zc->entropy->offcode_repeatMode = FSE_repeat_check;
-        } else if ((zc->entropy->offcode_repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
-            Offtype = set_repeat;
-        } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) {
-            FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
-            Offtype = set_basic;
-            zc->entropy->offcode_repeatMode = FSE_repeat_valid;
-        } else {
-            size_t nbSeq_1 = nbSeq;
-            const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
-            if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; }
-            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
-            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-              if (FSE_isError(NCountSize)) return NCountSize;
-              op += NCountSize; }
-            FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
-            Offtype = set_compressed;
-            zc->entropy->offcode_repeatMode = FSE_repeat_check;
+        Offtype = ZSTD_selectEncodingType(&zc->entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog);
+        {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
+                    count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, MaxOff,
+                    zc->entropy->workspace, sizeof(zc->entropy->workspace));
+            if (ZSTD_isError(countSize)) return countSize;
+            op += countSize;
     }   }
-
     /* CTable for MatchLengths */
     {   U32 max = MaxML;
         size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->entropy->workspace);
-        if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
-            *op++ = *mlCodeTable;
-            FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
-            MLtype = set_rle;
-            zc->entropy->matchlength_repeatMode = FSE_repeat_check;
-        } else if ((zc->entropy->matchlength_repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
-            MLtype = set_repeat;
-        } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) {
-            FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
-            MLtype = set_basic;
-            zc->entropy->matchlength_repeatMode = FSE_repeat_valid;
-        } else {
-            size_t nbSeq_1 = nbSeq;
-            const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
-            if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; }
-            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
-            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
-              if (FSE_isError(NCountSize)) return NCountSize;
-              op += NCountSize; }
-            FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
-            MLtype = set_compressed;
-            zc->entropy->matchlength_repeatMode = FSE_repeat_check;
+        MLtype = ZSTD_selectEncodingType(&zc->entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog);
+        {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
+                    count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
+                    zc->entropy->workspace, sizeof(zc->entropy->workspace));
+            if (ZSTD_isError(countSize)) return countSize;
+            op += countSize;
     }   }
 
     *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
 
-    /* Encoding Sequences */
-    {   BIT_CStream_t blockStream;
-        FSE_CState_t  stateMatchLength;
-        FSE_CState_t  stateOffsetBits;
-        FSE_CState_t  stateLitLength;
+    {   size_t const streamSize = ZSTD_encodeSequences(op, oend - op,
+                CTable_MatchLength, mlCodeTable,
+                CTable_OffsetBits, ofCodeTable,
+                CTable_LitLength, llCodeTable,
+                sequences, nbSeq, longOffsets);
+        if (ZSTD_isError(streamSize)) return streamSize;
+        op += streamSize;
+    }
 
-        CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */
-
-        /* first symbols */
-        FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
-        FSE_initCState2(&stateOffsetBits,  CTable_OffsetBits,  ofCodeTable[nbSeq-1]);
-        FSE_initCState2(&stateLitLength,   CTable_LitLength,   llCodeTable[nbSeq-1]);
-        BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
-        if (MEM_32bits()) BIT_flushBits(&blockStream);
-        BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
-        if (MEM_32bits()) BIT_flushBits(&blockStream);
-        if (longOffsets) {
-            U32 const ofBits = ofCodeTable[nbSeq-1];
-            int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
-            if (extraBits) {
-                BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
-                BIT_flushBits(&blockStream);
-            }
-            BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
-                        ofBits - extraBits);
-        } else {
-            BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
-        }
-        BIT_flushBits(&blockStream);
-
-        {   size_t n;
-            for (n=nbSeq-2 ; n<nbSeq ; n--) {      /* intentional underflow */
-                BYTE const llCode = llCodeTable[n];
-                BYTE const ofCode = ofCodeTable[n];
-                BYTE const mlCode = mlCodeTable[n];
-                U32  const llBits = LL_bits[llCode];
-                U32  const ofBits = ofCode;                                     /* 32b*/  /* 64b*/
-                U32  const mlBits = ML_bits[mlCode];
-                                                                                /* (7)*/  /* (7)*/
-                FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode);       /* 15 */  /* 15 */
-                FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode);      /* 24 */  /* 24 */
-                if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
-                FSE_encodeSymbol(&blockStream, &stateLitLength, llCode);        /* 16 */  /* 33 */
-                if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog)))
-                    BIT_flushBits(&blockStream);                                /* (7)*/
-                BIT_addBits(&blockStream, sequences[n].litLength, llBits);
-                if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
-                BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
-                if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
-                if (longOffsets) {
-                    int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
-                    if (extraBits) {
-                        BIT_addBits(&blockStream, sequences[n].offset, extraBits);
-                        BIT_flushBits(&blockStream);                            /* (7)*/
-                    }
-                    BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
-                                ofBits - extraBits);                            /* 31 */
-                } else {
-                    BIT_addBits(&blockStream, sequences[n].offset, ofBits);     /* 31 */
-                }
-                BIT_flushBits(&blockStream);                                    /* (7)*/
-        }   }
-
-        FSE_flushCState(&blockStream, &stateMatchLength);
-        FSE_flushCState(&blockStream, &stateOffsetBits);
-        FSE_flushCState(&blockStream, &stateLitLength);
-
-        {   size_t const streamSize = BIT_closeCStream(&blockStream);
-            if (streamSize==0) return ERROR(dstSize_tooSmall);   /* not enough space */
-            op += streamSize;
-    }   }
 
     /* check compressibility */
 _check_compressibility:

From 4bb42b02c190c606176332ee2a1f7cb540b194ab Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 17 Jul 2017 11:53:54 -0700
Subject: [PATCH 062/100] Add basic chaining table

---
 contrib/long_distance_matching/Makefile       | 10 +-
 contrib/long_distance_matching/basic_table.c  | 19 ++--
 .../long_distance_matching/chaining_table.c   | 92 +++++++++++++++++++
 contrib/long_distance_matching/ldm.c          | 40 +++++---
 contrib/long_distance_matching/ldm.h          |  4 +-
 .../long_distance_matching/ldm_hashtable.h    |  6 +-
 6 files changed, 144 insertions(+), 27 deletions(-)
 create mode 100644 contrib/long_distance_matching/chaining_table.c

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 0d4dea06..3159df75 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,13 +25,17 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-ldm
+all: main-basic main-chaining
 
-main-ldm : basic_table.c ldm.c main-ldm.c
+main-basic : basic_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+main-chaining : chaining_table.c ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main main-ldm
+	main-basic main-chaining
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
index 007086fe..c6a5040e 100644
--- a/contrib/long_distance_matching/basic_table.c
+++ b/contrib/long_distance_matching/basic_table.c
@@ -2,16 +2,19 @@
 #include <stdio.h>
 
 #include "ldm_hashtable.h"
+#include "mem.h"
 
 struct LDM_hashTable {
   U32 size;
   LDM_hashEntry *entries;
+  const BYTE *offsetBase;
 };
 
-LDM_hashTable *HASH_createTable(U32 size) {
+LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase) {
   LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
   table->size = size;
   table->entries = calloc(size, sizeof(LDM_hashEntry));
+  table->offsetBase = offsetBase;
   return table;
 }
 
@@ -20,15 +23,19 @@ void HASH_initializeTable(LDM_hashTable *table, U32 size) {
   table->entries = calloc(size, sizeof(LDM_hashEntry));
 }
 
+LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
+  return table->entries + hash;
+}
 
 LDM_hashEntry *HASH_getEntryFromHash(
-    const LDM_hashTable *table, const hash_t hash) {
-  return &(table->entries[hash]);
+    const LDM_hashTable *table, const hash_t hash, const U32 checksum) {
+  (void)checksum;
+  return getBucket(table, hash);
 }
 
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
-  *HASH_getEntryFromHash(table, hash) = entry;
+  *getBucket(table, hash) = entry;
 }
 
 U32 HASH_getSize(const LDM_hashTable *table) {
@@ -44,7 +51,7 @@ void HASH_outputTableOccupancy(const LDM_hashTable *hashTable) {
   U32 i = 0;
   U32 ctr = 0;
   for (; i < HASH_getSize(hashTable); i++) {
-    if (HASH_getEntryFromHash(hashTable, i)->offset == 0) {
+    if (getBucket(hashTable, i)->offset == 0) {
       ctr++;
     }
   }
@@ -52,5 +59,3 @@ void HASH_outputTableOccupancy(const LDM_hashTable *hashTable) {
          HASH_getSize(hashTable), ctr,
          100.0 * (double)(ctr) / (double)HASH_getSize(hashTable));
 }
-
-
diff --git a/contrib/long_distance_matching/chaining_table.c b/contrib/long_distance_matching/chaining_table.c
new file mode 100644
index 00000000..226f7822
--- /dev/null
+++ b/contrib/long_distance_matching/chaining_table.c
@@ -0,0 +1,92 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "ldm_hashtable.h"
+#include "mem.h"
+
+//TODO: move def somewhere else.
+//TODO: memory usage is currently no longer LDM_MEMORY_USAGE.
+//      refactor code to scale the number of elements appropriately.
+
+// Number of elements per hash bucket.
+#define HASH_BUCKET_SIZE_LOG 2  // MAX is 4 for now
+#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
+
+struct LDM_hashTable {
+  U32 size;
+  LDM_hashEntry *entries;  // 1-D array for now.
+
+  // Position corresponding to offset=0 in LDM_hashEntry.
+  const BYTE *offsetBase;
+  BYTE *bucketOffsets;     // Pointer to current insert position.
+                           // Last insert was at bucketOffsets - 1?
+};
+
+LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase) {
+  LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
+  table->size = size;
+  table->entries = calloc(size * HASH_BUCKET_SIZE, sizeof(LDM_hashEntry));
+  table->bucketOffsets = calloc(size, sizeof(BYTE));
+  table->offsetBase = offsetBase;
+  return table;
+}
+
+static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
+  return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
+}
+
+/*
+static LDM_hashEntry *getLastInsertFromHash(const LDM_hashTable *table,
+                                            const hash_t hash) {
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  BYTE offset = (table->bucketOffsets[hash] - 1) & (HASH_BUCKET_SIZE - 1);
+  return bucket + offset;
+}
+*/
+
+LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
+                                     const hash_t hash,
+                                     const U32 checksum) {
+  // Loop through bucket.
+  // TODO: in order of recency???
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  LDM_hashEntry *cur = bucket;
+  for(; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
+    if (cur->checksum == checksum) {
+      return cur;
+    }
+  }
+  return NULL;
+}
+
+void HASH_insert(LDM_hashTable *table,
+                 const hash_t hash, const LDM_hashEntry entry) {
+  *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
+  table->bucketOffsets[hash]++;
+  table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
+}
+
+U32 HASH_getSize(const LDM_hashTable *table) {
+  return table->size * HASH_BUCKET_SIZE;
+}
+
+void HASH_destroyTable(LDM_hashTable *table) {
+  free(table->entries);
+  free(table->bucketOffsets);
+  free(table);
+}
+
+void HASH_outputTableOccupancy(const LDM_hashTable *table) {
+  U32 ctr = 0;
+  LDM_hashEntry *cur = table->entries;
+  LDM_hashEntry *end = table->entries + (table->size * HASH_BUCKET_SIZE);
+  for (; cur < end; ++cur) {
+    if (cur->offset == 0) {
+      ctr++;
+    }
+  }
+
+  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
+         HASH_getSize(table), ctr,
+         100.0 * (double)(ctr) / (double)HASH_getSize(table));
+}
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 32da40f8..3cb82ea6 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -20,9 +20,8 @@
 #define CHECKSUM_CHAR_OFFSET 10
 //#define RUN_CHECKS
 //#define LDM_DEBUG
-//
-#include "ldm.h"
 
+#include "ldm.h"
 #include "ldm_hashtable.h"
 
 // TODO: Scanning speed
@@ -98,6 +97,7 @@ static int intLog2(U32 x) {
 
 // TODO: Maybe we would eventually prefer to have linear rather than
 // exponential buckets.
+/**
 void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
   U32 i = 0;
   int buckets[32] = { 0 };
@@ -119,6 +119,7 @@ void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
   }
   printf("\n");
 }
+*/
 
 void LDM_printCompressStats(const LDM_compressStats *stats) {
   int i = 0;
@@ -127,9 +128,11 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   //TODO: compute percentage matched?
   printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
           stats->windowSizeLog, stats->hashTableSizeLog);
-  printf("num matches, total match length: %u, %llu\n",
+  printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
           stats->numMatches,
-          stats->totalMatchLength);
+          stats->totalMatchLength,
+          100.0 * (double)stats->totalMatchLength /
+              (double)(stats->totalMatchLength + stats->totalLiteralLength));
   printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
                                          (double)stats->numMatches);
   printf("avg literal length, total literalLength: %.1f, %llu\n",
@@ -155,11 +158,13 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("Num invalid hashes, num valid hashes, %llu %llu\n",
          stats->numInvalidHashes, stats->numValidHashes);
   */
+  /*
   printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
          stats->numCollisions, stats->numHashInserts,
          stats->numHashInserts == 0 ?
             1.0 : (100.0 * (double)stats->numCollisions) /
                   (double)stats->numHashInserts);
+  */
   printf("=====================\n");
 
 }
@@ -173,6 +178,7 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
   */
 
   //TODO: This seems to be faster for some reason?
+
   U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
   const BYTE *curIn = pIn;
   const BYTE *curMatch = pMatch;
@@ -286,8 +292,9 @@ static void setNextHash(LDM_CCtx *cctx) {
 
 static void putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash, U32 sum) {
+  /*
 #ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
+  if (cctx->stats.numHashInserts < HASH_getSize(cctx->hashTable)) {
     U32 offset = HASH_getEntryFromHash(cctx->hashTable, hash)->offset;
     cctx->stats.numHashInserts++;
     if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
@@ -295,11 +302,13 @@ static void putHashOfCurrentPositionFromHash(
     }
   }
 #endif
+*/
 
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase ,
+                                  MEM_read32(cctx->ip) };
     HASH_insert(cctx->hashTable, hash, entry);
   }
 
@@ -393,7 +402,7 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U32);
+  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U32, cctx->ibase);
 
   //HASH_initializeTable(cctx->hashTable, LDM_HASHTABLESIZE_U32);
 
@@ -425,12 +434,13 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
  *
  */
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+
+  LDM_hashEntry *entry = NULL;
   cctx->nextIp = cctx->ip + cctx->step;
 
   do {
     hash_t h;
     U32 sum;
-    LDM_hashEntry *entry;
     setNextHash(cctx);
     h = cctx->nextHash;
     sum = cctx->nextSum;
@@ -441,13 +451,17 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
       return 1;
     }
 
-    entry = HASH_getEntryFromHash(cctx->hashTable, h);
-    *match = entry->offset + cctx->ibase;
+    entry = HASH_getEntryFromHash(cctx->hashTable, h, MEM_read32(cctx->ip));
+
+    if (entry != NULL) {
+      *match = entry->offset + cctx->ibase;
+    }
 
     putHashOfCurrentPositionFromHash(cctx, h, sum);
 
-  } while (cctx->ip - *match > LDM_WINDOW_SIZE ||
-           !LDM_isValidMatch(cctx->ip, *match));
+  } while (entry == NULL ||
+           (cctx->ip - *match > LDM_WINDOW_SIZE ||
+           !LDM_isValidMatch(cctx->ip, *match)));
   setNextHash(cctx);
   return 0;
 }
@@ -510,7 +524,7 @@ void LDM_outputBlock(LDM_CCtx *cctx,
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
-  const BYTE *match;
+  const BYTE *match = NULL;
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 18b64e37..6325d1b1 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -17,8 +17,8 @@
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 4
-#define LDM_HASH_LENGTH 4
+#define LDM_MIN_MATCH_LENGTH 1024
+#define LDM_HASH_LENGTH 1024
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 690c47a1..92add96f 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -7,6 +7,7 @@ typedef U32 hash_t;
 
 typedef struct LDM_hashEntry {
   U32 offset;
+  U32 checksum; // Not needed?
 } LDM_hashEntry;
 
 typedef struct LDM_hashTable LDM_hashTable;
@@ -14,10 +15,11 @@ typedef struct LDM_hashTable LDM_hashTable;
 // TODO: rename functions
 // TODO: comments
 
-LDM_hashTable *HASH_createTable(U32 size);
+LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase);
 
 LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
-                                     const hash_t hash);
+                                     const hash_t hash,
+                                     const U32 checksum);
 
 void HASH_insert(LDM_hashTable *table, const hash_t hash,
                         const LDM_hashEntry entry);

From e198230645b01678dde9599eb5a8be6eb3127bd3 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 17 Jul 2017 12:27:24 -0700
Subject: [PATCH 063/100] [libzstd] Remove ZSTD_CCtx* argument of
 ZSTD_compressSequences()

---
 lib/common/zstd_internal.h   |   2 +
 lib/compress/zstd_compress.c | 118 ++++++++++++++++++-----------------
 lib/compress/zstd_opt.h      |   8 +--
 3 files changed, 66 insertions(+), 62 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f3779e84..7f183c8c 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -246,6 +246,8 @@ typedef struct {
     BYTE* ofCode;
     U32   longLengthID;   /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
     U32   longLengthPos;
+    U32   rep[ZSTD_REP_NUM];
+    U32   repToConfirm[ZSTD_REP_NUM];
     /* opt */
     ZSTD_optimal_t* priceTable;
     ZSTD_match_t* matchTable;
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3c792a85..64f783c7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -82,8 +82,6 @@ struct ZSTD_CCtx_s {
     U32   loadedDictEnd;    /* index of end of dictionary */
     U32   forceWindow;      /* force back-references to respect limit of 1<<wLog, even for dictionary */
     ZSTD_compressionStage_e stage;
-    U32   rep[ZSTD_REP_NUM];
-    U32   repToConfirm[ZSTD_REP_NUM];
     U32   dictID;
     int   compressionLevel;
     ZSTD_parameters requestedParams;
@@ -595,7 +593,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 ple
     cctx->stage = ZSTDcs_init;
     cctx->dictID = 0;
     cctx->loadedDictEnd = 0;
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = repStartValue[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->seqStore.rep[i] = repStartValue[i]; }
     cctx->seqStore.litLengthSum = 0;  /* force reset of btopt stats */
     XXH64_reset(&cctx->xxhState, 0);
     return 0;
@@ -693,7 +691,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->dictBase = NULL;
         zc->dictLimit = 0;
         zc->lowLimit = 0;
-        { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; }
+        { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->seqStore.rep[i] = repStartValue[i]; }
         zc->hashLog3 = hashLog3;
         zc->seqStore.litLengthSum = 0;
 
@@ -747,7 +745,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
  *        do not use with extDict variant ! */
 void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
     int i;
-    for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0;
+    for (i=0; i<ZSTD_REP_NUM; i++) cctx->seqStore.rep[i] = 0;
 }
 
 
@@ -911,7 +909,8 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons
 
 static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
 
-static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
+static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,
+                                     ZSTD_strategy strategy,
                                      void* dst, size_t dstCapacity,
                                const void* src, size_t srcSize)
 {
@@ -925,28 +924,28 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
 
     /* small ? don't even attempt compression (speed opt) */
 #   define LITERAL_NOENTROPY 63
-    {   size_t const minLitSize = zc->entropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
+    {   size_t const minLitSize = entropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
         if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }
 
     if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
-    {   HUF_repeat repeat = zc->entropy->hufCTable_repeatMode;
-        int const preferRepeat = zc->appliedParams.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
+    {   HUF_repeat repeat = entropy->hufCTable_repeatMode;
+        int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
         cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
-                                      zc->entropy->workspace, sizeof(zc->entropy->workspace), (HUF_CElt*)zc->entropy->hufCTable, &repeat, preferRepeat)
+                                      entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat)
                                 : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
-                                      zc->entropy->workspace, sizeof(zc->entropy->workspace), (HUF_CElt*)zc->entropy->hufCTable, &repeat, preferRepeat);
+                                      entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat);
         if (repeat != HUF_repeat_none) { hType = set_repeat; }    /* reused the existing table */
-        else { zc->entropy->hufCTable_repeatMode = HUF_repeat_check; }       /* now have a table to reuse */
+        else { entropy->hufCTable_repeatMode = HUF_repeat_check; }       /* now have a table to reuse */
     }
 
     if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) {
-        zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
+        entropy->hufCTable_repeatMode = HUF_repeat_none;
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }
     if (cLitSize==1) {
-        zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
+        entropy->hufCTable_repeatMode = HUF_repeat_none;
         return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
     }
 
@@ -1155,16 +1154,17 @@ MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity,
     }
 }
 
-MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
+MEM_STATIC size_t ZSTD_compressSequences (seqStore_t* seqStorePtr,
+                              ZSTD_entropyCTables_t* entropy,
+                              ZSTD_compressionParameters const* cParams,
                               void* dst, size_t dstCapacity,
                               size_t srcSize)
 {
-    const int longOffsets = zc->appliedParams.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
-    const seqStore_t* seqStorePtr = &(zc->seqStore);
+    const int longOffsets = cParams->windowLog > STREAM_ACCUMULATOR_MIN;
     U32 count[MaxSeq+1];
-    FSE_CTable* CTable_LitLength = zc->entropy->litlengthCTable;
-    FSE_CTable* CTable_OffsetBits = zc->entropy->offcodeCTable;
-    FSE_CTable* CTable_MatchLength = zc->entropy->matchlengthCTable;
+    FSE_CTable* CTable_LitLength = entropy->litlengthCTable;
+    FSE_CTable* CTable_OffsetBits = entropy->offcodeCTable;
+    FSE_CTable* CTable_MatchLength = entropy->matchlengthCTable;
     U32 LLtype, Offtype, MLtype;   /* compressed, raw or rle */
     const seqDef* const sequences = seqStorePtr->sequencesStart;
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
@@ -1176,13 +1176,15 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
     size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
     BYTE* seqHead;
 
-    ZSTD_STATIC_ASSERT(sizeof(zc->entropy->workspace) >= (1<<MAX(MLFSELog,LLFSELog)));
+    ZSTD_STATIC_ASSERT(sizeof(entropy->workspace) >= (1<<MAX(MLFSELog,LLFSELog)));
 
     /* Compress literals */
     {   const BYTE* const literals = seqStorePtr->litStart;
         size_t const litSize = seqStorePtr->lit - literals;
-        size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize);
-        if (ZSTD_isError(cSize)) return cSize;
+        size_t const cSize = ZSTD_compressLiterals(
+                entropy, cParams->strategy, op, dstCapacity, literals, litSize);
+        if (ZSTD_isError(cSize))
+          return cSize;
         op += cSize;
     }
 
@@ -1200,31 +1202,31 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
     ZSTD_seqToCodes(seqStorePtr);
     /* CTable for Literal Lengths */
     {   U32 max = MaxLL;
-        size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->entropy->workspace);
-        LLtype = ZSTD_selectEncodingType(&zc->entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog);
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace);
+        LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog);
         {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
                     count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
-                    zc->entropy->workspace, sizeof(zc->entropy->workspace));
+                    entropy->workspace, sizeof(entropy->workspace));
             if (ZSTD_isError(countSize)) return countSize;
             op += countSize;
     }   }
     /* CTable for Offsets */
     {   U32 max = MaxOff;
-        size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->entropy->workspace);
-        Offtype = ZSTD_selectEncodingType(&zc->entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog);
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace);
+        Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog);
         {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
                     count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, MaxOff,
-                    zc->entropy->workspace, sizeof(zc->entropy->workspace));
+                    entropy->workspace, sizeof(entropy->workspace));
             if (ZSTD_isError(countSize)) return countSize;
             op += countSize;
     }   }
     /* CTable for MatchLengths */
     {   U32 max = MaxML;
-        size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->entropy->workspace);
-        MLtype = ZSTD_selectEncodingType(&zc->entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog);
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace);
+        MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog);
         {   size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
                     count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
-                    zc->entropy->workspace, sizeof(zc->entropy->workspace));
+                    entropy->workspace, sizeof(entropy->workspace));
             if (ZSTD_isError(countSize)) return countSize;
             op += countSize;
     }   }
@@ -1246,15 +1248,15 @@ _check_compressibility:
     {   size_t const minGain = ZSTD_minGain(srcSize);
         size_t const maxCSize = srcSize - minGain;
         if ((size_t)(op-ostart) >= maxCSize) {
-            zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
-            zc->entropy->offcode_repeatMode = FSE_repeat_none;
-            zc->entropy->matchlength_repeatMode = FSE_repeat_none;
-            zc->entropy->litlength_repeatMode = FSE_repeat_none;
+            entropy->hufCTable_repeatMode = HUF_repeat_none;
+            entropy->offcode_repeatMode = FSE_repeat_none;
+            entropy->matchlength_repeatMode = FSE_repeat_none;
+            entropy->litlength_repeatMode = FSE_repeat_none;
             return 0;
     }   }
 
     /* confirm repcodes */
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; }
 
     return op - ostart;
 }
@@ -1479,7 +1481,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
     const BYTE* const lowest = base + lowestIndex;
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - HASH_READ_SIZE;
-    U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1];
+    U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
     U32 offsetSaved = 0;
 
     /* init */
@@ -1540,8 +1542,8 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
     }   }   }
 
     /* save reps for next block */
-    cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
-    cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
+    seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+    seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1589,7 +1591,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
     const BYTE* const dictEnd = dictBase + dictLimit;
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - 8;
-    U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1];
+    U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
 
     /* Search Loop */
     while (ip < ilimit) {  /* < instead of <=, because (ip+1) */
@@ -1655,7 +1657,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
     }   }   }
 
     /* save reps for next block */
-    ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
+    seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1724,7 +1726,7 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
     const BYTE* const lowest = base + lowestIndex;
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - HASH_READ_SIZE;
-    U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1];
+    U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
     U32 offsetSaved = 0;
 
     /* init */
@@ -1811,8 +1813,8 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
     }   }   }
 
     /* save reps for next block */
-    cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
-    cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
+    seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+    seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1861,7 +1863,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
     const BYTE* const dictEnd = dictBase + dictLimit;
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - 8;
-    U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1];
+    U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1];
 
     /* Search Loop */
     while (ip < ilimit) {  /* < instead of <=, because (ip+1) */
@@ -1961,7 +1963,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
     }   }   }
 
     /* save reps for next block */
-    ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
+    seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -2397,7 +2399,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
                         size_t* offsetPtr,
                         U32 maxNbAttempts, U32 matchLengthSearch);
     searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
-    U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset=0;
+    U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1], savedOffset=0;
 
     /* init */
     ip += (ip==base);
@@ -2507,8 +2509,8 @@ _storeSequence:
     }   }
 
     /* Save reps for next block */
-    ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
-    ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
+    seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
+    seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -2566,7 +2568,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
                         U32 maxNbAttempts, U32 matchLengthSearch);
     searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;
 
-    U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
+    U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1];
 
     /* init */
     ctx->nextToUpdate3 = ctx->nextToUpdate;
@@ -2702,7 +2704,7 @@ _storeSequence:
     }   }
 
     /* Save reps for next block */
-    ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
+    seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -2811,7 +2813,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
     if (current > zc->nextToUpdate + 384)
         zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));   /* limited update after finding a very long match */
     blockCompressor(zc, src, srcSize);
-    return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
+    return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize);
 }
 
 
@@ -3141,9 +3143,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
     }
 
     if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
-    cctx->rep[0] = MEM_readLE32(dictPtr+0);
-    cctx->rep[1] = MEM_readLE32(dictPtr+4);
-    cctx->rep[2] = MEM_readLE32(dictPtr+8);
+    cctx->seqStore.rep[0] = MEM_readLE32(dictPtr+0);
+    cctx->seqStore.rep[1] = MEM_readLE32(dictPtr+4);
+    cctx->seqStore.rep[2] = MEM_readLE32(dictPtr+8);
     dictPtr += 12;
 
     {   size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
@@ -3157,8 +3159,8 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
         /* All repCodes must be <= dictContentSize and != 0*/
         {   U32 u;
             for (u=0; u<3; u++) {
-                if (cctx->rep[u] == 0) return ERROR(dictionary_corrupted);
-                if (cctx->rep[u] > dictContentSize) return ERROR(dictionary_corrupted);
+                if (cctx->seqStore.rep[u] == 0) return ERROR(dictionary_corrupted);
+                if (cctx->seqStore.rep[u] > dictContentSize) return ERROR(dictionary_corrupted);
         }   }
 
         cctx->entropy->hufCTable_repeatMode = HUF_repeat_valid;
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index e8e98915..d6e3449a 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -439,7 +439,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
     ctx->nextToUpdate3 = ctx->nextToUpdate;
     ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize);
     ip += (ip==prefixStart);
-    { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
+    { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
 
     /* Match Loop */
     while (ip < ilimit) {
@@ -651,7 +651,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
     }    }   /* for (cur=0; cur < last_pos; ) */
 
     /* Save reps for next block */
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->repToConfirm[i] = rep[i]; }
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -689,7 +689,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
 
     /* init */
     U32 offset, rep[ZSTD_REP_NUM];
-    { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
+    { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
 
     ctx->nextToUpdate3 = ctx->nextToUpdate;
     ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize);
@@ -924,7 +924,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
     }    }   /* for (cur=0; cur < last_pos; ) */
 
     /* Save reps for next block */
-    { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
+    { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->repToConfirm[i] = rep[i]; }
 
     /* Last Literals */
     {   size_t lastLLSize = iend - anchor;

From 15a041adbf8b59bc88838fe07297d8399319cec0 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 17 Jul 2017 15:16:58 -0700
Subject: [PATCH 064/100] Add function to get valid entries only from table

---
 contrib/long_distance_matching/Makefile       |  6 +--
 contrib/long_distance_matching/basic_table.c  | 17 ++++++
 ...aining_table.c => circular_buffer_table.c} | 21 +++++++-
 contrib/long_distance_matching/ldm.c          | 54 ++++++-------------
 contrib/long_distance_matching/ldm.h          | 11 ++--
 .../long_distance_matching/ldm_hashtable.h    |  9 +++-
 6 files changed, 70 insertions(+), 48 deletions(-)
 rename contrib/long_distance_matching/{chaining_table.c => circular_buffer_table.c} (79%)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 3159df75..47085022 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,17 +25,17 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-basic main-chaining
+all: main-basic main-circular-buffer
 
 main-basic : basic_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-chaining : chaining_table.c ldm.c main-ldm.c
+main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-basic main-chaining
+	main-basic main-circular-buffer
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
index c6a5040e..859bf061 100644
--- a/contrib/long_distance_matching/basic_table.c
+++ b/contrib/long_distance_matching/basic_table.c
@@ -27,12 +27,29 @@ LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + hash;
 }
 
+
 LDM_hashEntry *HASH_getEntryFromHash(
     const LDM_hashTable *table, const hash_t hash, const U32 checksum) {
   (void)checksum;
   return getBucket(table, hash);
 }
 
+LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
+                                  const hash_t hash,
+                                  const U32 checksum,
+                                  const BYTE *pIn,
+                                  int (*isValid)(const BYTE *pIn, const BYTE *pMatch)) {
+  LDM_hashEntry *entry = getBucket(table, hash);
+  (void)checksum;
+  if ((*isValid)(pIn, entry->offset + table->offsetBase)) {
+    return entry;
+  } else {
+    return NULL;
+  }
+}
+
+
+
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
   *getBucket(table, hash) = entry;
diff --git a/contrib/long_distance_matching/chaining_table.c b/contrib/long_distance_matching/circular_buffer_table.c
similarity index 79%
rename from contrib/long_distance_matching/chaining_table.c
rename to contrib/long_distance_matching/circular_buffer_table.c
index 226f7822..f45f945c 100644
--- a/contrib/long_distance_matching/chaining_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -9,7 +9,7 @@
 //      refactor code to scale the number of elements appropriately.
 
 // Number of elements per hash bucket.
-#define HASH_BUCKET_SIZE_LOG 2  // MAX is 4 for now
+#define HASH_BUCKET_SIZE_LOG 1  // MAX is 4 for now
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 
 struct LDM_hashTable {
@@ -44,6 +44,25 @@ static LDM_hashEntry *getLastInsertFromHash(const LDM_hashTable *table,
 }
 */
 
+LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
+                                  const hash_t hash,
+                                  const U32 checksum,
+                                  const BYTE *pIn,
+                                  int (*isValid)(const BYTE *pIn, const BYTE *pMatch)) {
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  LDM_hashEntry *cur = bucket;
+  // TODO: in order of recency?
+  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
+    // CHeck checksum for faster check.
+    if (cur->checksum == checksum &&
+        (*isValid)(pIn, cur->offset + table->offsetBase)) {
+      return cur;
+    }
+  }
+  return NULL;
+}
+
+
 LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
                                      const hash_t hash,
                                      const U32 checksum) {
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 3cb82ea6..bf54842f 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -5,7 +5,7 @@
 #include <string.h>
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY 0
+#define HASH_ONLY_EVERY 31
 
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
@@ -38,8 +38,6 @@ struct LDM_compressStats {
   U32 numCollisions;
   U32 numHashInserts;
 
-//  U64 numInvalidHashes, numValidHashes; // tmp
-
   U32 offsetHistogram[32];
 };
 
@@ -153,45 +151,25 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
                    (double) stats->numMatches);
   }
   printf("\n");
-
-  /*
-  printf("Num invalid hashes, num valid hashes, %llu %llu\n",
-         stats->numInvalidHashes, stats->numValidHashes);
-  */
-  /*
-  printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
-         stats->numCollisions, stats->numHashInserts,
-         stats->numHashInserts == 0 ?
-            1.0 : (100.0 * (double)stats->numCollisions) /
-                  (double)stats->numHashInserts);
-  */
   printf("=====================\n");
 
 }
 
 int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
-  /*
-  if (memcmp(pIn, pMatch, LDM_MIN_MATCH_LENGTH) == 0) {
-    return 1;
-  }
-  return 0;
-  */
-
-  //TODO: This seems to be faster for some reason?
-
   U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
   const BYTE *curIn = pIn;
   const BYTE *curMatch = pMatch;
 
-  for (; lengthLeft >= 8; lengthLeft -= 8) {
-    if (MEM_read64(curIn) != MEM_read64(curMatch)) {
+  if (pIn - pMatch > LDM_WINDOW_SIZE) {
+    return 0;
+  }
+
+  for (; lengthLeft >= 4; lengthLeft -= 4) {
+    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
       return 0;
     }
-    curIn += 8;
-    curMatch += 8;
-  }
-  if (lengthLeft > 0) {
-    return (MEM_read32(curIn) == MEM_read32(curMatch));
+    curIn += 4;
+    curMatch += 4;
   }
   return 1;
 }
@@ -307,8 +285,11 @@ static void putHashOfCurrentPositionFromHash(
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+    /**
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase ,
                                   MEM_read32(cctx->ip) };
+                                  */
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
     HASH_insert(cctx->hashTable, hash, entry);
   }
 
@@ -438,7 +419,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
   LDM_hashEntry *entry = NULL;
   cctx->nextIp = cctx->ip + cctx->step;
 
-  do {
+  while (entry == NULL) {
     hash_t h;
     U32 sum;
     setNextHash(cctx);
@@ -451,17 +432,14 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
       return 1;
     }
 
-    entry = HASH_getEntryFromHash(cctx->hashTable, h, MEM_read32(cctx->ip));
+    entry = HASH_getValidEntry(cctx->hashTable, h, sum, cctx->ip,
+                               &LDM_isValidMatch);
 
     if (entry != NULL) {
       *match = entry->offset + cctx->ibase;
     }
-
     putHashOfCurrentPositionFromHash(cctx, h, sum);
-
-  } while (entry == NULL ||
-           (cctx->ip - *match > LDM_WINDOW_SIZE ||
-           !LDM_isValidMatch(cctx->ip, *match)));
+  }
   setNextHash(cctx);
   return 0;
 }
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 6325d1b1..6d97bd56 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -11,14 +11,14 @@
 #define LDM_OFFSET_SIZE 4
 
 // Defines the size of the hash table.
-#define LDM_MEMORY_USAGE 16
+#define LDM_MEMORY_USAGE 20
 
-#define LDM_WINDOW_SIZE_LOG 25
+#define LDM_WINDOW_SIZE_LOG 30
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 1024
-#define LDM_HASH_LENGTH 1024
+#define LDM_MIN_MATCH_LENGTH 64
+#define LDM_HASH_LENGTH 64
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
@@ -82,7 +82,8 @@ void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx);
 void LDM_printCompressStats(const LDM_compressStats *stats);
 /**
  * Checks whether the LDM_MIN_MATCH_LENGTH bytes from p are the same as the
- * LDM_MIN_MATCH_LENGTH bytes from match.
+ * LDM_MIN_MATCH_LENGTH bytes from match and also if
+ * pIn - pMatch <= LDM_WINDOW_SIZE.
  *
  * This assumes LDM_MIN_MATCH_LENGTH is a multiple of four.
  *
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 92add96f..88d19ae2 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -7,7 +7,7 @@ typedef U32 hash_t;
 
 typedef struct LDM_hashEntry {
   U32 offset;
-  U32 checksum; // Not needed?
+  U32 checksum;
 } LDM_hashEntry;
 
 typedef struct LDM_hashTable LDM_hashTable;
@@ -17,10 +17,17 @@ typedef struct LDM_hashTable LDM_hashTable;
 
 LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase);
 
+//TODO: unneeded?
 LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
                                      const hash_t hash,
                                      const U32 checksum);
 
+LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
+                                  const hash_t hash,
+                                  const U32 checksum,
+                                  const BYTE *pIn,
+                                  int (*isValid)(const BYTE *pIn, const BYTE *pMatch));
+
 void HASH_insert(LDM_hashTable *table, const hash_t hash,
                         const LDM_hashEntry entry);
 

From a00e406231d2cb7fbeffd2a65d1a1044b0d7bcde Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 17 Jul 2017 15:17:32 -0700
Subject: [PATCH 065/100] Remove version archive

---
 .../versions/v0.3/Makefile                    |  30 -
 .../versions/v0.3/README                      |   3 -
 .../versions/v0.3/ldm.c                       | 464 ------------
 .../versions/v0.3/ldm.h                       |  19 -
 .../versions/v0.3/main-ldm.c                  | 479 ------------
 .../versions/v0.3/util.c                      |  64 --
 .../versions/v0.3/util.h                      |  23 -
 .../versions/v0.5/Makefile                    |  37 -
 .../versions/v0.5/README                      |   5 -
 .../versions/v0.5/ldm.c                       | 710 ------------------
 .../versions/v0.5/ldm.h                       | 159 ----
 .../versions/v0.5/main-ldm.c                  | 270 -------
 12 files changed, 2263 deletions(-)
 delete mode 100644 contrib/long_distance_matching/versions/v0.3/Makefile
 delete mode 100644 contrib/long_distance_matching/versions/v0.3/README
 delete mode 100644 contrib/long_distance_matching/versions/v0.3/ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.3/ldm.h
 delete mode 100644 contrib/long_distance_matching/versions/v0.3/main-ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.3/util.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.3/util.h
 delete mode 100644 contrib/long_distance_matching/versions/v0.5/Makefile
 delete mode 100644 contrib/long_distance_matching/versions/v0.5/README
 delete mode 100644 contrib/long_distance_matching/versions/v0.5/ldm.c
 delete mode 100644 contrib/long_distance_matching/versions/v0.5/ldm.h
 delete mode 100644 contrib/long_distance_matching/versions/v0.5/main-ldm.c

diff --git a/contrib/long_distance_matching/versions/v0.3/Makefile b/contrib/long_distance_matching/versions/v0.3/Makefile
deleted file mode 100644
index e5153970..00000000
--- a/contrib/long_distance_matching/versions/v0.3/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-# This Makefile presumes libzstd is installed, using `sudo make install`
-
-CFLAGS  ?= -O3
-DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls
-CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CFLAGS)
-
-LDFLAGS += -lzstd
-
-.PHONY: default all clean
-
-default: all
-
-all: main-ldm
-
-#main : ldm.c main.c
-#	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-main-ldm : util.c ldm.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-clean:
-	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main main-ldm
-	@echo Cleaning completed
-
diff --git a/contrib/long_distance_matching/versions/v0.3/README b/contrib/long_distance_matching/versions/v0.3/README
deleted file mode 100644
index 8699562e..00000000
--- a/contrib/long_distance_matching/versions/v0.3/README
+++ /dev/null
@@ -1,3 +0,0 @@
-This version uses simple lz4-style compression:
-- A 4-byte hash is inserted into the hash table for every position.
-- Hash table replacement policy: direct overwrite.
diff --git a/contrib/long_distance_matching/versions/v0.3/ldm.c b/contrib/long_distance_matching/versions/v0.3/ldm.c
deleted file mode 100644
index 1dedf5c3..00000000
--- a/contrib/long_distance_matching/versions/v0.3/ldm.c
+++ /dev/null
@@ -1,464 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-
-#include "ldm.h"
-#include "util.h"
-
-#define HASH_EVERY 1
-
-#define LDM_MEMORY_USAGE 16
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
-
-#define LDM_OFFSET_SIZE 4
-
-#define WINDOW_SIZE (1 << 20)
-#define MAX_WINDOW_SIZE 31
-#define HASH_SIZE 4
-#define LDM_HASH_LENGTH 4
-#define MINMATCH 4
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-//#define LDM_ROLLING_HASH
-//#define LDM_DEBUG
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-typedef uint32_t offset_t;
-typedef uint32_t hash_t;
-typedef signed char schar;
-
-typedef struct LDM_hashEntry {
-  offset_t offset;
-} LDM_hashEntry;
-
-typedef struct LDM_compressStats {
-  U32 numMatches;
-  U32 totalMatchLength;
-  U32 totalLiteralLength;
-  U64 totalOffset;
-} LDM_compressStats;
-
-static void LDM_printCompressStats(const LDM_compressStats *stats) {
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  printf("Total number of matches: %u\n", stats->numMatches);
-  printf("Average match length: %.1f\n", ((double)stats->totalMatchLength) /
-                                         (double)stats->numMatches);
-  printf("Average literal length: %.1f\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches);
-  printf("Average offset length: %.1f\n",
-         ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("=====================\n");
-}
-
-typedef struct LDM_CCtx {
-  size_t isize;             /* Input size */
-  size_t maxOSize;          /* Maximum output size */
-
-  const BYTE *ibase;        /* Base of input */
-  const BYTE *ip;           /* Current input position */
-  const BYTE *iend;         /* End of input */
-
-  // Maximum input position such that hashing at the position does not exceed
-  // end of input.
-  const BYTE *ihashLimit;
-
-  // Maximum input position such that finding a match of at least the minimum
-  // match length does not exceed end of input.
-  const BYTE *imatchLimit;
-
-  const BYTE *obase;        /* Base of output */
-  BYTE *op;                 /* Output */
-
-  const BYTE *anchor;       /* Anchor to start of current (match) block */
-
-  LDM_compressStats stats;            /* Compression statistics */
-
-  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
-
-  const BYTE *lastPosHashed;          /* Last position hashed */
-  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  const BYTE *nextIp;
-  hash_t nextHash;                    /* Hash corresponding to nextIp */
-
-  unsigned step;
-} LDM_CCtx;
-
-#ifdef LDM_ROLLING_HASH
-/**
- * Convert a sum computed from LDM_getRollingHash to a hash value in the range
- * of the hash table.
- */
-static hash_t LDM_sumToHash(U32 sum) {
-  return sum % (LDM_HASHTABLESIZE >> 2);
-//  return sum & (LDM_HASHTABLESIZE - 1);
-}
-
-static U32 LDM_getRollingHash(const char *data, U32 len) {
-  U32 i;
-  U32 s1, s2;
-  const schar *buf = (const schar *)data;
-
-  s1 = s2 = 0;
-  for (i = 0; i < (len - 4); i += 4) {
-    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3];
-  }
-  for(; i < len; i++) {
-    s1 += buf[i];
-    s2 += s1;
-  }
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_sumToHash(LDM_getRollingHash((const char *)p, LDM_HASH_LENGTH));
-}
-
-typedef struct LDM_sumStruct {
-  U16 s1, s2;
-} LDM_sumStruct;
-
-static void LDM_getRollingHashParts(U32 sum, LDM_sumStruct *sumStruct) {
-  sumStruct->s1 = sum & 0xffff;
-  sumStruct->s2 = sum >> 16;
-}
-
-#else
-static hash_t LDM_hash(U32 sequence) {
-  return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
-}
-
-static hash_t LDM_hashPosition(const void * const p) {
-  return LDM_hash(LDM_read32(p));
-}
-#endif
-
-/*
-static hash_t LDM_hash5(U64 sequence) {
-  static const U64 prime5bytes = 889523592379ULL;
-  static const U64 prime8bytes = 11400714785074694791ULL;
-  const U32 hashLog = LDM_HASHLOG;
-  if (LDM_isLittleEndian())
-    return (((sequence << 24) * prime5bytes) >> (64 - hashLog));
-  else
-    return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
-}
-*/
-
-static void LDM_putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash) {
-  if (((cctx->ip - cctx->ibase) & HASH_EVERY) != HASH_EVERY) {
-    return;
-  }
-  (cctx->hashTable)[hash] = (LDM_hashEntry){ (hash_t)(cctx->ip - cctx->ibase) };
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-}
-
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  hash_t hash = LDM_hashPosition(cctx->ip);
-  LDM_putHashOfCurrentPositionFromHash(cctx, hash);
-}
-
-static const BYTE *LDM_get_position_on_hash(
-    hash_t h, void *tableBase, const BYTE *srcBase) {
-  const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
-  return hashTable[h].offset + srcBase;
-}
-
-static BYTE LDM_read_byte(const void *memPtr) {
-  BYTE val;
-  memcpy(&val, memPtr, 1);
-  return val;
-}
-
-static unsigned LDM_count(const BYTE *pIn, const BYTE *pMatch,
-                          const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = LDM_read_byte(pMatch) ^ LDM_read_byte(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (unsigned)(pIn - pStart);
-  }
-  return (unsigned)(pIn - pStart);
-}
-
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize) {
-  const U32 *ip = (const U32 *)src;
-  *compressSize = *ip++;
-  *decompressSize = *ip;
-}
-
-static void LDM_initializeCCtx(LDM_CCtx *cctx,
-                               const void *src, size_t srcSize,
-                               void *dst, size_t maxDstSize) {
-  cctx->isize = srcSize;
-  cctx->maxOSize = maxDstSize;
-
-  cctx->ibase = (const BYTE *)src;
-  cctx->ip = cctx->ibase;
-  cctx->iend = cctx->ibase + srcSize;
-
-  cctx->ihashLimit = cctx->iend - HASH_SIZE;
-  cctx->imatchLimit = cctx->iend - MINMATCH;
-
-  cctx->obase = (BYTE *)dst;
-  cctx->op = (BYTE *)dst;
-
-  cctx->anchor = cctx->ibase;
-
-  memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
-
-  cctx->lastPosHashed = NULL;
-  cctx->nextIp = NULL;
-
-  cctx->step = 1;
-}
-
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
-  cctx->nextIp = cctx->ip;
-
-  do {
-    hash_t const h = cctx->nextHash;
-    cctx->ip = cctx->nextIp;
-    cctx->nextIp += cctx->step;
-
-    if (cctx->nextIp > cctx->imatchLimit) {
-      return 1;
-    }
-
-    *match = LDM_get_position_on_hash(h, cctx->hashTable, cctx->ibase);
-
-    cctx->nextHash = LDM_hashPosition(cctx->nextIp);
-    LDM_putHashOfCurrentPositionFromHash(cctx, h);
-  } while (cctx->ip - *match > WINDOW_SIZE ||
-           LDM_read64(*match) != LDM_read64(cctx->ip));
-  return 0;
-}
-
-// TODO: srcSize and maxDstSize is unused
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize) {
-  LDM_CCtx cctx;
-  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
-
-  /* Hash the first position and put it into the hash table. */
-  LDM_putHashOfCurrentPosition(&cctx);
-  cctx.ip++;
-  cctx.nextHash = LDM_hashPosition(cctx.ip);
-
-  // TODO: loop condition is not accurate.
-  while (1) {
-    const BYTE *match;
-
-    /**
-     * Find a match.
-     * If no more matches can be found (i.e. the length of the remaining input
-     * is less than the minimum match length), then stop searching for matches
-     * and encode the final literals.
-     */
-    if (LDM_findBestMatch(&cctx, &match) != 0) {
-      goto _last_literals;
-    }
-
-    cctx.stats.numMatches++;
-
-    /**
-     * Catch up: look back to extend the match backwards from the found match.
-     */
-    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
-           cctx.ip[-1] == match[-1]) {
-      cctx.ip--;
-      match--;
-    }
-
-    /**
-     * Write current block (literals, literal length, match offset, match
-     * length) and update pointers and hashes.
-     */
-    {
-      unsigned const literalLength = (unsigned)(cctx.ip - cctx.anchor);
-      unsigned const offset = cctx.ip - match;
-      unsigned const matchLength = LDM_count(
-          cctx.ip + MINMATCH, match + MINMATCH, cctx.ihashLimit);
-      BYTE *token = cctx.op++;
-
-      cctx.stats.totalLiteralLength += literalLength;
-      cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + MINMATCH;
-
-      /* Encode the literal length. */
-      if (literalLength >= RUN_MASK) {
-        int len = (int)literalLength - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for (; len >= 255; len -= 255) {
-          *(cctx.op)++ = 255;
-        }
-        *(cctx.op)++ = (BYTE)len;
-      } else {
-        *token = (BYTE)(literalLength << ML_BITS);
-      }
-
-      /* Encode the literals. */
-      memcpy(cctx.op, cctx.anchor, literalLength);
-      cctx.op += literalLength;
-
-      /* Encode the offset. */
-      LDM_write32(cctx.op, offset);
-      cctx.op += LDM_OFFSET_SIZE;
-
-      /* Encode match length */
-      if (matchLength >= ML_MASK) {
-        unsigned matchLengthRemaining = matchLength;
-        *token += ML_MASK;
-        matchLengthRemaining -= ML_MASK;
-        LDM_write32(cctx.op, 0xFFFFFFFF);
-        while (matchLengthRemaining >= 4*0xFF) {
-          cctx.op += 4;
-          LDM_write32(cctx.op, 0xffffffff);
-          matchLengthRemaining -= 4*0xFF;
-        }
-        cctx.op += matchLengthRemaining / 255;
-        *(cctx.op)++ = (BYTE)(matchLengthRemaining % 255);
-      } else {
-        *token += (BYTE)(matchLength);
-      }
-
-      /* Update input pointer, inserting hashes into hash table along the
-       * way.
-       */
-      while (cctx.ip < cctx.anchor + MINMATCH + matchLength + literalLength) {
-        LDM_putHashOfCurrentPosition(&cctx);
-        cctx.ip++;
-      }
-    }
-
-    // Set start of next block to current input pointer.
-    cctx.anchor = cctx.ip;
-    LDM_putHashOfCurrentPosition(&cctx);
-    cctx.nextHash = LDM_hashPosition(++cctx.ip);
-  }
-_last_literals:
-  /* Encode the last literals (no more matches). */
-  {
-    size_t const lastRun = (size_t)(cctx.iend - cctx.anchor);
-    if (lastRun >= RUN_MASK) {
-      size_t accumulator = lastRun - RUN_MASK;
-      *(cctx.op)++ = RUN_MASK << ML_BITS;
-      for(; accumulator >= 255; accumulator -= 255) {
-        *(cctx.op)++ = 255;
-      }
-      *(cctx.op)++ = (BYTE)accumulator;
-    } else {
-      *(cctx.op)++ = (BYTE)(lastRun << ML_BITS);
-    }
-    memcpy(cctx.op, cctx.anchor, lastRun);
-    cctx.op += lastRun;
-  }
-  LDM_printCompressStats(&cctx.stats);
-  return (cctx.op - (const BYTE *)cctx.obase);
-}
-
-typedef struct LDM_DCtx {
-  size_t compressSize;
-  size_t maxDecompressSize;
-
-  const BYTE *ibase;   /* Base of input */
-  const BYTE *ip;      /* Current input position */
-  const BYTE *iend;    /* End of source */
-
-  const BYTE *obase;   /* Base of output */
-  BYTE *op;            /* Current output position */
-  const BYTE *oend;    /* End of output */
-} LDM_DCtx;
-
-static void LDM_initializeDCtx(LDM_DCtx *dctx,
-                               const void *src, size_t compressSize,
-                               void *dst, size_t maxDecompressSize) {
-  dctx->compressSize = compressSize;
-  dctx->maxDecompressSize = maxDecompressSize;
-
-  dctx->ibase = src;
-  dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressSize;
-  dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressSize;
-
-}
-
-size_t LDM_decompress(const void *src, size_t compressSize,
-                      void *dst, size_t maxDecompressSize) {
-  LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
-
-  while (dctx.ip < dctx.iend) {
-    BYTE *cpy;
-    const BYTE *match;
-    size_t length, offset;
-
-    /* Get the literal length. */
-    unsigned const token = *(dctx.ip)++;
-    if ((length = (token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-
-    /* Copy literals. */
-    cpy = dctx.op + length;
-    memcpy(dctx.op, dctx.ip, length);
-    dctx.ip += length;
-    dctx.op = cpy;
-
-    //TODO : dynamic offset size
-    offset = LDM_read32(dctx.ip);
-    dctx.ip += LDM_OFFSET_SIZE;
-    match = dctx.op - offset;
-
-    /* Get the match length. */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-    length += MINMATCH;
-
-    /* Copy match. */
-    cpy = dctx.op + length;
-
-    // Inefficient for now
-    while (match < cpy - offset && dctx.op < dctx.oend) {
-      *(dctx.op)++ = *match++;
-    }
-  }
-  return dctx.op - (BYTE *)dst;
-}
-
-
diff --git a/contrib/long_distance_matching/versions/v0.3/ldm.h b/contrib/long_distance_matching/versions/v0.3/ldm.h
deleted file mode 100644
index 287d444d..00000000
--- a/contrib/long_distance_matching/versions/v0.3/ldm.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef LDM_H
-#define LDM_H
-
-#include <stddef.h>   /* size_t */
-
-#define LDM_COMPRESS_SIZE 4
-#define LDM_DECOMPRESS_SIZE 4
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
-
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize);
-
-size_t LDM_decompress(const void *src, size_t srcSize,
-                      void *dst, size_t maxDstSize);
-
-void LDM_readHeader(const void *src, size_t *compressSize,
-                    size_t *decompressSize);
-
-#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.3/main-ldm.c b/contrib/long_distance_matching/versions/v0.3/main-ldm.c
deleted file mode 100644
index 724d735d..00000000
--- a/contrib/long_distance_matching/versions/v0.3/main-ldm.c
+++ /dev/null
@@ -1,479 +0,0 @@
-// TODO: file size must fit into a U32
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zstd.h>
-
-#include <fcntl.h>
-#include "ldm.h"
-
-// #define BUF_SIZE 16*1024  // Block size
-#define DEBUG
-
-//#define ZSTD
-
-/* Compress file given by fname and output to oname.
- * Returns 0 if successful, error code otherwise.
- */
-static int compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-  size_t maxCompressSize, compressSize;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
-
- /* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */
-  if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
- /* Write a dummy byte at the last location. */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, maxCompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef ZSTD
-  compressSize = ZSTD_compress(dst, statbuf.st_size,
-                                  src, statbuf.st_size, 1);
-#else
-  compressSize = LDM_HEADER_SIZE +
-      LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, statbuf.st_size);
-
-    // Write compress and decompress size to header
-    // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressSize, 4);
-  memcpy(dst + 4, &(statbuf.st_size), 4);
-
-#ifdef DEBUG
-  printf("Compressed size: %zu\n", compressSize);
-  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
-#endif
-#endif
-
-  // Truncate file to compressSize.
-  ftruncate(fdout, compressSize);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressSize, oname,
-         (double)compressSize / (statbuf.st_size) * 100);
-
-  // Close files.
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Decompress file compressed using LDM_compress.
- * The input file should have the LDM_HEADER followed by payload.
- * Returns 0 if succesful, and an error code otherwise.
- */
-static int decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-  size_t compressSize, decompressSize, outSize;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* Read the header. */
-  LDM_readHeader(src, &compressSize, &decompressSize);
-
-#ifdef DEBUG
-  printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
-         (size_t)statbuf.st_size, compressSize, decompressSize);
-#endif
-
-  /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-#ifdef ZSTD
-  outSize = ZSTD_decompress(dst, decomrpessed_size,
-                                  src + LDM_HEADER_SIZE,
-                                  statbuf.st_size - LDM_HEADER_SIZE);
-#else
-  outSize = LDM_decompress(
-      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
-      dst, decompressSize);
-
-  printf("Ret size out: %zu\n", outSize);
-  #endif
-  ftruncate(fdout, outSize);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Compare two files.
- * Returns 0 iff they are the same.
- */
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) break;
-
-    if (0 == result) result = memcmp(b0, b1, r0);
-  }
-  return result;
-}
-
-/* Verify the input file is the same as the decompressed file. */
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-  {
-    const int cmp = compare(inpFp, decFp);
-    if(0 == cmp) {
-      printf("verify : OK\n");
-    } else {
-      printf("verify : NG\n");
-    }
-  }
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-
-  /* Compress */
-  {
-    struct timeval tv1, tv2;
-    gettimeofday(&tv1, NULL);
-    if (compress(inpFilename, ldmFilename)) {
-        printf("Compress error");
-        return 1;
-    }
-    gettimeofday(&tv2, NULL);
-    printf("Total time = %f seconds\n",
-           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-           (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-
-  /* Decompress */
-  {
-    struct timeval tv1, tv2;
-    gettimeofday(&tv1, NULL);
-    if (decompress(ldmFilename, decFilename)) {
-        printf("Decompress error");
-        return 1;
-    }
-    gettimeofday(&tv2, NULL);
-    printf("Total time = %f seconds\n",
-          (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-          (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-  /* verify */
-  verify(inpFilename, decFilename);
-  return 0;
-}
-
-
-#if 0
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in,
-                            size_t *size_out) {
-  char *src, *buf = NULL;
-  size_t r = 1;
-  size_t size, n, k, count_in = 0, count_out = 0, offset, frame_size = 0;
-
-  src = malloc(BUF_SIZE);
-  if (!src) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-  size = BUF_SIZE + LDM_HEADER_SIZE;
-  buf = malloc(size);
-  if (!buf) {
-    printf("Not enough memory\n");
-    goto cleanup;
-  }
-
-
-  for (;;) {
-    k = fread(src, 1, BUF_SIZE, in);
-    if (k == 0)
-      break;
-    count_in += k;
-
-    n = LDM_compress(src, buf, k, BUF_SIZE);
-
-    // n = k;
-    // offset += n;
-    offset = k;
-    count_out += k;
-
-//    k = fwrite(src, 1, offset, out);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-      if (ferror(out))
-        printf("Write failed\n");
-      else
-        printf("Short write\n");
-      goto cleanup;
-    }
-
-  }
-  *size_in = count_in;
-  *size_out = count_out;
-  r = 0;
- cleanup:
-  free(src);
-  free(buf);
-  return r;
-}
-
-static size_t decompress_file(FILE *in, FILE *out) {
-  void *src = malloc(BUF_SIZE);
-  void *dst = NULL;
-  size_t dst_capacity = BUF_SIZE;
-  size_t ret = 1;
-  size_t bytes_written = 0;
-
-  if (!src) {
-    perror("decompress_file(src)");
-    goto cleanup;
-  }
-
-  while (ret != 0) {
-    /* Load more input */
-    size_t src_size = fread(src, 1, BUF_SIZE, in);
-    void *src_ptr = src;
-    void *src_end = src_ptr + src_size;
-    if (src_size == 0 || ferror(in)) {
-      printf("(TODO): Decompress: not enough input or error reading file\n");
-      //TODO
-      ret = 0;
-      goto cleanup;
-    }
-
-    /* Allocate destination buffer if it hasn't been allocated already */
-    if (!dst) {
-      dst = malloc(dst_capacity);
-      if (!dst) {
-        perror("decompress_file(dst)");
-        goto cleanup;
-      }
-    }
-
-    // TODO
-
-    /* Decompress:
-     * Continue while there is more input to read.
-     */
-    while (src_ptr != src_end && ret != 0) {
-      // size_t dst_size = src_size;
-      size_t dst_size = LDM_decompress(src, dst, src_size, dst_capacity);
-      size_t written = fwrite(dst, 1, dst_size, out);
-//      printf("Writing %zu bytes\n", dst_size);
-      bytes_written += dst_size;
-      if (written != dst_size) {
-        printf("Decompress: Failed to write to file\n");
-        goto cleanup;
-      }
-      src_ptr += src_size;
-      src_size = src_end - src_ptr;
-    }
-
-    /* Update input */
-
-  }
-
-  printf("Wrote %zu bytes\n", bytes_written);
-
- cleanup:
-  free(src);
-  free(dst);
-
-  return ret;
-}
-
-int main2(int argc, char *argv[]) {
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Please specify input filename\n");
-    return 0;
-  }
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
-    printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-  /* compress */
-  {
-    FILE *inpFp = fopen(inpFilename, "rb");
-    FILE *outFp = fopen(ldmFilename, "wb");
-    size_t sizeIn = 0;
-    size_t sizeOut = 0;
-    size_t ret;
-		printf("compress : %s -> %s\n", inpFilename, ldmFilename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-  }
-
-	/* decompress */
-	{
-    FILE *inpFp = fopen(ldmFilename, "rb");
-		FILE *outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", ldmFilename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{
-    FILE *inpFp = fopen(inpFilename, "rb");
-		FILE *decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
-  return 0;
-}
-#endif
-
diff --git a/contrib/long_distance_matching/versions/v0.3/util.c b/contrib/long_distance_matching/versions/v0.3/util.c
deleted file mode 100644
index 9ea4ca1e..00000000
--- a/contrib/long_distance_matching/versions/v0.3/util.c
+++ /dev/null
@@ -1,64 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdio.h>
-
-#include "util.h"
-
-typedef  uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef  int32_t S32;
-typedef uint64_t U64;
-
-unsigned LDM_isLittleEndian(void) {
-    const union { U32 u; BYTE c[4]; } one = { 1 };
-    return one.c[0];
-}
-
-U16 LDM_read16(const void *memPtr) {
-  U16 val;
-  memcpy(&val, memPtr, sizeof(val));
-  return val;
-}
-
-U16 LDM_readLE16(const void *memPtr) {
-  if (LDM_isLittleEndian()) {
-    return LDM_read16(memPtr);
-  } else {
-    const BYTE *p = (const BYTE *)memPtr;
-    return (U16)((U16)p[0] + (p[1] << 8));
-  }
-}
-
-void LDM_write16(void *memPtr, U16 value){
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_write32(void *memPtr, U32 value) {
-  memcpy(memPtr, &value, sizeof(value));
-}
-
-void LDM_writeLE16(void *memPtr, U16 value) {
-  if (LDM_isLittleEndian()) {
-    LDM_write16(memPtr, value);
-  } else {
-    BYTE* p = (BYTE *)memPtr;
-    p[0] = (BYTE) value;
-    p[1] = (BYTE)(value>>8);
-  }
-}
-
-U32 LDM_read32(const void *ptr) {
-  return *(const U32 *)ptr;
-}
-
-U64 LDM_read64(const void *ptr) {
-  return *(const U64 *)ptr;
-}
-
-void LDM_copy8(void *dst, const void *src) {
-  memcpy(dst, src, 8);
-}
-
-
diff --git a/contrib/long_distance_matching/versions/v0.3/util.h b/contrib/long_distance_matching/versions/v0.3/util.h
deleted file mode 100644
index 90726412..00000000
--- a/contrib/long_distance_matching/versions/v0.3/util.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#ifndef LDM_UTIL_H
-#define LDM_UTIL_H
-
-unsigned LDM_isLittleEndian(void);
-
-uint16_t LDM_read16(const void *memPtr);
-
-uint16_t LDM_readLE16(const void *memPtr);
-
-void LDM_write16(void *memPtr, uint16_t value);
-
-void LDM_write32(void *memPtr, uint32_t value);
-
-void LDM_writeLE16(void *memPtr, uint16_t value);
-
-uint32_t LDM_read32(const void *ptr);
-
-uint64_t LDM_read64(const void *ptr);
-
-void LDM_copy8(void *dst, const void *src);
-
-
-#endif /* LDM_UTIL_H */
diff --git a/contrib/long_distance_matching/versions/v0.5/Makefile b/contrib/long_distance_matching/versions/v0.5/Makefile
deleted file mode 100644
index cff78644..00000000
--- a/contrib/long_distance_matching/versions/v0.5/Makefile
+++ /dev/null
@@ -1,37 +0,0 @@
-# ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
-
-# This Makefile presumes libzstd is installed, using `sudo make install`
-
-CPPFLAGS+= -I../../lib/common
-CFLAGS  ?= -O3
-DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls
-CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CFLAGS)
-
-LDFLAGS += -lzstd
-
-.PHONY: default all clean
-
-default: all
-
-all: main-ldm
-
-main-ldm : ldm.h ldm.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-clean:
-	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main main-ldm
-	@echo Cleaning completed
-
diff --git a/contrib/long_distance_matching/versions/v0.5/README b/contrib/long_distance_matching/versions/v0.5/README
deleted file mode 100644
index 7901ae76..00000000
--- a/contrib/long_distance_matching/versions/v0.5/README
+++ /dev/null
@@ -1,5 +0,0 @@
-This version uses simple lz4-style compression with a rolling hash.
-- A rolling checksum based on rsync's Adler-32 style checksum is used.
-- The checksum is hashed using lz4's hash function.
-- Hash table replacement policy: direct overwrite.
-- The length of input to the hash function can be set with LDM_HASH_LENGTH.
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.c b/contrib/long_distance_matching/versions/v0.5/ldm.c
deleted file mode 100644
index 06c97bc4..00000000
--- a/contrib/long_distance_matching/versions/v0.5/ldm.c
+++ /dev/null
@@ -1,710 +0,0 @@
-#include <limits.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "ldm.h"
-
-// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY 0
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-#define COMPUTE_STATS
-#define CHECKSUM_CHAR_OFFSET 10
-//#define RUN_CHECKS
-//#define LDM_DEBUG
-
-struct LDM_hashEntry {
-  offset_t offset;
-};
-
-typedef struct LDM_hashTable {
-  U32 numEntries;
-  U32 minimumTagMask;  // TODO: what if tag == offset?
-
-  // Maximum number of elements in the table.
-  U32 limit;
-
-  LDM_hashEntry *entries;
-} LDM_hashTable;
-
-// TODO: Add offset histogram by powers of two
-// TODO: Scanning speed
-// TODO: Memory usage
-struct LDM_compressStats {
-  U32 windowSizeLog, hashTableSizeLog;
-  U32 numMatches;
-  U64 totalMatchLength;
-  U64 totalLiteralLength;
-  U64 totalOffset;
-
-  U32 minOffset, maxOffset;
-
-  U32 numCollisions;
-  U32 numHashInserts;
-
-  U32 offsetHistogram[32];
-};
-
-struct LDM_CCtx {
-  U64 isize;             /* Input size */
-  U64 maxOSize;          /* Maximum output size */
-
-  const BYTE *ibase;        /* Base of input */
-  const BYTE *ip;           /* Current input position */
-  const BYTE *iend;         /* End of input */
-
-  // Maximum input position such that hashing at the position does not exceed
-  // end of input.
-  const BYTE *ihashLimit;
-
-  // Maximum input position such that finding a match of at least the minimum
-  // match length does not exceed end of input.
-  const BYTE *imatchLimit;
-
-  const BYTE *obase;        /* Base of output */
-  BYTE *op;                 /* Output */
-
-  const BYTE *anchor;       /* Anchor to start of current (match) block */
-
-  LDM_compressStats stats;            /* Compression statistics */
-
-  LDM_hashEntry *hashTable;
-
-//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
-
-  const BYTE *lastPosHashed;          /* Last position hashed */
-  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  U32 lastSum;
-
-  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
-  const BYTE *nextPosHashed;
-  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
-  U32 nextSum;
-
-  unsigned step;                      // ip step, should be 1.
-
-  // DEBUG
-  const BYTE *DEBUG_setNextHash;
-};
-
-void LDM_outputHashTableOccupancy(
-    const LDM_hashEntry *hashTable, U32 hashTableSize) {
-  U32 i = 0;
-  U32 ctr = 0;
-  for (; i < hashTableSize; i++) {
-    if (hashTable[i].offset == 0) {
-      ctr++;
-    }
-  }
-  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         hashTableSize, ctr,
-         100.0 * (double)(ctr) / (double)hashTableSize);
-}
-
-// TODO: This can be done more efficiently (but it is not that important as it
-// is only used for computing stats).
-static int intLog2(U32 x) {
-  int ret = 0;
-  while (x >>= 1) {
-    ret++;
-  }
-  return ret;
-}
-
-// TODO: Maybe we would eventually prefer to have linear rather than
-// exponential buckets.
-void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx) {
-  int i = 0;
-  int buckets[32] = { 0 };
-
-  printf("\n");
-  printf("Hash table histogram\n");
-  for (; i < LDM_HASHTABLESIZE_U32; i++) {
-    int offset = (cctx->ip - cctx->ibase) - cctx->hashTable[i].offset;
-    buckets[intLog2(offset)]++;
-  }
-
-  i = 0;
-  for (; i < 32; i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
-           buckets[i],
-           100.0 * (double) buckets[i] /
-                   (double) LDM_HASHTABLESIZE_U32);
-  }
-  printf("\n");
-}
-
-void LDM_printCompressStats(const LDM_compressStats *stats) {
-  int i = 0;
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  //TODO: compute percentage matched?
-  printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
-          stats->windowSizeLog, stats->hashTableSizeLog);
-  printf("num matches, total match length: %u, %llu\n",
-          stats->numMatches,
-          stats->totalMatchLength);
-  printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
-                                         (double)stats->numMatches);
-  printf("avg literal length, total literalLength: %.1f, %llu\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches,
-         stats->totalLiteralLength);
-  printf("avg offset length: %.1f\n",
-         ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("min offset, max offset: %u, %u\n",
-         stats->minOffset, stats->maxOffset);
-
-  printf("\n");
-  printf("offset histogram: offset, num matches, %% of matches\n");
-
-  for (; i <= intLog2(stats->maxOffset); i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
-           stats->offsetHistogram[i],
-           100.0 * (double) stats->offsetHistogram[i] /
-                   (double) stats->numMatches);
-  }
-  printf("\n");
-
-
-  printf("num collisions, num hash inserts, %% collisions: %u, %u, %.3f\n",
-         stats->numCollisions, stats->numHashInserts,
-         stats->numHashInserts == 0 ?
-            1.0 : (100.0 * (double)stats->numCollisions) /
-                  (double)stats->numHashInserts);
-  printf("=====================\n");
-
-}
-
-int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
-  /*
-  if (memcmp(pIn, pMatch, LDM_MIN_MATCH_LENGTH) == 0) {
-    return 1;
-  }
-  return 0;
-  */
-
-  //TODO: This seems to be faster for some reason?
-  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
-  const BYTE *curIn = pIn;
-  const BYTE *curMatch = pMatch;
-
-  for (; lengthLeft >= 8; lengthLeft -= 8) {
-    if (MEM_read64(curIn) != MEM_read64(curMatch)) {
-      return 0;
-    }
-    curIn += 8;
-    curMatch += 8;
-  }
-  if (lengthLeft > 0) {
-    return (MEM_read32(curIn) == MEM_read32(curMatch));
-  }
-  return 1;
-}
-
-/**
- * Convert a sum computed from getChecksum to a hash value in the range
- * of the hash table.
- */
-static hash_t checksumToHash(U32 sum) {
-  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
-}
-
-/**
- * Computes a checksum based on rsync's checksum.
- *
- * a(k,l) = \sum_{i = k}^l x_i (mod M)
- * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
- * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
-  */
-static U32 getChecksum(const BYTE *buf, U32 len) {
-  U32 i;
-  U32 s1, s2;
-
-  s1 = s2 = 0;
-  for (i = 0; i < (len - 4); i += 4) {
-    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]) +
-          (10 * CHECKSUM_CHAR_OFFSET);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
-          + (4 * CHECKSUM_CHAR_OFFSET);
-
-  }
-  for(; i < len; i++) {
-    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
-    s2 += s1;
-  }
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-/**
- * Update a checksum computed from getChecksum(data, len).
- *
- * The checksum can be updated along its ends as follows:
- * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
- * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
- *
- * Thus toRemove should correspond to data[0].
- */
-static U32 updateChecksum(U32 sum, U32 len,
-                          BYTE toRemove, BYTE toAdd) {
-  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
-  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
-
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-/**
- * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
- * based on cctx->lastSum and cctx->lastPosHashed.
- *
- * This uses a rolling hash and requires that the last position hashed
- * corresponds to cctx->nextIp - step.
- */
-static void setNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  U32 check;
-  if ((cctx->nextIp - cctx->ibase != 1) &&
-      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
-    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
-            cctx->DEBUG_setNextHash - cctx->ibase);
-  }
-
-  cctx->DEBUG_setNextHash = cctx->nextIp;
-#endif
-
-//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
-  cctx->nextSum = updateChecksum(
-      cctx->lastSum, LDM_HASH_LENGTH,
-      cctx->lastPosHashed[0],
-      cctx->lastPosHashed[LDM_HASH_LENGTH]);
-  cctx->nextPosHashed = cctx->nextIp;
-  cctx->nextHash = checksumToHash(cctx->nextSum);
-
-#ifdef RUN_CHECKS
-  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
-
-  if (check != cctx->nextSum) {
-    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
-  }
-
-  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
-    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
-            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
-            cctx->ip - cctx->ibase);
-  }
-#endif
-}
-
-static void putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash, U32 sum) {
-#ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < LDM_HASHTABLESIZE_U32) {
-    offset_t offset = cctx->hashTable[hash].offset;
-    cctx->stats.numHashInserts++;
-    if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
-      cctx->stats.numCollisions++;
-    }
-  }
-#endif
-
-  // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
-  // Note: this works only when cctx->step is 1.
-  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
-    cctx->hashTable[hash] = entry;
-  }
-
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-  cctx->lastSum = sum;
-}
-
-/**
- * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
- * fields from the "next" fields.
- *
- * This requires that cctx->ip == cctx->nextPosHashed.
- */
-static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  if (cctx->ip != cctx->nextPosHashed) {
-    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
-  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
-}
-
-/**
- * Insert hash of the current position into the hash table.
- */
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
-  hash_t hash = checksumToHash(sum);
-
-#ifdef RUN_CHECKS
-  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
-    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
-
-  putHashOfCurrentPositionFromHash(cctx, hash, sum);
-}
-
-/**
- * Returns the position of the entry at hashTable[hash].
- */
-static const BYTE *getPositionOnHash(LDM_CCtx *cctx, hash_t hash) {
-  return cctx->hashTable[hash].offset + cctx->ibase;
-}
-
-U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = (*pMatch) ^ *(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (U32)(pIn - pStart);
-  }
-  return (U32)(pIn - pStart);
-}
-
-void LDM_readHeader(const void *src, U64 *compressedSize,
-                    U64 *decompressedSize) {
-  const BYTE *ip = (const BYTE *)src;
-  *compressedSize = MEM_readLE64(ip);
-  ip += sizeof(U64);
-  *decompressedSize = MEM_readLE64(ip);
-  // ip += sizeof(U64);
-}
-
-void LDM_initializeCCtx(LDM_CCtx *cctx,
-                        const void *src, size_t srcSize,
-                        void *dst, size_t maxDstSize) {
-  cctx->isize = srcSize;
-  cctx->maxOSize = maxDstSize;
-
-  cctx->ibase = (const BYTE *)src;
-  cctx->ip = cctx->ibase;
-  cctx->iend = cctx->ibase + srcSize;
-
-  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
-  cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
-
-  cctx->obase = (BYTE *)dst;
-  cctx->op = (BYTE *)dst;
-
-  cctx->anchor = cctx->ibase;
-
-  memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  cctx->hashTable = calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
-//  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
-  cctx->stats.minOffset = UINT_MAX;
-  cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
-  cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
-
-
-  cctx->lastPosHashed = NULL;
-
-  cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
-  cctx->nextIp = cctx->ip + cctx->step;
-  cctx->nextPosHashed = 0;
-
-  cctx->DEBUG_setNextHash = 0;
-}
-
-void LDM_destroyCCtx(LDM_CCtx *cctx) {
-  free(cctx->hashTable);
-}
-
-/**
- * Finds the "best" match.
- *
- * Returns 0 if successful and 1 otherwise (i.e. no match can be found
- * in the remaining input that is long enough).
- *
- */
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
-  cctx->nextIp = cctx->ip + cctx->step;
-
-  do {
-    hash_t h;
-    U32 sum;
-    setNextHash(cctx);
-    h = cctx->nextHash;
-    sum = cctx->nextSum;
-    cctx->ip = cctx->nextIp;
-    cctx->nextIp += cctx->step;
-
-    if (cctx->ip > cctx->imatchLimit) {
-      return 1;
-    }
-
-    *match = getPositionOnHash(cctx, h);
-    putHashOfCurrentPositionFromHash(cctx, h, sum);
-
-  } while (cctx->ip - *match > LDM_WINDOW_SIZE ||
-           !LDM_isValidMatch(cctx->ip, *match));
-  setNextHash(cctx);
-  return 0;
-}
-
-void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
-  /* Encode the literal length. */
-  if (literalLength >= RUN_MASK) {
-    int len = (int)literalLength - RUN_MASK;
-    *pToken = (RUN_MASK << ML_BITS);
-    for (; len >= 255; len -= 255) {
-      *(cctx->op)++ = 255;
-    }
-    *(cctx->op)++ = (BYTE)len;
-  } else {
-    *pToken = (BYTE)(literalLength << ML_BITS);
-  }
-
-  /* Encode the literals. */
-  memcpy(cctx->op, cctx->anchor, literalLength);
-  cctx->op += literalLength;
-}
-
-void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U32 literalLength,
-                     const U32 offset,
-                     const U32 matchLength) {
-  BYTE *pToken = cctx->op++;
-
-  /* Encode the literal length and literals. */
-  LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
-
-  /* Encode the offset. */
-  MEM_write32(cctx->op, offset);
-  cctx->op += LDM_OFFSET_SIZE;
-
-  /* Encode the match length. */
-  if (matchLength >= ML_MASK) {
-    unsigned matchLengthRemaining = matchLength;
-    *pToken += ML_MASK;
-    matchLengthRemaining -= ML_MASK;
-    MEM_write32(cctx->op, 0xFFFFFFFF);
-    while (matchLengthRemaining >= 4*0xFF) {
-      cctx->op += 4;
-      MEM_write32(cctx->op, 0xffffffff);
-      matchLengthRemaining -= 4*0xFF;
-    }
-    cctx->op += matchLengthRemaining / 255;
-    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
-  } else {
-    *pToken += (BYTE)(matchLength);
-  }
-}
-
-// TODO: maxDstSize is unused. This function may seg fault when writing
-// beyond the size of dst, as it does not check maxDstSize. Writing to
-// a buffer and performing checks is a possible solution.
-//
-// This is based upon lz4.
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize) {
-  LDM_CCtx cctx;
-  const BYTE *match;
-  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
-
-  /* Hash the first position and put it into the hash table. */
-  LDM_putHashOfCurrentPosition(&cctx);
-
-  /**
-   * Find a match.
-   * If no more matches can be found (i.e. the length of the remaining input
-   * is less than the minimum match length), then stop searching for matches
-   * and encode the final literals.
-   */
-  while (LDM_findBestMatch(&cctx, &match) == 0) {
-#ifdef COMPUTE_STATS
-    cctx.stats.numMatches++;
-#endif
-
-    /**
-     * Catch up: look back to extend the match backwards from the found match.
-     */
-    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
-           cctx.ip[-1] == match[-1]) {
-      cctx.ip--;
-      match--;
-    }
-
-    /**
-     * Write current block (literals, literal length, match offset, match
-     * length) and update pointers and hashes.
-     */
-    {
-      const U32 literalLength = cctx.ip - cctx.anchor;
-      const U32 offset = cctx.ip - match;
-      const U32 matchLength = LDM_countMatchLength(
-          cctx.ip + LDM_MIN_MATCH_LENGTH, match + LDM_MIN_MATCH_LENGTH,
-          cctx.ihashLimit);
-
-      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
-
-#ifdef COMPUTE_STATS
-      cctx.stats.totalLiteralLength += literalLength;
-      cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
-      cctx.stats.minOffset =
-          offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
-      cctx.stats.maxOffset =
-          offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
-      cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
-#endif
-
-      // Move ip to end of block, inserting hashes at each position.
-      cctx.nextIp = cctx.ip + cctx.step;
-      while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
-                       matchLength + literalLength) {
-        if (cctx.ip > cctx.lastPosHashed) {
-          // TODO: Simplify.
-          LDM_updateLastHashFromNextHash(&cctx);
-          setNextHash(&cctx);
-        }
-        cctx.ip++;
-        cctx.nextIp++;
-      }
-    }
-
-    // Set start of next block to current input pointer.
-    cctx.anchor = cctx.ip;
-    LDM_updateLastHashFromNextHash(&cctx);
-  }
-
-  // LDM_outputHashTableOffsetHistogram(&cctx);
-
-  /* Encode the last literals (no more matches). */
-  {
-    const size_t lastRun = cctx.iend - cctx.anchor;
-    BYTE *pToken = cctx.op++;
-    LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
-  }
-
-#ifdef COMPUTE_STATS
-  LDM_printCompressStats(&cctx.stats);
-  LDM_outputHashTableOccupancy(cctx.hashTable, LDM_HASHTABLESIZE_U32);
-#endif
-
-  {
-    const size_t ret = cctx.op - cctx.obase;
-    LDM_destroyCCtx(&cctx);
-    return ret;
-  }
-}
-
-struct LDM_DCtx {
-  size_t compressedSize;
-  size_t maxDecompressedSize;
-
-  const BYTE *ibase;   /* Base of input */
-  const BYTE *ip;      /* Current input position */
-  const BYTE *iend;    /* End of source */
-
-  const BYTE *obase;   /* Base of output */
-  BYTE *op;            /* Current output position */
-  const BYTE *oend;    /* End of output */
-};
-
-void LDM_initializeDCtx(LDM_DCtx *dctx,
-                        const void *src, size_t compressedSize,
-                        void *dst, size_t maxDecompressedSize) {
-  dctx->compressedSize = compressedSize;
-  dctx->maxDecompressedSize = maxDecompressedSize;
-
-  dctx->ibase = src;
-  dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressedSize;
-  dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressedSize;
-}
-
-size_t LDM_decompress(const void *src, size_t compressedSize,
-                      void *dst, size_t maxDecompressedSize) {
-  LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
-
-  while (dctx.ip < dctx.iend) {
-    BYTE *cpy;
-    const BYTE *match;
-    size_t length, offset;
-
-    /* Get the literal length. */
-    const unsigned token = *(dctx.ip)++;
-    if ((length = (token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-
-    /* Copy the literals. */
-    cpy = dctx.op + length;
-    memcpy(dctx.op, dctx.ip, length);
-    dctx.ip += length;
-    dctx.op = cpy;
-
-    //TODO : dynamic offset size
-    offset = MEM_read32(dctx.ip);
-    dctx.ip += LDM_OFFSET_SIZE;
-    match = dctx.op - offset;
-
-    /* Get the match length. */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-    length += LDM_MIN_MATCH_LENGTH;
-
-    /* Copy match. */
-    cpy = dctx.op + length;
-
-    // Inefficient for now.
-    while (match < cpy - offset && dctx.op < dctx.oend) {
-      *(dctx.op)++ = *match++;
-    }
-  }
-  return dctx.op - (BYTE *)dst;
-}
-
-// TODO: implement and test hash function
-void LDM_test(void) {
-}
-
-/*
-void LDM_test(const void *src, size_t srcSize,
-              void *dst, size_t maxDstSize) {
-  const BYTE *ip = (const BYTE *)src + 1125;
-  U32 sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
-  U32 sum2;
-  ++ip;
-  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
-    sum2 = updateChecksum(sum, LDM_HASH_LENGTH,
-                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
-    sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
-    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
-  }
-}
-*/
-
-
diff --git a/contrib/long_distance_matching/versions/v0.5/ldm.h b/contrib/long_distance_matching/versions/v0.5/ldm.h
deleted file mode 100644
index 70cda8b8..00000000
--- a/contrib/long_distance_matching/versions/v0.5/ldm.h
+++ /dev/null
@@ -1,159 +0,0 @@
-#ifndef LDM_H
-#define LDM_H
-
-#include <stddef.h>   /* size_t */
-
-#include "mem.h"    // from /lib/common/mem.h
-
-#define LDM_COMPRESS_SIZE 8
-#define LDM_DECOMPRESS_SIZE 8
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
-#define LDM_OFFSET_SIZE 4
-
-// Defines the size of the hash table.
-#define LDM_MEMORY_USAGE 16
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-
-#define LDM_WINDOW_SIZE_LOG 25
-#define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
-
-//These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 4
-#define LDM_HASH_LENGTH 4
-
-typedef U32 offset_t;
-typedef U32 hash_t;
-typedef struct LDM_hashEntry LDM_hashEntry;
-typedef struct LDM_compressStats LDM_compressStats;
-typedef struct LDM_CCtx LDM_CCtx;
-typedef struct LDM_DCtx LDM_DCtx;
-
-/**
- *  Compresses src into dst.
- *
- *  NB: This currently ignores maxDstSize and assumes enough space is available.
- *
- *  Block format (see lz4 documentation for more information):
- *  github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
- *
- *  A block is composed of sequences. Each sequence begins with a token, which
- *  is a one-byte value separated into two 4-bit fields.
- *
- *  The first field uses the four high bits of the token and encodes the literal
- *  length. If the field value is 0, there is no literal. If it is 15,
- *  additional bytes are added (each ranging from 0 to 255) to the previous
- *  value to produce a total length.
- *
- *  Following the token and optional length bytes are the literals.
- *
- *  Next are the 4 bytes representing the offset of the match (2 in lz4),
- *  representing the position to copy the literals.
- *
- *  The lower four bits of the token encode the match length. With additional
- *  bytes added similarly to the additional literal length bytes after the offset.
- *
- *  The last sequence is incomplete and stops right after the lieterals.
- *
- */
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize);
-
-/**
- * Initialize the compression context.
- *
- * Allocates memory for the hash table.
- */
-void LDM_initializeCCtx(LDM_CCtx *cctx,
-                        const void *src, size_t srcSize,
-                        void *dst, size_t maxDstSize);
-
-/**
- * Frees up memory allocating in initializeCCtx
- */
-void LDM_destroyCCtx(LDM_CCtx *cctx);
-
-/**
- * Prints the percentage of the hash table occupied (where occupied is defined
- * as the entry being non-zero).
- */
-void LDM_outputHashTableOccupancy(const LDM_hashEntry *hashTable,
-                                  U32 hashTableSize);
-
-/**
- * Prints the distribution of offsets in the hash table.
- *
- * The offsets are defined as the distance of the hash table entry from the
- * current input position of the cctx.
- */
-void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx);
-
-/**
- * Outputs compression statistics to stdout.
- */
-void LDM_printCompressStats(const LDM_compressStats *stats);
-/**
- * Checks whether the LDM_MIN_MATCH_LENGTH bytes from p are the same as the
- * LDM_MIN_MATCH_LENGTH bytes from match.
- *
- * This assumes LDM_MIN_MATCH_LENGTH is a multiple of four.
- *
- * Return 1 if valid, 0 otherwise.
- */
-int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch);
-
-/**
- *  Counts the number of bytes that match from pIn and pMatch,
- *  up to pInLimit.
- */
-U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *pInLimit);
-
-/**
- * Encode the literal length followed by the literals.
- *
- * The literal length is written to the upper four bits of pToken, with
- * additional bytes written to the output as needed (see lz4).
- *
- * This is followed by literalLength bytes corresponding to the literals.
- */
-void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength);
-
-/**
- * Write current block (literals, literal length, match offset,
- * match length).
- */
-void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U32 literalLength,
-                     const U32 offset,
-                     const U32 matchLength);
-
-/**
- * Decompresses src into dst.
- *
- * Note: assumes src does not have a header.
- */
-size_t LDM_decompress(const void *src, size_t srcSize,
-                      void *dst, size_t maxDstSize);
-
-/**
- * Initialize the decompression context.
- */
-void LDM_initializeDCtx(LDM_DCtx *dctx,
-                        const void *src, size_t compressedSize,
-                        void *dst, size_t maxDecompressedSize);
-
-/**
- * Reads the header from src and writes the compressed size and
- * decompressed size into compressedSize and decompressedSize respectively.
- *
- * NB: LDM_compress and LDM_decompress currently do not add/read headers.
- */
-void LDM_readHeader(const void *src, U64 *compressedSize,
-                    U64 *decompressedSize);
-
-void LDM_test(void);
-
-#endif /* LDM_H */
diff --git a/contrib/long_distance_matching/versions/v0.5/main-ldm.c b/contrib/long_distance_matching/versions/v0.5/main-ldm.c
deleted file mode 100644
index ea6375ba..00000000
--- a/contrib/long_distance_matching/versions/v0.5/main-ldm.c
+++ /dev/null
@@ -1,270 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <zstd.h>
-
-#include <fcntl.h>
-#include "ldm.h"
-#include "zstd.h"
-
-#define DEBUG
-#define TEST
-
-/* Compress file given by fname and output to oname.
- * Returns 0 if successful, error code otherwise.
- *
- * TODO: This currently seg faults if the compressed size is > the decompress
- * size due to the mmapping and output file size allocated to be the input size.
- * The compress function should check before writing or buffer writes.
- */
-static int compress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-  size_t maxCompressedSize, compressedSize;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  maxCompressedSize = statbuf.st_size + LDM_HEADER_SIZE;
-
- /* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */
-  if (lseek(fdout, maxCompressedSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
- /* Write a dummy byte at the last location. */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, maxCompressedSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  compressedSize = LDM_HEADER_SIZE +
-      LDM_compress(src, statbuf.st_size,
-                   dst + LDM_HEADER_SIZE, maxCompressedSize);
-
-  // Write compress and decompress size to header
-  // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressedSize, 8);
-  memcpy(dst + 8, &(statbuf.st_size), 8);
-
-#ifdef DEBUG
-  printf("Compressed size: %zu\n", compressedSize);
-  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
-#endif
-
-  // Truncate file to compressedSize.
-  ftruncate(fdout, compressedSize);
-
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressedSize, oname,
-         (double)compressedSize / (statbuf.st_size) * 100);
-
-  // Close files.
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Decompress file compressed using LDM_compress.
- * The input file should have the LDM_HEADER followed by payload.
- * Returns 0 if succesful, and an error code otherwise.
- */
-static int decompress(const char *fname, const char *oname) {
-  int fdin, fdout;
-  struct stat statbuf;
-  char *src, *dst;
-  U64 compressedSize, decompressedSize;
-  size_t outSize;
-
-  /* Open the input file. */
-  if ((fdin = open(fname, O_RDONLY)) < 0) {
-    perror("Error in file opening");
-    return 1;
-  }
-
-  /* Open the output file. */
-  if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
-    perror("Can't create output file");
-    return 1;
-  }
-
-  /* Find the size of the input file. */
-  if (fstat (fdin, &statbuf) < 0) {
-    perror("Fstat error");
-    return 1;
-  }
-
-  /* mmap the input file. */
-  if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
-          == (caddr_t) - 1) {
-      perror("mmap error for input");
-      return 1;
-  }
-
-  /* Read the header. */
-  LDM_readHeader(src, &compressedSize, &decompressedSize);
-
-  /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressedSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
-
-  /* mmap the output file */
-  if ((dst = mmap(0, decompressedSize, PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
-      perror("mmap error for output");
-      return 1;
-  }
-
-  outSize = LDM_decompress(
-      src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
-      dst, decompressedSize);
-
-  printf("Ret size out: %zu\n", outSize);
-  ftruncate(fdout, outSize);
-
-  close(fdin);
-  close(fdout);
-  return 0;
-}
-
-/* Compare two files.
- * Returns 0 iff they are the same.
- */
-static int compare(FILE *fp0, FILE *fp1) {
-  int result = 0;
-  while (result == 0) {
-    char b0[1024];
-    char b1[1024];
-    const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-    const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
-
-    result = (int)r0 - (int)r1;
-
-    if (0 == r0 || 0 == r1) break;
-
-    if (0 == result) result = memcmp(b0, b1, r0);
-  }
-  return result;
-}
-
-/* Verify the input file is the same as the decompressed file. */
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
-
-  printf("verify : %s <-> %s\n", inpFilename, decFilename);
-  {
-    const int cmp = compare(inpFp, decFp);
-    if(0 == cmp) {
-      printf("verify : OK\n");
-    } else {
-      printf("verify : NG\n");
-    }
-  }
-
-	fclose(decFp);
-	fclose(inpFp);
-}
-
-int main(int argc, const char *argv[]) {
-  const char * const exeName = argv[0];
-  char inpFilename[256] = { 0 };
-  char ldmFilename[256] = { 0 };
-  char decFilename[256] = { 0 };
-
-  if (argc < 2) {
-    printf("Wrong arguments\n");
-    printf("Usage:\n");
-    printf("%s FILE\n", exeName);
-    return 1;
-  }
-
-  snprintf(inpFilename, 256, "%s", argv[1]);
-  snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
-  snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
-
- 	printf("inp = [%s]\n", inpFilename);
-	printf("ldm = [%s]\n", ldmFilename);
-	printf("dec = [%s]\n", decFilename);
-
-
-  /* Compress */
-  {
-    struct timeval tv1, tv2;
-    gettimeofday(&tv1, NULL);
-    if (compress(inpFilename, ldmFilename)) {
-        printf("Compress error");
-        return 1;
-    }
-    gettimeofday(&tv2, NULL);
-    printf("Total compress time = %f seconds\n",
-           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-           (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-
-  /* Decompress */
-  {
-    struct timeval tv1, tv2;
-    gettimeofday(&tv1, NULL);
-    if (decompress(ldmFilename, decFilename)) {
-        printf("Decompress error");
-        return 1;
-    }
-    gettimeofday(&tv2, NULL);
-    printf("Total decompress time = %f seconds\n",
-          (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-          (double) (tv2.tv_sec - tv1.tv_sec));
-  }
-  /* verify */
-  verify(inpFilename, decFilename);
-
-#ifdef TEST
-  LDM_test();
-#endif
-  return 0;
-}

From 7a28b9e4a3a6e2c7ab1a9a66beceea95e710560e Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 17 Jul 2017 15:29:11 -0700
Subject: [PATCH 066/100] [libzstd] Pull optimal parser state out of seqStore_t

---
 lib/common/zstd_internal.h   |  42 +++----
 lib/compress/zstd_compress.c |  23 ++--
 lib/compress/zstd_opt.h      | 208 ++++++++++++++++++-----------------
 3 files changed, 139 insertions(+), 134 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 7f183c8c..1621bca6 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -215,20 +215,6 @@ MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd)   /* s
 *********************************************/
 typedef struct ZSTD_stats_s ZSTD_stats_t;
 
-typedef struct {
-    U32 off;
-    U32 len;
-} ZSTD_match_t;
-
-typedef struct {
-    U32 price;
-    U32 off;
-    U32 mlen;
-    U32 litlen;
-    U32 rep[ZSTD_REP_NUM];
-} ZSTD_optimal_t;
-
-
 typedef struct seqDef_s {
     U32 offset;
     U16 litLength;
@@ -248,13 +234,29 @@ typedef struct {
     U32   longLengthPos;
     U32   rep[ZSTD_REP_NUM];
     U32   repToConfirm[ZSTD_REP_NUM];
-    /* opt */
-    ZSTD_optimal_t* priceTable;
-    ZSTD_match_t* matchTable;
-    U32* matchLengthFreq;
-    U32* litLengthFreq;
+} seqStore_t;
+
+typedef struct {
+    U32 off;
+    U32 len;
+} ZSTD_match_t;
+
+typedef struct {
+    U32 price;
+    U32 off;
+    U32 mlen;
+    U32 litlen;
+    U32 rep[ZSTD_REP_NUM];
+} ZSTD_optimal_t;
+
+typedef struct {
     U32* litFreq;
+    U32* litLengthFreq;
+    U32* matchLengthFreq;
     U32* offCodeFreq;
+    ZSTD_match_t* matchTable;
+    ZSTD_optimal_t* priceTable;
+
     U32  matchLengthSum;
     U32  matchSum;
     U32  litLengthSum;
@@ -270,7 +272,7 @@ typedef struct {
     U32  cachedPrice;
     U32  cachedLitLength;
     const BYTE* cachedLiterals;
-} seqStore_t;
+} optState_t;
 
 typedef struct {
     U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 64f783c7..5d8e2632 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -96,6 +96,7 @@ struct ZSTD_CCtx_s {
     size_t staticSize;
 
     seqStore_t seqStore;    /* sequences storage ptrs */
+    optState_t optState;
     U32* hashTable;
     U32* hashTable3;
     U32* chainTable;
@@ -594,7 +595,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 ple
     cctx->dictID = 0;
     cctx->loadedDictEnd = 0;
     { int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->seqStore.rep[i] = repStartValue[i]; }
-    cctx->seqStore.litLengthSum = 0;  /* force reset of btopt stats */
+    cctx->optState.litLengthSum = 0;  /* force reset of btopt stats */
     XXH64_reset(&cctx->xxhState, 0);
     return 0;
 }
@@ -693,7 +694,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->lowLimit = 0;
         { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->seqStore.rep[i] = repStartValue[i]; }
         zc->hashLog3 = hashLog3;
-        zc->seqStore.litLengthSum = 0;
+        zc->optState.litLengthSum = 0;
 
         ptr = zc->entropy + 1;
 
@@ -701,15 +702,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) {
             DEBUGLOG(5, "reserving optimal parser space");
             assert(((size_t)ptr & 3) == 0);  /* ensure ptr is properly aligned */
-            zc->seqStore.litFreq = (U32*)ptr;
-            zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
-            zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
-            zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
-            ptr = zc->seqStore.offCodeFreq + (MaxOff+1);
-            zc->seqStore.matchTable = (ZSTD_match_t*)ptr;
-            ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
-            zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr;
-            ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
+            zc->optState.litFreq = (U32*)ptr;
+            zc->optState.litLengthFreq = zc->optState.litFreq + (1<<Litbits);
+            zc->optState.matchLengthFreq = zc->optState.litLengthFreq + (MaxLL+1);
+            zc->optState.offCodeFreq = zc->optState.matchLengthFreq + (MaxML+1);
+            ptr = zc->optState.offCodeFreq + (MaxOff+1);
+            zc->optState.matchTable = (ZSTD_match_t*)ptr;
+            ptr = zc->optState.matchTable + ZSTD_OPT_NUM+1;
+            zc->optState.priceTable = (ZSTD_optimal_t*)ptr;
+            ptr = zc->optState.priceTable + ZSTD_OPT_NUM+1;
         }
 
         /* table Space */
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index d6e3449a..53e806eb 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -22,173 +22,173 @@
 /*-*************************************
 *  Price functions for optimal parser
 ***************************************/
-FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
+FORCE_INLINE void ZSTD_setLog2Prices(optState_t* optPtr)
 {
-    ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1);
-    ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1);
-    ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1);
-    ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1);
-    ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum));
+    optPtr->log2matchLengthSum = ZSTD_highbit32(optPtr->matchLengthSum+1);
+    optPtr->log2litLengthSum = ZSTD_highbit32(optPtr->litLengthSum+1);
+    optPtr->log2litSum = ZSTD_highbit32(optPtr->litSum+1);
+    optPtr->log2offCodeSum = ZSTD_highbit32(optPtr->offCodeSum+1);
+    optPtr->factor = 1 + ((optPtr->litSum>>5) / optPtr->litLengthSum) + ((optPtr->litSum<<1) / (optPtr->litSum + optPtr->matchSum));
 }
 
 
-MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t srcSize)
+MEM_STATIC void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSize)
 {
     unsigned u;
 
-    ssPtr->cachedLiterals = NULL;
-    ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
-    ssPtr->staticPrices = 0;
+    optPtr->cachedLiterals = NULL;
+    optPtr->cachedPrice = optPtr->cachedLitLength = 0;
+    optPtr->staticPrices = 0;
 
-    if (ssPtr->litLengthSum == 0) {
-        if (srcSize <= 1024) ssPtr->staticPrices = 1;
+    if (optPtr->litLengthSum == 0) {
+        if (srcSize <= 1024) optPtr->staticPrices = 1;
 
-        assert(ssPtr->litFreq!=NULL);
+        assert(optPtr->litFreq!=NULL);
         for (u=0; u<=MaxLit; u++)
-            ssPtr->litFreq[u] = 0;
+            optPtr->litFreq[u] = 0;
         for (u=0; u<srcSize; u++)
-            ssPtr->litFreq[src[u]]++;
+            optPtr->litFreq[src[u]]++;
 
-        ssPtr->litSum = 0;
-        ssPtr->litLengthSum = MaxLL+1;
-        ssPtr->matchLengthSum = MaxML+1;
-        ssPtr->offCodeSum = (MaxOff+1);
-        ssPtr->matchSum = (ZSTD_LITFREQ_ADD<<Litbits);
+        optPtr->litSum = 0;
+        optPtr->litLengthSum = MaxLL+1;
+        optPtr->matchLengthSum = MaxML+1;
+        optPtr->offCodeSum = (MaxOff+1);
+        optPtr->matchSum = (ZSTD_LITFREQ_ADD<<Litbits);
 
         for (u=0; u<=MaxLit; u++) {
-            ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV);
-            ssPtr->litSum += ssPtr->litFreq[u];
+            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->litSum += optPtr->litFreq[u];
         }
         for (u=0; u<=MaxLL; u++)
-            ssPtr->litLengthFreq[u] = 1;
+            optPtr->litLengthFreq[u] = 1;
         for (u=0; u<=MaxML; u++)
-            ssPtr->matchLengthFreq[u] = 1;
+            optPtr->matchLengthFreq[u] = 1;
         for (u=0; u<=MaxOff; u++)
-            ssPtr->offCodeFreq[u] = 1;
+            optPtr->offCodeFreq[u] = 1;
     } else {
-        ssPtr->matchLengthSum = 0;
-        ssPtr->litLengthSum = 0;
-        ssPtr->offCodeSum = 0;
-        ssPtr->matchSum = 0;
-        ssPtr->litSum = 0;
+        optPtr->matchLengthSum = 0;
+        optPtr->litLengthSum = 0;
+        optPtr->offCodeSum = 0;
+        optPtr->matchSum = 0;
+        optPtr->litSum = 0;
 
         for (u=0; u<=MaxLit; u++) {
-            ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1));
-            ssPtr->litSum += ssPtr->litFreq[u];
+            optPtr->litFreq[u] = 1 + (optPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1));
+            optPtr->litSum += optPtr->litFreq[u];
         }
         for (u=0; u<=MaxLL; u++) {
-            ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1));
-            ssPtr->litLengthSum += ssPtr->litLengthFreq[u];
+            optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1));
+            optPtr->litLengthSum += optPtr->litLengthFreq[u];
         }
         for (u=0; u<=MaxML; u++) {
-            ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV);
-            ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u];
-            ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3);
+            optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->matchLengthSum += optPtr->matchLengthFreq[u];
+            optPtr->matchSum += optPtr->matchLengthFreq[u] * (u + 3);
         }
-        ssPtr->matchSum *= ZSTD_LITFREQ_ADD;
+        optPtr->matchSum *= ZSTD_LITFREQ_ADD;
         for (u=0; u<=MaxOff; u++) {
-            ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV);
-            ssPtr->offCodeSum += ssPtr->offCodeFreq[u];
+            optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV);
+            optPtr->offCodeSum += optPtr->offCodeFreq[u];
         }
     }
 
-    ZSTD_setLog2Prices(ssPtr);
+    ZSTD_setLog2Prices(optPtr);
 }
 
 
-FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals)
+FORCE_INLINE U32 ZSTD_getLiteralPrice(optState_t* optPtr, U32 litLength, const BYTE* literals)
 {
     U32 price, u;
 
-    if (ssPtr->staticPrices)
+    if (optPtr->staticPrices)
         return ZSTD_highbit32((U32)litLength+1) + (litLength*6);
 
     if (litLength == 0)
-        return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1);
+        return optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[0]+1);
 
     /* literals */
-    if (ssPtr->cachedLiterals == literals) {
-        U32 const additional = litLength - ssPtr->cachedLitLength;
-        const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
-        price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
+    if (optPtr->cachedLiterals == literals) {
+        U32 const additional = litLength - optPtr->cachedLitLength;
+        const BYTE* literals2 = optPtr->cachedLiterals + optPtr->cachedLitLength;
+        price = optPtr->cachedPrice + additional * optPtr->log2litSum;
         for (u=0; u < additional; u++)
-            price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1);
-        ssPtr->cachedPrice = price;
-        ssPtr->cachedLitLength = litLength;
+            price -= ZSTD_highbit32(optPtr->litFreq[literals2[u]]+1);
+        optPtr->cachedPrice = price;
+        optPtr->cachedLitLength = litLength;
     } else {
-        price = litLength * ssPtr->log2litSum;
+        price = litLength * optPtr->log2litSum;
         for (u=0; u < litLength; u++)
-            price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1);
+            price -= ZSTD_highbit32(optPtr->litFreq[literals[u]]+1);
 
         if (litLength >= 12) {
-            ssPtr->cachedLiterals = literals;
-            ssPtr->cachedPrice = price;
-            ssPtr->cachedLitLength = litLength;
+            optPtr->cachedLiterals = literals;
+            optPtr->cachedPrice = price;
+            optPtr->cachedLitLength = litLength;
         }
     }
 
     /* literal Length */
     {   const BYTE LL_deltaCode = 19;
         const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
-        price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1);
+        price += LL_bits[llCode] + optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1);
     }
 
     return price;
 }
 
 
-FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
+FORCE_INLINE U32 ZSTD_getPrice(optState_t* optPtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
 {
     /* offset */
     U32 price;
     BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
 
-    if (seqStorePtr->staticPrices)
-        return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode;
+    if (optPtr->staticPrices)
+        return ZSTD_getLiteralPrice(optPtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode;
 
-    price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
+    price = offCode + optPtr->log2offCodeSum - ZSTD_highbit32(optPtr->offCodeFreq[offCode]+1);
     if (!ultra && offCode >= 20) price += (offCode-19)*2;
 
     /* match Length */
     {   const BYTE ML_deltaCode = 36;
         const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
-        price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1);
+        price += ML_bits[mlCode] + optPtr->log2matchLengthSum - ZSTD_highbit32(optPtr->matchLengthFreq[mlCode]+1);
     }
 
-    return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
+    return price + ZSTD_getLiteralPrice(optPtr, litLength, literals) + optPtr->factor;
 }
 
 
-MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
+MEM_STATIC void ZSTD_updatePrice(optState_t* optPtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
 {
     U32 u;
 
     /* literals */
-    seqStorePtr->litSum += litLength*ZSTD_LITFREQ_ADD;
+    optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
     for (u=0; u < litLength; u++)
-        seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
+        optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
 
     /* literal Length */
     {   const BYTE LL_deltaCode = 19;
         const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
-        seqStorePtr->litLengthFreq[llCode]++;
-        seqStorePtr->litLengthSum++;
+        optPtr->litLengthFreq[llCode]++;
+        optPtr->litLengthSum++;
     }
 
     /* match offset */
     {   BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
-        seqStorePtr->offCodeSum++;
-        seqStorePtr->offCodeFreq[offCode]++;
+        optPtr->offCodeSum++;
+        optPtr->offCodeFreq[offCode]++;
     }
 
     /* match Length */
     {   const BYTE ML_deltaCode = 36;
         const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
-        seqStorePtr->matchLengthFreq[mlCode]++;
-        seqStorePtr->matchLengthSum++;
+        optPtr->matchLengthFreq[mlCode]++;
+        optPtr->matchLengthSum++;
     }
 
-    ZSTD_setLog2Prices(seqStorePtr);
+    ZSTD_setLog2Prices(optPtr);
 }
 
 
@@ -417,6 +417,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                                     const void* src, size_t srcSize, const int ultra)
 {
     seqStore_t* seqStorePtr = &(ctx->seqStore);
+    optState_t* optStatePtr = &(ctx->optState);
     const BYTE* const istart = (const BYTE*)src;
     const BYTE* ip = istart;
     const BYTE* anchor = istart;
@@ -430,14 +431,14 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
     const U32 mls = ctx->appliedParams.cParams.searchLength;
     const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
 
-    ZSTD_optimal_t* opt = seqStorePtr->priceTable;
-    ZSTD_match_t* matches = seqStorePtr->matchTable;
+    ZSTD_optimal_t* opt = optStatePtr->priceTable;
+    ZSTD_match_t* matches = optStatePtr->matchTable;
     const BYTE* inr;
     U32 offset, rep[ZSTD_REP_NUM];
 
     /* init */
     ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize);
+    ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize);
     ip += (ip==prefixStart);
     { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
 
@@ -462,7 +463,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                     }
                     best_off = i - (ip == anchor);
                     do {
-                        price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+                        price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                         if (mlen > last_pos || price < opt[mlen].price)
                             SET_PRICE(mlen, mlen, i, litlen, price);   /* note : macro modifies last_pos */
                         mlen--;
@@ -487,7 +488,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
             mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
             best_mlen = matches[u].len;
             while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
+                price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                 if (mlen > last_pos || price < opt[mlen].price)
                     SET_PRICE(mlen, mlen, matches[u].off, litlen, price);   /* note : macro modifies last_pos */
                 mlen++;
@@ -507,12 +508,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
            if (opt[cur-1].mlen == 1) {
                 litlen = opt[cur-1].litlen + 1;
                 if (cur > litlen) {
-                    price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen);
+                    price = opt[cur - litlen].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-litlen);
                 } else
-                    price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
+                    price = ZSTD_getLiteralPrice(optStatePtr, litlen, anchor);
            } else {
                 litlen = 1;
-                price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
+                price = opt[cur - 1].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-1);
            }
 
            if (cur > last_pos || price <= opt[cur].price)
@@ -554,12 +555,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                            if (opt[cur].mlen == 1) {
                                 litlen = opt[cur].litlen;
                                 if (cur > litlen) {
-                                    price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
+                                    price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
                                 } else
-                                    price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+                                    price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                             } else {
                                 litlen = 0;
-                                price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+                                price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
                             }
 
                             if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
@@ -586,12 +587,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
                         if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
+                            price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
                         else
-                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
+                            price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                     } else {
                         litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
+                        price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
                     }
 
                     if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
@@ -645,7 +646,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
                 if (litLength==0) offset--;
             }
 
-            ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
+            ZSTD_updatePrice(optStatePtr, litLength, anchor, offset, mlen-MINMATCH);
             ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             anchor = ip = ip + mlen;
     }    }   /* for (cur=0; cur < last_pos; ) */
@@ -666,6 +667,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                                      const void* src, size_t srcSize, const int ultra)
 {
     seqStore_t* seqStorePtr = &(ctx->seqStore);
+    optState_t* optStatePtr = &(ctx->optState);
     const BYTE* const istart = (const BYTE*)src;
     const BYTE* ip = istart;
     const BYTE* anchor = istart;
@@ -683,8 +685,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
     const U32 mls = ctx->appliedParams.cParams.searchLength;
     const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
 
-    ZSTD_optimal_t* opt = seqStorePtr->priceTable;
-    ZSTD_match_t* matches = seqStorePtr->matchTable;
+    ZSTD_optimal_t* opt = optStatePtr->priceTable;
+    ZSTD_match_t* matches = optStatePtr->matchTable;
     const BYTE* inr;
 
     /* init */
@@ -692,7 +694,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
     { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=seqStorePtr->rep[i]; }
 
     ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize);
+    ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize);
     ip += (ip==prefixStart);
 
     /* Match Loop */
@@ -726,7 +728,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                     best_off = i - (ip==anchor);
                     litlen = opt[0].litlen;
                     do {
-                        price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+                        price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                         if (mlen > last_pos || price < opt[mlen].price)
                             SET_PRICE(mlen, mlen, i, litlen, price);   /* note : macro modifies last_pos */
                         mlen--;
@@ -756,7 +758,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
             best_mlen = matches[u].len;
             litlen = opt[0].litlen;
             while (mlen <= best_mlen) {
-                price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
+                price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                 if (mlen > last_pos || price < opt[mlen].price)
                     SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
                 mlen++;
@@ -773,12 +775,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
             if (opt[cur-1].mlen == 1) {
                 litlen = opt[cur-1].litlen + 1;
                 if (cur > litlen) {
-                    price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen);
+                    price = opt[cur - litlen].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-litlen);
                 } else
-                    price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
+                    price = ZSTD_getLiteralPrice(optStatePtr, litlen, anchor);
             } else {
                 litlen = 1;
-                price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
+                price = opt[cur - 1].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-1);
             }
 
             if (cur > last_pos || price <= opt[cur].price)
@@ -826,12 +828,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                             if (opt[cur].mlen == 1) {
                                 litlen = opt[cur].litlen;
                                 if (cur > litlen) {
-                                    price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
+                                    price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
                                 } else
-                                    price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+                                    price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
                             } else {
                                 litlen = 0;
-                                price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+                                price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
                             }
 
                             if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
@@ -858,12 +860,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
                         if (cur > litlen)
-                            price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
+                            price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
                         else
-                            price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
+                            price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
                     } else {
                         litlen = 0;
-                        price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
+                        price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
                     }
 
                     if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
@@ -918,7 +920,7 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
                 if (litLength==0) offset--;
             }
 
-            ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
+            ZSTD_updatePrice(optStatePtr, litLength, anchor, offset, mlen-MINMATCH);
             ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             anchor = ip = ip + mlen;
     }    }   /* for (cur=0; cur < last_pos; ) */

From fc41a8796493063c26103ca7ed32561dd2e649d0 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 17 Jul 2017 18:13:09 -0700
Subject: [PATCH 067/100] Experiment with using a lag when hashing

---
 contrib/long_distance_matching/Makefile       |  6 +-
 contrib/long_distance_matching/basic_table.c  |  6 +-
 .../circular_buffer_table.c                   | 20 +++---
 contrib/long_distance_matching/ldm.c          | 71 ++++++++++++-------
 contrib/long_distance_matching/ldm.h          |  4 +-
 .../long_distance_matching/ldm_hashtable.h    | 32 +++++++--
 contrib/long_distance_matching/main-ldm.c     |  1 -
 7 files changed, 88 insertions(+), 52 deletions(-)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 47085022..df439015 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,7 +25,7 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-basic main-circular-buffer
+all: main-basic main-circular-buffer main-lag
 
 main-basic : basic_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -33,9 +33,11 @@ main-basic : basic_table.c ldm.c main-ldm.c
 main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+main-lag: lag_table.c ldm.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-basic main-circular-buffer
+	main-basic main-circular-buffer main-lag
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
index 859bf061..893a4caf 100644
--- a/contrib/long_distance_matching/basic_table.c
+++ b/contrib/long_distance_matching/basic_table.c
@@ -27,7 +27,6 @@ LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + hash;
 }
 
-
 LDM_hashEntry *HASH_getEntryFromHash(
     const LDM_hashTable *table, const hash_t hash, const U32 checksum) {
   (void)checksum;
@@ -43,13 +42,10 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
   (void)checksum;
   if ((*isValid)(pIn, entry->offset + table->offsetBase)) {
     return entry;
-  } else {
-    return NULL;
   }
+  return NULL;
 }
 
-
-
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
   *getBucket(table, hash) = entry;
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index f45f945c..b578d2bf 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -9,7 +9,7 @@
 //      refactor code to scale the number of elements appropriately.
 
 // Number of elements per hash bucket.
-#define HASH_BUCKET_SIZE_LOG 1  // MAX is 4 for now
+#define HASH_BUCKET_SIZE_LOG 0 // MAX is 4 for now
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 
 struct LDM_hashTable {
@@ -19,6 +19,7 @@ struct LDM_hashTable {
   // Position corresponding to offset=0 in LDM_hashEntry.
   const BYTE *offsetBase;
   BYTE *bucketOffsets;     // Pointer to current insert position.
+
                            // Last insert was at bucketOffsets - 1?
 };
 
@@ -35,15 +36,6 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
 }
 
-/*
-static LDM_hashEntry *getLastInsertFromHash(const LDM_hashTable *table,
-                                            const hash_t hash) {
-  LDM_hashEntry *bucket = getBucket(table, hash);
-  BYTE offset = (table->bucketOffsets[hash] - 1) & (HASH_BUCKET_SIZE - 1);
-  return bucket + offset;
-}
-*/
-
 LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const hash_t hash,
                                   const U32 checksum,
@@ -53,7 +45,12 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
   LDM_hashEntry *cur = bucket;
   // TODO: in order of recency?
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    // CHeck checksum for faster check.
+    /*
+    if (cur->checksum == 0 && cur->offset == 0) {
+      return NULL;
+    }
+    */
+    // Check checksum for faster check.
     if (cur->checksum == checksum &&
         (*isValid)(pIn, cur->offset + table->offsetBase)) {
       return cur;
@@ -62,7 +59,6 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
   return NULL;
 }
 
-
 LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
                                      const hash_t hash,
                                      const U32 checksum) {
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index bf54842f..dedbf79a 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -5,7 +5,7 @@
 #include <string.h>
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY 31
+#define HASH_ONLY_EVERY 15
 
 #define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
@@ -18,6 +18,10 @@
 
 #define COMPUTE_STATS
 #define CHECKSUM_CHAR_OFFSET 10
+
+#define LAG 0
+
+//#define HASH_CHECK
 //#define RUN_CHECKS
 //#define LDM_DEBUG
 
@@ -79,6 +83,10 @@ struct LDM_CCtx {
 
   unsigned step;                      // ip step, should be 1.
 
+  const BYTE *lagIp;
+  hash_t lagHash;
+  U32 lagSum;
+
   // DEBUG
   const BYTE *DEBUG_setNextHash;
 };
@@ -253,6 +261,17 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->nextPosHashed = cctx->nextIp;
   cctx->nextHash = checksumToHash(cctx->nextSum);
 
+#if LAG
+  if (cctx->ip - cctx->ibase > LAG) {
+//    printf("LAG %zu\n", cctx->ip - cctx->lagIp);
+    cctx->lagSum = updateChecksum(
+      cctx->lagSum, LDM_HASH_LENGTH,
+      cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
+    cctx->lagIp++;
+    cctx->lagHash = checksumToHash(cctx->lagSum);
+  }
+#endif
+
 #ifdef RUN_CHECKS
   check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
 
@@ -270,18 +289,6 @@ static void setNextHash(LDM_CCtx *cctx) {
 
 static void putHashOfCurrentPositionFromHash(
     LDM_CCtx *cctx, hash_t hash, U32 sum) {
-  /*
-#ifdef COMPUTE_STATS
-  if (cctx->stats.numHashInserts < HASH_getSize(cctx->hashTable)) {
-    U32 offset = HASH_getEntryFromHash(cctx->hashTable, hash)->offset;
-    cctx->stats.numHashInserts++;
-    if (offset != 0 && !LDM_isValidMatch(cctx->ip, offset + cctx->ibase)) {
-      cctx->stats.numCollisions++;
-    }
-  }
-#endif
-*/
-
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
@@ -289,8 +296,19 @@ static void putHashOfCurrentPositionFromHash(
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase ,
                                   MEM_read32(cctx->ip) };
                                   */
+#if LAG
+    // TODO: off by 1, but whatever
+    if (cctx->lagIp - cctx->ibase > 0) {
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
+      HASH_insert(cctx->hashTable, cctx->lagHash, entry);
+    } else {
+      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+      HASH_insert(cctx->hashTable, hash, entry);
+    }
+#else
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
     HASH_insert(cctx->hashTable, hash, entry);
+#endif
   }
 
   cctx->lastPosHashed = cctx->ip;
@@ -331,15 +349,6 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   putHashOfCurrentPositionFromHash(cctx, hash, sum);
 }
 
-/**
- * Returns the position of the entry at hashTable[hash].
- */
-/*
-static const BYTE *getPositionOnHash(const LDM_CCtx *cctx, const hash_t hash) {
-  return HASH_getEntryFromHash(cctx->hashTable, hash)->offset + cctx->ibase;
-}
-*/
-
 U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
                          const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
@@ -431,12 +440,20 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
     if (cctx->ip > cctx->imatchLimit) {
       return 1;
     }
-
+#ifdef HASH_CHECK
+    entry = HASH_getEntryFromHash(cctx->hashTable, h, sum);
+#else
     entry = HASH_getValidEntry(cctx->hashTable, h, sum, cctx->ip,
                                &LDM_isValidMatch);
+#endif
 
     if (entry != NULL) {
       *match = entry->offset + cctx->ibase;
+#ifdef HASH_CHECK
+      if (!LDM_isValidMatch(cctx->ip, *match)) {
+        entry = NULL;
+      }
+#endif
     }
     putHashOfCurrentPositionFromHash(cctx, h, sum);
   }
@@ -508,6 +525,12 @@ size_t LDM_compress(const void *src, size_t srcSize,
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
 
+#if LAG
+  cctx.lagIp = cctx.ip;
+  cctx.lagHash = cctx.lastHash;
+  cctx.lagSum = cctx.lastSum;
+#endif
+
   /**
    * Find a match.
    * If no more matches can be found (i.e. the length of the remaining input
@@ -575,7 +598,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
   /* Encode the last literals (no more matches). */
   {
-    const size_t lastRun = cctx.iend - cctx.anchor;
+    const U32 lastRun = cctx.iend - cctx.anchor;
     BYTE *pToken = cctx.op++;
     LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
   }
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 6d97bd56..6d7c4af2 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -10,8 +10,8 @@
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
 #define LDM_OFFSET_SIZE 4
 
-// Defines the size of the hash table.
-#define LDM_MEMORY_USAGE 20
+// Defines the size of the hash table (currently the number of elements).
+#define LDM_MEMORY_USAGE 12
 
 #define LDM_WINDOW_SIZE_LOG 30
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 88d19ae2..83a9ed27 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -3,34 +3,54 @@
 
 #include "mem.h"
 
+// TODO: clean up comments
+
 typedef U32 hash_t;
 
 typedef struct LDM_hashEntry {
-  U32 offset;
+  U32 offset;   // TODO: Replace with pointer?
   U32 checksum;
 } LDM_hashEntry;
 
 typedef struct LDM_hashTable LDM_hashTable;
 
-// TODO: rename functions
-// TODO: comments
-
+/**
+ * Create a hash table with size hash buckets.
+ * LDM_hashEntry.offset is added to offsetBase to calculate pMatch in
+ * HASH_getValidEntry.
+ */
 LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase);
 
-//TODO: unneeded?
+/**
+ * Returns an LDM_hashEntry from the table that matches the checksum.
+ * Returns NULL if one does not exist.
+ */
 LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
                                      const hash_t hash,
                                      const U32 checksum);
 
+/**
+ *  Gets a valid entry that matches the checksum. A valid entry is defined by
+ *  *isValid.
+ *
+ *  The function finds an entry matching the checksum, computes pMatch as
+ *  offset + table.offsetBase, and calls isValid.
+ */
 LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const hash_t hash,
                                   const U32 checksum,
                                   const BYTE *pIn,
                                   int (*isValid)(const BYTE *pIn, const BYTE *pMatch));
 
+/**
+ * Insert an LDM_hashEntry into the bucket corresponding to hash.
+ */
 void HASH_insert(LDM_hashTable *table, const hash_t hash,
-                        const LDM_hashEntry entry);
+                 const LDM_hashEntry entry);
 
+/**
+ * Return the number of distinct hash buckets.
+ */
 U32 HASH_getSize(const LDM_hashTable *table);
 
 void HASH_destroyTable(LDM_hashTable *table);
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index ea6375ba..a379d3a6 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -163,7 +163,6 @@ static int decompress(const char *fname, const char *oname) {
   outSize = LDM_decompress(
       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
       dst, decompressedSize);
-
   printf("Ret size out: %zu\n", outSize);
   ftruncate(fdout, outSize);
 

From cc1522351f37e4a5b7a69ed05388003cd5c28ff9 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 18 Jul 2017 11:21:19 -0700
Subject: [PATCH 068/100] [libzstd] Fix bug in Huffman encoding

Summary:
Huffman encoding with a bad dictionary can encode worse than the
HUF_BLOCKBOUND(srcSize), since we don't filter out incompressible
input, and even if we did, the dictionaries Huffman table could be
ill suited to compressing actual data.

The fast optimization doesn't seem to improve compression speed,
even when I hard coded fast = 1, the speed didn't improve over hard coding
it to 0.

Benchmarks:
$ ./zstd.dev -b1e5
Benchmarking levels from 1 to 5
 1#Synthetic 50%     :  10000000 ->   3139163 (3.186), 524.8 MB/s ,1890.0 MB/s
 2#Synthetic 50%     :  10000000 ->   3115138 (3.210), 372.6 MB/s ,1830.2 MB/s
 3#Synthetic 50%     :  10000000 ->   3222672 (3.103), 223.3 MB/s ,1400.2 MB/s
 4#Synthetic 50%     :  10000000 ->   3276678 (3.052), 198.0 MB/s ,1280.1 MB/s
 5#Synthetic 50%     :  10000000 ->   3271570 (3.057), 107.8 MB/s ,1200.0 MB/s
$ ./zstd -b1e5
Benchmarking levels from 1 to 5
 1#Synthetic 50%     :  10000000 ->   3139163 (3.186), 524.8 MB/s ,1870.2 MB/s
 2#Synthetic 50%     :  10000000 ->   3115138 (3.210), 370.0 MB/s ,1810.3 MB/s
 3#Synthetic 50%     :  10000000 ->   3222672 (3.103), 223.3 MB/s ,1380.1 MB/s
 4#Synthetic 50%     :  10000000 ->   3276678 (3.052), 196.1 MB/s ,1270.0 MB/s
 5#Synthetic 50%     :  10000000 ->   3271570 (3.057), 106.8 MB/s ,1180.1 MB/s
$ ./zstd.dev -b1e5 ../silesia.tar
Benchmarking levels from 1 to 5
 1#silesia.tar       : 211988480 ->  73651685 (2.878), 429.7 MB/s ,1096.5 MB/s
 2#silesia.tar       : 211988480 ->  70158785 (3.022), 321.2 MB/s ,1029.1 MB/s
 3#silesia.tar       : 211988480 ->  66993813 (3.164), 243.7 MB/s , 981.4 MB/s
 4#silesia.tar       : 211988480 ->  66306481 (3.197), 226.7 MB/s , 972.4 MB/s
 5#silesia.tar       : 211988480 ->  64757852 (3.274), 150.3 MB/s , 963.6 MB/s
$ ./zstd -b1e5 ../silesia.tar
Benchmarking levels from 1 to 5
 1#silesia.tar       : 211988480 ->  73651685 (2.878), 429.7 MB/s ,1087.1 MB/s
 2#silesia.tar       : 211988480 ->  70158785 (3.022), 318.8 MB/s ,1029.1 MB/s
 3#silesia.tar       : 211988480 ->  66993813 (3.164), 246.5 MB/s , 981.4 MB/s
 4#silesia.tar       : 211988480 ->  66306481 (3.197), 229.2 MB/s , 972.4 MB/s
 5#silesia.tar       : 211988480 ->  64757852 (3.274), 149.3 MB/s , 963.6 MB/s

Test Plan:
I added a test case to the fuzzer which crashed with ASAN before the patch
and succeeded after.
---
 lib/compress/huf_compress.c |  3 +--
 tests/fuzzer.c              | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 7af0789a..beb4fdb6 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -436,7 +436,7 @@ static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt*
 
 size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
 
-#define HUF_FLUSHBITS(s)  (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
+#define HUF_FLUSHBITS(s)  BIT_flushBits(s)
 
 #define HUF_FLUSHBITS_1(stream) \
     if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
@@ -451,7 +451,6 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
     BYTE* const oend = ostart + dstSize;
     BYTE* op = ostart;
     size_t n;
-    const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
     BIT_CStream_t bitC;
 
     /* init */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 904ce6fd..3099f346 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1003,6 +1003,42 @@ static int basicUnitTests(U32 seed, double compressibility)
       if (r != _3BYTESTESTLENGTH) goto _output_error; }
     DISPLAYLEVEL(4, "OK \n");
 
+    DISPLAYLEVEL(4, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
+    RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
+    {   /* Train a dictionary on low characters */
+        size_t dictSize = 16 KB;
+        void* const dictBuffer = malloc(dictSize);
+        size_t const totalSampleSize = 1 MB;
+        size_t const sampleUnitSize = 8 KB;
+        U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
+        size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
+        if (!dictBuffer || !samplesSizes) goto _output_error;
+        { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
+        dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
+        if (ZDICT_isError(dictSize)) goto _output_error;
+        /* Reverse the characters to make the dictionary ill suited */
+        {   U32 u;
+            for (u = 0; u < CNBuffSize; ++u) {
+              ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
+            }
+        }
+        {   /* Compress the data */
+            size_t const inputSize = 500;
+            size_t const outputSize = ZSTD_compressBound(inputSize);
+            void* const outputBuffer = malloc(outputSize);
+            ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+            if (!outputBuffer || !cctx) goto _output_error;
+            CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
+            free(outputBuffer);
+            ZSTD_freeCCtx(cctx);
+        }
+
+        free(dictBuffer);
+        free(samplesSizes);
+    }
+    DISPLAYLEVEL(4, "OK \n");
+
+
     /* findFrameCompressedSize on skippable frames */
     DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++);
     {   const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";

From 19258f51c1d4e9b0e10ae0488e457887b4c383cb Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Tue, 18 Jul 2017 14:25:39 -0700
Subject: [PATCH 069/100] Make the meaning of LDM_MEMORY_USAGE consistent
 across tables

---
 contrib/long_distance_matching/Makefile       |  7 +--
 contrib/long_distance_matching/basic_table.c  |  7 +++
 .../circular_buffer_table.c                   | 34 +++++++------
 contrib/long_distance_matching/ldm.c          | 50 ++++++++++++-------
 contrib/long_distance_matching/ldm.h          | 21 +++++---
 .../long_distance_matching/ldm_hashtable.h    |  3 +-
 contrib/long_distance_matching/main-ldm.c     | 25 +++++++---
 7 files changed, 94 insertions(+), 53 deletions(-)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index df439015..131638fd 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,7 +25,7 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-basic main-circular-buffer main-lag
+all: main-basic main-circular-buffer 
 
 main-basic : basic_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -33,11 +33,8 @@ main-basic : basic_table.c ldm.c main-ldm.c
 main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-lag: lag_table.c ldm.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-basic main-circular-buffer main-lag
+	main-basic main-circular-buffer 
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
index 893a4caf..8b3588e8 100644
--- a/contrib/long_distance_matching/basic_table.c
+++ b/contrib/long_distance_matching/basic_table.c
@@ -1,9 +1,12 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+#include "ldm.h"
 #include "ldm_hashtable.h"
 #include "mem.h"
 
+#define LDM_HASHLOG ((LDM_MEMORY_USAGE) - 4)
+
 struct LDM_hashTable {
   U32 size;
   LDM_hashEntry *entries;
@@ -46,6 +49,10 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
   return NULL;
 }
 
+hash_t HASH_hashU32(U32 value) {
+  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
+}
+
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
   *getBucket(table, hash) = entry;
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index b578d2bf..bc7503f1 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -1,33 +1,36 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+#include "ldm.h"
 #include "ldm_hashtable.h"
 #include "mem.h"
 
 //TODO: move def somewhere else.
-//TODO: memory usage is currently no longer LDM_MEMORY_USAGE.
-//      refactor code to scale the number of elements appropriately.
 
 // Number of elements per hash bucket.
+// HASH_BUCKET_SIZE_LOG defined in ldm.h
 #define HASH_BUCKET_SIZE_LOG 0 // MAX is 4 for now
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 
+#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
+
 struct LDM_hashTable {
-  U32 size;
+  U32 size;  // Number of buckets
+  U32 maxEntries;  // Rename...
   LDM_hashEntry *entries;  // 1-D array for now.
 
   // Position corresponding to offset=0 in LDM_hashEntry.
   const BYTE *offsetBase;
   BYTE *bucketOffsets;     // Pointer to current insert position.
-
                            // Last insert was at bucketOffsets - 1?
 };
 
 LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase) {
   LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
-  table->size = size;
-  table->entries = calloc(size * HASH_BUCKET_SIZE, sizeof(LDM_hashEntry));
-  table->bucketOffsets = calloc(size, sizeof(BYTE));
+  table->size = size >> HASH_BUCKET_SIZE_LOG;
+  table->maxEntries = size;
+  table->entries = calloc(size, sizeof(LDM_hashEntry));
+  table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
   table->offsetBase = offsetBase;
   return table;
 }
@@ -45,11 +48,6 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
   LDM_hashEntry *cur = bucket;
   // TODO: in order of recency?
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    /*
-    if (cur->checksum == 0 && cur->offset == 0) {
-      return NULL;
-    }
-    */
     // Check checksum for faster check.
     if (cur->checksum == checksum &&
         (*isValid)(pIn, cur->offset + table->offsetBase)) {
@@ -59,6 +57,11 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
   return NULL;
 }
 
+hash_t HASH_hashU32(U32 value) {
+  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
+}
+
+
 LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
                                      const hash_t hash,
                                      const U32 checksum) {
@@ -82,7 +85,7 @@ void HASH_insert(LDM_hashTable *table,
 }
 
 U32 HASH_getSize(const LDM_hashTable *table) {
-  return table->size * HASH_BUCKET_SIZE;
+  return table->size;
 }
 
 void HASH_destroyTable(LDM_hashTable *table) {
@@ -101,7 +104,8 @@ void HASH_outputTableOccupancy(const LDM_hashTable *table) {
     }
   }
 
+  printf("Num buckets, bucket size: %d, %d\n", table->size, HASH_BUCKET_SIZE);
   printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         HASH_getSize(table), ctr,
-         100.0 * (double)(ctr) / (double)HASH_getSize(table));
+         table->maxEntries, ctr,
+         100.0 * (double)(ctr) / table->maxEntries);
 }
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index dedbf79a..4d8ca40b 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -4,12 +4,16 @@
 #include <stdlib.h>
 #include <string.h>
 
-// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY 15
 
-#define LDM_HASHLOG (LDM_MEMORY_USAGE-2)
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+//#define LDM_HASH_ENTRY_SIZE 4
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 4)
+
+// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - 4))
+#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
+
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
@@ -17,13 +21,13 @@
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
 #define COMPUTE_STATS
+#define OUTPUT_CONFIGURATION
 #define CHECKSUM_CHAR_OFFSET 10
 
-#define LAG 0
+//#define LDM_LAG 0
 
 //#define HASH_CHECK
 //#define RUN_CHECKS
-//#define LDM_DEBUG
 
 #include "ldm.h"
 #include "ldm_hashtable.h"
@@ -187,7 +191,8 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
  * of the hash table.
  */
 static hash_t checksumToHash(U32 sum) {
-  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
+  return HASH_hashU32(sum);
+//  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
 }
 
 /**
@@ -261,9 +266,9 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->nextPosHashed = cctx->nextIp;
   cctx->nextHash = checksumToHash(cctx->nextSum);
 
-#if LAG
-  if (cctx->ip - cctx->ibase > LAG) {
-//    printf("LAG %zu\n", cctx->ip - cctx->lagIp);
+#if LDM_LAG
+//  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
+  if (cctx->ip - cctx->ibase > LDM_LAG) {
     cctx->lagSum = updateChecksum(
       cctx->lagSum, LDM_HASH_LENGTH,
       cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
@@ -296,7 +301,7 @@ static void putHashOfCurrentPositionFromHash(
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase ,
                                   MEM_read32(cctx->ip) };
                                   */
-#if LAG
+#if LDM_LAG
     // TODO: off by 1, but whatever
     if (cctx->lagIp - cctx->ibase > 0) {
       const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
@@ -364,6 +369,18 @@ U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
   return (U32)(pIn - pStart);
 }
 
+void LDM_outputConfiguration(void) {
+  printf("=====================\n");
+  printf("Configuration\n");
+  printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("Min match, hash length: %d, %d\n",
+         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
+  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
+  printf("HASH_ONLY_EVERY: %d\n", HASH_ONLY_EVERY);
+  printf("LDM_LAG %d\n", LDM_LAG);
+  printf("=====================\n");
+}
+
 void LDM_readHeader(const void *src, U64 *compressedSize,
                     U64 *decompressedSize) {
   const BYTE *ip = (const BYTE *)src;
@@ -392,12 +409,8 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U32, cctx->ibase);
+  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64, cctx->ibase);
 
-  //HASH_initializeTable(cctx->hashTable, LDM_HASHTABLESIZE_U32);
-
-//    calloc(LDM_HASHTABLESIZE_U32, sizeof(LDM_hashEntry));
-//  memset(cctx->hashTable, 0, sizeof(cctx->hashTable));
   cctx->stats.minOffset = UINT_MAX;
   cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
   cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
@@ -520,17 +533,19 @@ size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
   const BYTE *match = NULL;
+//  printf("TST: %d\n", LDM_WINDOW_SIZE / LDM_HASHTABLESIZE_U64);
+  printf("HASH LOG: %d\n", HASH_ONLY_EVERY_LOG);
+
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
 
-#if LAG
+#if LDM_LAG
   cctx.lagIp = cctx.ip;
   cctx.lagHash = cctx.lastHash;
   cctx.lagSum = cctx.lastSum;
 #endif
-
   /**
    * Find a match.
    * If no more matches can be found (i.e. the length of the remaining input
@@ -542,6 +557,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
     cctx.stats.numMatches++;
 #endif
 
+//    printf("HERE %zu\n", cctx.ip - cctx.ibase);
     /**
      * Catch up: look back to extend the match backwards from the found match.
      */
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 6d7c4af2..2d4ff9cf 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -10,15 +10,20 @@
 #define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
 #define LDM_OFFSET_SIZE 4
 
-// Defines the size of the hash table (currently the number of elements).
-#define LDM_MEMORY_USAGE 12
+// Defines the size of the hash table.
+// Currently this should be less than WINDOW_SIZE_LOG + 4?
+#define LDM_MEMORY_USAGE 24
 
-#define LDM_WINDOW_SIZE_LOG 30
+//#define LDM_LAG (1 << 23)
+//#define LDM_LAG (1 << 20)
+#define LDM_LAG 0
+
+#define LDM_WINDOW_SIZE_LOG 28
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
-//These should be multiples of four.
-#define LDM_MIN_MATCH_LENGTH 64
-#define LDM_HASH_LENGTH 64
+//These should be multiples of four (and perhaps set to the same values?).
+#define LDM_MIN_MATCH_LENGTH 512
+#define LDM_HASH_LENGTH 512
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
@@ -48,7 +53,7 @@ typedef struct LDM_DCtx LDM_DCtx;
  *  The lower four bits of the token encode the match length. With additional
  *  bytes added similarly to the additional literal length bytes after the offset.
  *
- *  The last sequence is incomplete and stops right after the lieterals.
+ *  The last sequence is incomplete and stops right after the literals.
  *
  */
 size_t LDM_compress(const void *src, size_t srcSize,
@@ -142,6 +147,8 @@ void LDM_initializeDCtx(LDM_DCtx *dctx,
 void LDM_readHeader(const void *src, U64 *compressedSize,
                     U64 *decompressedSize);
 
+void LDM_outputConfiguration(void);
+
 void LDM_test(void);
 
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 83a9ed27..4fef6621 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -42,6 +42,8 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const BYTE *pIn,
                                   int (*isValid)(const BYTE *pIn, const BYTE *pMatch));
 
+hash_t HASH_hashU32(U32 value);
+
 /**
  * Insert an LDM_hashEntry into the bucket corresponding to hash.
  */
@@ -61,5 +63,4 @@ void HASH_destroyTable(LDM_hashTable *table);
  */
 void HASH_outputTableOccupancy(const LDM_hashTable *hashTable);
 
-
 #endif /* LDM_HASHTABLE_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index a379d3a6..a43ec000 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -18,7 +18,7 @@
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
  *
- * TODO: This currently seg faults if the compressed size is > the decompress
+ * TODO: This might seg fault if the compressed size is > the decompress
  * size due to the mmapping and output file size allocated to be the input size.
  * The compress function should check before writing or buffer writes.
  */
@@ -28,6 +28,8 @@ static int compress(const char *fname, const char *oname) {
   char *src, *dst;
   size_t maxCompressedSize, compressedSize;
 
+  struct timeval tv1, tv2;
+
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
     perror("Error in file opening");
@@ -46,7 +48,10 @@ static int compress(const char *fname, const char *oname) {
     return 1;
   }
 
-  maxCompressedSize = statbuf.st_size + LDM_HEADER_SIZE;
+  maxCompressedSize = (statbuf.st_size + LDM_HEADER_SIZE);
+  // Handle case where compressed size is > decompressed size.
+  // The compress function should check before writing or buffer writes.
+  maxCompressedSize += statbuf.st_size / 255;
 
  /* Go to the location corresponding to the last byte. */
  /* TODO: fallocate? */
@@ -74,10 +79,12 @@ static int compress(const char *fname, const char *oname) {
       perror("mmap error for output");
       return 1;
   }
+  gettimeofday(&tv1, NULL);
 
   compressedSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
                    dst + LDM_HEADER_SIZE, maxCompressedSize);
+  gettimeofday(&tv2, NULL);
 
   // Write compress and decompress size to header
   // TODO: should depend on LDM_DECOMPRESS_SIZE write32
@@ -96,6 +103,14 @@ static int compress(const char *fname, const char *oname) {
          (unsigned)statbuf.st_size, (unsigned)compressedSize, oname,
          (double)compressedSize / (statbuf.st_size) * 100);
 
+  printf("Total compress time = %.3f seconds, Average compression speed: %.3f MB/s\n",
+         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+         (double) (tv2.tv_sec - tv1.tv_sec),
+         ((double)statbuf.st_size / (double) (1 << 20)) /
+            ((double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+            (double) (tv2.tv_sec - tv1.tv_sec)));
+
+
   // Close files.
   close(fdin);
   close(fdout);
@@ -234,16 +249,10 @@ int main(int argc, const char *argv[]) {
 
   /* Compress */
   {
-    struct timeval tv1, tv2;
-    gettimeofday(&tv1, NULL);
     if (compress(inpFilename, ldmFilename)) {
         printf("Compress error");
         return 1;
     }
-    gettimeofday(&tv2, NULL);
-    printf("Total compress time = %f seconds\n",
-           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-           (double) (tv2.tv_sec - tv1.tv_sec));
   }
 
   /* Decompress */

From d0b27483ae0eb3d8ab11df998e6e0901f297778b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 18 Jul 2017 14:45:49 -0700
Subject: [PATCH 070/100] [zstdcli] Fix -t in streaming mode

---
 programs/zstdcli.c | 5 ++++-
 tests/playTests.sh | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 35772e0a..b1268c1f 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -719,6 +719,10 @@ int main(int argCount, const char* argv[])
         goto _end;
     }
 
+#ifndef ZSTD_NODECOMPRESS
+    if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
+#endif
+
     /* No input filename ==> use stdin and stdout */
     filenameIdx += !filenameIdx;   /* filenameTable[0] is stdin by default */
     if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark;   /* when input is stdin, default output is stdout */
@@ -763,7 +767,6 @@ int main(int argCount, const char* argv[])
 #endif
     } else {  /* decompression or test */
 #ifndef ZSTD_NODECOMPRESS
-        if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
         FIO_setMemLimit(memLimit);
         if (filenameIdx==1 && outFileName)
             operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index dd0f2dbf..77853b1a 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -383,6 +383,7 @@ $ZSTD -t --rm tmp1.zst
 test -f tmp1.zst   # check file is still present
 split -b16384 tmp1.zst tmpSplit.
 $ZSTD -t tmpSplit.* && die "bad file not detected !"
+./datagen | $ZSTD -c | $ZSTD -t
 
 
 $ECHO "\n**** benchmark mode tests **** "

From 1fa223859fb7f53d5a03c8b30dfd74d40898b05f Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Tue, 18 Jul 2017 18:05:10 -0700
Subject: [PATCH 071/100] Switch to using ZSTD_count instead of function
 pointer

---
 contrib/long_distance_matching/basic_table.c  |  28 +++-
 .../circular_buffer_table.c                   | 151 +++++++++++++++++-
 contrib/long_distance_matching/ldm.c          |  22 +--
 contrib/long_distance_matching/ldm.h          |   9 +-
 .../long_distance_matching/ldm_hashtable.h    |   4 +-
 5 files changed, 194 insertions(+), 20 deletions(-)

diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
index 8b3588e8..6c12b508 100644
--- a/contrib/long_distance_matching/basic_table.c
+++ b/contrib/long_distance_matching/basic_table.c
@@ -36,14 +36,38 @@ LDM_hashEntry *HASH_getEntryFromHash(
   return getBucket(table, hash);
 }
 
+static int isValidMatch(const BYTE *pIn, const BYTE *pMatch,
+                            U32 minMatchLength, U32 maxWindowSize) {
+  U32 lengthLeft = minMatchLength;
+  const BYTE *curIn = pIn;
+  const BYTE *curMatch = pMatch;
+
+  if (pIn - pMatch > maxWindowSize) {
+    return 0;
+  }
+
+  for (; lengthLeft >= 4; lengthLeft -= 4) {
+    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
+      return 0;
+    }
+    curIn += 4;
+    curMatch += 4;
+  }
+  return 1;
+}
+
 LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const hash_t hash,
                                   const U32 checksum,
                                   const BYTE *pIn,
-                                  int (*isValid)(const BYTE *pIn, const BYTE *pMatch)) {
+                                  const BYTE *pEnd,
+                                  U32 minMatchLength,
+                                  U32 maxWindowSize) {
   LDM_hashEntry *entry = getBucket(table, hash);
   (void)checksum;
-  if ((*isValid)(pIn, entry->offset + table->offsetBase)) {
+  (void)pEnd;
+  if (isValidMatch(pIn, entry->offset + table->offsetBase,
+                   minMatchLength, maxWindowSize)) {
     return entry;
   }
   return NULL;
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index bc7503f1..653d9e51 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -9,11 +9,14 @@
 
 // Number of elements per hash bucket.
 // HASH_BUCKET_SIZE_LOG defined in ldm.h
-#define HASH_BUCKET_SIZE_LOG 0 // MAX is 4 for now
+#define HASH_BUCKET_SIZE_LOG 2 // MAX is 4 for now
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 
+// TODO: rename. Number of hash buckets.
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
 
+//#define TMP_ZSTDTOGGLE
+
 struct LDM_hashTable {
   U32 size;  // Number of buckets
   U32 maxEntries;  // Rename...
@@ -39,20 +42,162 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
 }
 
+#ifdef TMP_ZSTDTOGGLE
+static unsigned ZSTD_NbCommonBytes (register size_t val)
+{
+    if (MEM_isLittleEndian()) {
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanForward64( &r, (U64)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctzll((U64)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+                                                     0, 3, 1, 3, 1, 4, 2, 7,
+                                                     0, 2, 3, 6, 1, 5, 3, 5,
+                                                     1, 3, 4, 4, 2, 5, 6, 7,
+                                                     7, 0, 1, 2, 3, 3, 4, 6,
+                                                     2, 6, 5, 5, 3, 4, 5, 6,
+                                                     7, 1, 2, 4, 6, 4, 4, 5,
+                                                     7, 2, 6, 5, 7, 6, 7, 7 };
+            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+#       endif
+        } else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r=0;
+            _BitScanForward( &r, (U32)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctz((U32)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+                                                     3, 2, 2, 1, 3, 2, 0, 1,
+                                                     3, 3, 1, 2, 2, 2, 2, 0,
+                                                     3, 1, 2, 0, 1, 0, 1, 1 };
+            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+#       endif
+        }
+    } else {  /* Big Endian CPU */
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanReverse64( &r, val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clzll(val) >> 3);
+#       else
+            unsigned r;
+            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
+            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+        } else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r = 0;
+            _BitScanReverse( &r, (unsigned long)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clz((U32)val) >> 3);
+#       else
+            unsigned r;
+            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+    }   }
+}
+
+// From lib/compress/zstd_compress.c
+static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *const pInLimit) {
+    const BYTE * const pStart = pIn;
+    const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
+
+    while (pIn < pInLoopLimit) {
+        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+        if (!diff) {
+          pIn += sizeof(size_t);
+          pMatch += sizeof(size_t);
+          continue;
+        }
+        pIn += ZSTD_NbCommonBytes(diff);
+        return (size_t)(pIn - pStart);
+    }
+
+    if (MEM_64bits()) {
+      if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
+        pIn += 4;
+        pMatch += 4;
+      }
+    }
+    if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
+      pIn += 2;
+      pMatch += 2;
+    }
+    if ((pIn < pInLimit) && (*pMatch == *pIn)) {
+      pIn++;
+    }
+    return (size_t)(pIn - pStart);
+}
+
+#else
+
+static int isValidMatch(const BYTE *pIn, const BYTE *pMatch,
+                        U32 minMatchLength, U32 maxWindowSize) {
+  U32 lengthLeft = minMatchLength;
+  const BYTE *curIn = pIn;
+  const BYTE *curMatch = pMatch;
+
+  if (pIn - pMatch > maxWindowSize) {
+    return 0;
+  }
+
+  for (; lengthLeft >= 4; lengthLeft -= 4) {
+    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
+      return 0;
+    }
+    curIn += 4;
+    curMatch += 4;
+  }
+  return 1;
+}
+
+#endif // TMP_ZSTDTOGGLE
+
 LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const hash_t hash,
                                   const U32 checksum,
                                   const BYTE *pIn,
-                                  int (*isValid)(const BYTE *pIn, const BYTE *pMatch)) {
+                                  const BYTE *pEnd,
+                                  U32 minMatchLength,
+                                  U32 maxWindowSize) {
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
   // TODO: in order of recency?
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
     // Check checksum for faster check.
+    const BYTE *pMatch = cur->offset + table->offsetBase;
+#ifdef TMP_ZSTDTOGGLE
+    if (cur->checksum == checksum && pIn - pMatch <= maxWindowSize) {
+      U32 matchLength = ZSTD_count(pIn, pMatch, pEnd);
+      if (matchLength >= minMatchLength) {
+        return cur;
+      }
+    }
+#else
+    (void)pEnd;
+    (void)minMatchLength;
+    (void)maxWindowSize;
+
     if (cur->checksum == checksum &&
-        (*isValid)(pIn, cur->offset + table->offsetBase)) {
+        isValidMatch(pIn, pMatch, minMatchLength, maxWindowSize)) {
       return cur;
     }
+#endif
   }
   return NULL;
 }
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 4d8ca40b..56b22d28 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -91,6 +91,7 @@ struct LDM_CCtx {
   hash_t lagHash;
   U32 lagSum;
 
+  U64 numHashInserts;
   // DEBUG
   const BYTE *DEBUG_setNextHash;
 };
@@ -164,7 +165,6 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   }
   printf("\n");
   printf("=====================\n");
-
 }
 
 int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
@@ -376,7 +376,7 @@ void LDM_outputConfiguration(void) {
   printf("Min match, hash length: %d, %d\n",
          LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
   printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
-  printf("HASH_ONLY_EVERY: %d\n", HASH_ONLY_EVERY);
+  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
   printf("LDM_LAG %d\n", LDM_LAG);
   printf("=====================\n");
 }
@@ -456,8 +456,10 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
 #ifdef HASH_CHECK
     entry = HASH_getEntryFromHash(cctx->hashTable, h, sum);
 #else
-    entry = HASH_getValidEntry(cctx->hashTable, h, sum, cctx->ip,
-                               &LDM_isValidMatch);
+    entry = HASH_getValidEntry(cctx->hashTable, h, sum,
+                               cctx->ip, cctx->iend,
+                               LDM_MIN_MATCH_LENGTH,
+                               LDM_WINDOW_SIZE);
 #endif
 
     if (entry != NULL) {
@@ -534,9 +536,10 @@ size_t LDM_compress(const void *src, size_t srcSize,
   LDM_CCtx cctx;
   const BYTE *match = NULL;
 //  printf("TST: %d\n", LDM_WINDOW_SIZE / LDM_HASHTABLESIZE_U64);
-  printf("HASH LOG: %d\n", HASH_ONLY_EVERY_LOG);
+//  printf("HASH LOG: %d\n", HASH_ONLY_EVERY_LOG);
 
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+  LDM_outputConfiguration();
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
@@ -553,11 +556,10 @@ size_t LDM_compress(const void *src, size_t srcSize,
    * and encode the final literals.
    */
   while (LDM_findBestMatch(&cctx, &match) == 0) {
+    U32 backwardsMatchLen = 0;
 #ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
 #endif
-
-//    printf("HERE %zu\n", cctx.ip - cctx.ibase);
     /**
      * Catch up: look back to extend the match backwards from the found match.
      */
@@ -565,6 +567,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
            cctx.ip[-1] == match[-1]) {
       cctx.ip--;
       match--;
+      backwardsMatchLen++;
     }
 
     /**
@@ -575,8 +578,9 @@ size_t LDM_compress(const void *src, size_t srcSize,
       const U32 literalLength = cctx.ip - cctx.anchor;
       const U32 offset = cctx.ip - match;
       const U32 matchLength = LDM_countMatchLength(
-          cctx.ip + LDM_MIN_MATCH_LENGTH, match + LDM_MIN_MATCH_LENGTH,
-          cctx.ihashLimit);
+          cctx.ip + LDM_MIN_MATCH_LENGTH + backwardsMatchLen,
+          match + LDM_MIN_MATCH_LENGTH + backwardsMatchLen,
+          cctx.ihashLimit) + backwardsMatchLen;
 
       LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 2d4ff9cf..735435e8 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -12,18 +12,17 @@
 
 // Defines the size of the hash table.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
-#define LDM_MEMORY_USAGE 24
+#define LDM_MEMORY_USAGE 23
 
-//#define LDM_LAG (1 << 23)
 //#define LDM_LAG (1 << 20)
-#define LDM_LAG 0
+#define LDM_LAG (0)
 
 #define LDM_WINDOW_SIZE_LOG 28
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four (and perhaps set to the same values?).
-#define LDM_MIN_MATCH_LENGTH 512
-#define LDM_HASH_LENGTH 512
+#define LDM_MIN_MATCH_LENGTH 64
+#define LDM_HASH_LENGTH 64
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 4fef6621..7566751d 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -40,7 +40,9 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const hash_t hash,
                                   const U32 checksum,
                                   const BYTE *pIn,
-                                  int (*isValid)(const BYTE *pIn, const BYTE *pMatch));
+                                  const BYTE *pEnd,
+                                  U32 minMatchLength,
+                                  U32 maxWindowSize);
 
 hash_t HASH_hashU32(U32 value);
 

From 4352e09cb002873f3c2eec5d79eddeefca28160f Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Tue, 18 Jul 2017 18:35:25 -0700
Subject: [PATCH 072/100] Avoid recounting match lengths with ZSTD_count

---
 contrib/long_distance_matching/basic_table.c   |  6 +++++-
 .../circular_buffer_table.c                    | 13 +++++++------
 contrib/long_distance_matching/ldm.c           | 18 +++++++++++++-----
 contrib/long_distance_matching/ldm_hashtable.h |  5 +++--
 4 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
index 6c12b508..30c548d2 100644
--- a/contrib/long_distance_matching/basic_table.c
+++ b/contrib/long_distance_matching/basic_table.c
@@ -62,12 +62,16 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const BYTE *pIn,
                                   const BYTE *pEnd,
                                   U32 minMatchLength,
-                                  U32 maxWindowSize) {
+                                  U32 maxWindowSize,
+                                  U32 *matchLength) {
   LDM_hashEntry *entry = getBucket(table, hash);
   (void)checksum;
   (void)pEnd;
+  (void)matchLength;
+  // TODO: Count the entire forward match length rather than check if valid.
   if (isValidMatch(pIn, entry->offset + table->offsetBase,
                    minMatchLength, maxWindowSize)) {
+
     return entry;
   }
   return NULL;
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index 653d9e51..104d1b33 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -15,17 +15,16 @@
 // TODO: rename. Number of hash buckets.
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
 
-//#define TMP_ZSTDTOGGLE
+#define TMP_ZSTDTOGGLE
 
 struct LDM_hashTable {
   U32 size;  // Number of buckets
   U32 maxEntries;  // Rename...
   LDM_hashEntry *entries;  // 1-D array for now.
+  BYTE *bucketOffsets;     // Pointer to current insert position.
 
   // Position corresponding to offset=0 in LDM_hashEntry.
   const BYTE *offsetBase;
-  BYTE *bucketOffsets;     // Pointer to current insert position.
-                           // Last insert was at bucketOffsets - 1?
 };
 
 LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase) {
@@ -174,7 +173,8 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const BYTE *pIn,
                                   const BYTE *pEnd,
                                   U32 minMatchLength,
-                                  U32 maxWindowSize) {
+                                  U32 maxWindowSize,
+                                  U32 *matchLength) {
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
   // TODO: in order of recency?
@@ -183,8 +183,9 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
     const BYTE *pMatch = cur->offset + table->offsetBase;
 #ifdef TMP_ZSTDTOGGLE
     if (cur->checksum == checksum && pIn - pMatch <= maxWindowSize) {
-      U32 matchLength = ZSTD_count(pIn, pMatch, pEnd);
-      if (matchLength >= minMatchLength) {
+      U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
+      if (forwardMatchLength >= minMatchLength) {
+        *matchLength = forwardMatchLength;
         return cur;
       }
     }
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 56b22d28..1512ab8c 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -28,6 +28,7 @@
 
 //#define HASH_CHECK
 //#define RUN_CHECKS
+//#define TMP_RECOMPUTE_LENGTHS
 
 #include "ldm.h"
 #include "ldm_hashtable.h"
@@ -435,8 +436,10 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
  * Returns 0 if successful and 1 otherwise (i.e. no match can be found
  * in the remaining input that is long enough).
  *
+ * matchLength contains the forward length of the match.
  */
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
+                             U32 *matchLength) {
 
   LDM_hashEntry *entry = NULL;
   cctx->nextIp = cctx->ip + cctx->step;
@@ -459,7 +462,8 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match) {
     entry = HASH_getValidEntry(cctx->hashTable, h, sum,
                                cctx->ip, cctx->iend,
                                LDM_MIN_MATCH_LENGTH,
-                               LDM_WINDOW_SIZE);
+                               LDM_WINDOW_SIZE,
+                               matchLength);
 #endif
 
     if (entry != NULL) {
@@ -535,8 +539,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
   const BYTE *match = NULL;
-//  printf("TST: %d\n", LDM_WINDOW_SIZE / LDM_HASHTABLESIZE_U64);
-//  printf("HASH LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  U32 forwardMatchLength = 0;
 
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
   LDM_outputConfiguration();
@@ -555,7 +558,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
    * is less than the minimum match length), then stop searching for matches
    * and encode the final literals.
    */
-  while (LDM_findBestMatch(&cctx, &match) == 0) {
+  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength) == 0) {
     U32 backwardsMatchLen = 0;
 #ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
@@ -577,10 +580,15 @@ size_t LDM_compress(const void *src, size_t srcSize,
     {
       const U32 literalLength = cctx.ip - cctx.anchor;
       const U32 offset = cctx.ip - match;
+#ifdef TMP_RECOMPUTE_LENGTHS
       const U32 matchLength = LDM_countMatchLength(
           cctx.ip + LDM_MIN_MATCH_LENGTH + backwardsMatchLen,
           match + LDM_MIN_MATCH_LENGTH + backwardsMatchLen,
           cctx.ihashLimit) + backwardsMatchLen;
+#else
+      const U32 matchLength = forwardMatchLength + backwardsMatchLen -
+                              LDM_MIN_MATCH_LENGTH;
+#endif
 
       LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 7566751d..2ea159f7 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -41,8 +41,9 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const U32 checksum,
                                   const BYTE *pIn,
                                   const BYTE *pEnd,
-                                  U32 minMatchLength,
-                                  U32 maxWindowSize);
+                                  const U32 minMatchLength,
+                                  const U32 maxWindowSize,
+                                  U32 *matchLength);
 
 hash_t HASH_hashU32(U32 value);
 

From b71363b967376f676bcfff842bf2c75bc9e2daec Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 19 Jul 2017 01:05:40 -0700
Subject: [PATCH 073/100] check pthread_*_init() success condition

---
 lib/common/bitstream.h         | 81 +++++++++++++++++-----------------
 lib/common/threading.h         |  6 +--
 lib/compress/zstdmt_compress.c | 20 +++++++--
 lib/dictBuilder/cover.c        |  8 ++--
 4 files changed, 62 insertions(+), 53 deletions(-)

diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index 07b85026..06121f21 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -80,9 +80,9 @@ extern "C" {
 *  bitStream encoding API (write forward)
 ********************************************/
 /* bitStream can mix input from multiple sources.
-*  A critical property of these streams is that they encode and decode in **reverse** direction.
-*  So the first bit sequence you add will be the last to be read, like a LIFO stack.
-*/
+ * A critical property of these streams is that they encode and decode in **reverse** direction.
+ * So the first bit sequence you add will be the last to be read, like a LIFO stack.
+ */
 typedef struct
 {
     size_t bitContainer;
@@ -203,7 +203,7 @@ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F,
 /*! BIT_initCStream() :
  *  `dstCapacity` must be > sizeof(size_t)
  *  @return : 0 if success,
-              otherwise an error code (can be tested using ERR_isError() ) */
+ *            otherwise an error code (can be tested using ERR_isError()) */
 MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
                                   void* startPtr, size_t dstCapacity)
 {
@@ -217,8 +217,8 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
 }
 
 /*! BIT_addBits() :
-    can add up to 26 bits into `bitC`.
-    Does not check for register overflow ! */
+ *  can add up to 26 bits into `bitC`.
+ *  Note : does not check for register overflow ! */
 MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
                             size_t value, unsigned nbBits)
 {
@@ -268,7 +268,7 @@ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
 
 /*! BIT_closeCStream() :
  *  @return : size of CStream, in bytes,
-              or 0 if it could not fit into dstBuffer */
+ *            or 0 if it could not fit into dstBuffer */
 MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
 {
     BIT_addBitsFast(bitC, 1, 1);   /* endMark */
@@ -279,14 +279,14 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
 
 
 /*-********************************************************
-* bitStream decoding
+*  bitStream decoding
 **********************************************************/
 /*! BIT_initDStream() :
-*   Initialize a BIT_DStream_t.
-*   `bitD` : a pointer to an already allocated BIT_DStream_t structure.
-*   `srcSize` must be the *exact* size of the bitStream, in bytes.
-*   @return : size of stream (== srcSize) or an errorCode if a problem is detected
-*/
+ *  Initialize a BIT_DStream_t.
+ * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
+ * `srcSize` must be the *exact* size of the bitStream, in bytes.
+ * @return : size of stream (== srcSize), or an errorCode if a problem is detected
+ */
 MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
 {
     if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
@@ -305,29 +305,30 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         bitD->bitContainer = *(const BYTE*)(bitD->start);
         switch(srcSize)
         {
-	    case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
-	            /* fall-through */
+        case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+                /* fall-through */
 
-	    case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
-	            /* fall-through */
+        case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+                /* fall-through */
 
-	    case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
-	            /* fall-through */
+        case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+                /* fall-through */
 
-	    case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
-	            /* fall-through */
+        case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
+                /* fall-through */
 
-	    case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
-	            /* fall-through */
+        case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
+                /* fall-through */
 
-	    case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
-	            /* fall-through */
+        case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
+                /* fall-through */
 
-            default: break;
+        default: break;
+        }
+        {   BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+            bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+            if (lastByte == 0) return ERROR(corruption_detected);  /* endMark not present */
         }
-        { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
-          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
-          if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
         bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
     }
 
@@ -363,9 +364,8 @@ MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
  *  local register is not modified.
  *  On 32-bits, maxNbBits==24.
  *  On 64-bits, maxNbBits==56.
- *  @return : value extracted
- */
- MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
+ * @return : value extracted */
+MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
 {
 #if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
     return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
@@ -392,8 +392,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
 /*! BIT_readBits() :
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
- *  @return : extracted value.
- */
+ * @return : extracted value. */
 MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 {
     size_t const value = BIT_lookBits(bitD, nbBits);
@@ -402,7 +401,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 }
 
 /*! BIT_readBitsFast() :
-*   unsafe version; only works only if nbBits >= 1 */
+ *  unsafe version; only works only if nbBits >= 1 */
 MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
 {
     size_t const value = BIT_lookBitsFast(bitD, nbBits);
@@ -412,10 +411,10 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
 }
 
 /*! BIT_reloadDStream() :
-*   Refill `bitD` from buffer previously set in BIT_initDStream() .
-*   This function is safe, it guarantees it will not read beyond src buffer.
-*   @return : status of `BIT_DStream_t` internal register.
-              if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
+ *  Refill `bitD` from buffer previously set in BIT_initDStream() .
+ *  This function is safe, it guarantees it will not read beyond src buffer.
+ * @return : status of `BIT_DStream_t` internal register.
+ *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
     if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* overflow detected, like end of stream */
@@ -446,8 +445,8 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 }
 
 /*! BIT_endOfDStream() :
-*   @return Tells if DStream has exactly reached its end (all bits consumed).
-*/
+ * @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
+ */
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
 {
     return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
diff --git a/lib/common/threading.h b/lib/common/threading.h
index c0086139..ec9e07a9 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -80,14 +80,14 @@ int _pthread_join(pthread_t* thread, void** value_ptr);
 #else  /* ZSTD_MULTITHREAD not defined */
 /* No multithreading support */
 
-#define pthread_mutex_t int   /* #define rather than typedef, as sometimes pthread support is implicit, resulting in duplicated symbols */
-#define pthread_mutex_init(a,b)
+#define pthread_mutex_t int   /* #define rather than typedef, because sometimes pthread support is implicit, resulting in duplicated symbols */
+#define pthread_mutex_init(a,b)    ((void)a, 0)
 #define pthread_mutex_destroy(a)
 #define pthread_mutex_lock(a)
 #define pthread_mutex_unlock(a)
 
 #define pthread_cond_t int
-#define pthread_cond_init(a,b)
+#define pthread_cond_init(a,b)     ((void)a, 0)
 #define pthread_cond_destroy(a)
 #define pthread_cond_wait(a,b)
 #define pthread_cond_signal(a)
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index ccc20c78..234ced9d 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -98,7 +98,10 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_custo
     ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
         sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
     if (bufPool==NULL) return NULL;
-    pthread_mutex_init(&bufPool->poolMutex, NULL);
+    if (pthread_mutex_init(&bufPool->poolMutex, NULL)) {
+        ZSTD_free(bufPool, cMem);
+        return NULL;
+    }
     bufPool->bufferSize = 64 KB;
     bufPool->totalBuffers = maxNbBuffers;
     bufPool->nbBuffers = 0;
@@ -213,7 +216,10 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
     ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
         sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
     if (!cctxPool) return NULL;
-    pthread_mutex_init(&cctxPool->poolMutex, NULL);
+    if (pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
+        ZSTD_free(cctxPool, cMem);
+        return NULL;
+    }
     cctxPool->cMem = cMem;
     cctxPool->totalCCtx = nbThreads;
     cctxPool->availCCtx = 1;   /* at least one cctx for single-thread mode */
@@ -429,8 +435,14 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
         ZSTDMT_freeCCtx(mtctx);
         return NULL;
     }
-    pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL);   /* Todo : check init function return */
-    pthread_cond_init(&mtctx->jobCompleted_cond, NULL);
+    if (pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) {
+        ZSTDMT_freeCCtx(mtctx);
+        return NULL;
+    }
+    if (pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) {
+        ZSTDMT_freeCCtx(mtctx);
+        return NULL;
+    }
     DEBUGLOG(3, "mt_cctx created, for %u threads", nbThreads);
     return mtctx;
 }
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 06c1b9fa..38376d08 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -714,11 +714,9 @@ typedef struct COVER_best_s {
  * Initialize the `COVER_best_t`.
  */
 static void COVER_best_init(COVER_best_t *best) {
-  if (!best) {
-    return;
-  }
-  pthread_mutex_init(&best->mutex, NULL);
-  pthread_cond_init(&best->cond, NULL);
+  if (best==NULL) return; /* compatible with init on NULL */
+  (void)pthread_mutex_init(&best->mutex, NULL);
+  (void)pthread_cond_init(&best->cond, NULL);
   best->liveJobs = 0;
   best->dict = NULL;
   best->dictSize = 0;

From 3974d2b38a8069fdc24d09f5a4adcc84971dfaa1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 19 Jul 2017 13:33:21 -0700
Subject: [PATCH 074/100] blind fix for Windows Multithreading module

adds a fake 0 return value for mutex/cond init
---
 lib/common/threading.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/common/threading.h b/lib/common/threading.h
index ec9e07a9..ee786455 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -42,14 +42,14 @@ extern "C" {
 
 /* mutex */
 #define pthread_mutex_t           CRITICAL_SECTION
-#define pthread_mutex_init(a,b)   InitializeCriticalSection((a))
+#define pthread_mutex_init(a,b)   (InitializeCriticalSection((a)), 0)
 #define pthread_mutex_destroy(a)  DeleteCriticalSection((a))
 #define pthread_mutex_lock(a)     EnterCriticalSection((a))
 #define pthread_mutex_unlock(a)   LeaveCriticalSection((a))
 
 /* condition variable */
 #define pthread_cond_t             CONDITION_VARIABLE
-#define pthread_cond_init(a, b)    InitializeConditionVariable((a))
+#define pthread_cond_init(a, b)    (InitializeConditionVariable((a)), 0)
 #define pthread_cond_destroy(a)    /* No delete */
 #define pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
 #define pthread_cond_signal(a)     WakeConditionVariable((a))

From 030264ca51814d3bef8debcedace65f56779f691 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 19 Jul 2017 14:14:26 -0700
Subject: [PATCH 075/100] Experiment with integrating ZSTD_count with
 findBestMatch

---
 contrib/long_distance_matching/Makefile       |  12 +-
 .../circular_buffer_table.c                   | 117 ++-
 contrib/long_distance_matching/ldm.c          |  37 +-
 contrib/long_distance_matching/ldm.h          |   9 +-
 .../long_distance_matching/ldm_hashtable.h    |   9 +-
 .../long_distance_matching/ldm_with_table.c   | 959 ++++++++++++++++++
 6 files changed, 1093 insertions(+), 50 deletions(-)
 create mode 100644 contrib/long_distance_matching/ldm_with_table.c

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 131638fd..3aa3f8bd 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,16 +25,20 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-basic main-circular-buffer 
+all: main-circular-buffer main-integrated
 
-main-basic : basic_table.c ldm.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+#main-basic : basic_table.c ldm.c main-ldm.c
+#	      $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+main-integrated: ldm_with_table.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-basic main-circular-buffer 
+	main-basic main-circular-buffer  main-integrated
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index 104d1b33..9b7ad088 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -14,8 +14,8 @@
 
 // TODO: rename. Number of hash buckets.
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
-
-#define TMP_ZSTDTOGGLE
+#define ZSTD_SKIP
+//#define TMP_TST
 
 struct LDM_hashTable {
   U32 size;  // Number of buckets
@@ -25,15 +25,20 @@ struct LDM_hashTable {
 
   // Position corresponding to offset=0 in LDM_hashEntry.
   const BYTE *offsetBase;
+  U32 minMatchLength;
+  U32 maxWindowSize;
 };
 
-LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase) {
+LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
+                                U32 minMatchLength, U32 maxWindowSize) {
   LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
   table->size = size >> HASH_BUCKET_SIZE_LOG;
   table->maxEntries = size;
   table->entries = calloc(size, sizeof(LDM_hashEntry));
   table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
   table->offsetBase = offsetBase;
+  table->minMatchLength = minMatchLength;
+  table->maxWindowSize = maxWindowSize;
   return table;
 }
 
@@ -41,7 +46,7 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
 }
 
-#ifdef TMP_ZSTDTOGGLE
+#if TMP_ZSTDTOGGLE
 static unsigned ZSTD_NbCommonBytes (register size_t val)
 {
     if (MEM_isLittleEndian()) {
@@ -143,10 +148,85 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
     return (size_t)(pIn - pStart);
 }
 
+U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+                        const BYTE *pMatch, const BYTE *pBase) {
+  U32 matchLength = 0;
+  while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
+    pIn--;
+    pMatch--;
+    matchLength++;
+  }
+  return matchLength;
+}
+
+LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
+                                  const hash_t hash,
+                                  const U32 checksum,
+                                  const BYTE *pIn,
+                                  const BYTE *pEnd,
+                                  U32 *matchLength,
+                                  U32 *backwardsMatchLength,
+                                  const BYTE *pAnchor) {
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  LDM_hashEntry *cur = bucket;
+  LDM_hashEntry *bestEntry = NULL;
+  U32 bestMatchLength = 0;
+  U32 forwardMatch = 0;
+  U32 backwardMatch = 0;
+#ifdef TMP_TST
+  U32 numBetter = 0;
+#endif
+  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
+    // Check checksum for faster check.
+    const BYTE *pMatch = cur->offset + table->offsetBase;
+    if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
+      U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
+      U32 backwardMatchLength, totalMatchLength;
+      if (forwardMatchLength < table->minMatchLength) {
+        continue;
+      }
+      backwardMatchLength =
+          countBackwardsMatch(pIn, pAnchor, cur->offset + table->offsetBase,
+                              table->offsetBase);
+
+      totalMatchLength = forwardMatchLength + backwardMatchLength;
+
+      if (totalMatchLength >= bestMatchLength) {
+        bestMatchLength = totalMatchLength;
+        forwardMatch = forwardMatchLength;
+        backwardMatch = backwardMatchLength;
+        bestEntry = cur;
+#ifdef TMP_TST
+        numBetter++;
+#endif
+
+#ifdef ZSTD_SKIP
+        *matchLength = forwardMatchLength;
+        *backwardsMatchLength = backwardMatchLength;
+
+        return cur;
+#endif
+//        *matchLength = forwardMatchLength;
+//        return cur;
+      }
+    }
+  }
+  if (bestEntry != NULL && bestMatchLength > table->minMatchLength) {
+#ifdef TMP_TST
+    printf("Num better %u\n", numBetter - 1);
+#endif
+    *matchLength = forwardMatch;
+    *backwardsMatchLength = backwardMatch;
+    return bestEntry;
+  }
+  return NULL;
+}
+
 #else
 
 static int isValidMatch(const BYTE *pIn, const BYTE *pMatch,
                         U32 minMatchLength, U32 maxWindowSize) {
+  printf("HERE\n");
   U32 lengthLeft = minMatchLength;
   const BYTE *curIn = pIn;
   const BYTE *curMatch = pMatch;
@@ -165,44 +245,33 @@ static int isValidMatch(const BYTE *pIn, const BYTE *pMatch,
   return 1;
 }
 
-#endif // TMP_ZSTDTOGGLE
-
+//TODO: clean up function call. This is not at all decoupled from LDM.
 LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const hash_t hash,
                                   const U32 checksum,
                                   const BYTE *pIn,
                                   const BYTE *pEnd,
-                                  U32 minMatchLength,
-                                  U32 maxWindowSize,
-                                  U32 *matchLength) {
+                                  U32 *matchLength,
+                                  U32 *backwardsMatchLength,
+                                  const BYTE *pAnchor) {
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
-  // TODO: in order of recency?
-  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
+  (void)matchLength;
+  (void)backwardsMatchLength;
+  (void)pAnchor; for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
     // Check checksum for faster check.
     const BYTE *pMatch = cur->offset + table->offsetBase;
-#ifdef TMP_ZSTDTOGGLE
-    if (cur->checksum == checksum && pIn - pMatch <= maxWindowSize) {
-      U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
-      if (forwardMatchLength >= minMatchLength) {
-        *matchLength = forwardMatchLength;
-        return cur;
-      }
-    }
-#else
     (void)pEnd;
-    (void)minMatchLength;
-    (void)maxWindowSize;
 
     if (cur->checksum == checksum &&
-        isValidMatch(pIn, pMatch, minMatchLength, maxWindowSize)) {
+        isValidMatch(pIn, pMatch, table->minMatchLength, table->maxWindowSize)) {
       return cur;
     }
-#endif
   }
   return NULL;
 }
 
+#endif
 hash_t HASH_hashU32(U32 value) {
   return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
 }
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 1512ab8c..a116af70 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -24,8 +24,6 @@
 #define OUTPUT_CONFIGURATION
 #define CHECKSUM_CHAR_OFFSET 10
 
-//#define LDM_LAG 0
-
 //#define HASH_CHECK
 //#define RUN_CHECKS
 //#define TMP_RECOMPUTE_LENGTHS
@@ -410,7 +408,8 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64, cctx->ibase);
+  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64, cctx->ibase,
+                                     LDM_MIN_MATCH_LENGTH, LDM_WINDOW_SIZE);
 
   cctx->stats.minOffset = UINT_MAX;
   cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
@@ -439,7 +438,7 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
  * matchLength contains the forward length of the match.
  */
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
-                             U32 *matchLength) {
+                             U32 *matchLength, U32 *backwardMatchLength) {
 
   LDM_hashEntry *entry = NULL;
   cctx->nextIp = cctx->ip + cctx->step;
@@ -461,9 +460,8 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
 #else
     entry = HASH_getValidEntry(cctx->hashTable, h, sum,
                                cctx->ip, cctx->iend,
-                               LDM_MIN_MATCH_LENGTH,
-                               LDM_WINDOW_SIZE,
-                               matchLength);
+                               matchLength, backwardMatchLength,
+                               cctx->anchor);
 #endif
 
     if (entry != NULL) {
@@ -540,6 +538,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
   LDM_CCtx cctx;
   const BYTE *match = NULL;
   U32 forwardMatchLength = 0;
+  U32 backwardsMatchLength = 0;
 
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
   LDM_outputConfiguration();
@@ -558,11 +557,14 @@ size_t LDM_compress(const void *src, size_t srcSize,
    * is less than the minimum match length), then stop searching for matches
    * and encode the final literals.
    */
-  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength) == 0) {
-    U32 backwardsMatchLen = 0;
+  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
+         &backwardsMatchLength) == 0) {
 #ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
 #endif
+
+#if TMP_RECOMPUTE_LENGTHS
+    backwardsMatchLength = 0;
     /**
      * Catch up: look back to extend the match backwards from the found match.
      */
@@ -570,8 +572,12 @@ size_t LDM_compress(const void *src, size_t srcSize,
            cctx.ip[-1] == match[-1]) {
       cctx.ip--;
       match--;
-      backwardsMatchLen++;
+      backwardsMatchLength++;
     }
+#else
+     cctx.ip -= backwardsMatchLength;
+     match -= backwardsMatchLength;
+#endif
 
     /**
      * Write current block (literals, literal length, match offset, match
@@ -580,13 +586,14 @@ size_t LDM_compress(const void *src, size_t srcSize,
     {
       const U32 literalLength = cctx.ip - cctx.anchor;
       const U32 offset = cctx.ip - match;
-#ifdef TMP_RECOMPUTE_LENGTHS
+#if TMP_RECOMPUTE_LENGTHS
       const U32 matchLength = LDM_countMatchLength(
-          cctx.ip + LDM_MIN_MATCH_LENGTH + backwardsMatchLen,
-          match + LDM_MIN_MATCH_LENGTH + backwardsMatchLen,
-          cctx.ihashLimit) + backwardsMatchLen;
+          cctx.ip + LDM_MIN_MATCH_LENGTH + backwardsMatchLength,
+          match + LDM_MIN_MATCH_LENGTH + backwardsMatchLength,
+          cctx.ihashLimit) + backwardsMatchLength;
 #else
-      const U32 matchLength = forwardMatchLength + backwardsMatchLen -
+      const U32 matchLength = forwardMatchLength +
+                              backwardsMatchLength -
                               LDM_MIN_MATCH_LENGTH;
 #endif
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 735435e8..2396227d 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -20,9 +20,12 @@
 #define LDM_WINDOW_SIZE_LOG 28
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
-//These should be multiples of four (and perhaps set to the same values?).
-#define LDM_MIN_MATCH_LENGTH 64
-#define LDM_HASH_LENGTH 64
+//These should be multiples of four (and perhaps set to the same value?).
+#define LDM_MIN_MATCH_LENGTH 1024
+#define LDM_HASH_LENGTH 1024
+
+#define TMP_ZSTDTOGGLE 1
+#define TMP_RECOMPUTE_LENGTHS (!(TMP_ZSTDTOGGLE))
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 2ea159f7..51d82525 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -19,7 +19,8 @@ typedef struct LDM_hashTable LDM_hashTable;
  * LDM_hashEntry.offset is added to offsetBase to calculate pMatch in
  * HASH_getValidEntry.
  */
-LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase);
+LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
+                                U32 minMatchLength, U32 maxWindowSize);
 
 /**
  * Returns an LDM_hashEntry from the table that matches the checksum.
@@ -41,9 +42,9 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
                                   const U32 checksum,
                                   const BYTE *pIn,
                                   const BYTE *pEnd,
-                                  const U32 minMatchLength,
-                                  const U32 maxWindowSize,
-                                  U32 *matchLength);
+                                  U32 *matchLength,
+                                  U32 *backwardsMatchLength,
+                                  const BYTE *pAnchor);
 
 hash_t HASH_hashU32(U32 value);
 
diff --git a/contrib/long_distance_matching/ldm_with_table.c b/contrib/long_distance_matching/ldm_with_table.c
new file mode 100644
index 00000000..68a33d0f
--- /dev/null
+++ b/contrib/long_distance_matching/ldm_with_table.c
@@ -0,0 +1,959 @@
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+//#define LDM_HASH_ENTRY_SIZE 4
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 4)
+
+// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - 4))
+#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
+
+/* Hash table stuff. */
+#define HASH_BUCKET_SIZE_LOG 3 // MAX is 4 for now
+#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
+#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+#define COMPUTE_STATS
+#define OUTPUT_CONFIGURATION
+#define CHECKSUM_CHAR_OFFSET 10
+
+// Take first match only.
+#define ZSTD_SKIP
+
+//#define RUN_CHECKS
+
+#include "ldm.h"
+
+/* Hash table stuff */
+
+typedef U32 hash_t;
+
+typedef struct LDM_hashEntry {
+  U32 offset;   // TODO: Replace with pointer?
+  U32 checksum;
+} LDM_hashEntry;
+
+// TODO: Scanning speed
+// TODO: Memory usage
+struct LDM_compressStats {
+  U32 windowSizeLog, hashTableSizeLog;
+  U32 numMatches;
+  U64 totalMatchLength;
+  U64 totalLiteralLength;
+  U64 totalOffset;
+
+  U32 minOffset, maxOffset;
+
+  U32 numCollisions;
+  U32 numHashInserts;
+
+  U32 offsetHistogram[32];
+};
+
+typedef struct LDM_hashTable LDM_hashTable;
+
+struct LDM_CCtx {
+  U64 isize;             /* Input size */
+  U64 maxOSize;          /* Maximum output size */
+
+  const BYTE *ibase;        /* Base of input */
+  const BYTE *ip;           /* Current input position */
+  const BYTE *iend;         /* End of input */
+
+  // Maximum input position such that hashing at the position does not exceed
+  // end of input.
+  const BYTE *ihashLimit;
+
+  // Maximum input position such that finding a match of at least the minimum
+  // match length does not exceed end of input.
+  const BYTE *imatchLimit;
+
+  const BYTE *obase;        /* Base of output */
+  BYTE *op;                 /* Output */
+
+  const BYTE *anchor;       /* Anchor to start of current (match) block */
+
+  LDM_compressStats stats;            /* Compression statistics */
+
+  LDM_hashTable *hashTable;
+
+//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+
+  const BYTE *lastPosHashed;          /* Last position hashed */
+  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
+  U32 lastSum;
+
+  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
+  const BYTE *nextPosHashed;
+  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
+  U32 nextSum;
+
+  unsigned step;                      // ip step, should be 1.
+
+  const BYTE *lagIp;
+  hash_t lagHash;
+  U32 lagSum;
+
+  U64 numHashInserts;
+  // DEBUG
+  const BYTE *DEBUG_setNextHash;
+};
+
+struct LDM_hashTable {
+  U32 size;  // Number of buckets
+  U32 maxEntries;  // Rename...
+  LDM_hashEntry *entries;  // 1-D array for now.
+
+  BYTE *bucketOffsets;
+  // Position corresponding to offset=0 in LDM_hashEntry.
+};
+
+LDM_hashTable *HASH_createTable(U32 size) {
+  LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
+  table->size = size >> HASH_BUCKET_SIZE_LOG;
+  table->maxEntries = size;
+  table->entries = calloc(size, sizeof(LDM_hashEntry));
+  table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
+  return table;
+}
+
+static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
+  return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
+}
+
+
+
+static unsigned ZSTD_NbCommonBytes (register size_t val)
+{
+    if (MEM_isLittleEndian()) {
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanForward64( &r, (U64)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctzll((U64)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+                                                     0, 3, 1, 3, 1, 4, 2, 7,
+                                                     0, 2, 3, 6, 1, 5, 3, 5,
+                                                     1, 3, 4, 4, 2, 5, 6, 7,
+                                                     7, 0, 1, 2, 3, 3, 4, 6,
+                                                     2, 6, 5, 5, 3, 4, 5, 6,
+                                                     7, 1, 2, 4, 6, 4, 4, 5,
+                                                     7, 2, 6, 5, 7, 6, 7, 7 };
+            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+#       endif
+        } else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r=0;
+            _BitScanForward( &r, (U32)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctz((U32)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+                                                     3, 2, 2, 1, 3, 2, 0, 1,
+                                                     3, 3, 1, 2, 2, 2, 2, 0,
+                                                     3, 1, 2, 0, 1, 0, 1, 1 };
+            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+#       endif
+        }
+    } else {  /* Big Endian CPU */
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanReverse64( &r, val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clzll(val) >> 3);
+#       else
+            unsigned r;
+            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
+            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+        } else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r = 0;
+            _BitScanReverse( &r, (unsigned long)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clz((U32)val) >> 3);
+#       else
+            unsigned r;
+            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+    }   }
+}
+
+// From lib/compress/zstd_compress.c
+static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *const pInLimit) {
+    const BYTE * const pStart = pIn;
+    const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
+
+    while (pIn < pInLoopLimit) {
+        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+        if (!diff) {
+          pIn += sizeof(size_t);
+          pMatch += sizeof(size_t);
+          continue;
+        }
+        pIn += ZSTD_NbCommonBytes(diff);
+        return (size_t)(pIn - pStart);
+    }
+
+    if (MEM_64bits()) {
+      if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
+        pIn += 4;
+        pMatch += 4;
+      }
+    }
+    if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
+      pIn += 2;
+      pMatch += 2;
+    }
+    if ((pIn < pInLimit) && (*pMatch == *pIn)) {
+      pIn++;
+    }
+    return (size_t)(pIn - pStart);
+}
+
+U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+                        const BYTE *pMatch, const BYTE *pBase) {
+  U32 matchLength = 0;
+  while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
+    pIn--;
+    pMatch--;
+    matchLength++;
+  }
+  return matchLength;
+}
+
+LDM_hashEntry *HASH_getValidEntry(const LDM_CCtx *cctx,
+                                  const hash_t hash,
+                                  const U32 checksum,
+                                  U32 *matchLength,
+                                  U32 *backwardsMatchLength) {
+  LDM_hashTable *table = cctx->hashTable;
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  LDM_hashEntry *cur = bucket;
+  LDM_hashEntry *bestEntry = NULL;
+  U32 bestMatchLength = 0;
+  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
+    // Check checksum for faster check.
+    const BYTE *pMatch = cur->offset + cctx->ibase;
+
+    if (cur->checksum == checksum &&
+        cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
+      U32 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
+      U32 backwardMatchLength, totalMatchLength;
+
+      // For speed.
+      if (forwardMatchLength < LDM_MIN_MATCH_LENGTH) {
+        continue;
+      }
+
+      backwardMatchLength =
+          countBackwardsMatch(cctx->ip, cctx->anchor,
+                              cur->offset + cctx->ibase,
+                              cctx->ibase);
+
+      totalMatchLength = forwardMatchLength + backwardMatchLength;
+
+      if (totalMatchLength >= bestMatchLength &&
+          totalMatchLength >= LDM_MIN_MATCH_LENGTH) {
+        bestMatchLength = totalMatchLength;
+        *matchLength = forwardMatchLength;
+        *backwardsMatchLength = backwardMatchLength;
+
+        bestEntry = cur;
+#ifdef ZSTD_SKIP
+        return cur;
+#endif
+      }
+    }
+  }
+  if (bestEntry != NULL && bestMatchLength > LDM_MIN_MATCH_LENGTH) {
+    return bestEntry;
+  }
+  return NULL;
+}
+
+void HASH_insert(LDM_hashTable *table,
+                 const hash_t hash, const LDM_hashEntry entry) {
+  *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
+  table->bucketOffsets[hash]++;
+  table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
+}
+
+U32 HASH_getSize(const LDM_hashTable *table) {
+  return table->size;
+}
+
+void HASH_destroyTable(LDM_hashTable *table) {
+  free(table->entries);
+  free(table->bucketOffsets);
+  free(table);
+}
+
+void HASH_outputTableOccupancy(const LDM_hashTable *table) {
+  U32 ctr = 0;
+  LDM_hashEntry *cur = table->entries;
+  LDM_hashEntry *end = table->entries + (table->size * HASH_BUCKET_SIZE);
+  for (; cur < end; ++cur) {
+    if (cur->offset == 0) {
+      ctr++;
+    }
+  }
+
+  printf("Num buckets, bucket size: %d, %d\n", table->size, HASH_BUCKET_SIZE);
+  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
+         table->maxEntries, ctr,
+         100.0 * (double)(ctr) / table->maxEntries);
+}
+
+
+// TODO: This can be done more efficiently (but it is not that important as it
+// is only used for computing stats).
+static int intLog2(U32 x) {
+  int ret = 0;
+  while (x >>= 1) {
+    ret++;
+  }
+  return ret;
+}
+
+// TODO: Maybe we would eventually prefer to have linear rather than
+// exponential buckets.
+/**
+void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
+  U32 i = 0;
+  int buckets[32] = { 0 };
+
+  printf("\n");
+  printf("Hash table histogram\n");
+  for (; i < HASH_getSize(cctx->hashTable); i++) {
+    int offset = (cctx->ip - cctx->ibase) -
+                 HASH_getEntryFromHash(cctx->hashTable, i)->offset;
+    buckets[intLog2(offset)]++;
+  }
+
+  i = 0;
+  for (; i < 32; i++) {
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           buckets[i],
+           100.0 * (double) buckets[i] /
+                   (double) HASH_getSize(cctx->hashTable));
+  }
+  printf("\n");
+}
+*/
+
+void LDM_printCompressStats(const LDM_compressStats *stats) {
+  int i = 0;
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  //TODO: compute percentage matched?
+  printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
+          stats->windowSizeLog, stats->hashTableSizeLog);
+  printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
+          stats->numMatches,
+          stats->totalMatchLength,
+          100.0 * (double)stats->totalMatchLength /
+              (double)(stats->totalMatchLength + stats->totalLiteralLength));
+  printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
+                                         (double)stats->numMatches);
+  printf("avg literal length, total literalLength: %.1f, %llu\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches,
+         stats->totalLiteralLength);
+  printf("avg offset length: %.1f\n",
+         ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("min offset, max offset: %u, %u\n",
+         stats->minOffset, stats->maxOffset);
+
+  printf("\n");
+  printf("offset histogram: offset, num matches, %% of matches\n");
+
+  for (; i <= intLog2(stats->maxOffset); i++) {
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           stats->offsetHistogram[i],
+           100.0 * (double) stats->offsetHistogram[i] /
+                   (double) stats->numMatches);
+  }
+  printf("\n");
+  printf("=====================\n");
+}
+
+int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
+  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
+  const BYTE *curIn = pIn;
+  const BYTE *curMatch = pMatch;
+
+  if (pIn - pMatch > LDM_WINDOW_SIZE) {
+    return 0;
+  }
+
+  for (; lengthLeft >= 4; lengthLeft -= 4) {
+    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
+      return 0;
+    }
+    curIn += 4;
+    curMatch += 4;
+  }
+  return 1;
+}
+
+hash_t HASH_hashU32(U32 value) {
+  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
+}
+
+/**
+ * Convert a sum computed from getChecksum to a hash value in the range
+ * of the hash table.
+ */
+static hash_t checksumToHash(U32 sum) {
+  return HASH_hashU32(sum);
+//  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
+}
+
+/**
+ * Computes a checksum based on rsync's checksum.
+ *
+ * a(k,l) = \sum_{i = k}^l x_i (mod M)
+ * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
+ * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
+  */
+static U32 getChecksum(const BYTE *buf, U32 len) {
+  U32 i;
+  U32 s1, s2;
+
+  s1 = s2 = 0;
+  for (i = 0; i < (len - 4); i += 4) {
+    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
+          (2 * buf[i + 2]) + (buf[i + 3]) +
+          (10 * CHECKSUM_CHAR_OFFSET);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
+          + (4 * CHECKSUM_CHAR_OFFSET);
+
+  }
+  for(; i < len; i++) {
+    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
+    s2 += s1;
+  }
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+/**
+ * Update a checksum computed from getChecksum(data, len).
+ *
+ * The checksum can be updated along its ends as follows:
+ * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
+ * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
+ *
+ * Thus toRemove should correspond to data[0].
+ */
+static U32 updateChecksum(U32 sum, U32 len,
+                          BYTE toRemove, BYTE toAdd) {
+  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
+  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
+
+  return (s1 & 0xffff) + (s2 << 16);
+}
+
+/**
+ * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
+ * based on cctx->lastSum and cctx->lastPosHashed.
+ *
+ * This uses a rolling hash and requires that the last position hashed
+ * corresponds to cctx->nextIp - step.
+ */
+static void setNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  U32 check;
+  if ((cctx->nextIp - cctx->ibase != 1) &&
+      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
+    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
+            cctx->DEBUG_setNextHash - cctx->ibase);
+  }
+
+  cctx->DEBUG_setNextHash = cctx->nextIp;
+#endif
+
+//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  cctx->nextSum = updateChecksum(
+      cctx->lastSum, LDM_HASH_LENGTH,
+      cctx->lastPosHashed[0],
+      cctx->lastPosHashed[LDM_HASH_LENGTH]);
+  cctx->nextPosHashed = cctx->nextIp;
+  cctx->nextHash = checksumToHash(cctx->nextSum);
+
+#if LDM_LAG
+//  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
+  if (cctx->ip - cctx->ibase > LDM_LAG) {
+    cctx->lagSum = updateChecksum(
+      cctx->lagSum, LDM_HASH_LENGTH,
+      cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
+    cctx->lagIp++;
+    cctx->lagHash = checksumToHash(cctx->lagSum);
+  }
+#endif
+
+#ifdef RUN_CHECKS
+  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
+
+  if (check != cctx->nextSum) {
+    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
+  }
+
+  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
+    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
+            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
+            cctx->ip - cctx->ibase);
+  }
+#endif
+}
+
+static void putHashOfCurrentPositionFromHash(
+    LDM_CCtx *cctx, hash_t hash, U32 sum) {
+  // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
+  // Note: this works only when cctx->step is 1.
+  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+    /**
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase ,
+                                  MEM_read32(cctx->ip) };
+                                  */
+#if LDM_LAG
+    // TODO: off by 1, but whatever
+    if (cctx->lagIp - cctx->ibase > 0) {
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
+      HASH_insert(cctx->hashTable, cctx->lagHash, entry);
+    } else {
+      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+      HASH_insert(cctx->hashTable, hash, entry);
+    }
+#else
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+    HASH_insert(cctx->hashTable, hash, entry);
+#endif
+  }
+
+  cctx->lastPosHashed = cctx->ip;
+  cctx->lastHash = hash;
+  cctx->lastSum = sum;
+}
+
+/**
+ * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
+ * fields from the "next" fields.
+ *
+ * This requires that cctx->ip == cctx->nextPosHashed.
+ */
+static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  if (cctx->ip != cctx->nextPosHashed) {
+    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
+           cctx->ip - cctx->ibase);
+  }
+#endif
+  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
+}
+
+/**
+ * Insert hash of the current position into the hash table.
+ */
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
+  hash_t hash = checksumToHash(sum);
+
+#ifdef RUN_CHECKS
+  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
+    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
+           cctx->ip - cctx->ibase);
+  }
+#endif
+
+  putHashOfCurrentPositionFromHash(cctx, hash, sum);
+}
+
+U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = (*pMatch) ^ *(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (U32)(pIn - pStart);
+  }
+  return (U32)(pIn - pStart);
+}
+
+void LDM_outputConfiguration(void) {
+  printf("=====================\n");
+  printf("Configuration\n");
+  printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("Min match, hash length: %d, %d\n",
+         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
+  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
+  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  printf("LDM_LAG %d\n", LDM_LAG);
+  printf("=====================\n");
+}
+
+void LDM_readHeader(const void *src, U64 *compressedSize,
+                    U64 *decompressedSize) {
+  const BYTE *ip = (const BYTE *)src;
+  *compressedSize = MEM_readLE64(ip);
+  ip += sizeof(U64);
+  *decompressedSize = MEM_readLE64(ip);
+  // ip += sizeof(U64);
+}
+
+void LDM_initializeCCtx(LDM_CCtx *cctx,
+                        const void *src, size_t srcSize,
+                        void *dst, size_t maxDstSize) {
+  cctx->isize = srcSize;
+  cctx->maxOSize = maxDstSize;
+
+  cctx->ibase = (const BYTE *)src;
+  cctx->ip = cctx->ibase;
+  cctx->iend = cctx->ibase + srcSize;
+
+  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
+  cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
+
+  cctx->obase = (BYTE *)dst;
+  cctx->op = (BYTE *)dst;
+
+  cctx->anchor = cctx->ibase;
+
+  memset(&(cctx->stats), 0, sizeof(cctx->stats));
+  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64);
+
+  cctx->stats.minOffset = UINT_MAX;
+  cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
+  cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
+
+
+  cctx->lastPosHashed = NULL;
+
+  cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
+  cctx->nextIp = cctx->ip + cctx->step;
+  cctx->nextPosHashed = 0;
+
+  cctx->DEBUG_setNextHash = 0;
+}
+
+void LDM_destroyCCtx(LDM_CCtx *cctx) {
+  HASH_destroyTable(cctx->hashTable);
+}
+
+/**
+ * Finds the "best" match.
+ *
+ * Returns 0 if successful and 1 otherwise (i.e. no match can be found
+ * in the remaining input that is long enough).
+ *
+ * matchLength contains the forward length of the match.
+ */
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
+                             U32 *matchLength, U32 *backwardMatchLength) {
+
+  LDM_hashEntry *entry = NULL;
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  while (entry == NULL) {
+    hash_t h;
+    U32 sum;
+    setNextHash(cctx);
+    h = cctx->nextHash;
+    sum = cctx->nextSum;
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
+
+    if (cctx->ip > cctx->imatchLimit) {
+      return 1;
+    }
+
+    entry = HASH_getValidEntry(cctx, h, sum,
+                               matchLength, backwardMatchLength);
+
+    if (entry != NULL) {
+      *match = entry->offset + cctx->ibase;
+    }
+    putHashOfCurrentPositionFromHash(cctx, h, sum);
+  }
+  setNextHash(cctx);
+  return 0;
+}
+
+void LDM_encodeLiteralLengthAndLiterals(
+    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+  /* Encode the literal length. */
+  if (literalLength >= RUN_MASK) {
+    int len = (int)literalLength - RUN_MASK;
+    *pToken = (RUN_MASK << ML_BITS);
+    for (; len >= 255; len -= 255) {
+      *(cctx->op)++ = 255;
+    }
+    *(cctx->op)++ = (BYTE)len;
+  } else {
+    *pToken = (BYTE)(literalLength << ML_BITS);
+  }
+
+  /* Encode the literals. */
+  memcpy(cctx->op, cctx->anchor, literalLength);
+  cctx->op += literalLength;
+}
+
+void LDM_outputBlock(LDM_CCtx *cctx,
+                     const U32 literalLength,
+                     const U32 offset,
+                     const U32 matchLength) {
+  BYTE *pToken = cctx->op++;
+
+  /* Encode the literal length and literals. */
+  LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
+
+  /* Encode the offset. */
+  MEM_write32(cctx->op, offset);
+  cctx->op += LDM_OFFSET_SIZE;
+
+  /* Encode the match length. */
+  if (matchLength >= ML_MASK) {
+    unsigned matchLengthRemaining = matchLength;
+    *pToken += ML_MASK;
+    matchLengthRemaining -= ML_MASK;
+    MEM_write32(cctx->op, 0xFFFFFFFF);
+    while (matchLengthRemaining >= 4*0xFF) {
+      cctx->op += 4;
+      MEM_write32(cctx->op, 0xffffffff);
+      matchLengthRemaining -= 4*0xFF;
+    }
+    cctx->op += matchLengthRemaining / 255;
+    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
+  } else {
+    *pToken += (BYTE)(matchLength);
+  }
+}
+
+// TODO: maxDstSize is unused. This function may seg fault when writing
+// beyond the size of dst, as it does not check maxDstSize. Writing to
+// a buffer and performing checks is a possible solution.
+//
+// This is based upon lz4.
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  LDM_CCtx cctx;
+  const BYTE *match = NULL;
+  U32 forwardMatchLength = 0;
+  U32 backwardsMatchLength = 0;
+
+  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+  LDM_outputConfiguration();
+
+  /* Hash the first position and put it into the hash table. */
+  LDM_putHashOfCurrentPosition(&cctx);
+
+#if LDM_LAG
+  cctx.lagIp = cctx.ip;
+  cctx.lagHash = cctx.lastHash;
+  cctx.lagSum = cctx.lastSum;
+#endif
+  /**
+   * Find a match.
+   * If no more matches can be found (i.e. the length of the remaining input
+   * is less than the minimum match length), then stop searching for matches
+   * and encode the final literals.
+   */
+  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
+         &backwardsMatchLength) == 0) {
+#ifdef COMPUTE_STATS
+    cctx.stats.numMatches++;
+#endif
+
+     cctx.ip -= backwardsMatchLength;
+     match -= backwardsMatchLength;
+
+    /**
+     * Write current block (literals, literal length, match offset, match
+     * length) and update pointers and hashes.
+     */
+    {
+      const U32 literalLength = cctx.ip - cctx.anchor;
+      const U32 offset = cctx.ip - match;
+      const U32 matchLength = forwardMatchLength +
+                              backwardsMatchLength -
+                              LDM_MIN_MATCH_LENGTH;
+
+      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
+
+#ifdef COMPUTE_STATS
+      cctx.stats.totalLiteralLength += literalLength;
+      cctx.stats.totalOffset += offset;
+      cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
+      cctx.stats.minOffset =
+          offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
+      cctx.stats.maxOffset =
+          offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
+      cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
+#endif
+
+      // Move ip to end of block, inserting hashes at each position.
+      cctx.nextIp = cctx.ip + cctx.step;
+      while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
+                       matchLength + literalLength) {
+        if (cctx.ip > cctx.lastPosHashed) {
+          // TODO: Simplify.
+          LDM_updateLastHashFromNextHash(&cctx);
+          setNextHash(&cctx);
+        }
+        cctx.ip++;
+        cctx.nextIp++;
+      }
+    }
+
+    // Set start of next block to current input pointer.
+    cctx.anchor = cctx.ip;
+    LDM_updateLastHashFromNextHash(&cctx);
+  }
+
+  // HASH_outputTableOffsetHistogram(&cctx);
+
+  /* Encode the last literals (no more matches). */
+  {
+    const U32 lastRun = cctx.iend - cctx.anchor;
+    BYTE *pToken = cctx.op++;
+    LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
+  }
+
+#ifdef COMPUTE_STATS
+  LDM_printCompressStats(&cctx.stats);
+  HASH_outputTableOccupancy(cctx.hashTable);
+#endif
+
+  {
+    const size_t ret = cctx.op - cctx.obase;
+    LDM_destroyCCtx(&cctx);
+    return ret;
+  }
+}
+
+struct LDM_DCtx {
+  size_t compressedSize;
+  size_t maxDecompressedSize;
+
+  const BYTE *ibase;   /* Base of input */
+  const BYTE *ip;      /* Current input position */
+  const BYTE *iend;    /* End of source */
+
+  const BYTE *obase;   /* Base of output */
+  BYTE *op;            /* Current output position */
+  const BYTE *oend;    /* End of output */
+};
+
+void LDM_initializeDCtx(LDM_DCtx *dctx,
+                        const void *src, size_t compressedSize,
+                        void *dst, size_t maxDecompressedSize) {
+  dctx->compressedSize = compressedSize;
+  dctx->maxDecompressedSize = maxDecompressedSize;
+
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + dctx->compressedSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + dctx->maxDecompressedSize;
+}
+
+size_t LDM_decompress(const void *src, size_t compressedSize,
+                      void *dst, size_t maxDecompressedSize) {
+  LDM_DCtx dctx;
+  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
+
+  while (dctx.ip < dctx.iend) {
+    BYTE *cpy;
+    const BYTE *match;
+    size_t length, offset;
+
+    /* Get the literal length. */
+    const unsigned token = *(dctx.ip)++;
+    if ((length = (token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+
+    /* Copy the literals. */
+    cpy = dctx.op + length;
+    memcpy(dctx.op, dctx.ip, length);
+    dctx.ip += length;
+    dctx.op = cpy;
+
+    //TODO : dynamic offset size
+    offset = MEM_read32(dctx.ip);
+    dctx.ip += LDM_OFFSET_SIZE;
+    match = dctx.op - offset;
+
+    /* Get the match length. */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+    length += LDM_MIN_MATCH_LENGTH;
+
+    /* Copy match. */
+    cpy = dctx.op + length;
+
+    // Inefficient for now.
+    while (match < cpy - offset && dctx.op < dctx.oend) {
+      *(dctx.op)++ = *match++;
+    }
+  }
+  return dctx.op - (BYTE *)dst;
+}
+
+// TODO: implement and test hash function
+void LDM_test(void) {
+}
+
+/*
+void LDM_test(const void *src, size_t srcSize,
+              void *dst, size_t maxDstSize) {
+  const BYTE *ip = (const BYTE *)src + 1125;
+  U32 sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
+  U32 sum2;
+  ++ip;
+  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
+    sum2 = updateChecksum(sum, LDM_HASH_LENGTH,
+                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
+    sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
+    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
+  }
+}
+*/
+
+

From 1ca128868912a978d815663bce9583520e47ace5 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 19 Jul 2017 16:01:16 -0700
Subject: [PATCH 076/100] added --memtest=# command to fuzzer

to jump directly to relevant test section
---
 tests/fuzzer.c | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 3099f346..06f98408 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -139,7 +139,7 @@ static void FUZ_freeDebug(void* counter, void* address)
 
 static void FUZ_displayMallocStats(mallocCounter_t count)
 {
-    DISPLAYLEVEL(3, "peak:%u KB,  nbMallocs:%u, total:%u KB \n",
+    DISPLAYLEVEL(3, "peak:%6u KB,  nbMallocs:%2u, total:%6u KB \n",
         (U32)(count.peakMalloc >> 10),
         count.nbMalloc,
         (U32)(count.totalMalloc >> 10));
@@ -153,7 +153,7 @@ static void FUZ_displayMallocStats(mallocCounter_t count)
         exit(1);                                   \
 }   }
 
-static int FUZ_mallocTests(unsigned seed, double compressibility)
+static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 {
     size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
     size_t const outSize = ZSTD_compressBound(inSize);
@@ -171,6 +171,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
     RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
 
     /* simple compression tests */
+    if (part <= 1)
     {   int compressionLevel;
         for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
             mallocCounter_t malcount = INIT_MALLOC_COUNTER;
@@ -183,6 +184,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
     }   }
 
     /* streaming compression tests */
+    if (part <= 2)
     {   int compressionLevel;
         for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
             mallocCounter_t malcount = INIT_MALLOC_COUNTER;
@@ -199,6 +201,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
     }   }
 
     /* advanced MT API test */
+    if (part <= 3)
     {   U32 nbThreads;
         for (nbThreads=1; nbThreads<=4; nbThreads++) {
             int compressionLevel;
@@ -218,6 +221,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility)
     }   }   }
 
     /* advanced MT streaming API test */
+    if (part <= 4)
     {   U32 nbThreads;
         for (nbThreads=1; nbThreads<=4; nbThreads++) {
             int compressionLevel;
@@ -1442,6 +1446,19 @@ static unsigned readU32FromChar(const char** stringPtr)
     return result;
 }
 
+/** longCommandWArg() :
+ *  check if *stringPtr is the same as longCommand.
+ *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
+ *  @return 0 and doesn't modify *stringPtr otherwise.
+ */
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+    size_t const comSize = strlen(longCommand);
+    int const result = !strncmp(*stringPtr, longCommand, comSize);
+    if (result) *stringPtr += comSize;
+    return result;
+}
+
 int main(int argc, const char** argv)
 {
     U32 seed = 0;
@@ -1465,6 +1482,8 @@ int main(int argc, const char** argv)
         /* Handle commands. Aggregated commands are allowed */
         if (argument[0]=='-') {
 
+            if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
+
             if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
             if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
 
@@ -1539,7 +1558,7 @@ int main(int argc, const char** argv)
 
     if (memTestsOnly) {
         g_displayLevel = MAX(3, g_displayLevel);
-        return FUZ_mallocTests(seed, ((double)proba) / 100);
+        return FUZ_mallocTests(seed, ((double)proba) / 100, memTestsOnly);
     }
 
     if (nbTests < testNb) nbTests = testNb;

From 2427a154cb6a5af622bdbe679f4b4c5b906b4821 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 19 Jul 2017 16:56:28 -0700
Subject: [PATCH 077/100] Minor refactoring

---
 contrib/long_distance_matching/basic_table.c  | 109 --------------
 .../circular_buffer_table.c                   | 137 ++++--------------
 contrib/long_distance_matching/ldm.c          |  42 +-----
 contrib/long_distance_matching/ldm.h          |  12 +-
 .../long_distance_matching/ldm_hashtable.h    |  36 +----
 .../long_distance_matching/ldm_with_table.c   |  84 ++++++-----
 contrib/long_distance_matching/main-ldm.c     |  36 +----
 7 files changed, 102 insertions(+), 354 deletions(-)
 delete mode 100644 contrib/long_distance_matching/basic_table.c

diff --git a/contrib/long_distance_matching/basic_table.c b/contrib/long_distance_matching/basic_table.c
deleted file mode 100644
index 30c548d2..00000000
--- a/contrib/long_distance_matching/basic_table.c
+++ /dev/null
@@ -1,109 +0,0 @@
-#include <stdlib.h>
-#include <stdio.h>
-
-#include "ldm.h"
-#include "ldm_hashtable.h"
-#include "mem.h"
-
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE) - 4)
-
-struct LDM_hashTable {
-  U32 size;
-  LDM_hashEntry *entries;
-  const BYTE *offsetBase;
-};
-
-LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase) {
-  LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
-  table->size = size;
-  table->entries = calloc(size, sizeof(LDM_hashEntry));
-  table->offsetBase = offsetBase;
-  return table;
-}
-
-void HASH_initializeTable(LDM_hashTable *table, U32 size) {
-  table->size = size;
-  table->entries = calloc(size, sizeof(LDM_hashEntry));
-}
-
-LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
-  return table->entries + hash;
-}
-
-LDM_hashEntry *HASH_getEntryFromHash(
-    const LDM_hashTable *table, const hash_t hash, const U32 checksum) {
-  (void)checksum;
-  return getBucket(table, hash);
-}
-
-static int isValidMatch(const BYTE *pIn, const BYTE *pMatch,
-                            U32 minMatchLength, U32 maxWindowSize) {
-  U32 lengthLeft = minMatchLength;
-  const BYTE *curIn = pIn;
-  const BYTE *curMatch = pMatch;
-
-  if (pIn - pMatch > maxWindowSize) {
-    return 0;
-  }
-
-  for (; lengthLeft >= 4; lengthLeft -= 4) {
-    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
-      return 0;
-    }
-    curIn += 4;
-    curMatch += 4;
-  }
-  return 1;
-}
-
-LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
-                                  const hash_t hash,
-                                  const U32 checksum,
-                                  const BYTE *pIn,
-                                  const BYTE *pEnd,
-                                  U32 minMatchLength,
-                                  U32 maxWindowSize,
-                                  U32 *matchLength) {
-  LDM_hashEntry *entry = getBucket(table, hash);
-  (void)checksum;
-  (void)pEnd;
-  (void)matchLength;
-  // TODO: Count the entire forward match length rather than check if valid.
-  if (isValidMatch(pIn, entry->offset + table->offsetBase,
-                   minMatchLength, maxWindowSize)) {
-
-    return entry;
-  }
-  return NULL;
-}
-
-hash_t HASH_hashU32(U32 value) {
-  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
-}
-
-void HASH_insert(LDM_hashTable *table,
-                 const hash_t hash, const LDM_hashEntry entry) {
-  *getBucket(table, hash) = entry;
-}
-
-U32 HASH_getSize(const LDM_hashTable *table) {
-  return table->size;
-}
-
-void HASH_destroyTable(LDM_hashTable *table) {
-  free(table->entries);
-  free(table);
-}
-
-void HASH_outputTableOccupancy(const LDM_hashTable *hashTable) {
-  U32 i = 0;
-  U32 ctr = 0;
-  for (; i < HASH_getSize(hashTable); i++) {
-    if (getBucket(hashTable, i)->offset == 0) {
-      ctr++;
-    }
-  }
-  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         HASH_getSize(hashTable), ctr,
-         100.0 * (double)(ctr) / (double)HASH_getSize(hashTable));
-}
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index 9b7ad088..9429fbcd 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -5,22 +5,19 @@
 #include "ldm_hashtable.h"
 #include "mem.h"
 
-//TODO: move def somewhere else.
 
 // Number of elements per hash bucket.
 // HASH_BUCKET_SIZE_LOG defined in ldm.h
-#define HASH_BUCKET_SIZE_LOG 2 // MAX is 4 for now
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 
 // TODO: rename. Number of hash buckets.
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
-#define ZSTD_SKIP
-//#define TMP_TST
+//#define ZSTD_SKIP
 
 struct LDM_hashTable {
-  U32 size;  // Number of buckets
-  U32 maxEntries;  // Rename...
-  LDM_hashEntry *entries;  // 1-D array for now.
+  U32 numBuckets;
+  U32 numEntries;
+  LDM_hashEntry *entries;
   BYTE *bucketOffsets;     // Pointer to current insert position.
 
   // Position corresponding to offset=0 in LDM_hashEntry.
@@ -32,8 +29,8 @@ struct LDM_hashTable {
 LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
                                 U32 minMatchLength, U32 maxWindowSize) {
   LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
-  table->size = size >> HASH_BUCKET_SIZE_LOG;
-  table->maxEntries = size;
+  table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
+  table->numEntries = size;
   table->entries = calloc(size, sizeof(LDM_hashEntry));
   table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
   table->offsetBase = offsetBase;
@@ -46,7 +43,6 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
 }
 
-#if TMP_ZSTDTOGGLE
 static unsigned ZSTD_NbCommonBytes (register size_t val)
 {
     if (MEM_isLittleEndian()) {
@@ -159,26 +155,22 @@ U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
   return matchLength;
 }
 
-LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
-                                  const hash_t hash,
-                                  const U32 checksum,
-                                  const BYTE *pIn,
-                                  const BYTE *pEnd,
-                                  U32 *matchLength,
-                                  U32 *backwardsMatchLength,
-                                  const BYTE *pAnchor) {
+LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
+                                 const hash_t hash,
+                                 const U32 checksum,
+                                 const BYTE *pIn,
+                                 const BYTE *pEnd,
+                                 const BYTE *pAnchor,
+                                 U32 *pForwardMatchLength,
+                                 U32 *pBackwardMatchLength) {
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
   LDM_hashEntry *bestEntry = NULL;
   U32 bestMatchLength = 0;
-  U32 forwardMatch = 0;
-  U32 backwardMatch = 0;
-#ifdef TMP_TST
-  U32 numBetter = 0;
-#endif
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    // Check checksum for faster check.
     const BYTE *pMatch = cur->offset + table->offsetBase;
+
+    // Check checksum for faster check.
     if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
       U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
       U32 backwardMatchLength, totalMatchLength;
@@ -193,105 +185,27 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
 
       if (totalMatchLength >= bestMatchLength) {
         bestMatchLength = totalMatchLength;
-        forwardMatch = forwardMatchLength;
-        backwardMatch = backwardMatchLength;
+        *pForwardMatchLength = forwardMatchLength;
+        *pBackwardMatchLength = backwardMatchLength;
+
         bestEntry = cur;
-#ifdef TMP_TST
-        numBetter++;
-#endif
 
 #ifdef ZSTD_SKIP
-        *matchLength = forwardMatchLength;
-        *backwardsMatchLength = backwardMatchLength;
-
         return cur;
 #endif
-//        *matchLength = forwardMatchLength;
-//        return cur;
       }
     }
   }
-  if (bestEntry != NULL && bestMatchLength > table->minMatchLength) {
-#ifdef TMP_TST
-    printf("Num better %u\n", numBetter - 1);
-#endif
-    *matchLength = forwardMatch;
-    *backwardsMatchLength = backwardMatch;
+  if (bestEntry != NULL) {
     return bestEntry;
   }
   return NULL;
 }
 
-#else
-
-static int isValidMatch(const BYTE *pIn, const BYTE *pMatch,
-                        U32 minMatchLength, U32 maxWindowSize) {
-  printf("HERE\n");
-  U32 lengthLeft = minMatchLength;
-  const BYTE *curIn = pIn;
-  const BYTE *curMatch = pMatch;
-
-  if (pIn - pMatch > maxWindowSize) {
-    return 0;
-  }
-
-  for (; lengthLeft >= 4; lengthLeft -= 4) {
-    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
-      return 0;
-    }
-    curIn += 4;
-    curMatch += 4;
-  }
-  return 1;
-}
-
-//TODO: clean up function call. This is not at all decoupled from LDM.
-LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
-                                  const hash_t hash,
-                                  const U32 checksum,
-                                  const BYTE *pIn,
-                                  const BYTE *pEnd,
-                                  U32 *matchLength,
-                                  U32 *backwardsMatchLength,
-                                  const BYTE *pAnchor) {
-  LDM_hashEntry *bucket = getBucket(table, hash);
-  LDM_hashEntry *cur = bucket;
-  (void)matchLength;
-  (void)backwardsMatchLength;
-  (void)pAnchor; for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    // Check checksum for faster check.
-    const BYTE *pMatch = cur->offset + table->offsetBase;
-    (void)pEnd;
-
-    if (cur->checksum == checksum &&
-        isValidMatch(pIn, pMatch, table->minMatchLength, table->maxWindowSize)) {
-      return cur;
-    }
-  }
-  return NULL;
-}
-
-#endif
 hash_t HASH_hashU32(U32 value) {
   return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
 }
 
-
-LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
-                                     const hash_t hash,
-                                     const U32 checksum) {
-  // Loop through bucket.
-  // TODO: in order of recency???
-  LDM_hashEntry *bucket = getBucket(table, hash);
-  LDM_hashEntry *cur = bucket;
-  for(; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    if (cur->checksum == checksum) {
-      return cur;
-    }
-  }
-  return NULL;
-}
-
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
   *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
@@ -300,7 +214,7 @@ void HASH_insert(LDM_hashTable *table,
 }
 
 U32 HASH_getSize(const LDM_hashTable *table) {
-  return table->size;
+  return table->numBuckets;
 }
 
 void HASH_destroyTable(LDM_hashTable *table) {
@@ -312,15 +226,16 @@ void HASH_destroyTable(LDM_hashTable *table) {
 void HASH_outputTableOccupancy(const LDM_hashTable *table) {
   U32 ctr = 0;
   LDM_hashEntry *cur = table->entries;
-  LDM_hashEntry *end = table->entries + (table->size * HASH_BUCKET_SIZE);
+  LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
   for (; cur < end; ++cur) {
     if (cur->offset == 0) {
       ctr++;
     }
   }
 
-  printf("Num buckets, bucket size: %d, %d\n", table->size, HASH_BUCKET_SIZE);
+  printf("Num buckets, bucket size: %d, %d\n",
+         table->numBuckets, HASH_BUCKET_SIZE);
   printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         table->maxEntries, ctr,
-         100.0 * (double)(ctr) / table->maxEntries);
+         table->numEntries, ctr,
+         100.0 * (double)(ctr) / table->numEntries);
 }
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index a116af70..6e9addf7 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -14,7 +14,6 @@
 #define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - 4))
 #define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
 
-
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)
 #define RUN_BITS (8-ML_BITS)
@@ -24,7 +23,6 @@
 #define OUTPUT_CONFIGURATION
 #define CHECKSUM_CHAR_OFFSET 10
 
-//#define HASH_CHECK
 //#define RUN_CHECKS
 //#define TMP_RECOMPUTE_LENGTHS
 
@@ -135,7 +133,6 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   int i = 0;
   printf("=====================\n");
   printf("Compression statistics\n");
-  //TODO: compute percentage matched?
   printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
           stats->windowSizeLog, stats->hashTableSizeLog);
   printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
@@ -191,7 +188,6 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
  */
 static hash_t checksumToHash(U32 sum) {
   return HASH_hashU32(sum);
-//  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
 }
 
 /**
@@ -455,22 +451,14 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     if (cctx->ip > cctx->imatchLimit) {
       return 1;
     }
-#ifdef HASH_CHECK
-    entry = HASH_getEntryFromHash(cctx->hashTable, h, sum);
-#else
-    entry = HASH_getValidEntry(cctx->hashTable, h, sum,
-                               cctx->ip, cctx->iend,
-                               matchLength, backwardMatchLength,
-                               cctx->anchor);
-#endif
+
+    entry = HASH_getBestEntry(cctx->hashTable, h, sum,
+                              cctx->ip, cctx->iend,
+                              cctx->anchor,
+                              matchLength, backwardMatchLength);
 
     if (entry != NULL) {
       *match = entry->offset + cctx->ibase;
-#ifdef HASH_CHECK
-      if (!LDM_isValidMatch(cctx->ip, *match)) {
-        entry = NULL;
-      }
-#endif
     }
     putHashOfCurrentPositionFromHash(cctx, h, sum);
   }
@@ -563,21 +551,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
     cctx.stats.numMatches++;
 #endif
 
-#if TMP_RECOMPUTE_LENGTHS
-    backwardsMatchLength = 0;
-    /**
-     * Catch up: look back to extend the match backwards from the found match.
-     */
-    while (cctx.ip > cctx.anchor && match > cctx.ibase &&
-           cctx.ip[-1] == match[-1]) {
-      cctx.ip--;
-      match--;
-      backwardsMatchLength++;
-    }
-#else
      cctx.ip -= backwardsMatchLength;
      match -= backwardsMatchLength;
-#endif
 
     /**
      * Write current block (literals, literal length, match offset, match
@@ -586,16 +561,9 @@ size_t LDM_compress(const void *src, size_t srcSize,
     {
       const U32 literalLength = cctx.ip - cctx.anchor;
       const U32 offset = cctx.ip - match;
-#if TMP_RECOMPUTE_LENGTHS
-      const U32 matchLength = LDM_countMatchLength(
-          cctx.ip + LDM_MIN_MATCH_LENGTH + backwardsMatchLength,
-          match + LDM_MIN_MATCH_LENGTH + backwardsMatchLength,
-          cctx.ihashLimit) + backwardsMatchLength;
-#else
       const U32 matchLength = forwardMatchLength +
                               backwardsMatchLength -
                               LDM_MIN_MATCH_LENGTH;
-#endif
 
       LDM_outputBlock(&cctx, literalLength, offset, matchLength);
 
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 2396227d..1d5b2f13 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -11,21 +11,21 @@
 #define LDM_OFFSET_SIZE 4
 
 // Defines the size of the hash table.
+// Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
 #define LDM_MEMORY_USAGE 23
+#define HASH_BUCKET_SIZE_LOG 3 // MAX is 4 for now
 
-//#define LDM_LAG (1 << 20)
-#define LDM_LAG (0)
+// Defines the lag in inserting elements into the hash table.
+#define LDM_LAG 0
 
 #define LDM_WINDOW_SIZE_LOG 28
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four (and perhaps set to the same value?).
-#define LDM_MIN_MATCH_LENGTH 1024
-#define LDM_HASH_LENGTH 1024
+#define LDM_MIN_MATCH_LENGTH 64
+#define LDM_HASH_LENGTH 64
 
-#define TMP_ZSTDTOGGLE 1
-#define TMP_RECOMPUTE_LENGTHS (!(TMP_ZSTDTOGGLE))
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 51d82525..df9dcd78 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -14,37 +14,17 @@ typedef struct LDM_hashEntry {
 
 typedef struct LDM_hashTable LDM_hashTable;
 
-/**
- * Create a hash table with size hash buckets.
- * LDM_hashEntry.offset is added to offsetBase to calculate pMatch in
- * HASH_getValidEntry.
- */
 LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
                                 U32 minMatchLength, U32 maxWindowSize);
 
-/**
- * Returns an LDM_hashEntry from the table that matches the checksum.
- * Returns NULL if one does not exist.
- */
-LDM_hashEntry *HASH_getEntryFromHash(const LDM_hashTable *table,
-                                     const hash_t hash,
-                                     const U32 checksum);
-
-/**
- *  Gets a valid entry that matches the checksum. A valid entry is defined by
- *  *isValid.
- *
- *  The function finds an entry matching the checksum, computes pMatch as
- *  offset + table.offsetBase, and calls isValid.
- */
-LDM_hashEntry *HASH_getValidEntry(const LDM_hashTable *table,
-                                  const hash_t hash,
-                                  const U32 checksum,
-                                  const BYTE *pIn,
-                                  const BYTE *pEnd,
-                                  U32 *matchLength,
-                                  U32 *backwardsMatchLength,
-                                  const BYTE *pAnchor);
+LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
+                                 const hash_t hash,
+                                 const U32 checksum,
+                                 const BYTE *pIn,
+                                 const BYTE *pEnd,
+                                 const BYTE *pAnchor,
+                                 U32 *matchLength,
+                                 U32 *backwardsMatchLength);
 
 hash_t HASH_hashU32(U32 value);
 
diff --git a/contrib/long_distance_matching/ldm_with_table.c b/contrib/long_distance_matching/ldm_with_table.c
index 68a33d0f..5919d588 100644
--- a/contrib/long_distance_matching/ldm_with_table.c
+++ b/contrib/long_distance_matching/ldm_with_table.c
@@ -4,6 +4,8 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "ldm.h"
+
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 //#define LDM_HASH_ENTRY_SIZE 4
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
@@ -14,7 +16,6 @@
 #define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
 
 /* Hash table stuff. */
-#define HASH_BUCKET_SIZE_LOG 3 // MAX is 4 for now
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
 
@@ -32,18 +33,15 @@
 
 //#define RUN_CHECKS
 
-#include "ldm.h"
-
 /* Hash table stuff */
 
 typedef U32 hash_t;
 
 typedef struct LDM_hashEntry {
-  U32 offset;   // TODO: Replace with pointer?
+  U32 offset;
   U32 checksum;
 } LDM_hashEntry;
 
-// TODO: Scanning speed
 // TODO: Memory usage
 struct LDM_compressStats {
   U32 windowSizeLog, hashTableSizeLog;
@@ -110,18 +108,22 @@ struct LDM_CCtx {
 };
 
 struct LDM_hashTable {
-  U32 size;  // Number of buckets
-  U32 maxEntries;  // Rename...
-  LDM_hashEntry *entries;  // 1-D array for now.
+  U32 numBuckets;  // Number of buckets
+  U32 numEntries;  // Rename...
+  LDM_hashEntry *entries;
 
   BYTE *bucketOffsets;
   // Position corresponding to offset=0 in LDM_hashEntry.
 };
 
+/**
+ * Create a hash table that can contain size elements.
+ * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
+ */
 LDM_hashTable *HASH_createTable(U32 size) {
   LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
-  table->size = size >> HASH_BUCKET_SIZE_LOG;
-  table->maxEntries = size;
+  table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
+  table->numEntries = size;
   table->entries = calloc(size, sizeof(LDM_hashEntry));
   table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
   return table;
@@ -131,10 +133,7 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
 }
 
-
-
-static unsigned ZSTD_NbCommonBytes (register size_t val)
-{
+static unsigned ZSTD_NbCommonBytes (register size_t val) {
     if (MEM_isLittleEndian()) {
         if (MEM_64bits()) {
 #       if defined(_MSC_VER) && defined(_WIN64)
@@ -234,6 +233,11 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
     return (size_t)(pIn - pStart);
 }
 
+/**
+ * Count number of bytes that match backwards before pIn and pMatch.
+ *
+ * We count only bytes where pMatch > pBaes and pIn > pAnchor.
+ */
 U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
                         const BYTE *pMatch, const BYTE *pBase) {
   U32 matchLength = 0;
@@ -245,20 +249,32 @@ U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
   return matchLength;
 }
 
-LDM_hashEntry *HASH_getValidEntry(const LDM_CCtx *cctx,
-                                  const hash_t hash,
-                                  const U32 checksum,
-                                  U32 *matchLength,
-                                  U32 *backwardsMatchLength) {
+/**
+ * Returns a pointer to the entry in the hash table matching the hash and
+ * checksum with the "longest match length" as defined below. The forward and
+ * backward match lengths are written to *pForwardMatchLength and
+ * *pBackwardMatchLength.
+ *
+ * The match length is defined based on cctx->ip and the entry's offset.
+ * The forward match is computed from cctx->ip and entry->offset + cctx->ibase.
+ * The backward match is computed backwards from cctx->ip and
+ * cctx->ibase only if the forward match is longer than LDM_MIN_MATCH_LENGTH.
+ *
+ */
+LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
+                                 const hash_t hash,
+                                 const U32 checksum,
+                                 U32 *pForwardMatchLength,
+                                 U32 *pBackwardMatchLength) {
   LDM_hashTable *table = cctx->hashTable;
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
   LDM_hashEntry *bestEntry = NULL;
   U32 bestMatchLength = 0;
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    // Check checksum for faster check.
     const BYTE *pMatch = cur->offset + cctx->ibase;
 
+    // Check checksum for faster check.
     if (cur->checksum == checksum &&
         cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
       U32 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
@@ -279,8 +295,8 @@ LDM_hashEntry *HASH_getValidEntry(const LDM_CCtx *cctx,
       if (totalMatchLength >= bestMatchLength &&
           totalMatchLength >= LDM_MIN_MATCH_LENGTH) {
         bestMatchLength = totalMatchLength;
-        *matchLength = forwardMatchLength;
-        *backwardsMatchLength = backwardMatchLength;
+        *pForwardMatchLength = forwardMatchLength;
+        *pBackwardMatchLength = backwardMatchLength;
 
         bestEntry = cur;
 #ifdef ZSTD_SKIP
@@ -303,7 +319,7 @@ void HASH_insert(LDM_hashTable *table,
 }
 
 U32 HASH_getSize(const LDM_hashTable *table) {
-  return table->size;
+  return table->numBuckets;
 }
 
 void HASH_destroyTable(LDM_hashTable *table) {
@@ -315,20 +331,20 @@ void HASH_destroyTable(LDM_hashTable *table) {
 void HASH_outputTableOccupancy(const LDM_hashTable *table) {
   U32 ctr = 0;
   LDM_hashEntry *cur = table->entries;
-  LDM_hashEntry *end = table->entries + (table->size * HASH_BUCKET_SIZE);
+  LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
   for (; cur < end; ++cur) {
     if (cur->offset == 0) {
       ctr++;
     }
   }
 
-  printf("Num buckets, bucket size: %d, %d\n", table->size, HASH_BUCKET_SIZE);
+  printf("Num buckets, bucket size: %d, %d\n",
+         table->numBuckets, HASH_BUCKET_SIZE);
   printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         table->maxEntries, ctr,
-         100.0 * (double)(ctr) / table->maxEntries);
+         table->numEntries, ctr,
+         100.0 * (double)(ctr) / table->numEntries);
 }
 
-
 // TODO: This can be done more efficiently (but it is not that important as it
 // is only used for computing stats).
 static int intLog2(U32 x) {
@@ -339,7 +355,7 @@ static int intLog2(U32 x) {
   return ret;
 }
 
-// TODO: Maybe we would eventually prefer to have linear rather than
+// Maybe we would eventually prefer to have linear rather than
 // exponential buckets.
 /**
 void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
@@ -369,7 +385,6 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   int i = 0;
   printf("=====================\n");
   printf("Compression statistics\n");
-  //TODO: compute percentage matched?
   printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
           stats->windowSizeLog, stats->hashTableSizeLog);
   printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
@@ -429,7 +444,6 @@ hash_t HASH_hashU32(U32 value) {
  */
 static hash_t checksumToHash(U32 sum) {
   return HASH_hashU32(sum);
-//  return ((sum * 2654435761U) >> (32 - LDM_HASHLOG));
 }
 
 /**
@@ -672,10 +686,10 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
  * Returns 0 if successful and 1 otherwise (i.e. no match can be found
  * in the remaining input that is long enough).
  *
- * matchLength contains the forward length of the match.
+ * forwardMatchLength contains the forward length of the match.
  */
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
-                             U32 *matchLength, U32 *backwardMatchLength) {
+                             U32 *forwardMatchLength, U32 *backwardMatchLength) {
 
   LDM_hashEntry *entry = NULL;
   cctx->nextIp = cctx->ip + cctx->step;
@@ -693,8 +707,8 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
       return 1;
     }
 
-    entry = HASH_getValidEntry(cctx, h, sum,
-                               matchLength, backwardMatchLength);
+    entry = HASH_getBestEntry(cctx, h, sum,
+                              forwardMatchLength, backwardMatchLength);
 
     if (entry != NULL) {
       *match = entry->offset + cctx->ibase;
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index a43ec000..96db0c22 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -29,6 +29,7 @@ static int compress(const char *fname, const char *oname) {
   size_t maxCompressedSize, compressedSize;
 
   struct timeval tv1, tv2;
+  double timeTaken;
 
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
@@ -53,18 +54,7 @@ static int compress(const char *fname, const char *oname) {
   // The compress function should check before writing or buffer writes.
   maxCompressedSize += statbuf.st_size / 255;
 
- /* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */
-  if (lseek(fdout, maxCompressedSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
- /* Write a dummy byte at the last location. */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
+  ftruncate(fdout, maxCompressedSize);
 
   /* mmap the input file. */
   if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
@@ -103,12 +93,12 @@ static int compress(const char *fname, const char *oname) {
          (unsigned)statbuf.st_size, (unsigned)compressedSize, oname,
          (double)compressedSize / (statbuf.st_size) * 100);
 
+  timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+              (double) (tv2.tv_sec - tv1.tv_sec),
+
   printf("Total compress time = %.3f seconds, Average compression speed: %.3f MB/s\n",
-         (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-         (double) (tv2.tv_sec - tv1.tv_sec),
-         ((double)statbuf.st_size / (double) (1 << 20)) /
-            ((double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
-            (double) (tv2.tv_sec - tv1.tv_sec)));
+         timeTaken,
+         ((double)statbuf.st_size / (double) (1 << 20)) / timeTaken);
 
 
   // Close files.
@@ -156,17 +146,7 @@ static int decompress(const char *fname, const char *oname) {
   /* Read the header. */
   LDM_readHeader(src, &compressedSize, &decompressedSize);
 
-  /* Go to the location corresponding to the last byte. */
-  if (lseek(fdout, decompressedSize - 1, SEEK_SET) == -1) {
-    perror("lseek error");
-    return 1;
-  }
-
-  /* write a dummy byte at the last location */
-  if (write(fdout, "", 1) != 1) {
-    perror("write error");
-    return 1;
-  }
+  ftruncate(fdout, decompressedSize);
 
   /* mmap the output file */
   if ((dst = mmap(0, decompressedSize, PROT_READ | PROT_WRITE,

From 13a01ffb27f58d24dae8dc56c1ef75f1743cca27 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 19 Jul 2017 17:24:09 -0700
Subject: [PATCH 078/100] Fix off-by-one in size calculations

---
 contrib/long_distance_matching/circular_buffer_table.c | 3 ++-
 contrib/long_distance_matching/ldm.c                   | 6 +++---
 contrib/long_distance_matching/ldm.h                   | 2 +-
 contrib/long_distance_matching/ldm_with_table.c        | 8 ++++----
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index 9429fbcd..ad7ae9e1 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -11,7 +11,8 @@
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 
 // TODO: rename. Number of hash buckets.
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
+// TODO: Link to HASH_ENTRY_SIZE_LOG
+#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-3-(HASH_BUCKET_SIZE_LOG))
 //#define ZSTD_SKIP
 
 struct LDM_hashTable {
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 6e9addf7..9ffbab48 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -6,12 +6,12 @@
 
 
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-//#define LDM_HASH_ENTRY_SIZE 4
+#define LDM_HASH_ENTRY_SIZE_LOG 3
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 4)
+#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - 4))
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
 #define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
 
 #define ML_BITS 4
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 1d5b2f13..04b6410c 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -13,7 +13,7 @@
 // Defines the size of the hash table.
 // Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
-#define LDM_MEMORY_USAGE 23
+#define LDM_MEMORY_USAGE 22
 #define HASH_BUCKET_SIZE_LOG 3 // MAX is 4 for now
 
 // Defines the lag in inserting elements into the hash table.
diff --git a/contrib/long_distance_matching/ldm_with_table.c b/contrib/long_distance_matching/ldm_with_table.c
index 5919d588..813ead6a 100644
--- a/contrib/long_distance_matching/ldm_with_table.c
+++ b/contrib/long_distance_matching/ldm_with_table.c
@@ -7,17 +7,17 @@
 #include "ldm.h"
 
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-//#define LDM_HASH_ENTRY_SIZE 4
+#define LDM_HASH_ENTRY_SIZE_LOG 3
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 4)
+#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - 4))
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
 #define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
 
 /* Hash table stuff. */
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-4-HASH_BUCKET_SIZE_LOG)
+#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
 #define ML_BITS 4
 #define ML_MASK ((1U<<ML_BITS)-1)

From b4d460f32cc42d237a44172f3f9975fd39637c5f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 20 Jul 2017 01:13:14 -0700
Subject: [PATCH 079/100] pool.c : blindfix for Visual warnings

---
 lib/common/threading.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/common/threading.h b/lib/common/threading.h
index ee786455..99e39b55 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -42,14 +42,14 @@ extern "C" {
 
 /* mutex */
 #define pthread_mutex_t           CRITICAL_SECTION
-#define pthread_mutex_init(a,b)   (InitializeCriticalSection((a)), 0)
+#define pthread_mutex_init(a,b)   (InitializeCriticalSection((a)), (void)0)
 #define pthread_mutex_destroy(a)  DeleteCriticalSection((a))
 #define pthread_mutex_lock(a)     EnterCriticalSection((a))
 #define pthread_mutex_unlock(a)   LeaveCriticalSection((a))
 
 /* condition variable */
 #define pthread_cond_t             CONDITION_VARIABLE
-#define pthread_cond_init(a, b)    (InitializeConditionVariable((a)), 0)
+#define pthread_cond_init(a, b)    (InitializeConditionVariable((a)), (void)0)
 #define pthread_cond_destroy(a)    /* No delete */
 #define pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
 #define pthread_cond_signal(a)     WakeConditionVariable((a))

From 7d3ac0710d4371e2d05874b655a0a11dbdb25421 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 20 Jul 2017 13:33:55 -0700
Subject: [PATCH 080/100] [linux] Update patches for v3

---
 contrib/linux-kernel/0000-cover-letter.patch  | 21 +++--
 .../0001-lib-Add-xxhash-module.patch          | 14 ++--
 .../0002-lib-Add-zstd-modules.patch           | 62 ++++++---------
 .../0003-btrfs-Add-zstd-support.patch         | 76 ++++++++++---------
 .../0004-squashfs-Add-zstd-support.patch      |  6 +-
 contrib/linux-kernel/fs/btrfs/zstd.c          | 20 ++---
 contrib/linux-kernel/lib/zstd/huf_compress.c  |  3 +-
 contrib/linux-kernel/lib/zstd/zstd_internal.h | 29 ++-----
 8 files changed, 105 insertions(+), 126 deletions(-)

diff --git a/contrib/linux-kernel/0000-cover-letter.patch b/contrib/linux-kernel/0000-cover-letter.patch
index 763b5a9c..33a5189b 100644
--- a/contrib/linux-kernel/0000-cover-letter.patch
+++ b/contrib/linux-kernel/0000-cover-letter.patch
@@ -1,7 +1,7 @@
-From 8bc9a0ae5c86a6d02d9a5274b9965ddac0e8d330 Mon Sep 17 00:00:00 2001
+From 0cd63464d182bb9708f8b25f7da3dc8e5ec6b4fa Mon Sep 17 00:00:00 2001
 From: Nick Terrell <terrelln@fb.com>
-Date: Wed, 28 Jun 2017 22:00:00 -0700
-Subject: [PATCH v2 0/4] Add xxhash and zstd modules
+Date: Thu, 20 Jul 2017 13:18:30 -0700
+Subject: [PATCH v3 0/4] Add xxhash and zstd modules
 
 Hi all,
 
@@ -24,6 +24,13 @@ v1 -> v2:
   HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4() (2/4)
 - No zstd function uses more than 400 B of stack space (2/4)
 
+v2 -> v3:
+- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388
+  (2/4)
+- Fix bug in dictionary compression from upstream commit cc1522351f (2/4)
+- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff (3/4)
+- Change default compression level for BtrFS to 3 (3/4)
+
 Nick Terrell (4):
   lib: Add xxhash module
   lib: Add zstd modules
@@ -40,7 +47,7 @@ Nick Terrell (4):
  fs/btrfs/props.c           |    6 +
  fs/btrfs/super.c           |   12 +-
  fs/btrfs/sysfs.c           |    2 +
- fs/btrfs/zstd.c            |  433 ++++++
+ fs/btrfs/zstd.c            |  435 ++++++
  fs/squashfs/Kconfig        |   14 +
  fs/squashfs/Makefile       |    1 +
  fs/squashfs/decompressor.c |    7 +
@@ -63,13 +70,13 @@ Nick Terrell (4):
  lib/zstd/fse_compress.c    |  795 ++++++++++
  lib/zstd/fse_decompress.c  |  332 +++++
  lib/zstd/huf.h             |  212 +++
- lib/zstd/huf_compress.c    |  771 ++++++++++
+ lib/zstd/huf_compress.c    |  770 ++++++++++
  lib/zstd/huf_decompress.c  |  960 ++++++++++++
  lib/zstd/mem.h             |  151 ++
  lib/zstd/zstd_common.c     |   75 +
- lib/zstd/zstd_internal.h   |  269 ++++
+ lib/zstd/zstd_internal.h   |  250 ++++
  lib/zstd/zstd_opt.h        | 1014 +++++++++++++
- 39 files changed, 14400 insertions(+), 12 deletions(-)
+ 39 files changed, 14382 insertions(+), 12 deletions(-)
  create mode 100644 fs/btrfs/zstd.c
  create mode 100644 fs/squashfs/zstd_wrapper.c
  create mode 100644 include/linux/xxhash.h
diff --git a/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch b/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch
index 84a2c53c..f86731ce 100644
--- a/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch
+++ b/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch
@@ -1,7 +1,7 @@
-From 5ac909c415ab4a18fd90794793c96e450795e8c6 Mon Sep 17 00:00:00 2001
+From fc7f26acbabda35f1c61dfc357dbb207dc8ed23d Mon Sep 17 00:00:00 2001
 From: Nick Terrell <terrelln@fb.com>
-Date: Wed, 21 Jun 2017 17:37:36 -0700
-Subject: [PATCH v2 1/4] lib: Add xxhash module
+Date: Mon, 17 Jul 2017 17:07:18 -0700
+Subject: [PATCH v3 1/4] lib: Add xxhash module
 
 Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
 extremely fast non-cryptographic hash algorithm for checksumming.
@@ -327,10 +327,10 @@ index 0000000..9e1f42c
 +
 +#endif /* XXHASH_H */
 diff --git a/lib/Kconfig b/lib/Kconfig
-index 0c8b78a..b6009d7 100644
+index 6762529..5e7541f 100644
 --- a/lib/Kconfig
 +++ b/lib/Kconfig
-@@ -184,6 +184,9 @@ config CRC8
+@@ -192,6 +192,9 @@ config CRC8
  	  when they need to do cyclic redundancy check according CRC8
  	  algorithm. Module will be called crc8.
 
@@ -341,10 +341,10 @@ index 0c8b78a..b6009d7 100644
  	bool
  	depends on AUDIT && !AUDIT_ARCH
 diff --git a/lib/Makefile b/lib/Makefile
-index 0166fbc..1338226 100644
+index 40c1837..d06b68a 100644
 --- a/lib/Makefile
 +++ b/lib/Makefile
-@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC32_SELFTEST)	+= crc32test.o
+@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC4)	+= crc4.o
  obj-$(CONFIG_CRC7)	+= crc7.o
  obj-$(CONFIG_LIBCRC32C)	+= libcrc32c.o
  obj-$(CONFIG_CRC8)	+= crc8.o
diff --git a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
index 97109399..268307cf 100644
--- a/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
+++ b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch
@@ -1,7 +1,7 @@
-From d2626127c6d6e60e940dd9a3ed58323bdcdc4930 Mon Sep 17 00:00:00 2001
+From 686a6149b98250d66b5951e3ae05e79063e9de98 Mon Sep 17 00:00:00 2001
 From: Nick Terrell <terrelln@fb.com>
-Date: Tue, 16 May 2017 14:55:36 -0700
-Subject: [PATCH v2 2/4] lib: Add zstd modules
+Date: Mon, 17 Jul 2017 17:08:19 -0700
+Subject: [PATCH v3 2/4] lib: Add zstd modules
 
 Add zstd compression and decompression kernel modules.
 zstd offers a wide varity of compression speed and quality trade-offs.
@@ -110,6 +110,10 @@ v1 -> v2:
   HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4()
 - No function uses more than 400 B of stack space
 
+v2 -> v3:
+- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388
+- Fix bug in dictionary compression from upstream commit cc1522351f
+
  include/linux/zstd.h      | 1157 +++++++++++++++
  lib/Kconfig               |    8 +
  lib/Makefile              |    2 +
@@ -123,13 +127,13 @@ v1 -> v2:
  lib/zstd/fse_compress.c   |  795 +++++++++++
  lib/zstd/fse_decompress.c |  332 +++++
  lib/zstd/huf.h            |  212 +++
- lib/zstd/huf_compress.c   |  771 ++++++++++
+ lib/zstd/huf_compress.c   |  770 ++++++++++
  lib/zstd/huf_decompress.c |  960 +++++++++++++
  lib/zstd/mem.h            |  151 ++
  lib/zstd/zstd_common.c    |   75 +
- lib/zstd/zstd_internal.h  |  269 ++++
+ lib/zstd/zstd_internal.h  |  250 ++++
  lib/zstd/zstd_opt.h       | 1014 +++++++++++++
- 19 files changed, 13014 insertions(+)
+ 19 files changed, 12994 insertions(+)
  create mode 100644 include/linux/zstd.h
  create mode 100644 lib/zstd/Makefile
  create mode 100644 lib/zstd/bitstream.h
@@ -1312,10 +1316,10 @@ index 0000000..249575e
 +
 +#endif  /* ZSTD_H */
 diff --git a/lib/Kconfig b/lib/Kconfig
-index b6009d7..f00ddab 100644
+index 5e7541f..0d49ed0 100644
 --- a/lib/Kconfig
 +++ b/lib/Kconfig
-@@ -241,6 +241,14 @@ config LZ4HC_COMPRESS
+@@ -249,6 +249,14 @@ config LZ4HC_COMPRESS
  config LZ4_DECOMPRESS
  	tristate
 
@@ -1331,7 +1335,7 @@ index b6009d7..f00ddab 100644
 
  #
 diff --git a/lib/Makefile b/lib/Makefile
-index 1338226..4fcef16 100644
+index d06b68a..d5c8a4f 100644
 --- a/lib/Makefile
 +++ b/lib/Makefile
 @@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
@@ -10012,10 +10016,10 @@ index 0000000..2143da2
 +#endif /* HUF_H_298734234 */
 diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c
 new file mode 100644
-index 0000000..0361f38
+index 0000000..40055a7
 --- /dev/null
 +++ b/lib/zstd/huf_compress.c
-@@ -0,0 +1,771 @@
+@@ -0,0 +1,770 @@
 +/*
 + * Huffman encoder, part of New Generation Entropy library
 + * Copyright (C) 2013-2016, Yann Collet.
@@ -10543,7 +10547,7 @@ index 0000000..0361f38
 +
 +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
 +
-+#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
++#define HUF_FLUSHBITS(s)  BIT_flushBits(s)
 +
 +#define HUF_FLUSHBITS_1(stream)                                            \
 +	if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
@@ -10560,7 +10564,6 @@ index 0000000..0361f38
 +	BYTE *const oend = ostart + dstSize;
 +	BYTE *op = ostart;
 +	size_t n;
-+	const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
 +	BIT_CStream_t bitC;
 +
 +	/* init */
@@ -11993,10 +11996,10 @@ index 0000000..a282624
 +}
 diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h
 new file mode 100644
-index 0000000..6748719
+index 0000000..f0ba474
 --- /dev/null
 +++ b/lib/zstd/zstd_internal.h
-@@ -0,0 +1,269 @@
+@@ -0,0 +1,250 @@
 +/**
 + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 + * All rights reserved.
@@ -12125,35 +12128,16 @@ index 0000000..6748719
 +/*-*******************************************
 +*  Shared functions to include for inlining
 +*********************************************/
-+static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); }
-+#define COPY8(d, s)               \
-+	{                         \
-+		ZSTD_copy8(d, s); \
-+		d += 8;           \
-+		s += 8;           \
-+	}
-+
++static void ZSTD_copy8(void *dst, const void *src) {
++	memcpy(dst, src, 8);
++}
 +/*! ZSTD_wildcopy() :
 +*   custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
 +#define WILDCOPY_OVERLENGTH 8
 +ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
 +{
-+	const BYTE *ip = (const BYTE *)src;
-+	BYTE *op = (BYTE *)dst;
-+	BYTE *const oend = op + length;
-+	do
-+		COPY8(op, ip)
-+	while (op < oend);
-+}
-+
-+ZSTD_STATIC void ZSTD_wildcopy_e(void *dst, const void *src, void *dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
-+{
-+	const BYTE *ip = (const BYTE *)src;
-+	BYTE *op = (BYTE *)dst;
-+	BYTE *const oend = (BYTE *)dstEnd;
-+	do
-+		COPY8(op, ip)
-+	while (op < oend);
++	if (length > 0)
++		memcpy(dst, src, length);
 +}
 +
 +/*-*******************************************
diff --git a/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch b/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch
index abc8326c..5578fa38 100644
--- a/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch
+++ b/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch
@@ -1,7 +1,7 @@
-From 599f8f2aaace3df939cb145368574a52268d82d0 Mon Sep 17 00:00:00 2001
+From b0ef8fc63c9ca251ceca632f53aa1de8f1f17772 Mon Sep 17 00:00:00 2001
 From: Nick Terrell <terrelln@fb.com>
-Date: Wed, 21 Jun 2017 17:31:39 -0700
-Subject: [PATCH v2 3/4] btrfs: Add zstd support
+Date: Mon, 17 Jul 2017 17:08:39 -0700
+Subject: [PATCH v3 3/4] btrfs: Add zstd support
 
 Add zstd compression and decompression support to BtrFS. zstd at its
 fastest level compresses almost as well as zlib, while offering much
@@ -63,6 +63,10 @@ zstd source repository: https://github.com/facebook/zstd
 
 Signed-off-by: Nick Terrell <terrelln@fb.com>
 ---
+v2 -> v3:
+- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff
+- Change default compression level for BtrFS to 3
+
  fs/btrfs/Kconfig           |   2 +
  fs/btrfs/Makefile          |   2 +-
  fs/btrfs/compression.c     |   1 +
@@ -73,9 +77,9 @@ Signed-off-by: Nick Terrell <terrelln@fb.com>
  fs/btrfs/props.c           |   6 +
  fs/btrfs/super.c           |  12 +-
  fs/btrfs/sysfs.c           |   2 +
- fs/btrfs/zstd.c            | 433 +++++++++++++++++++++++++++++++++++++++++++++
+ fs/btrfs/zstd.c            | 435 +++++++++++++++++++++++++++++++++++++++++++++
  include/uapi/linux/btrfs.h |   8 +-
- 12 files changed, 469 insertions(+), 12 deletions(-)
+ 12 files changed, 471 insertions(+), 12 deletions(-)
  create mode 100644 fs/btrfs/zstd.c
 
 diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
@@ -105,10 +109,10 @@ index 128ce17..962a95a 100644
  	   reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
  	   uuid-tree.o props.o hash.o free-space-tree.o
 diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
-index 10e6b28..3beb0d0 100644
+index d2ef9ac..4ff42d1 100644
 --- a/fs/btrfs/compression.c
 +++ b/fs/btrfs/compression.c
-@@ -761,6 +761,7 @@ static struct {
+@@ -704,6 +704,7 @@ static struct {
  static const struct btrfs_compress_op * const btrfs_compress_op[] = {
  	&btrfs_zlib_compress,
  	&btrfs_lzo_compress,
@@ -117,10 +121,10 @@ index 10e6b28..3beb0d0 100644
 
  void __init btrfs_init_compress(void)
 diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
-index 39ec43a..d99fc21 100644
+index 87f6d33..2269e00 100644
 --- a/fs/btrfs/compression.h
 +++ b/fs/btrfs/compression.h
-@@ -60,8 +60,9 @@ enum btrfs_compression_type {
+@@ -99,8 +99,9 @@ enum btrfs_compression_type {
  	BTRFS_COMPRESS_NONE  = 0,
  	BTRFS_COMPRESS_ZLIB  = 1,
  	BTRFS_COMPRESS_LZO   = 2,
@@ -132,7 +136,7 @@ index 39ec43a..d99fc21 100644
  };
 
  struct btrfs_compress_op {
-@@ -92,5 +93,6 @@ struct btrfs_compress_op {
+@@ -128,5 +129,6 @@ struct btrfs_compress_op {
 
  extern const struct btrfs_compress_op btrfs_zlib_compress;
  extern const struct btrfs_compress_op btrfs_lzo_compress;
@@ -140,10 +144,10 @@ index 39ec43a..d99fc21 100644
 
  #endif
 diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
-index 4f8f75d..61dd3dd 100644
+index 3f3eb7b..845d77c 100644
 --- a/fs/btrfs/ctree.h
 +++ b/fs/btrfs/ctree.h
-@@ -271,6 +271,7 @@ struct btrfs_super_block {
+@@ -270,6 +270,7 @@ struct btrfs_super_block {
  	 BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS |		\
  	 BTRFS_FEATURE_INCOMPAT_BIG_METADATA |		\
  	 BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO |		\
@@ -152,10 +156,10 @@ index 4f8f75d..61dd3dd 100644
  	 BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF |		\
  	 BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA |	\
 diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
-index 5f678dc..49c0e91 100644
+index 080e2eb..04632f4 100644
 --- a/fs/btrfs/disk-io.c
 +++ b/fs/btrfs/disk-io.c
-@@ -2831,6 +2831,8 @@ int open_ctree(struct super_block *sb,
+@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb,
  	features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
  	if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
  		features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
@@ -165,7 +169,7 @@ index 5f678dc..49c0e91 100644
  	if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
  		btrfs_info(fs_info, "has skinny extents");
 diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
-index e176375..f732cfd 100644
+index fa1b78c..b9963d9 100644
 --- a/fs/btrfs/ioctl.c
 +++ b/fs/btrfs/ioctl.c
 @@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
@@ -180,7 +184,7 @@ index e176375..f732cfd 100644
  		ret = btrfs_set_prop(inode, "btrfs.compression",
  				     comp, strlen(comp), 0);
  		if (ret)
-@@ -1463,6 +1465,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
+@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
 
  	if (range->compress_type == BTRFS_COMPRESS_LZO) {
  		btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
@@ -190,10 +194,10 @@ index e176375..f732cfd 100644
 
  	ret = defrag_count;
 diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
-index d6cb155..162105f 100644
+index 4b23ae5..20631e9 100644
 --- a/fs/btrfs/props.c
 +++ b/fs/btrfs/props.c
-@@ -383,6 +383,8 @@ static int prop_compression_validate(const char *value, size_t len)
+@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len)
  		return 0;
  	else if (!strncmp("zlib", value, len))
  		return 0;
@@ -202,7 +206,7 @@ index d6cb155..162105f 100644
 
  	return -EINVAL;
  }
-@@ -405,6 +407,8 @@ static int prop_compression_apply(struct inode *inode,
+@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode,
  		type = BTRFS_COMPRESS_LZO;
  	else if (!strncmp("zlib", value, len))
  		type = BTRFS_COMPRESS_ZLIB;
@@ -211,7 +215,7 @@ index d6cb155..162105f 100644
  	else
  		return -EINVAL;
 
-@@ -422,6 +426,8 @@ static const char *prop_compression_extract(struct inode *inode)
+@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode)
  		return "zlib";
  	case BTRFS_COMPRESS_LZO:
  		return "lzo";
@@ -221,7 +225,7 @@ index d6cb155..162105f 100644
 
  	return NULL;
 diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
-index 4f1cdd5..4f792d5 100644
+index 12540b6..c370dea 100644
 --- a/fs/btrfs/super.c
 +++ b/fs/btrfs/super.c
 @@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
@@ -239,7 +243,7 @@ index 4f1cdd5..4f792d5 100644
  			} else if (strncmp(args[0].from, "no", 2) == 0) {
  				compress_type = "no";
  				btrfs_clear_opt(info->mount_opt, COMPRESS);
-@@ -1240,8 +1248,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
+@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
  	if (btrfs_test_opt(info, COMPRESS)) {
  		if (info->compress_type == BTRFS_COMPRESS_ZLIB)
  			compress_type = "zlib";
@@ -252,7 +256,7 @@ index 4f1cdd5..4f792d5 100644
  			seq_printf(seq, ",compress-force=%s", compress_type);
  		else
 diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
-index 1f157fb..b0dec90 100644
+index c2d5f35..2b6d37c 100644
 --- a/fs/btrfs/sysfs.c
 +++ b/fs/btrfs/sysfs.c
 @@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
@@ -273,10 +277,10 @@ index 1f157fb..b0dec90 100644
  	BTRFS_FEAT_ATTR_PTR(raid56),
 diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c
 new file mode 100644
-index 0000000..838741b
+index 0000000..1822068
 --- /dev/null
 +++ b/fs/btrfs/zstd.c
-@@ -0,0 +1,433 @@
+@@ -0,0 +1,435 @@
 +/*
 + * Copyright (c) 2016-present, Facebook, Inc.
 + * All rights reserved.
@@ -308,10 +312,11 @@ index 0000000..838741b
 +
 +#define ZSTD_BTRFS_MAX_WINDOWLOG 17
 +#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
++#define ZSTD_BTRFS_DEFAULT_LEVEL 3
 +
 +static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
 +{
-+	ZSTD_parameters params = ZSTD_getParams(1, src_len, 0);
++	ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, src_len, 0);
 +
 +	if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
 +		params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
@@ -330,7 +335,7 @@ index 0000000..838741b
 +{
 +	struct workspace *workspace = list_entry(ws, struct workspace, list);
 +
-+	vfree(workspace->mem);
++	kvfree(workspace->mem);
 +	kfree(workspace->buf);
 +	kfree(workspace);
 +}
@@ -341,15 +346,15 @@ index 0000000..838741b
 +			zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
 +	struct workspace *workspace;
 +
-+	workspace = kzalloc(sizeof(*workspace), GFP_NOFS);
++	workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
 +	if (!workspace)
 +		return ERR_PTR(-ENOMEM);
 +
 +	workspace->size = max_t(size_t,
 +			ZSTD_CStreamWorkspaceBound(params.cParams),
 +			ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
-+	workspace->mem = vmalloc(workspace->size);
-+	workspace->buf = kmalloc(PAGE_SIZE, GFP_NOFS);
++	workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
++	workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 +	if (!workspace->mem || !workspace->buf)
 +		goto fail;
 +
@@ -541,12 +546,13 @@ index 0000000..838741b
 +	return ret;
 +}
 +
-+static int zstd_decompress_bio(struct list_head *ws, struct page **pages_in,
-+		u64 disk_start,
-+		struct bio *orig_bio,
-+		size_t srclen)
++static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 +{
 +	struct workspace *workspace = list_entry(ws, struct workspace, list);
++	struct page **pages_in = cb->compressed_pages;
++	u64 disk_start = cb->start;
++	struct bio *orig_bio = cb->orig_bio;
++	size_t srclen = cb->compressed_len;
 +	ZSTD_DStream *stream;
 +	int ret = 0;
 +	unsigned long page_in_index = 0;
@@ -711,7 +717,7 @@ index 0000000..838741b
 +	.decompress = zstd_decompress,
 +};
 diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
-index a456e53..992c150 100644
+index 9aa74f3..378230c 100644
 --- a/include/uapi/linux/btrfs.h
 +++ b/include/uapi/linux/btrfs.h
 @@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
diff --git a/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch b/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch
index b638194f..02bd1073 100644
--- a/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch
+++ b/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch
@@ -1,7 +1,7 @@
-From 5ff6a64abaea7b7f11d37cb0fdf08642316a3a90 Mon Sep 17 00:00:00 2001
+From 0cd63464d182bb9708f8b25f7da3dc8e5ec6b4fa Mon Sep 17 00:00:00 2001
 From: Nick Terrell <terrelln@fb.com>
-Date: Mon, 12 Jun 2017 12:18:23 -0700
-Subject: [PATCH v2 4/4] squashfs: Add zstd support
+Date: Mon, 17 Jul 2017 17:08:59 -0700
+Subject: [PATCH v3 4/4] squashfs: Add zstd support
 
 Add zstd compression and decompression support to SquashFS. zstd is a
 great fit for SquashFS because it can compress at ratios approaching xz,
diff --git a/contrib/linux-kernel/fs/btrfs/zstd.c b/contrib/linux-kernel/fs/btrfs/zstd.c
index 838741b3..18220687 100644
--- a/contrib/linux-kernel/fs/btrfs/zstd.c
+++ b/contrib/linux-kernel/fs/btrfs/zstd.c
@@ -29,10 +29,11 @@
 
 #define ZSTD_BTRFS_MAX_WINDOWLOG 17
 #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
+#define ZSTD_BTRFS_DEFAULT_LEVEL 3
 
 static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
 {
-	ZSTD_parameters params = ZSTD_getParams(1, src_len, 0);
+	ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL, src_len, 0);
 
 	if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
 		params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
@@ -51,7 +52,7 @@ static void zstd_free_workspace(struct list_head *ws)
 {
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
 
-	vfree(workspace->mem);
+	kvfree(workspace->mem);
 	kfree(workspace->buf);
 	kfree(workspace);
 }
@@ -62,15 +63,15 @@ static struct list_head *zstd_alloc_workspace(void)
 			zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
 	struct workspace *workspace;
 
-	workspace = kzalloc(sizeof(*workspace), GFP_NOFS);
+	workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
 	if (!workspace)
 		return ERR_PTR(-ENOMEM);
 
 	workspace->size = max_t(size_t,
 			ZSTD_CStreamWorkspaceBound(params.cParams),
 			ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
-	workspace->mem = vmalloc(workspace->size);
-	workspace->buf = kmalloc(PAGE_SIZE, GFP_NOFS);
+	workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
+	workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!workspace->mem || !workspace->buf)
 		goto fail;
 
@@ -262,12 +263,13 @@ out:
 	return ret;
 }
 
-static int zstd_decompress_bio(struct list_head *ws, struct page **pages_in,
-		u64 disk_start,
-		struct bio *orig_bio,
-		size_t srclen)
+static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 {
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
+	struct page **pages_in = cb->compressed_pages;
+	u64 disk_start = cb->start;
+	struct bio *orig_bio = cb->orig_bio;
+	size_t srclen = cb->compressed_len;
 	ZSTD_DStream *stream;
 	int ret = 0;
 	unsigned long page_in_index = 0;
diff --git a/contrib/linux-kernel/lib/zstd/huf_compress.c b/contrib/linux-kernel/lib/zstd/huf_compress.c
index 0361f387..40055a70 100644
--- a/contrib/linux-kernel/lib/zstd/huf_compress.c
+++ b/contrib/linux-kernel/lib/zstd/huf_compress.c
@@ -525,7 +525,7 @@ static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt
 
 size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
 
-#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
+#define HUF_FLUSHBITS(s)  BIT_flushBits(s)
 
 #define HUF_FLUSHBITS_1(stream)                                            \
 	if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
@@ -542,7 +542,6 @@ size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, si
 	BYTE *const oend = ostart + dstSize;
 	BYTE *op = ostart;
 	size_t n;
-	const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
 	BIT_CStream_t bitC;
 
 	/* init */
diff --git a/contrib/linux-kernel/lib/zstd/zstd_internal.h b/contrib/linux-kernel/lib/zstd/zstd_internal.h
index 67487199..f0ba4744 100644
--- a/contrib/linux-kernel/lib/zstd/zstd_internal.h
+++ b/contrib/linux-kernel/lib/zstd/zstd_internal.h
@@ -126,35 +126,16 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
 /*-*******************************************
 *  Shared functions to include for inlining
 *********************************************/
-static void ZSTD_copy8(void *dst, const void *src) { memcpy(dst, src, 8); }
-#define COPY8(d, s)               \
-	{                         \
-		ZSTD_copy8(d, s); \
-		d += 8;           \
-		s += 8;           \
-	}
-
+static void ZSTD_copy8(void *dst, const void *src) {
+	memcpy(dst, src, 8);
+}
 /*! ZSTD_wildcopy() :
 *   custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
 #define WILDCOPY_OVERLENGTH 8
 ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
 {
-	const BYTE *ip = (const BYTE *)src;
-	BYTE *op = (BYTE *)dst;
-	BYTE *const oend = op + length;
-	do
-		COPY8(op, ip)
-	while (op < oend);
-}
-
-ZSTD_STATIC void ZSTD_wildcopy_e(void *dst, const void *src, void *dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
-{
-	const BYTE *ip = (const BYTE *)src;
-	BYTE *op = (BYTE *)dst;
-	BYTE *const oend = (BYTE *)dstEnd;
-	do
-		COPY8(op, ip)
-	while (op < oend);
+	if (length > 0)
+		memcpy(dst, src, length);
 }
 
 /*-*******************************************

From 5e6c5203f3370445da2903d302bde1a0a4fa7230 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 20 Jul 2017 15:11:56 -0700
Subject: [PATCH 081/100] fixed fuzzer test for non OS-X platforms

---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 06f98408..d63f1352 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -246,7 +246,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 
 #else
 
-static int FUZ_mallocTests(unsigned seed, double compressibility)
+static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 {
     (void)seed; (void)compressibility;
     return 0;

From a90b16e150d393b76210dc8c2c0b8f32fbd39826 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 20 Jul 2017 15:57:55 -0700
Subject: [PATCH 082/100] Visual blind fix 2

---
 lib/common/pool.c      | 6 +++---
 lib/common/threading.h | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index 06d8a5f5..aeaca7e7 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -95,9 +95,9 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
     ctx->queue = (POOL_job*) malloc(ctx->queueSize * sizeof(POOL_job));
     ctx->queueHead = 0;
     ctx->queueTail = 0;
-    pthread_mutex_init(&ctx->queueMutex, NULL);
-    pthread_cond_init(&ctx->queuePushCond, NULL);
-    pthread_cond_init(&ctx->queuePopCond, NULL);
+    (void)pthread_mutex_init(&ctx->queueMutex, NULL);
+    (void)pthread_cond_init(&ctx->queuePushCond, NULL);
+    (void)pthread_cond_init(&ctx->queuePopCond, NULL);
     ctx->shutdown = 0;
     /* Allocate space for the thread handles */
     ctx->threads = (pthread_t*)malloc(numThreads * sizeof(pthread_t));
diff --git a/lib/common/threading.h b/lib/common/threading.h
index 99e39b55..ee786455 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -42,14 +42,14 @@ extern "C" {
 
 /* mutex */
 #define pthread_mutex_t           CRITICAL_SECTION
-#define pthread_mutex_init(a,b)   (InitializeCriticalSection((a)), (void)0)
+#define pthread_mutex_init(a,b)   (InitializeCriticalSection((a)), 0)
 #define pthread_mutex_destroy(a)  DeleteCriticalSection((a))
 #define pthread_mutex_lock(a)     EnterCriticalSection((a))
 #define pthread_mutex_unlock(a)   LeaveCriticalSection((a))
 
 /* condition variable */
 #define pthread_cond_t             CONDITION_VARIABLE
-#define pthread_cond_init(a, b)    (InitializeConditionVariable((a)), (void)0)
+#define pthread_cond_init(a, b)    (InitializeConditionVariable((a)), 0)
 #define pthread_cond_destroy(a)    /* No delete */
 #define pthread_cond_wait(a, b)    SleepConditionVariableCS((a), (b), INFINITE)
 #define pthread_cond_signal(a)     WakeConditionVariable((a))

From 38ba7002f2c9156842c5042990c322fc9546dd86 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 20 Jul 2017 18:39:04 -0700
Subject: [PATCH 083/100] fixed minor warning on unused variable in shell
 function

---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index d63f1352..046c67ea 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -248,7 +248,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 
 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 {
-    (void)seed; (void)compressibility;
+    (void)seed; (void)compressibility; (void)part;
     return 0;
 }
 

From 273c17b350ac329d2db72f83ded00ba5477ec084 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 20 Jul 2017 16:50:06 -0700
Subject: [PATCH 084/100] Experiment with 64-bit hash and checksum

---
 contrib/long_distance_matching/Makefile       |    8 +-
 .../circular_buffer_table.c                   |    7 +-
 contrib/long_distance_matching/ldm.c          |   65 +-
 contrib/long_distance_matching/ldm.h          |    3 +-
 .../long_distance_matching/ldm_hashtable.h    |    2 +
 contrib/long_distance_matching/ldm_hf_test.c  | 1048 +++++++++++++++++
 .../long_distance_matching/ldm_with_table.c   |   26 +-
 contrib/long_distance_matching/main-ldm.c     |   12 +-
 8 files changed, 1096 insertions(+), 75 deletions(-)
 create mode 100644 contrib/long_distance_matching/ldm_hf_test.c

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 3aa3f8bd..5119f464 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,7 +25,7 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-circular-buffer main-integrated
+all: main-circular-buffer main-integrated main-hf
 
 #main-basic : basic_table.c ldm.c main-ldm.c
 #	      $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -33,12 +33,14 @@ all: main-circular-buffer main-integrated
 main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+main-hf: ldm_hf_test.c main-ldm.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
 main-integrated: ldm_with_table.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-basic main-circular-buffer  main-integrated
+	main-basic main-circular-buffer  main-integrated main-hf
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index ad7ae9e1..66107e06 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -5,14 +5,16 @@
 #include "ldm_hashtable.h"
 #include "mem.h"
 
-
 // Number of elements per hash bucket.
 // HASH_BUCKET_SIZE_LOG defined in ldm.h
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
+#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
+
+
 
 // TODO: rename. Number of hash buckets.
 // TODO: Link to HASH_ENTRY_SIZE_LOG
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-3-(HASH_BUCKET_SIZE_LOG))
+
 //#define ZSTD_SKIP
 
 struct LDM_hashTable {
@@ -175,6 +177,7 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
     if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
       U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
       U32 backwardMatchLength, totalMatchLength;
+
       if (forwardMatchLength < table->minMatchLength) {
         continue;
       }
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 9ffbab48..ab2de7c1 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -4,14 +4,15 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "ldm.h"
+#include "ldm_hashtable.h"
 
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASH_ENTRY_SIZE_LOG 3
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
 #define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
 
 #define ML_BITS 4
@@ -26,8 +27,7 @@
 //#define RUN_CHECKS
 //#define TMP_RECOMPUTE_LENGTHS
 
-#include "ldm.h"
-#include "ldm_hashtable.h"
+typedef U32 checksum_t;
 
 // TODO: Scanning speed
 // TODO: Memory usage
@@ -71,22 +71,22 @@ struct LDM_CCtx {
 
   LDM_hashTable *hashTable;
 
-//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
-
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  U32 lastSum;
+  checksum_t lastSum;
 
   const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
   const BYTE *nextPosHashed;
   hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
-  U32 nextSum;
+  checksum_t nextSum;
+
+
 
   unsigned step;                      // ip step, should be 1.
 
   const BYTE *lagIp;
   hash_t lagHash;
-  U32 lagSum;
+  checksum_t lagSum;
 
   U64 numHashInserts;
   // DEBUG
@@ -191,15 +191,15 @@ static hash_t checksumToHash(U32 sum) {
 }
 
 /**
- * Computes a checksum based on rsync's checksum.
+ * Computes a 32-bit checksum based on rsync's checksum.
  *
  * a(k,l) = \sum_{i = k}^l x_i (mod M)
  * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
  * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
   */
-static U32 getChecksum(const BYTE *buf, U32 len) {
+static checksum_t getChecksum(const BYTE *buf, U32 len) {
   U32 i;
-  U32 s1, s2;
+  checksum_t s1, s2;
 
   s1 = s2 = 0;
   for (i = 0; i < (len - 4); i += 4) {
@@ -226,8 +226,8 @@ static U32 getChecksum(const BYTE *buf, U32 len) {
  *
  * Thus toRemove should correspond to data[0].
  */
-static U32 updateChecksum(U32 sum, U32 len,
-                          BYTE toRemove, BYTE toAdd) {
+static checksum_t updateChecksum(checksum_t sum, U32 len,
+                                 BYTE toRemove, BYTE toAdd) {
   U32 s1 = (sum & 0xffff) - toRemove + toAdd;
   U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
 
@@ -262,7 +262,6 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->nextHash = checksumToHash(cctx->nextSum);
 
 #if LDM_LAG
-//  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
   if (cctx->ip - cctx->ibase > LDM_LAG) {
     cctx->lagSum = updateChecksum(
       cctx->lagSum, LDM_HASH_LENGTH,
@@ -288,32 +287,28 @@ static void setNextHash(LDM_CCtx *cctx) {
 }
 
 static void putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash, U32 sum) {
+    LDM_CCtx *cctx, hash_t hash, U32 checksum) {
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    /**
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase ,
-                                  MEM_read32(cctx->ip) };
-                                  */
 #if LDM_LAG
     // TODO: off by 1, but whatever
     if (cctx->lagIp - cctx->ibase > 0) {
       const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
       HASH_insert(cctx->hashTable, cctx->lagHash, entry);
     } else {
-      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
       HASH_insert(cctx->hashTable, hash, entry);
     }
 #else
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
     HASH_insert(cctx->hashTable, hash, entry);
 #endif
   }
 
   cctx->lastPosHashed = cctx->ip;
   cctx->lastHash = hash;
-  cctx->lastSum = sum;
+  cctx->lastSum = checksum;
 }
 
 /**
@@ -336,7 +331,7 @@ static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
  * Insert hash of the current position into the hash table.
  */
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
+  checksum_t sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
   hash_t hash = checksumToHash(sum);
 
 #ifdef RUN_CHECKS
@@ -441,7 +436,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
 
   while (entry == NULL) {
     hash_t h;
-    U32 sum;
+    checksum_t sum;
     setNextHash(cctx);
     h = cctx->nextHash;
     sum = cctx->nextSum;
@@ -698,23 +693,7 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
 }
 
 // TODO: implement and test hash function
-void LDM_test(void) {
+void LDM_test(const BYTE *src) {
+  (void)src;
 }
 
-/*
-void LDM_test(const void *src, size_t srcSize,
-              void *dst, size_t maxDstSize) {
-  const BYTE *ip = (const BYTE *)src + 1125;
-  U32 sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
-  U32 sum2;
-  ++ip;
-  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
-    sum2 = updateChecksum(sum, LDM_HASH_LENGTH,
-                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
-    sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
-    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
-  }
-}
-*/
-
-
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 04b6410c..c420e60c 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -31,6 +31,7 @@ typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
 typedef struct LDM_DCtx LDM_DCtx;
 
+
 /**
  *  Compresses src into dst.
  *
@@ -151,6 +152,6 @@ void LDM_readHeader(const void *src, U64 *compressedSize,
 
 void LDM_outputConfiguration(void);
 
-void LDM_test(void);
+void LDM_test(const BYTE *src);
 
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index df9dcd78..9d5ba0e2 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -3,6 +3,8 @@
 
 #include "mem.h"
 
+#define LDM_HASH_ENTRY_SIZE_LOG 3
+
 // TODO: clean up comments
 
 typedef U32 hash_t;
diff --git a/contrib/long_distance_matching/ldm_hf_test.c b/contrib/long_distance_matching/ldm_hf_test.c
new file mode 100644
index 00000000..63be82d1
--- /dev/null
+++ b/contrib/long_distance_matching/ldm_hf_test.c
@@ -0,0 +1,1048 @@
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ldm.h"
+
+#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASH_ENTRY_SIZE_LOG 3
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
+#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
+
+// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
+#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
+
+/* Hash table stuff. */
+#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
+#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+#define COMPUTE_STATS
+#define OUTPUT_CONFIGURATION
+#define CHECKSUM_CHAR_OFFSET 10
+
+// Take first match only.
+//#define ZSTD_SKIP
+
+//#define RUN_CHECKS
+
+/* Hash table stuff */
+
+typedef U32 hash_t;
+
+typedef struct LDM_hashEntry {
+  U32 offset;
+  U32 checksum;
+} LDM_hashEntry;
+
+// TODO: Memory usage
+struct LDM_compressStats {
+  U32 windowSizeLog, hashTableSizeLog;
+  U32 numMatches;
+  U64 totalMatchLength;
+  U64 totalLiteralLength;
+  U64 totalOffset;
+
+  U32 minOffset, maxOffset;
+
+  U32 numCollisions;
+  U32 numHashInserts;
+
+  U32 offsetHistogram[32];
+};
+
+typedef struct LDM_hashTable LDM_hashTable;
+
+struct LDM_CCtx {
+  U64 isize;             /* Input size */
+  U64 maxOSize;          /* Maximum output size */
+
+  const BYTE *ibase;        /* Base of input */
+  const BYTE *ip;           /* Current input position */
+  const BYTE *iend;         /* End of input */
+
+  // Maximum input position such that hashing at the position does not exceed
+  // end of input.
+  const BYTE *ihashLimit;
+
+  // Maximum input position such that finding a match of at least the minimum
+  // match length does not exceed end of input.
+  const BYTE *imatchLimit;
+
+  const BYTE *obase;        /* Base of output */
+  BYTE *op;                 /* Output */
+
+  const BYTE *anchor;       /* Anchor to start of current (match) block */
+
+  LDM_compressStats stats;            /* Compression statistics */
+
+  LDM_hashTable *hashTable;
+
+//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
+
+  const BYTE *lastPosHashed;          /* Last position hashed */
+  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
+  U32 lastSum;
+
+  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
+  const BYTE *nextPosHashed;
+  U64 nextSum;
+
+//  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
+//  U32 nextSum;
+
+  unsigned step;                      // ip step, should be 1.
+
+  const BYTE *lagIp;
+  U64 lagSum;
+//  hash_t lagHash;
+//  U32 lagSum;
+
+  U64 numHashInserts;
+  // DEBUG
+  const BYTE *DEBUG_setNextHash;
+};
+
+struct LDM_hashTable {
+  U32 numBuckets;  // Number of buckets
+  U32 numEntries;  // Rename...
+  LDM_hashEntry *entries;
+
+  BYTE *bucketOffsets;
+  // Position corresponding to offset=0 in LDM_hashEntry.
+};
+
+/**
+ * Create a hash table that can contain size elements.
+ * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
+ */
+LDM_hashTable *HASH_createTable(U32 size) {
+  LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
+  table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
+  table->numEntries = size;
+  table->entries = calloc(size, sizeof(LDM_hashEntry));
+  table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
+  return table;
+}
+
+static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
+  return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
+}
+
+static unsigned ZSTD_NbCommonBytes (register size_t val) {
+    if (MEM_isLittleEndian()) {
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanForward64( &r, (U64)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctzll((U64)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+                                                     0, 3, 1, 3, 1, 4, 2, 7,
+                                                     0, 2, 3, 6, 1, 5, 3, 5,
+                                                     1, 3, 4, 4, 2, 5, 6, 7,
+                                                     7, 0, 1, 2, 3, 3, 4, 6,
+                                                     2, 6, 5, 5, 3, 4, 5, 6,
+                                                     7, 1, 2, 4, 6, 4, 4, 5,
+                                                     7, 2, 6, 5, 7, 6, 7, 7 };
+            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+#       endif
+        } else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r=0;
+            _BitScanForward( &r, (U32)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctz((U32)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+                                                     3, 2, 2, 1, 3, 2, 0, 1,
+                                                     3, 3, 1, 2, 2, 2, 2, 0,
+                                                     3, 1, 2, 0, 1, 0, 1, 1 };
+            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+#       endif
+        }
+    } else {  /* Big Endian CPU */
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanReverse64( &r, val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clzll(val) >> 3);
+#       else
+            unsigned r;
+            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
+            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+        } else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r = 0;
+            _BitScanReverse( &r, (unsigned long)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clz((U32)val) >> 3);
+#       else
+            unsigned r;
+            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+    }   }
+}
+
+// From lib/compress/zstd_compress.c
+static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *const pInLimit) {
+    const BYTE * const pStart = pIn;
+    const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
+
+    while (pIn < pInLoopLimit) {
+        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+        if (!diff) {
+          pIn += sizeof(size_t);
+          pMatch += sizeof(size_t);
+          continue;
+        }
+        pIn += ZSTD_NbCommonBytes(diff);
+        return (size_t)(pIn - pStart);
+    }
+
+    if (MEM_64bits()) {
+      if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
+        pIn += 4;
+        pMatch += 4;
+      }
+    }
+    if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
+      pIn += 2;
+      pMatch += 2;
+    }
+    if ((pIn < pInLimit) && (*pMatch == *pIn)) {
+      pIn++;
+    }
+    return (size_t)(pIn - pStart);
+}
+
+/**
+ * Count number of bytes that match backwards before pIn and pMatch.
+ *
+ * We count only bytes where pMatch > pBaes and pIn > pAnchor.
+ */
+U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+                        const BYTE *pMatch, const BYTE *pBase) {
+  U32 matchLength = 0;
+  while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
+    pIn--;
+    pMatch--;
+    matchLength++;
+  }
+  return matchLength;
+}
+
+/**
+ * Returns a pointer to the entry in the hash table matching the hash and
+ * checksum with the "longest match length" as defined below. The forward and
+ * backward match lengths are written to *pForwardMatchLength and
+ * *pBackwardMatchLength.
+ *
+ * The match length is defined based on cctx->ip and the entry's offset.
+ * The forward match is computed from cctx->ip and entry->offset + cctx->ibase.
+ * The backward match is computed backwards from cctx->ip and
+ * cctx->ibase only if the forward match is longer than LDM_MIN_MATCH_LENGTH.
+ *
+ */
+LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
+                                 const hash_t hash,
+                                 const U32 checksum,
+                                 U32 *pForwardMatchLength,
+                                 U32 *pBackwardMatchLength) {
+  LDM_hashTable *table = cctx->hashTable;
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  LDM_hashEntry *cur = bucket;
+  LDM_hashEntry *bestEntry = NULL;
+  U32 bestMatchLength = 0;
+  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
+    const BYTE *pMatch = cur->offset + cctx->ibase;
+
+    // Check checksum for faster check.
+    if (cur->checksum == checksum &&
+        cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
+      U32 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
+      U32 backwardMatchLength, totalMatchLength;
+
+      // For speed.
+      if (forwardMatchLength < LDM_MIN_MATCH_LENGTH) {
+        continue;
+      }
+
+      backwardMatchLength =
+          countBackwardsMatch(cctx->ip, cctx->anchor,
+                              cur->offset + cctx->ibase,
+                              cctx->ibase);
+
+      totalMatchLength = forwardMatchLength + backwardMatchLength;
+
+      if (totalMatchLength >= bestMatchLength) {
+        bestMatchLength = totalMatchLength;
+        *pForwardMatchLength = forwardMatchLength;
+        *pBackwardMatchLength = backwardMatchLength;
+
+        bestEntry = cur;
+#ifdef ZSTD_SKIP
+        return cur;
+#endif
+      }
+    }
+  }
+  if (bestEntry != NULL) {
+    return bestEntry;
+  }
+  return NULL;
+}
+
+void HASH_insert(LDM_hashTable *table,
+                 const hash_t hash, const LDM_hashEntry entry) {
+  *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
+  table->bucketOffsets[hash]++;
+  table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
+}
+
+U32 HASH_getSize(const LDM_hashTable *table) {
+  return table->numBuckets;
+}
+
+void HASH_destroyTable(LDM_hashTable *table) {
+  free(table->entries);
+  free(table->bucketOffsets);
+  free(table);
+}
+
+void HASH_outputTableOccupancy(const LDM_hashTable *table) {
+  U32 ctr = 0;
+  LDM_hashEntry *cur = table->entries;
+  LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
+  for (; cur < end; ++cur) {
+    if (cur->offset == 0) {
+      ctr++;
+    }
+  }
+
+  printf("Num buckets, bucket size: %d, %d\n",
+         table->numBuckets, HASH_BUCKET_SIZE);
+  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
+         table->numEntries, ctr,
+         100.0 * (double)(ctr) / table->numEntries);
+}
+
+// TODO: This can be done more efficiently (but it is not that important as it
+// is only used for computing stats).
+static int intLog2(U32 x) {
+  int ret = 0;
+  while (x >>= 1) {
+    ret++;
+  }
+  return ret;
+}
+
+// Maybe we would eventually prefer to have linear rather than
+// exponential buckets.
+/**
+void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
+  U32 i = 0;
+  int buckets[32] = { 0 };
+
+  printf("\n");
+  printf("Hash table histogram\n");
+  for (; i < HASH_getSize(cctx->hashTable); i++) {
+    int offset = (cctx->ip - cctx->ibase) -
+                 HASH_getEntryFromHash(cctx->hashTable, i)->offset;
+    buckets[intLog2(offset)]++;
+  }
+
+  i = 0;
+  for (; i < 32; i++) {
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           buckets[i],
+           100.0 * (double) buckets[i] /
+                   (double) HASH_getSize(cctx->hashTable));
+  }
+  printf("\n");
+}
+*/
+
+void LDM_printCompressStats(const LDM_compressStats *stats) {
+  int i = 0;
+  printf("=====================\n");
+  printf("Compression statistics\n");
+  printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
+          stats->windowSizeLog, stats->hashTableSizeLog);
+  printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
+          stats->numMatches,
+          stats->totalMatchLength,
+          100.0 * (double)stats->totalMatchLength /
+              (double)(stats->totalMatchLength + stats->totalLiteralLength));
+  printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
+                                         (double)stats->numMatches);
+  printf("avg literal length, total literalLength: %.1f, %llu\n",
+         ((double)stats->totalLiteralLength) / (double)stats->numMatches,
+         stats->totalLiteralLength);
+  printf("avg offset length: %.1f\n",
+         ((double)stats->totalOffset) / (double)stats->numMatches);
+  printf("min offset, max offset: %u, %u\n",
+         stats->minOffset, stats->maxOffset);
+
+  printf("\n");
+  printf("offset histogram: offset, num matches, %% of matches\n");
+
+  for (; i <= intLog2(stats->maxOffset); i++) {
+    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+           stats->offsetHistogram[i],
+           100.0 * (double) stats->offsetHistogram[i] /
+                   (double) stats->numMatches);
+  }
+  printf("\n");
+  printf("=====================\n");
+}
+
+int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
+  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
+  const BYTE *curIn = pIn;
+  const BYTE *curMatch = pMatch;
+
+  if (pIn - pMatch > LDM_WINDOW_SIZE) {
+    return 0;
+  }
+
+  for (; lengthLeft >= 4; lengthLeft -= 4) {
+    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
+      return 0;
+    }
+    curIn += 4;
+    curMatch += 4;
+  }
+  return 1;
+}
+
+#if 0
+hash_t HASH_hashU32(U32 value) {
+  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
+}
+#endif
+
+/**
+ * Convert a sum computed from getChecksum to a hash value in the range
+ * of the hash table.
+ */
+#if 0
+static hash_t checksumToHash(U32 sum) {
+  return HASH_hashU32(sum);
+}
+#endif
+
+// Upper LDM_HASH_LOG bits.
+static hash_t checksumToHash(U64 sum) {
+  return sum >> (64 - LDM_HASHLOG);
+}
+
+// 32 bits after LDM_HASH_LOG bits.
+static U32 checksumFromHfHash(U64 hfHash) {
+  return (hfHash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
+}
+
+#if 0
+/**
+ * Computes a checksum based on rsync's checksum.
+ *
+ * a(k,l) = \sum_{i = k}^l x_i (mod M)
+ * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
+ * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
+  */
+static U32 getChecksum(const BYTE *buf, U32 len) {
+  U32 i;
+  U32 s1, s2;
+
+  s1 = s2 = 0;
+  for (i = 0; i < (len - 4); i += 4) {
+    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
+          (2 * buf[i + 2]) + (buf[i + 3]) +
+          (10 * CHECKSUM_CHAR_OFFSET);
+    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
+          + (4 * CHECKSUM_CHAR_OFFSET);
+
+  }
+  for(; i < len; i++) {
+    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
+    s2 += s1;
+  }
+  return (s1 & 0xffff) + (s2 << 16);
+}
+#endif
+
+static U64 getChecksum(const BYTE *buf, U32 len) {
+  static const U64 prime8bytes = 11400714785074694791ULL;
+
+//  static const U64 prime8bytes = 5;
+  U64 ret = 0;
+  U32 i;
+  for (i = 0; i < len; i++) {
+    ret *= prime8bytes;
+    ret += buf[i] + CHECKSUM_CHAR_OFFSET;
+//    printf("HERE %llu\n", ret);
+  }
+  return ret;
+
+}
+
+#if 0
+/**
+ * Update a checksum computed from getChecksum(data, len).
+ *
+ * The checksum can be updated along its ends as follows:
+ * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
+ * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
+ *
+ * Thus toRemove should correspond to data[0].
+ */
+static U32 updateChecksum(U32 sum, U32 len,
+                          BYTE toRemove, BYTE toAdd) {
+  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
+  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
+
+  return (s1 & 0xffff) + (s2 << 16);
+}
+#endif
+
+static U64 ipow(U64 base, U64 exp) {
+  U64 ret = 1;
+  while (exp) {
+    if (exp & 1) {
+      ret *= base;
+    }
+    exp >>= 1;
+    base *= base;
+  }
+  return ret;
+}
+
+static U64 updateChecksum(U64 sum, U32 len,
+                          BYTE toRemove, BYTE toAdd) {
+  // TODO: deduplicate.
+  static const U64 prime8bytes = 11400714785074694791ULL;
+//  static const U64 prime8bytes = 5;
+  sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
+          ipow(prime8bytes, len - 1));
+  sum *= prime8bytes;
+  sum += toAdd + CHECKSUM_CHAR_OFFSET;
+  return sum;
+}
+
+/**
+ * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
+ * based on cctx->lastSum and cctx->lastPosHashed.
+ *
+ * This uses a rolling hash and requires that the last position hashed
+ * corresponds to cctx->nextIp - step.
+ */
+static void setNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  U32 check;
+  if ((cctx->nextIp - cctx->ibase != 1) &&
+      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
+    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
+            cctx->DEBUG_setNextHash - cctx->ibase);
+  }
+
+  cctx->DEBUG_setNextHash = cctx->nextIp;
+#endif
+
+//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
+  cctx->nextSum = updateChecksum(
+      cctx->lastSum, LDM_HASH_LENGTH,
+      cctx->lastPosHashed[0],
+      cctx->lastPosHashed[LDM_HASH_LENGTH]);
+  cctx->nextPosHashed = cctx->nextIp;
+#if 0
+  cctx->nextHash = checksumToHash(cctx->nextSum);
+#endif
+
+
+#if LDM_LAG
+//  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
+  if (cctx->ip - cctx->ibase > LDM_LAG) {
+    cctx->lagSum = updateChecksum(
+      cctx->lagSum, LDM_HASH_LENGTH,
+      cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
+    cctx->lagIp++;
+#if 0
+    cctx->lagHash = checksumToHash(cctx->lagSum);
+#endif
+  }
+#endif
+
+#ifdef RUN_CHECKS
+  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
+
+  if (check != cctx->nextSum) {
+    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
+  }
+
+  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
+    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
+            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
+            cctx->ip - cctx->ibase);
+  }
+#endif
+}
+
+static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hfHash) {
+  // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
+  // Note: this works only when cctx->step is 1.
+  U32 hash = checksumToHash(hfHash);
+  U32 sum = checksumFromHfHash(hfHash);
+//  printf("TMP %u %u %llu\n", hash, sum, hfHash);
+
+  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+
+#if LDM_LAG
+    // TODO: off by 1, but whatever
+    if (cctx->lagIp - cctx->ibase > 0) {
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
+      HASH_insert(cctx->hashTable, cctx->lagHash, entry);
+    } else {
+      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+      HASH_insert(cctx->hashTable, hash, entry);
+    }
+#else
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+    HASH_insert(cctx->hashTable, hash, entry);
+#endif
+  }
+
+  cctx->lastPosHashed = cctx->ip;
+#if 0
+  cctx->lastHash = hash;
+#endif
+  cctx->lastSum = hfHash;
+}
+
+/**
+ * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
+ * fields from the "next" fields.
+ *
+ * This requires that cctx->ip == cctx->nextPosHashed.
+ */
+static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
+#ifdef RUN_CHECKS
+  if (cctx->ip != cctx->nextPosHashed) {
+    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
+           cctx->ip - cctx->ibase);
+  }
+#endif
+#if 0
+  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
+#endif
+  putHashOfCurrentPositionFromHash(cctx, cctx->nextSum);
+}
+
+/**
+ * Insert hash of the current position into the hash table.
+ */
+static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
+  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
+#if 0
+  hash_t hash = checksumToHash(sum);
+#endif
+
+#ifdef RUN_CHECKS
+  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
+    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
+           cctx->ip - cctx->ibase);
+  }
+#endif
+#if 0
+  putHashOfCurrentPositionFromHash(cctx, hash, sum);
+#endif
+  putHashOfCurrentPositionFromHash(cctx, sum);
+}
+
+U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+                         const BYTE *pInLimit) {
+  const BYTE * const pStart = pIn;
+  while (pIn < pInLimit - 1) {
+    BYTE const diff = (*pMatch) ^ *(pIn);
+    if (!diff) {
+      pIn++;
+      pMatch++;
+      continue;
+    }
+    return (U32)(pIn - pStart);
+  }
+  return (U32)(pIn - pStart);
+}
+
+void LDM_outputConfiguration(void) {
+  printf("=====================\n");
+  printf("Configuration\n");
+  printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("Min match, hash length: %d, %d\n",
+         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
+  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
+  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  printf("LDM_LAG %d\n", LDM_LAG);
+  printf("=====================\n");
+}
+
+void LDM_readHeader(const void *src, U64 *compressedSize,
+                    U64 *decompressedSize) {
+  const BYTE *ip = (const BYTE *)src;
+  *compressedSize = MEM_readLE64(ip);
+  ip += sizeof(U64);
+  *decompressedSize = MEM_readLE64(ip);
+  // ip += sizeof(U64);
+}
+
+void LDM_initializeCCtx(LDM_CCtx *cctx,
+                        const void *src, size_t srcSize,
+                        void *dst, size_t maxDstSize) {
+  cctx->isize = srcSize;
+  cctx->maxOSize = maxDstSize;
+
+  cctx->ibase = (const BYTE *)src;
+  cctx->ip = cctx->ibase;
+  cctx->iend = cctx->ibase + srcSize;
+
+  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
+  cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
+
+  cctx->obase = (BYTE *)dst;
+  cctx->op = (BYTE *)dst;
+
+  cctx->anchor = cctx->ibase;
+
+  memset(&(cctx->stats), 0, sizeof(cctx->stats));
+  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64);
+
+  cctx->stats.minOffset = UINT_MAX;
+  cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
+  cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
+
+
+  cctx->lastPosHashed = NULL;
+
+  cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
+  cctx->nextIp = cctx->ip + cctx->step;
+  cctx->nextPosHashed = 0;
+
+  cctx->DEBUG_setNextHash = 0;
+}
+
+void LDM_destroyCCtx(LDM_CCtx *cctx) {
+  HASH_destroyTable(cctx->hashTable);
+}
+
+/**
+ * Finds the "best" match.
+ *
+ * Returns 0 if successful and 1 otherwise (i.e. no match can be found
+ * in the remaining input that is long enough).
+ *
+ * forwardMatchLength contains the forward length of the match.
+ */
+static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
+                             U32 *forwardMatchLength, U32 *backwardMatchLength) {
+
+  LDM_hashEntry *entry = NULL;
+  cctx->nextIp = cctx->ip + cctx->step;
+
+  while (entry == NULL) {
+    hash_t h;
+    U64 hash;
+    U32 sum;
+    setNextHash(cctx);
+#if 0
+    h = cctx->nextHash;
+    sum = cctx->nextSum;
+#endif
+    hash = cctx->nextSum;
+    h = checksumToHash(hash);
+    sum = checksumFromHfHash(hash);
+
+    cctx->ip = cctx->nextIp;
+    cctx->nextIp += cctx->step;
+
+    if (cctx->ip > cctx->imatchLimit) {
+      return 1;
+    }
+
+    entry = HASH_getBestEntry(cctx, h, sum,
+                              forwardMatchLength, backwardMatchLength);
+
+    if (entry != NULL) {
+      *match = entry->offset + cctx->ibase;
+    }
+    putHashOfCurrentPositionFromHash(cctx, hash);
+  }
+  setNextHash(cctx);
+  return 0;
+}
+
+void LDM_encodeLiteralLengthAndLiterals(
+    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+  /* Encode the literal length. */
+  if (literalLength >= RUN_MASK) {
+    int len = (int)literalLength - RUN_MASK;
+    *pToken = (RUN_MASK << ML_BITS);
+    for (; len >= 255; len -= 255) {
+      *(cctx->op)++ = 255;
+    }
+    *(cctx->op)++ = (BYTE)len;
+  } else {
+    *pToken = (BYTE)(literalLength << ML_BITS);
+  }
+
+  /* Encode the literals. */
+  memcpy(cctx->op, cctx->anchor, literalLength);
+  cctx->op += literalLength;
+}
+
+void LDM_outputBlock(LDM_CCtx *cctx,
+                     const U32 literalLength,
+                     const U32 offset,
+                     const U32 matchLength) {
+  BYTE *pToken = cctx->op++;
+
+  /* Encode the literal length and literals. */
+  LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
+
+  /* Encode the offset. */
+  MEM_write32(cctx->op, offset);
+  cctx->op += LDM_OFFSET_SIZE;
+
+  /* Encode the match length. */
+  if (matchLength >= ML_MASK) {
+    unsigned matchLengthRemaining = matchLength;
+    *pToken += ML_MASK;
+    matchLengthRemaining -= ML_MASK;
+    MEM_write32(cctx->op, 0xFFFFFFFF);
+    while (matchLengthRemaining >= 4*0xFF) {
+      cctx->op += 4;
+      MEM_write32(cctx->op, 0xffffffff);
+      matchLengthRemaining -= 4*0xFF;
+    }
+    cctx->op += matchLengthRemaining / 255;
+    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
+  } else {
+    *pToken += (BYTE)(matchLength);
+  }
+}
+
+// TODO: maxDstSize is unused. This function may seg fault when writing
+// beyond the size of dst, as it does not check maxDstSize. Writing to
+// a buffer and performing checks is a possible solution.
+//
+// This is based upon lz4.
+size_t LDM_compress(const void *src, size_t srcSize,
+                    void *dst, size_t maxDstSize) {
+  LDM_CCtx cctx;
+  const BYTE *match = NULL;
+  U32 forwardMatchLength = 0;
+  U32 backwardsMatchLength = 0;
+
+  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+  LDM_outputConfiguration();
+
+  /* Hash the first position and put it into the hash table. */
+  LDM_putHashOfCurrentPosition(&cctx);
+
+#if LDM_LAG
+  cctx.lagIp = cctx.ip;
+//  cctx.lagHash = cctx.lastHash;
+  cctx.lagSum = cctx.lastSum;
+#endif
+  /**
+   * Find a match.
+   * If no more matches can be found (i.e. the length of the remaining input
+   * is less than the minimum match length), then stop searching for matches
+   * and encode the final literals.
+   */
+  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
+         &backwardsMatchLength) == 0) {
+#ifdef COMPUTE_STATS
+    cctx.stats.numMatches++;
+#endif
+
+     cctx.ip -= backwardsMatchLength;
+     match -= backwardsMatchLength;
+
+    /**
+     * Write current block (literals, literal length, match offset, match
+     * length) and update pointers and hashes.
+     */
+    {
+      const U32 literalLength = cctx.ip - cctx.anchor;
+      const U32 offset = cctx.ip - match;
+      const U32 matchLength = forwardMatchLength +
+                              backwardsMatchLength -
+                              LDM_MIN_MATCH_LENGTH;
+
+      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
+
+#ifdef COMPUTE_STATS
+      cctx.stats.totalLiteralLength += literalLength;
+      cctx.stats.totalOffset += offset;
+      cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
+      cctx.stats.minOffset =
+          offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
+      cctx.stats.maxOffset =
+          offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
+      cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
+#endif
+
+      // Move ip to end of block, inserting hashes at each position.
+      cctx.nextIp = cctx.ip + cctx.step;
+      while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
+                       matchLength + literalLength) {
+        if (cctx.ip > cctx.lastPosHashed) {
+          // TODO: Simplify.
+          LDM_updateLastHashFromNextHash(&cctx);
+          setNextHash(&cctx);
+        }
+        cctx.ip++;
+        cctx.nextIp++;
+      }
+    }
+
+    // Set start of next block to current input pointer.
+    cctx.anchor = cctx.ip;
+    LDM_updateLastHashFromNextHash(&cctx);
+  }
+
+  // HASH_outputTableOffsetHistogram(&cctx);
+
+  /* Encode the last literals (no more matches). */
+  {
+    const U32 lastRun = cctx.iend - cctx.anchor;
+    BYTE *pToken = cctx.op++;
+    LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
+  }
+
+#ifdef COMPUTE_STATS
+  LDM_printCompressStats(&cctx.stats);
+  HASH_outputTableOccupancy(cctx.hashTable);
+#endif
+
+  {
+    const size_t ret = cctx.op - cctx.obase;
+    LDM_destroyCCtx(&cctx);
+    return ret;
+  }
+}
+
+struct LDM_DCtx {
+  size_t compressedSize;
+  size_t maxDecompressedSize;
+
+  const BYTE *ibase;   /* Base of input */
+  const BYTE *ip;      /* Current input position */
+  const BYTE *iend;    /* End of source */
+
+  const BYTE *obase;   /* Base of output */
+  BYTE *op;            /* Current output position */
+  const BYTE *oend;    /* End of output */
+};
+
+void LDM_initializeDCtx(LDM_DCtx *dctx,
+                        const void *src, size_t compressedSize,
+                        void *dst, size_t maxDecompressedSize) {
+  dctx->compressedSize = compressedSize;
+  dctx->maxDecompressedSize = maxDecompressedSize;
+
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + dctx->compressedSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + dctx->maxDecompressedSize;
+}
+
+size_t LDM_decompress(const void *src, size_t compressedSize,
+                      void *dst, size_t maxDecompressedSize) {
+  LDM_DCtx dctx;
+  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
+
+  while (dctx.ip < dctx.iend) {
+    BYTE *cpy;
+    const BYTE *match;
+    size_t length, offset;
+
+    /* Get the literal length. */
+    const unsigned token = *(dctx.ip)++;
+    if ((length = (token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+
+    /* Copy the literals. */
+    cpy = dctx.op + length;
+    memcpy(dctx.op, dctx.ip, length);
+    dctx.ip += length;
+    dctx.op = cpy;
+
+    //TODO : dynamic offset size
+    offset = MEM_read32(dctx.ip);
+    dctx.ip += LDM_OFFSET_SIZE;
+    match = dctx.op - offset;
+
+    /* Get the match length. */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+    length += LDM_MIN_MATCH_LENGTH;
+
+    /* Copy match. */
+    cpy = dctx.op + length;
+
+    // Inefficient for now.
+    while (match < cpy - offset && dctx.op < dctx.oend) {
+      *(dctx.op)++ = *match++;
+    }
+  }
+  return dctx.op - (BYTE *)dst;
+}
+
+// TODO: implement and test hash function
+void LDM_test(const BYTE *src) {
+  const U32 diff = 100;
+  const BYTE *pCur = src + diff;
+  U64 checksum = getChecksum(pCur, LDM_HASH_LENGTH);
+
+  for (; pCur < src + diff + 60; ++pCur) {
+    U64 nextSum = getChecksum(pCur + 1, LDM_HASH_LENGTH);
+    U64 updateSum = updateChecksum(checksum, LDM_HASH_LENGTH,
+                                   pCur[0], pCur[LDM_HASH_LENGTH]);
+    checksum = nextSum;
+    printf("%llu %llu\n", nextSum, updateSum);
+  }
+}
+
+
diff --git a/contrib/long_distance_matching/ldm_with_table.c b/contrib/long_distance_matching/ldm_with_table.c
index 813ead6a..c727616a 100644
--- a/contrib/long_distance_matching/ldm_with_table.c
+++ b/contrib/long_distance_matching/ldm_with_table.c
@@ -29,7 +29,7 @@
 #define CHECKSUM_CHAR_OFFSET 10
 
 // Take first match only.
-#define ZSTD_SKIP
+//#define ZSTD_SKIP
 
 //#define RUN_CHECKS
 
@@ -292,8 +292,7 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
 
       totalMatchLength = forwardMatchLength + backwardMatchLength;
 
-      if (totalMatchLength >= bestMatchLength &&
-          totalMatchLength >= LDM_MIN_MATCH_LENGTH) {
+      if (totalMatchLength >= bestMatchLength) {
         bestMatchLength = totalMatchLength;
         *pForwardMatchLength = forwardMatchLength;
         *pBackwardMatchLength = backwardMatchLength;
@@ -305,7 +304,7 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
       }
     }
   }
-  if (bestEntry != NULL && bestMatchLength > LDM_MIN_MATCH_LENGTH) {
+  if (bestEntry != NULL) {
     return bestEntry;
   }
   return NULL;
@@ -951,23 +950,8 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
 }
 
 // TODO: implement and test hash function
-void LDM_test(void) {
+void LDM_test(const BYTE *src) {
+  (void)src;
 }
 
-/*
-void LDM_test(const void *src, size_t srcSize,
-              void *dst, size_t maxDstSize) {
-  const BYTE *ip = (const BYTE *)src + 1125;
-  U32 sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
-  U32 sum2;
-  ++ip;
-  for (; ip < (const BYTE *)src + 1125 + 100; ip++) {
-    sum2 = updateChecksum(sum, LDM_HASH_LENGTH,
-                                 ip[-1], ip[LDM_HASH_LENGTH - 1]);
-    sum = getChecksum((const char *)ip, LDM_HASH_LENGTH);
-    printf("TEST HASH: %zu %u %u\n", ip - (const BYTE *)src, sum, sum2);
-  }
-}
-*/
-
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 96db0c22..3582d5a2 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -13,13 +13,13 @@
 #include "zstd.h"
 
 #define DEBUG
-#define TEST
+//#define TEST
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
  *
  * TODO: This might seg fault if the compressed size is > the decompress
- * size due to the mmapping and output file size allocated to be the input size.
+ * size due to the mmapping and output file size allocated to be the input size
  * The compress function should check before writing or buffer writes.
  */
 static int compress(const char *fname, const char *oname) {
@@ -69,6 +69,11 @@ static int compress(const char *fname, const char *oname) {
       perror("mmap error for output");
       return 1;
   }
+
+#ifdef TEST
+  LDM_test((const BYTE *)src);
+#endif
+
   gettimeofday(&tv1, NULL);
 
   compressedSize = LDM_HEADER_SIZE +
@@ -251,8 +256,5 @@ int main(int argc, const char *argv[]) {
   /* verify */
   verify(inpFilename, decFilename);
 
-#ifdef TEST
-  LDM_test();
-#endif
   return 0;
 }

From 0b8fb1703b39ae1f07aea1eaac46b97372d9238d Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 20 Jul 2017 16:51:01 -0700
Subject: [PATCH 085/100] Experiment with 64-bit hash insertion policy

---
 contrib/long_distance_matching/Makefile       |  6 +--
 .../circular_buffer_table.c                   | 10 ++---
 contrib/long_distance_matching/ldm.c          | 10 ++---
 contrib/long_distance_matching/ldm.h          |  8 ++--
 .../{ldm_hf_test.c => ldm_64_hash.c}          | 38 ++++++++++---------
 .../long_distance_matching/ldm_hashtable.h    |  4 +-
 .../long_distance_matching/ldm_with_table.c   | 12 +++---
 contrib/long_distance_matching/main-ldm.c     |  7 ++--
 8 files changed, 48 insertions(+), 47 deletions(-)
 rename contrib/long_distance_matching/{ldm_hf_test.c => ldm_64_hash.c} (97%)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 5119f464..9dc33fae 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,7 +25,7 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-circular-buffer main-integrated main-hf
+all: main-circular-buffer main-integrated main-64
 
 #main-basic : basic_table.c ldm.c main-ldm.c
 #	      $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -33,7 +33,7 @@ all: main-circular-buffer main-integrated main-hf
 main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-hf: ldm_hf_test.c main-ldm.c
+main-64: ldm_64_hash.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 main-integrated: ldm_with_table.c main-ldm.c
@@ -41,6 +41,6 @@ main-integrated: ldm_with_table.c main-ldm.c
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-basic main-circular-buffer  main-integrated main-hf
+	main-basic main-circular-buffer  main-integrated main-64
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index 66107e06..fb6c19d2 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -164,19 +164,19 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
                                  const BYTE *pIn,
                                  const BYTE *pEnd,
                                  const BYTE *pAnchor,
-                                 U32 *pForwardMatchLength,
-                                 U32 *pBackwardMatchLength) {
+                                 U64 *pForwardMatchLength,
+                                 U64 *pBackwardMatchLength) {
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
   LDM_hashEntry *bestEntry = NULL;
-  U32 bestMatchLength = 0;
+  U64 bestMatchLength = 0;
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
     const BYTE *pMatch = cur->offset + table->offsetBase;
 
     // Check checksum for faster check.
     if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
-      U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
-      U32 backwardMatchLength, totalMatchLength;
+      U64 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
+      U64 backwardMatchLength, totalMatchLength;
 
       if (forwardMatchLength < table->minMatchLength) {
         continue;
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index ab2de7c1..b018c475 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -429,7 +429,7 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
  * matchLength contains the forward length of the match.
  */
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
-                             U32 *matchLength, U32 *backwardMatchLength) {
+                             U64 *matchLength, U64 *backwardMatchLength) {
 
   LDM_hashEntry *entry = NULL;
   cctx->nextIp = cctx->ip + cctx->step;
@@ -462,7 +462,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
 }
 
 void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+    LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
   /* Encode the literal length. */
   if (literalLength >= RUN_MASK) {
     int len = (int)literalLength - RUN_MASK;
@@ -481,9 +481,9 @@ void LDM_encodeLiteralLengthAndLiterals(
 }
 
 void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U32 literalLength,
+                     const U64 literalLength,
                      const U32 offset,
-                     const U32 matchLength) {
+                     const U64 matchLength) {
   BYTE *pToken = cctx->op++;
 
   /* Encode the literal length and literals. */
@@ -495,7 +495,7 @@ void LDM_outputBlock(LDM_CCtx *cctx,
 
   /* Encode the match length. */
   if (matchLength >= ML_MASK) {
-    unsigned matchLengthRemaining = matchLength;
+    U64 matchLengthRemaining = matchLength;
     *pToken += ML_MASK;
     matchLengthRemaining -= ML_MASK;
     MEM_write32(cctx->op, 0xFFFFFFFF);
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index c420e60c..83cd3623 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -14,7 +14,7 @@
 // Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
 #define LDM_MEMORY_USAGE 22
-#define HASH_BUCKET_SIZE_LOG 3 // MAX is 4 for now
+#define HASH_BUCKET_SIZE_LOG 1 // MAX is 4 for now
 
 // Defines the lag in inserting elements into the hash table.
 #define LDM_LAG 0
@@ -115,16 +115,16 @@ U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
  * This is followed by literalLength bytes corresponding to the literals.
  */
 void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength);
+    LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength);
 
 /**
  * Write current block (literals, literal length, match offset,
  * match length).
  */
 void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U32 literalLength,
+                     const U64 literalLength,
                      const U32 offset,
-                     const U32 matchLength);
+                     const U64 matchLength);
 
 /**
  * Decompresses src into dst.
diff --git a/contrib/long_distance_matching/ldm_hf_test.c b/contrib/long_distance_matching/ldm_64_hash.c
similarity index 97%
rename from contrib/long_distance_matching/ldm_hf_test.c
rename to contrib/long_distance_matching/ldm_64_hash.c
index 63be82d1..a72c283f 100644
--- a/contrib/long_distance_matching/ldm_hf_test.c
+++ b/contrib/long_distance_matching/ldm_64_hash.c
@@ -241,9 +241,9 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
  *
  * We count only bytes where pMatch > pBaes and pIn > pAnchor.
  */
-U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+U64 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
                         const BYTE *pMatch, const BYTE *pBase) {
-  U32 matchLength = 0;
+  U64 matchLength = 0;
   while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
     pIn--;
     pMatch--;
@@ -267,8 +267,8 @@ U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
 LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
                                  const hash_t hash,
                                  const U32 checksum,
-                                 U32 *pForwardMatchLength,
-                                 U32 *pBackwardMatchLength) {
+                                 U64 *pForwardMatchLength,
+                                 U64 *pBackwardMatchLength) {
   LDM_hashTable *table = cctx->hashTable;
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
@@ -541,7 +541,7 @@ static U64 updateChecksum(U64 sum, U32 len,
                           BYTE toRemove, BYTE toAdd) {
   // TODO: deduplicate.
   static const U64 prime8bytes = 11400714785074694791ULL;
-//  static const U64 prime8bytes = 5;
+
   sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
           ipow(prime8bytes, len - 1));
   sum *= prime8bytes;
@@ -696,11 +696,12 @@ U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
 void LDM_outputConfiguration(void) {
   printf("=====================\n");
   printf("Configuration\n");
-  printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG);
-  printf("Min match, hash length: %d, %d\n",
+  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
          LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
   printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
   printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
   printf("LDM_LAG %d\n", LDM_LAG);
   printf("=====================\n");
 }
@@ -762,7 +763,7 @@ void LDM_destroyCCtx(LDM_CCtx *cctx) {
  * forwardMatchLength contains the forward length of the match.
  */
 static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
-                             U32 *forwardMatchLength, U32 *backwardMatchLength) {
+                             U64 *forwardMatchLength, U64 *backwardMatchLength) {
 
   LDM_hashEntry *entry = NULL;
   cctx->nextIp = cctx->ip + cctx->step;
@@ -800,10 +801,10 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
 }
 
 void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+    LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
   /* Encode the literal length. */
   if (literalLength >= RUN_MASK) {
-    int len = (int)literalLength - RUN_MASK;
+    U64 len = (U64)literalLength - RUN_MASK;
     *pToken = (RUN_MASK << ML_BITS);
     for (; len >= 255; len -= 255) {
       *(cctx->op)++ = 255;
@@ -819,9 +820,9 @@ void LDM_encodeLiteralLengthAndLiterals(
 }
 
 void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U32 literalLength,
+                     const U64 literalLength,
                      const U32 offset,
-                     const U32 matchLength) {
+                     const U64 matchLength) {
   BYTE *pToken = cctx->op++;
 
   /* Encode the literal length and literals. */
@@ -833,7 +834,7 @@ void LDM_outputBlock(LDM_CCtx *cctx,
 
   /* Encode the match length. */
   if (matchLength >= ML_MASK) {
-    unsigned matchLengthRemaining = matchLength;
+    U64 matchLengthRemaining = matchLength;
     *pToken += ML_MASK;
     matchLengthRemaining -= ML_MASK;
     MEM_write32(cctx->op, 0xFFFFFFFF);
@@ -858,8 +859,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
   const BYTE *match = NULL;
-  U32 forwardMatchLength = 0;
-  U32 backwardsMatchLength = 0;
+  U64 forwardMatchLength = 0;
+  U64 backwardsMatchLength = 0;
 
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
   LDM_outputConfiguration();
@@ -892,9 +893,9 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * length) and update pointers and hashes.
      */
     {
-      const U32 literalLength = cctx.ip - cctx.anchor;
+      const U64 literalLength = cctx.ip - cctx.anchor;
       const U32 offset = cctx.ip - match;
-      const U32 matchLength = forwardMatchLength +
+      const U64 matchLength = forwardMatchLength +
                               backwardsMatchLength -
                               LDM_MIN_MATCH_LENGTH;
 
@@ -934,7 +935,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
   /* Encode the last literals (no more matches). */
   {
-    const U32 lastRun = cctx.iend - cctx.anchor;
+    const U64 lastRun = cctx.iend - cctx.anchor;
     BYTE *pToken = cctx.op++;
     LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
   }
@@ -979,6 +980,7 @@ void LDM_initializeDCtx(LDM_DCtx *dctx,
 
 size_t LDM_decompress(const void *src, size_t compressedSize,
                       void *dst, size_t maxDecompressedSize) {
+
   LDM_DCtx dctx;
   LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
 
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index 9d5ba0e2..d59f401e 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -25,8 +25,8 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
                                  const BYTE *pIn,
                                  const BYTE *pEnd,
                                  const BYTE *pAnchor,
-                                 U32 *matchLength,
-                                 U32 *backwardsMatchLength);
+                                 U64 *matchLength,
+                                 U64 *backwardsMatchLength);
 
 hash_t HASH_hashU32(U32 value);
 
diff --git a/contrib/long_distance_matching/ldm_with_table.c b/contrib/long_distance_matching/ldm_with_table.c
index c727616a..babfdf3f 100644
--- a/contrib/long_distance_matching/ldm_with_table.c
+++ b/contrib/long_distance_matching/ldm_with_table.c
@@ -719,10 +719,10 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
 }
 
 void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+    LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
   /* Encode the literal length. */
   if (literalLength >= RUN_MASK) {
-    int len = (int)literalLength - RUN_MASK;
+    U64 len = (U64)literalLength - RUN_MASK;
     *pToken = (RUN_MASK << ML_BITS);
     for (; len >= 255; len -= 255) {
       *(cctx->op)++ = 255;
@@ -738,9 +738,9 @@ void LDM_encodeLiteralLengthAndLiterals(
 }
 
 void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U32 literalLength,
+                     const U64 literalLength,
                      const U32 offset,
-                     const U32 matchLength) {
+                     const U64 matchLength) {
   BYTE *pToken = cctx->op++;
 
   /* Encode the literal length and literals. */
@@ -811,9 +811,9 @@ size_t LDM_compress(const void *src, size_t srcSize,
      * length) and update pointers and hashes.
      */
     {
-      const U32 literalLength = cctx.ip - cctx.anchor;
+      const U64 literalLength = cctx.ip - cctx.anchor;
       const U32 offset = cctx.ip - match;
-      const U32 matchLength = forwardMatchLength +
+      const U64 matchLength = forwardMatchLength +
                               backwardsMatchLength -
                               LDM_MIN_MATCH_LENGTH;
 
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 3582d5a2..b6788c67 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -94,8 +94,8 @@ static int compress(const char *fname, const char *oname) {
   // Truncate file to compressedSize.
   ftruncate(fdout, compressedSize);
 
-  printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
-         (unsigned)statbuf.st_size, (unsigned)compressedSize, oname,
+  printf("%25s : %10lu -> %10lu - %s (%.1f%%)\n", fname,
+         (size_t)statbuf.st_size, (size_t)compressedSize, oname,
          (double)compressedSize / (statbuf.st_size) * 100);
 
   timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
@@ -164,7 +164,7 @@ static int decompress(const char *fname, const char *oname) {
       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
       dst, decompressedSize);
   printf("Ret size out: %zu\n", outSize);
-  ftruncate(fdout, outSize);
+//  ftruncate(fdout, decompressedSize);
 
   close(fdin);
   close(fdout);
@@ -231,7 +231,6 @@ int main(int argc, const char *argv[]) {
 	printf("ldm = [%s]\n", ldmFilename);
 	printf("dec = [%s]\n", decFilename);
 
-
   /* Compress */
   {
     if (compress(inpFilename, ldmFilename)) {

From 1a188fe864c67673621b0b20a9911e3d23ef935a Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Fri, 21 Jul 2017 10:44:39 -0700
Subject: [PATCH 086/100] Fix overflow bug when calculating hash

---
 contrib/long_distance_matching/Makefile      |   4 +-
 contrib/long_distance_matching/ldm.c         |   4 +-
 contrib/long_distance_matching/ldm.h         |  11 +-
 contrib/long_distance_matching/ldm_64_hash.c | 172 ++++---------------
 contrib/long_distance_matching/main-ldm.c    |   5 +-
 5 files changed, 46 insertions(+), 150 deletions(-)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 9dc33fae..16844297 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,7 +25,7 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-circular-buffer main-integrated main-64
+all: main-circular-buffer main-integrated main-64 
 
 #main-basic : basic_table.c ldm.c main-ldm.c
 #	      $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -41,6 +41,6 @@ main-integrated: ldm_with_table.c main-ldm.c
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-basic main-circular-buffer  main-integrated main-64
+	main-circular-buffer  main-integrated main-64
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index b018c475..bfaff1f5 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -520,8 +520,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize) {
   LDM_CCtx cctx;
   const BYTE *match = NULL;
-  U32 forwardMatchLength = 0;
-  U32 backwardsMatchLength = 0;
+  U64 forwardMatchLength = 0;
+  U64 backwardsMatchLength = 0;
 
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
   LDM_outputConfiguration();
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 83cd3623..e2f78697 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -14,7 +14,7 @@
 // Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
 #define LDM_MEMORY_USAGE 22
-#define HASH_BUCKET_SIZE_LOG 1 // MAX is 4 for now
+#define HASH_BUCKET_SIZE_LOG 2 // MAX is 4 for now
 
 // Defines the lag in inserting elements into the hash table.
 #define LDM_LAG 0
@@ -26,7 +26,8 @@
 #define LDM_MIN_MATCH_LENGTH 64
 #define LDM_HASH_LENGTH 64
 
-
+#define TMP_EVICTION
+#define TMP_TAG_INSERT
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
 typedef struct LDM_DCtx LDM_DCtx;
@@ -99,12 +100,6 @@ void LDM_printCompressStats(const LDM_compressStats *stats);
  */
 int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch);
 
-/**
- *  Counts the number of bytes that match from pIn and pMatch,
- *  up to pInLimit.
- */
-U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *pInLimit);
 
 /**
  * Encode the literal length followed by the literals.
diff --git a/contrib/long_distance_matching/ldm_64_hash.c b/contrib/long_distance_matching/ldm_64_hash.c
index a72c283f..7a813534 100644
--- a/contrib/long_distance_matching/ldm_64_hash.c
+++ b/contrib/long_distance_matching/ldm_64_hash.c
@@ -89,21 +89,16 @@ struct LDM_CCtx {
 
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  U32 lastSum;
+  U64 lastSum;
 
   const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
   const BYTE *nextPosHashed;
   U64 nextSum;
 
-//  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
-//  U32 nextSum;
-
   unsigned step;                      // ip step, should be 1.
 
   const BYTE *lagIp;
   U64 lagSum;
-//  hash_t lagHash;
-//  U32 lagSum;
 
   U64 numHashInserts;
   // DEBUG
@@ -273,15 +268,15 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
   LDM_hashEntry *bestEntry = NULL;
-  U32 bestMatchLength = 0;
+  U64 bestMatchLength = 0;
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
     const BYTE *pMatch = cur->offset + cctx->ibase;
 
     // Check checksum for faster check.
     if (cur->checksum == checksum &&
         cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
-      U32 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
-      U32 backwardMatchLength, totalMatchLength;
+      U64 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
+      U64 backwardMatchLength, totalMatchLength;
 
       // For speed.
       if (forwardMatchLength < LDM_MIN_MATCH_LENGTH) {
@@ -313,6 +308,8 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
   return NULL;
 }
 
+#ifdef TMP_EVICTION
+
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
   *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
@@ -320,6 +317,17 @@ void HASH_insert(LDM_hashTable *table,
   table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
 }
 
+#else
+
+void HASH_insert(LDM_hashTable *table,
+                 const hash_t hash, const LDM_hashEntry entry) {
+  *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
+  table->bucketOffsets[hash]++;
+  table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
+}
+#endif // TMP_EVICTION
+
+
 U32 HASH_getSize(const LDM_hashTable *table) {
   return table->numBuckets;
 }
@@ -349,7 +357,7 @@ void HASH_outputTableOccupancy(const LDM_hashTable *table) {
 
 // TODO: This can be done more efficiently (but it is not that important as it
 // is only used for computing stats).
-static int intLog2(U32 x) {
+static int intLog2(U64 x) {
   int ret = 0;
   while (x >>= 1) {
     ret++;
@@ -357,32 +365,6 @@ static int intLog2(U32 x) {
   return ret;
 }
 
-// Maybe we would eventually prefer to have linear rather than
-// exponential buckets.
-/**
-void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
-  U32 i = 0;
-  int buckets[32] = { 0 };
-
-  printf("\n");
-  printf("Hash table histogram\n");
-  for (; i < HASH_getSize(cctx->hashTable); i++) {
-    int offset = (cctx->ip - cctx->ibase) -
-                 HASH_getEntryFromHash(cctx->hashTable, i)->offset;
-    buckets[intLog2(offset)]++;
-  }
-
-  i = 0;
-  for (; i < 32; i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
-           buckets[i],
-           100.0 * (double) buckets[i] /
-                   (double) HASH_getSize(cctx->hashTable));
-  }
-  printf("\n");
-}
-*/
-
 void LDM_printCompressStats(const LDM_compressStats *stats) {
   int i = 0;
   printf("=====================\n");
@@ -436,22 +418,6 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
   return 1;
 }
 
-#if 0
-hash_t HASH_hashU32(U32 value) {
-  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
-}
-#endif
-
-/**
- * Convert a sum computed from getChecksum to a hash value in the range
- * of the hash table.
- */
-#if 0
-static hash_t checksumToHash(U32 sum) {
-  return HASH_hashU32(sum);
-}
-#endif
-
 // Upper LDM_HASH_LOG bits.
 static hash_t checksumToHash(U64 sum) {
   return sum >> (64 - LDM_HASHLOG);
@@ -462,69 +428,19 @@ static U32 checksumFromHfHash(U64 hfHash) {
   return (hfHash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
 }
 
-#if 0
-/**
- * Computes a checksum based on rsync's checksum.
- *
- * a(k,l) = \sum_{i = k}^l x_i (mod M)
- * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
- * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
-  */
-static U32 getChecksum(const BYTE *buf, U32 len) {
-  U32 i;
-  U32 s1, s2;
-
-  s1 = s2 = 0;
-  for (i = 0; i < (len - 4); i += 4) {
-    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]) +
-          (10 * CHECKSUM_CHAR_OFFSET);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
-          + (4 * CHECKSUM_CHAR_OFFSET);
-
-  }
-  for(; i < len; i++) {
-    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
-    s2 += s1;
-  }
-  return (s1 & 0xffff) + (s2 << 16);
-}
-#endif
-
 static U64 getChecksum(const BYTE *buf, U32 len) {
   static const U64 prime8bytes = 11400714785074694791ULL;
 
-//  static const U64 prime8bytes = 5;
   U64 ret = 0;
   U32 i;
   for (i = 0; i < len; i++) {
     ret *= prime8bytes;
     ret += buf[i] + CHECKSUM_CHAR_OFFSET;
-//    printf("HERE %llu\n", ret);
   }
   return ret;
 
 }
 
-#if 0
-/**
- * Update a checksum computed from getChecksum(data, len).
- *
- * The checksum can be updated along its ends as follows:
- * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
- * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
- *
- * Thus toRemove should correspond to data[0].
- */
-static U32 updateChecksum(U32 sum, U32 len,
-                          BYTE toRemove, BYTE toAdd) {
-  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
-  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
-
-  return (s1 & 0xffff) + (s2 << 16);
-}
-#endif
-
 static U64 ipow(U64 base, U64 exp) {
   U64 ret = 1;
   while (exp) {
@@ -542,6 +458,8 @@ static U64 updateChecksum(U64 sum, U32 len,
   // TODO: deduplicate.
   static const U64 prime8bytes = 11400714785074694791ULL;
 
+  // TODO: relying on compiler optimization here.
+  // The exponential can be calculated explicitly.
   sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
           ipow(prime8bytes, len - 1));
   sum *= prime8bytes;
@@ -558,7 +476,7 @@ static U64 updateChecksum(U64 sum, U32 len,
  */
 static void setNextHash(LDM_CCtx *cctx) {
 #ifdef RUN_CHECKS
-  U32 check;
+  U64 check;
   if ((cctx->nextIp - cctx->ibase != 1) &&
       (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
     printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
@@ -568,16 +486,11 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->DEBUG_setNextHash = cctx->nextIp;
 #endif
 
-//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
   cctx->nextSum = updateChecksum(
       cctx->lastSum, LDM_HASH_LENGTH,
       cctx->lastPosHashed[0],
       cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
-#if 0
-  cctx->nextHash = checksumToHash(cctx->nextSum);
-#endif
-
 
 #if LDM_LAG
 //  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
@@ -586,9 +499,6 @@ static void setNextHash(LDM_CCtx *cctx) {
       cctx->lagSum, LDM_HASH_LENGTH,
       cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
     cctx->lagIp++;
-#if 0
-    cctx->lagHash = checksumToHash(cctx->lagSum);
-#endif
   }
 #endif
 
@@ -596,7 +506,7 @@ static void setNextHash(LDM_CCtx *cctx) {
   check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
 
   if (check != cctx->nextSum) {
-    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
+    printf("CHECK: setNextHash failed %llu %llu\n", check, cctx->nextSum);
   }
 
   if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
@@ -610,8 +520,6 @@ static void setNextHash(LDM_CCtx *cctx) {
 static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hfHash) {
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
-  U32 hash = checksumToHash(hfHash);
-  U32 sum = checksumFromHfHash(hfHash);
 //  printf("TMP %u %u %llu\n", hash, sum, hfHash);
 
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
@@ -619,22 +527,26 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hfHash) {
 #if LDM_LAG
     // TODO: off by 1, but whatever
     if (cctx->lagIp - cctx->ibase > 0) {
-      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
-      HASH_insert(cctx->hashTable, cctx->lagHash, entry);
+      U32 hash = checksumToHash(cctx->lagSum);
+      U32 sum = checksumFromHfHash(cctx->lagSum);
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, sum };
+      HASH_insert(cctx->hashTable, hash, entry);
     } else {
+      U32 hash = checksumToHash(hfHash);
+      U32 sum = checksumFromHfHash(hfHash);
+
       const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
       HASH_insert(cctx->hashTable, hash, entry);
     }
 #else
+    U32 hash = checksumToHash(hfHash);
+    U32 sum = checksumFromHfHash(hfHash);
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
     HASH_insert(cctx->hashTable, hash, entry);
 #endif
   }
 
   cctx->lastPosHashed = cctx->ip;
-#if 0
-  cctx->lastHash = hash;
-#endif
   cctx->lastSum = hfHash;
 }
 
@@ -650,9 +562,6 @@ static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
     printf("CHECK failed: updateLastHashFromNextHash %zu\n",
            cctx->ip - cctx->ibase);
   }
-#endif
-#if 0
-  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
 #endif
   putHashOfCurrentPositionFromHash(cctx, cctx->nextSum);
 }
@@ -661,10 +570,7 @@ static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
  * Insert hash of the current position into the hash table.
  */
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
-#if 0
-  hash_t hash = checksumToHash(sum);
-#endif
+  U64 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
 
 #ifdef RUN_CHECKS
   if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
@@ -672,13 +578,11 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
            cctx->ip - cctx->ibase);
   }
 #endif
-#if 0
-  putHashOfCurrentPositionFromHash(cctx, hash, sum);
-#endif
+
   putHashOfCurrentPositionFromHash(cctx, sum);
 }
 
-U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
+U64 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
                          const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
   while (pIn < pInLimit - 1) {
@@ -688,9 +592,9 @@ U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
       pMatch++;
       continue;
     }
-    return (U32)(pIn - pStart);
+    return (U64)(pIn - pStart);
   }
-  return (U32)(pIn - pStart);
+  return (U64)(pIn - pStart);
 }
 
 void LDM_outputConfiguration(void) {
@@ -773,10 +677,6 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     U64 hash;
     U32 sum;
     setNextHash(cctx);
-#if 0
-    h = cctx->nextHash;
-    sum = cctx->nextSum;
-#endif
     hash = cctx->nextSum;
     h = checksumToHash(hash);
     sum = checksumFromHfHash(hash);
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index b6788c67..9769f10e 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -94,9 +94,10 @@ static int compress(const char *fname, const char *oname) {
   // Truncate file to compressedSize.
   ftruncate(fdout, compressedSize);
 
-  printf("%25s : %10lu -> %10lu - %s (%.1f%%)\n", fname,
+  printf("%25s : %10lu -> %10lu - %s (%.2fx --- %.1f%%)\n", fname,
          (size_t)statbuf.st_size, (size_t)compressedSize, oname,
-         (double)compressedSize / (statbuf.st_size) * 100);
+         (statbuf.st_size) / (double)compressedSize,
+         (double)compressedSize / (double)(statbuf.st_size) * 100.0);
 
   timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
               (double) (tv2.tv_sec - tv1.tv_sec),

From eb16da647d38df0f2180c9c335ddadc559624e85 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 24 Jul 2017 10:18:58 -0700
Subject: [PATCH 087/100] Minor clean up

---
 contrib/long_distance_matching/ldm.h         |  13 +-
 contrib/long_distance_matching/ldm_64_hash.c | 199 +++++++++++++++++--
 2 files changed, 188 insertions(+), 24 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index e2f78697..840824c4 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -13,21 +13,26 @@
 // Defines the size of the hash table.
 // Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
-#define LDM_MEMORY_USAGE 22
-#define HASH_BUCKET_SIZE_LOG 2 // MAX is 4 for now
+#define LDM_MEMORY_USAGE 24
+#define HASH_BUCKET_SIZE_LOG 0 // MAX is 4 for now
 
 // Defines the lag in inserting elements into the hash table.
 #define LDM_LAG 0
 
-#define LDM_WINDOW_SIZE_LOG 28
+#define LDM_WINDOW_SIZE_LOG 28 // Max value is 30
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four (and perhaps set to the same value?).
 #define LDM_MIN_MATCH_LENGTH 64
 #define LDM_HASH_LENGTH 64
 
-#define TMP_EVICTION
+// Experimental.
+//:w
+//#define TMP_EVICTION
 #define TMP_TAG_INSERT
+//#define TMP_SIMPLE_LOWER
+//#define TMP_FORCE_HASH_ONLY
+
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
 typedef struct LDM_DCtx LDM_DCtx;
diff --git a/contrib/long_distance_matching/ldm_64_hash.c b/contrib/long_distance_matching/ldm_64_hash.c
index 7a813534..bdbdd199 100644
--- a/contrib/long_distance_matching/ldm_64_hash.c
+++ b/contrib/long_distance_matching/ldm_64_hash.c
@@ -12,7 +12,11 @@
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
 // Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
+#ifdef TMP_FORCE_HASH_ONLY
+  #define HASH_ONLY_EVERY_LOG 7
+#else
+  #define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
+#endif
 #define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
 
 /* Hash table stuff. */
@@ -26,12 +30,15 @@
 
 #define COMPUTE_STATS
 #define OUTPUT_CONFIGURATION
-#define CHECKSUM_CHAR_OFFSET 10
+#define CHECKSUM_CHAR_OFFSET 1
 
 // Take first match only.
 //#define ZSTD_SKIP
 
 //#define RUN_CHECKS
+//
+//
+static const U64 prime8bytes = 11400714785074694791ULL;
 
 /* Hash table stuff */
 
@@ -56,6 +63,14 @@ struct LDM_compressStats {
   U32 numHashInserts;
 
   U32 offsetHistogram[32];
+
+  U64 TMP_hashCount[1 << HASH_ONLY_EVERY_LOG];
+  U64 TMP_totalHashCount;
+
+  U64 TMP_totalInWindow;
+  U64 TMP_totalInserts;
+
+  U64 TMP_matchCount;
 };
 
 typedef struct LDM_hashTable LDM_hashTable;
@@ -311,10 +326,80 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
 #ifdef TMP_EVICTION
 
 void HASH_insert(LDM_hashTable *table,
-                 const hash_t hash, const LDM_hashEntry entry) {
+                 const hash_t hash, const LDM_hashEntry entry,
+                 LDM_CCtx *cctx) {
+  // Overwrite based on part of checksum.
+  /*
+  LDM_hashEntry *toOverwrite =
+    getBucket(table, hash) + table->bucketOffsets[hash];
+  const BYTE *pMatch = toOverwrite->offset + cctx->ibase;
+  if (toOverwrite->offset != 0 &&
+      cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
+    cctx->stats.TMP_totalInWindow++;
+  }
+
+  cctx->stats.TMP_totalInserts++;
+  *(toOverwrite) = entry;
+  */
+
+  /*
+  int i;
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  for (i = 0; i < HASH_BUCKET_SIZE; i++) {
+    if (bucket[i].checksum == entry.checksum) {
+      bucket[i] = entry;
+      cctx->stats.TMP_matchCount++;
+      return;
+    }
+  }
+  */
+
+  // Find entry beyond window size, replace. Else, random.
+  int i;
+  LDM_hashEntry *bucket = getBucket(table, hash);
+  for (i = 0; i < HASH_BUCKET_SIZE; i++) {
+    if (cctx->ip - cctx->ibase - bucket[i].offset > LDM_WINDOW_SIZE) {
+      bucket[i] = entry;
+      return;
+    }
+  }
+
+  i = rand() & (HASH_BUCKET_SIZE - 1);
+  *(bucket + i) = entry;
+
+
+  /**
+   * Sliding buffer style pointer
+   * Keep old entry as temporary. If the old entry is outside the window,
+   * overwrite and we are done.
+   *
+   * Backwards (insert at x):
+   * x, a, b b, c c c c, d d d d d d d d
+   * x, d d d d d d d d, c c c c, b b, a
+   *
+   * Else, find something to evict.
+   * If old entry has more ones, it takes
+   * the next spot. <-- reversed order?
+   *
+   * If window size > LDM_WINDOW_SIZE,
+   * overwrite,
+   *
+   * Insert forwards. If > tag, keep. Else evict.
+   *
+   *
+   *
+   *
+   */
+
+
+  /*
   *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
   table->bucketOffsets[hash]++;
   table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
+  */
+
+//  U16 mask = entry.checksum & (HASH_BUCKET_SIZE - 1);
+//  *(getBucket(table, hash) + mask) = entry;
 }
 
 #else
@@ -348,8 +433,9 @@ void HASH_outputTableOccupancy(const LDM_hashTable *table) {
     }
   }
 
-  printf("Num buckets, bucket size: %d, %d\n",
-         table->numBuckets, HASH_BUCKET_SIZE);
+  // TODO: repeat numBuckets as a check for now.
+  printf("Num buckets, bucket size: %d (2^%d), %d\n",
+         table->numBuckets, LDM_HASHLOG, HASH_BUCKET_SIZE);
   printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
          table->numEntries, ctr,
          100.0 * (double)(ctr) / table->numEntries);
@@ -396,6 +482,24 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
                    (double) stats->numMatches);
   }
   printf("\n");
+#ifdef TMP_TAG_INSERT
+/*
+  printf("Lower bit distribution\n");
+  for (i = 0; i < (1 << HASH_ONLY_EVERY_LOG); i++) {
+    printf("%5d %5llu %6.3f\n", i, stats->TMP_hashCount[i],
+           100.0 * (double) stats->TMP_hashCount[i] /
+                   (double) stats->TMP_totalHashCount);
+  }
+*/
+#endif
+
+#ifdef TMP_EVICTION
+  printf("Evicted something in window: %llu %6.3f\n",
+         stats->TMP_totalInWindow,
+         100.0 * (double)stats->TMP_totalInWindow /
+                 (double)stats->TMP_totalInserts);
+  printf("Match count: %llu\n", stats->TMP_matchCount);
+#endif
   printf("=====================\n");
 }
 
@@ -418,7 +522,7 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
   return 1;
 }
 
-// Upper LDM_HASH_LOG bits.
+// Upper LDM_HASHLOG bits.
 static hash_t checksumToHash(U64 sum) {
   return sum >> (64 - LDM_HASHLOG);
 }
@@ -428,9 +532,30 @@ static U32 checksumFromHfHash(U64 hfHash) {
   return (hfHash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
 }
 
-static U64 getChecksum(const BYTE *buf, U32 len) {
-  static const U64 prime8bytes = 11400714785074694791ULL;
+#ifdef TMP_TAG_INSERT
+static U32 lowerBitsFromHfHash(U64 hfHash) {
+  // The number of bits used so far is LDM_HASHLOG + 32.
+  // So there are 32 - LDM_HASHLOG bits left.
+  // Occasional hashing requires HASH_ONLY_EVERY_LOG bits.
+  // So if 32 - LDMHASHLOG < HASH_ONLY_EVERY_LOG, just return lower bits
+  // allowing for reuse of bits.
+#ifdef TMP_SIMPLE_LOWER
+  return hfHash & HASH_ONLY_EVERY;
+#else
+  if (32 - LDM_HASHLOG < HASH_ONLY_EVERY_LOG) {
+    return hfHash & HASH_ONLY_EVERY;
+  } else {
+    // Otherwise shift by (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG) bits first.
+    return (hfHash >> (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG)) &
+           HASH_ONLY_EVERY;
+  }
+#endif
+}
+#endif
 
+
+
+static U64 getChecksum(const BYTE *buf, U32 len) {
   U64 ret = 0;
   U32 i;
   for (i = 0; i < len; i++) {
@@ -455,11 +580,8 @@ static U64 ipow(U64 base, U64 exp) {
 
 static U64 updateChecksum(U64 sum, U32 len,
                           BYTE toRemove, BYTE toAdd) {
-  // TODO: deduplicate.
-  static const U64 prime8bytes = 11400714785074694791ULL;
-
   // TODO: relying on compiler optimization here.
-  // The exponential can be calculated explicitly.
+  // The exponential can (should?) be calculated explicitly.
   sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
           ipow(prime8bytes, len - 1));
   sum *= prime8bytes;
@@ -492,6 +614,14 @@ static void setNextHash(LDM_CCtx *cctx) {
       cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
 
+#ifdef TMP_TAG_INSERT
+  {
+    U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextSum);
+    cctx->stats.TMP_totalHashCount++;
+    cctx->stats.TMP_hashCount[hashEveryMask]++;
+  }
+#endif
+
 #if LDM_LAG
 //  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
   if (cctx->ip - cctx->ibase > LDM_LAG) {
@@ -520,31 +650,48 @@ static void setNextHash(LDM_CCtx *cctx) {
 static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hfHash) {
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
-//  printf("TMP %u %u %llu\n", hash, sum, hfHash);
-
-  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-
 #if LDM_LAG
-    // TODO: off by 1, but whatever
+  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+    // TODO: off by 1, but whatever.
     if (cctx->lagIp - cctx->ibase > 0) {
       U32 hash = checksumToHash(cctx->lagSum);
       U32 sum = checksumFromHfHash(cctx->lagSum);
       const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, sum };
+#ifdef TMP_EVICTION
+      HASH_insert(cctx->hashTable, hash, entry, cctx);
+#else
       HASH_insert(cctx->hashTable, hash, entry);
+#endif
     } else {
       U32 hash = checksumToHash(hfHash);
       U32 sum = checksumFromHfHash(hfHash);
 
       const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
-      HASH_insert(cctx->hashTable, hash, entry);
-    }
+#ifdef TMP_EVICTION
+      HASH_insert(cctx->hashTable, hash, entry, cctx);
 #else
+      HASH_insert(cctx->hashTable, hash, entry);
+#endif
+    }
+  }
+#else
+#ifdef TMP_TAG_INSERT
+  U32 hashEveryMask = lowerBitsFromHfHash(hfHash);
+  // TODO: look at stats.
+  if (hashEveryMask == HASH_ONLY_EVERY) {
+#else
+  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+#endif
     U32 hash = checksumToHash(hfHash);
     U32 sum = checksumFromHfHash(hfHash);
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+#ifdef TMP_EVICTION
+    HASH_insert(cctx->hashTable, hash, entry, cctx);
+#else
     HASH_insert(cctx->hashTable, hash, entry);
 #endif
   }
+#endif
 
   cctx->lastPosHashed = cctx->ip;
   cctx->lastSum = hfHash;
@@ -676,10 +823,16 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     hash_t h;
     U64 hash;
     U32 sum;
+#ifdef TMP_TAG_INSERT
+    U32 hashEveryMask;
+#endif
     setNextHash(cctx);
     hash = cctx->nextSum;
     h = checksumToHash(hash);
     sum = checksumFromHfHash(hash);
+#ifdef TMP_TAG_INSERT
+    hashEveryMask = lowerBitsFromHfHash(hash);
+#endif
 
     cctx->ip = cctx->nextIp;
     cctx->nextIp += cctx->step;
@@ -687,9 +840,15 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     if (cctx->ip > cctx->imatchLimit) {
       return 1;
     }
-
+#ifdef TMP_TAG_INSERT
+    if (hashEveryMask == HASH_ONLY_EVERY) {
+      entry = HASH_getBestEntry(cctx, h, sum,
+                                forwardMatchLength, backwardMatchLength);
+    }
+#else
     entry = HASH_getBestEntry(cctx, h, sum,
                               forwardMatchLength, backwardMatchLength);
+#endif
 
     if (entry != NULL) {
       *match = entry->offset + cctx->ibase;

From 8ed92201024e889333a9ac89dd10b188d28d8647 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 24 Jul 2017 12:05:43 -0700
Subject: [PATCH 088/100] Experiment with eviction policies and minor code
 cleanup

---
 contrib/long_distance_matching/Makefile       |  7 +-
 .../circular_buffer_table.c                   | 43 +++++++-----
 contrib/long_distance_matching/ldm.c          | 51 ++++-----------
 contrib/long_distance_matching/ldm.h          | 44 ++++++-------
 contrib/long_distance_matching/ldm_64_hash.c  | 32 ++++-----
 .../long_distance_matching/ldm_hashtable.h    | 49 ++++++++++++--
 .../{ldm_with_table.c => ldm_integrated.c}    | 65 +++----------------
 contrib/long_distance_matching/main-ldm.c     | 17 ++---
 8 files changed, 139 insertions(+), 169 deletions(-)
 rename contrib/long_distance_matching/{ldm_with_table.c => ldm_integrated.c} (94%)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 16844297..c8129f67 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -27,20 +27,17 @@ default: all
 
 all: main-circular-buffer main-integrated main-64 
 
-#main-basic : basic_table.c ldm.c main-ldm.c
-#	      $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
 main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 main-64: ldm_64_hash.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-integrated: ldm_with_table.c main-ldm.c
+main-integrated: ldm_integrated.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-circular-buffer  main-integrated main-64
+	main-circular-buffer main-64 main-integrated 
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
index fb6c19d2..92ffc55b 100644
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ b/contrib/long_distance_matching/circular_buffer_table.c
@@ -5,26 +5,24 @@
 #include "ldm_hashtable.h"
 #include "mem.h"
 
-// Number of elements per hash bucket.
-// HASH_BUCKET_SIZE_LOG defined in ldm.h
+// THe number of elements per hash bucket.
+// HASH_BUCKET_SIZE_LOG is defined in ldm.h.
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
+
+// The number of hash buckets.
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
-
-
-// TODO: rename. Number of hash buckets.
-// TODO: Link to HASH_ENTRY_SIZE_LOG
-
+// If ZSTD_SKIP is defined, then the first entry is returned in HASH_getBestEntry
+// (without looking at other entries in the bucket).
 //#define ZSTD_SKIP
 
 struct LDM_hashTable {
-  U32 numBuckets;
-  U32 numEntries;
+  U32 numBuckets;          // The number of buckets.
+  U32 numEntries;          // numBuckets * HASH_BUCKET_SIZE.
   LDM_hashEntry *entries;
-  BYTE *bucketOffsets;     // Pointer to current insert position.
+  BYTE *bucketOffsets;     // A pointer (per bucket) to the next insert position.
 
-  // Position corresponding to offset=0 in LDM_hashEntry.
-  const BYTE *offsetBase;
+  const BYTE *offsetBase;  // Corresponds to offset=0 in LDM_hashEntry.
   U32 minMatchLength;
   U32 maxWindowSize;
 };
@@ -46,6 +44,7 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
   return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
 }
 
+// From lib/compress/zstd_compress.c
 static unsigned ZSTD_NbCommonBytes (register size_t val)
 {
     if (MEM_isLittleEndian()) {
@@ -114,7 +113,11 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
     }   }
 }
 
-// From lib/compress/zstd_compress.c
+/**
+ * From lib/compress/zstd_compress.c
+ * Returns the number of bytes (consecutively) in common between pIn and pMatch
+ * up to pInLimit.
+ */
 static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
                          const BYTE *const pInLimit) {
     const BYTE * const pStart = pIn;
@@ -147,9 +150,14 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
     return (size_t)(pIn - pStart);
 }
 
-U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
-                        const BYTE *pMatch, const BYTE *pBase) {
-  U32 matchLength = 0;
+/**
+ * Returns the number of bytes in common between pIn and pMatch,
+ * counting backwards, with pIn having a lower limit of pAnchor and
+ * pMatch having a lower limit of pBase.
+ */
+static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+                                  const BYTE *pMatch, const BYTE *pBase) {
+  size_t matchLength = 0;
   while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
     pIn--;
     pMatch--;
@@ -178,6 +186,8 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
       U64 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
       U64 backwardMatchLength, totalMatchLength;
 
+      // Only take matches where the forwardMatchLength is large enough
+      // for speed.
       if (forwardMatchLength < table->minMatchLength) {
         continue;
       }
@@ -212,6 +222,7 @@ hash_t HASH_hashU32(U32 value) {
 
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
+  // Circular buffer.
   *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
   table->bucketOffsets[hash]++;
   table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index bfaff1f5..a5594ff6 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -29,7 +29,6 @@
 
 typedef U32 checksum_t;
 
-// TODO: Scanning speed
 // TODO: Memory usage
 struct LDM_compressStats {
   U32 windowSizeLog, hashTableSizeLog;
@@ -40,9 +39,6 @@ struct LDM_compressStats {
 
   U32 minOffset, maxOffset;
 
-  U32 numCollisions;
-  U32 numHashInserts;
-
   U32 offsetHistogram[32];
 };
 
@@ -80,15 +76,12 @@ struct LDM_CCtx {
   hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
   checksum_t nextSum;
 
-
-
   unsigned step;                      // ip step, should be 1.
 
   const BYTE *lagIp;
   hash_t lagHash;
   checksum_t lagSum;
 
-  U64 numHashInserts;
   // DEBUG
   const BYTE *DEBUG_setNextHash;
 };
@@ -103,32 +96,6 @@ static int intLog2(U32 x) {
   return ret;
 }
 
-// TODO: Maybe we would eventually prefer to have linear rather than
-// exponential buckets.
-/**
-void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
-  U32 i = 0;
-  int buckets[32] = { 0 };
-
-  printf("\n");
-  printf("Hash table histogram\n");
-  for (; i < HASH_getSize(cctx->hashTable); i++) {
-    int offset = (cctx->ip - cctx->ibase) -
-                 HASH_getEntryFromHash(cctx->hashTable, i)->offset;
-    buckets[intLog2(offset)]++;
-  }
-
-  i = 0;
-  for (; i < 32; i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
-           buckets[i],
-           100.0 * (double) buckets[i] /
-                   (double) HASH_getSize(cctx->hashTable));
-  }
-  printf("\n");
-}
-*/
-
 void LDM_printCompressStats(const LDM_compressStats *stats) {
   int i = 0;
   printf("=====================\n");
@@ -163,7 +130,8 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("=====================\n");
 }
 
-int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
+/*
+static int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
   U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
   const BYTE *curIn = pIn;
   const BYTE *curMatch = pMatch;
@@ -181,6 +149,7 @@ int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
   }
   return 1;
 }
+*/
 
 /**
  * Convert a sum computed from getChecksum to a hash value in the range
@@ -253,7 +222,6 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->DEBUG_setNextHash = cctx->nextIp;
 #endif
 
-//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
   cctx->nextSum = updateChecksum(
       cctx->lastSum, LDM_HASH_LENGTH,
       cctx->lastPosHashed[0],
@@ -292,7 +260,7 @@ static void putHashOfCurrentPositionFromHash(
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
 #if LDM_LAG
-    // TODO: off by 1, but whatever
+    // Off by 1, but whatever
     if (cctx->lagIp - cctx->ibase > 0) {
       const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
       HASH_insert(cctx->hashTable, cctx->lagHash, entry);
@@ -344,6 +312,7 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   putHashOfCurrentPositionFromHash(cctx, hash, sum);
 }
 
+/*
 U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
                          const BYTE *pInLimit) {
   const BYTE * const pStart = pIn;
@@ -358,6 +327,7 @@ U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
   }
   return (U32)(pIn - pStart);
 }
+*/
 
 void LDM_outputConfiguration(void) {
   printf("=====================\n");
@@ -380,6 +350,12 @@ void LDM_readHeader(const void *src, U64 *compressedSize,
   // ip += sizeof(U64);
 }
 
+void LDM_writeHeader(void *memPtr, U64 compressedSize,
+                     U64 decompressedSize) {
+  MEM_write64(memPtr, compressedSize);
+  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
+}
+
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
@@ -592,8 +568,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
     LDM_updateLastHashFromNextHash(&cctx);
   }
 
-  // HASH_outputTableOffsetHistogram(&cctx);
-
   /* Encode the last literals (no more matches). */
   {
     const U32 lastRun = cctx.iend - cctx.anchor;
@@ -692,7 +666,6 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
   return dctx.op - (BYTE *)dst;
 }
 
-// TODO: implement and test hash function
 void LDM_test(const BYTE *src) {
   (void)src;
 }
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 840824c4..adbe35bf 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -1,20 +1,24 @@
 #ifndef LDM_H
 #define LDM_H
 
-#include <stddef.h>   /* size_t */
-
 #include "mem.h"    // from /lib/common/mem.h
 
-#define LDM_COMPRESS_SIZE 8
-#define LDM_DECOMPRESS_SIZE 8
-#define LDM_HEADER_SIZE ((LDM_COMPRESS_SIZE)+(LDM_DECOMPRESS_SIZE))
+// The number of bytes storing the compressed and decompressed size
+// in the header.
+#define LDM_COMPRESSED_SIZE 8
+#define LDM_DECOMPRESSED_SIZE 8
+#define LDM_HEADER_SIZE ((LDM_COMPRESSED_SIZE)+(LDM_DECOMPRESSED_SIZE))
+
+// THe number of bytes storing the offset.
 #define LDM_OFFSET_SIZE 4
 
 // Defines the size of the hash table.
 // Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
-#define LDM_MEMORY_USAGE 24
-#define HASH_BUCKET_SIZE_LOG 0 // MAX is 4 for now
+#define LDM_MEMORY_USAGE 23
+
+// The number of entries in a hash bucket.
+#define HASH_BUCKET_SIZE_LOG 0 // The maximum is 4 for now.
 
 // Defines the lag in inserting elements into the hash table.
 #define LDM_LAG 0
@@ -23,11 +27,10 @@
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four (and perhaps set to the same value?).
-#define LDM_MIN_MATCH_LENGTH 64
-#define LDM_HASH_LENGTH 64
+#define LDM_MIN_MATCH_LENGTH 16
+#define LDM_HASH_LENGTH 16
 
 // Experimental.
-//:w
 //#define TMP_EVICTION
 #define TMP_TAG_INSERT
 //#define TMP_SIMPLE_LOWER
@@ -37,7 +40,6 @@ typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
 typedef struct LDM_DCtx LDM_DCtx;
 
-
 /**
  *  Compresses src into dst.
  *
@@ -94,17 +96,6 @@ void LDM_outputHashTableOffsetHistogram(const LDM_CCtx *cctx);
  * Outputs compression statistics to stdout.
  */
 void LDM_printCompressStats(const LDM_compressStats *stats);
-/**
- * Checks whether the LDM_MIN_MATCH_LENGTH bytes from p are the same as the
- * LDM_MIN_MATCH_LENGTH bytes from match and also if
- * pIn - pMatch <= LDM_WINDOW_SIZE.
- *
- * This assumes LDM_MIN_MATCH_LENGTH is a multiple of four.
- *
- * Return 1 if valid, 0 otherwise.
- */
-int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch);
-
 
 /**
  * Encode the literal length followed by the literals.
@@ -150,6 +141,15 @@ void LDM_initializeDCtx(LDM_DCtx *dctx,
 void LDM_readHeader(const void *src, U64 *compressedSize,
                     U64 *decompressedSize);
 
+/**
+ * Write the compressed and decompressed size.
+ */
+void LDM_writeHeader(void *memPtr, U64 compressedSize,
+                     U64 decompressedSize);
+
+/**
+ * Output the configuration used.
+ */
 void LDM_outputConfiguration(void);
 
 void LDM_test(const BYTE *src);
diff --git a/contrib/long_distance_matching/ldm_64_hash.c b/contrib/long_distance_matching/ldm_64_hash.c
index bdbdd199..d0080efd 100644
--- a/contrib/long_distance_matching/ldm_64_hash.c
+++ b/contrib/long_distance_matching/ldm_64_hash.c
@@ -36,8 +36,7 @@
 //#define ZSTD_SKIP
 
 //#define RUN_CHECKS
-//
-//
+
 static const U64 prime8bytes = 11400714785074694791ULL;
 
 /* Hash table stuff */
@@ -49,7 +48,6 @@ typedef struct LDM_hashEntry {
   U32 checksum;
 } LDM_hashEntry;
 
-// TODO: Memory usage
 struct LDM_compressStats {
   U32 windowSizeLog, hashTableSizeLog;
   U32 numMatches;
@@ -59,9 +57,6 @@ struct LDM_compressStats {
 
   U32 minOffset, maxOffset;
 
-  U32 numCollisions;
-  U32 numHashInserts;
-
   U32 offsetHistogram[32];
 
   U64 TMP_hashCount[1 << HASH_ONLY_EVERY_LOG];
@@ -115,20 +110,19 @@ struct LDM_CCtx {
   const BYTE *lagIp;
   U64 lagSum;
 
-  U64 numHashInserts;
   // DEBUG
   const BYTE *DEBUG_setNextHash;
 };
 
 struct LDM_hashTable {
-  U32 numBuckets;  // Number of buckets
-  U32 numEntries;  // Rename...
-  LDM_hashEntry *entries;
+  U32 numBuckets;          // The number of buckets.
+  U32 numEntries;          // numBuckets * HASH_BUCKET_SIZE.
 
-  BYTE *bucketOffsets;
-  // Position corresponding to offset=0 in LDM_hashEntry.
+  LDM_hashEntry *entries;
+  BYTE *bucketOffsets;     // A pointer (per bucket) to the next insert position.
 };
 
+
 /**
  * Create a hash table that can contain size elements.
  * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
@@ -251,9 +245,9 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
  *
  * We count only bytes where pMatch > pBaes and pIn > pAnchor.
  */
-U64 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
                         const BYTE *pMatch, const BYTE *pBase) {
-  U64 matchLength = 0;
+  size_T matchLength = 0;
   while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
     pIn--;
     pMatch--;
@@ -293,7 +287,8 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
       U64 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
       U64 backwardMatchLength, totalMatchLength;
 
-      // For speed.
+      // Only take matches where the forward match length is large enough
+      // for speed.
       if (forwardMatchLength < LDM_MIN_MATCH_LENGTH) {
         continue;
       }
@@ -766,6 +761,13 @@ void LDM_readHeader(const void *src, U64 *compressedSize,
   // ip += sizeof(U64);
 }
 
+void LDM_writeHeader(void *memPtr, U64 compressedSize,
+                     U64 decompressedSize) {
+  MEM_write64(memPtr, compressedSize);
+  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
+}
+
+
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
index d59f401e..6093197d 100644
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ b/contrib/long_distance_matching/ldm_hashtable.h
@@ -1,37 +1,73 @@
+/**
+ * A "hash" table used in LDM compression.
+ *
+ * This is not exactly a hash table in the sense that inserted entries
+ * are not guaranteed to remain in the hash table.
+ */
+
 #ifndef LDM_HASHTABLE_H
 #define LDM_HASHTABLE_H
 
 #include "mem.h"
 
+// The log size of LDM_hashEntry in bytes.
 #define LDM_HASH_ENTRY_SIZE_LOG 3
 
-// TODO: clean up comments
-
 typedef U32 hash_t;
 
 typedef struct LDM_hashEntry {
-  U32 offset;   // TODO: Replace with pointer?
-  U32 checksum;
+  U32 offset;       // Represents the offset of the entry from offsetBase.
+  U32 checksum;     // A checksum to select entries with the same hash value.
 } LDM_hashEntry;
 
 typedef struct LDM_hashTable LDM_hashTable;
 
+/**
+ * Create a table that can contain size elements. This does not necessarily
+ * correspond to the number of hash buckets. The number of hash buckets
+ * is size / (1 << HASH_BUCKET_SIZE_LOG)
+ *
+ * minMatchLength is the minimum match length required in HASH_getBestEntry.
+ *
+ * maxWindowSize is the maximum distance from pIn in HASH_getBestEntry.
+ * The window is defined to be (pIn - offsetBase - offset).
+ */
 LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
                                 U32 minMatchLength, U32 maxWindowSize);
 
+/**
+ * Return the "best" entry from the table with the same hash and checksum.
+ *
+ * pIn: a pointer to the current input position.
+ * pEnd: a pointer to the maximum input position.
+ * pAnchor: a pointer to the minimum input position.
+ *
+ * This function computes the forward and backward match length from pIn
+ * and writes it to forwardMatchLength and backwardsMatchLength.
+ *
+ * E.g. for the two strings "aaabbbb" "aaabbbb" with pIn and the
+ * entry pointing at the first "b", the forward match length would be
+ * four (representing the "b" matches) and the backward match length would
+ * three (representing the "a" matches before the pointer).
+ */
 LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
                                  const hash_t hash,
                                  const U32 checksum,
                                  const BYTE *pIn,
                                  const BYTE *pEnd,
                                  const BYTE *pAnchor,
-                                 U64 *matchLength,
+                                 U64 *forwardMatchLength,
                                  U64 *backwardsMatchLength);
 
+/**
+ * Return a hash of the value.
+ */
 hash_t HASH_hashU32(U32 value);
 
 /**
  * Insert an LDM_hashEntry into the bucket corresponding to hash.
+ *
+ * An entry may be evicted in the process.
  */
 void HASH_insert(LDM_hashTable *table, const hash_t hash,
                  const LDM_hashEntry entry);
@@ -41,6 +77,9 @@ void HASH_insert(LDM_hashTable *table, const hash_t hash,
  */
 U32 HASH_getSize(const LDM_hashTable *table);
 
+/**
+ * Destroy the table.
+ */
 void HASH_destroyTable(LDM_hashTable *table);
 
 /**
diff --git a/contrib/long_distance_matching/ldm_with_table.c b/contrib/long_distance_matching/ldm_integrated.c
similarity index 94%
rename from contrib/long_distance_matching/ldm_with_table.c
rename to contrib/long_distance_matching/ldm_integrated.c
index babfdf3f..7733d4e9 100644
--- a/contrib/long_distance_matching/ldm_with_table.c
+++ b/contrib/long_distance_matching/ldm_integrated.c
@@ -33,8 +33,6 @@
 
 //#define RUN_CHECKS
 
-/* Hash table stuff */
-
 typedef U32 hash_t;
 
 typedef struct LDM_hashEntry {
@@ -42,7 +40,6 @@ typedef struct LDM_hashEntry {
   U32 checksum;
 } LDM_hashEntry;
 
-// TODO: Memory usage
 struct LDM_compressStats {
   U32 windowSizeLog, hashTableSizeLog;
   U32 numMatches;
@@ -52,9 +49,6 @@ struct LDM_compressStats {
 
   U32 minOffset, maxOffset;
 
-  U32 numCollisions;
-  U32 numHashInserts;
-
   U32 offsetHistogram[32];
 };
 
@@ -85,8 +79,6 @@ struct LDM_CCtx {
 
   LDM_hashTable *hashTable;
 
-//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
-
   const BYTE *lastPosHashed;          /* Last position hashed */
   hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
   U32 lastSum;
@@ -109,11 +101,10 @@ struct LDM_CCtx {
 
 struct LDM_hashTable {
   U32 numBuckets;  // Number of buckets
-  U32 numEntries;  // Rename...
+  U32 numEntries;
   LDM_hashEntry *entries;
 
   BYTE *bucketOffsets;
-  // Position corresponding to offset=0 in LDM_hashEntry.
 };
 
 /**
@@ -354,32 +345,6 @@ static int intLog2(U32 x) {
   return ret;
 }
 
-// Maybe we would eventually prefer to have linear rather than
-// exponential buckets.
-/**
-void HASH_outputTableOffsetHistogram(const LDM_CCtx *cctx) {
-  U32 i = 0;
-  int buckets[32] = { 0 };
-
-  printf("\n");
-  printf("Hash table histogram\n");
-  for (; i < HASH_getSize(cctx->hashTable); i++) {
-    int offset = (cctx->ip - cctx->ibase) -
-                 HASH_getEntryFromHash(cctx->hashTable, i)->offset;
-    buckets[intLog2(offset)]++;
-  }
-
-  i = 0;
-  for (; i < 32; i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
-           buckets[i],
-           100.0 * (double) buckets[i] /
-                   (double) HASH_getSize(cctx->hashTable));
-  }
-  printf("\n");
-}
-*/
-
 void LDM_printCompressStats(const LDM_compressStats *stats) {
   int i = 0;
   printf("=====================\n");
@@ -508,7 +473,6 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->DEBUG_setNextHash = cctx->nextIp;
 #endif
 
-//  cctx->nextSum = getChecksum((const char *)cctx->nextIp, LDM_HASH_LENGTH);
   cctx->nextSum = updateChecksum(
       cctx->lastSum, LDM_HASH_LENGTH,
       cctx->lastPosHashed[0],
@@ -517,7 +481,6 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->nextHash = checksumToHash(cctx->nextSum);
 
 #if LDM_LAG
-//  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
   if (cctx->ip - cctx->ibase > LDM_LAG) {
     cctx->lagSum = updateChecksum(
       cctx->lagSum, LDM_HASH_LENGTH,
@@ -547,10 +510,6 @@ static void putHashOfCurrentPositionFromHash(
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    /**
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase ,
-                                  MEM_read32(cctx->ip) };
-                                  */
 #if LDM_LAG
     // TODO: off by 1, but whatever
     if (cctx->lagIp - cctx->ibase > 0) {
@@ -604,21 +563,6 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   putHashOfCurrentPositionFromHash(cctx, hash, sum);
 }
 
-U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = (*pMatch) ^ *(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (U32)(pIn - pStart);
-  }
-  return (U32)(pIn - pStart);
-}
-
 void LDM_outputConfiguration(void) {
   printf("=====================\n");
   printf("Configuration\n");
@@ -640,6 +584,13 @@ void LDM_readHeader(const void *src, U64 *compressedSize,
   // ip += sizeof(U64);
 }
 
+void LDM_writeHeader(void *memPtr, U64 compressedSize,
+                     U64 decompressedSize) {
+  MEM_write64(memPtr, compressedSize);
+  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
+}
+
+
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main-ldm.c
index 9769f10e..232c14a2 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main-ldm.c
@@ -12,13 +12,12 @@
 #include "ldm.h"
 #include "zstd.h"
 
-#define DEBUG
 //#define TEST
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
  *
- * TODO: This might seg fault if the compressed size is > the decompress
+ * This might seg fault if the compressed size is > the decompress
  * size due to the mmapping and output file size allocated to be the input size
  * The compress function should check before writing or buffer writes.
  */
@@ -31,6 +30,7 @@ static int compress(const char *fname, const char *oname) {
   struct timeval tv1, tv2;
   double timeTaken;
 
+
   /* Open the input file. */
   if ((fdin = open(fname, O_RDONLY)) < 0) {
     perror("Error in file opening");
@@ -50,6 +50,7 @@ static int compress(const char *fname, const char *oname) {
   }
 
   maxCompressedSize = (statbuf.st_size + LDM_HEADER_SIZE);
+
   // Handle case where compressed size is > decompressed size.
   // The compress function should check before writing or buffer writes.
   maxCompressedSize += statbuf.st_size / 255;
@@ -79,21 +80,17 @@ static int compress(const char *fname, const char *oname) {
   compressedSize = LDM_HEADER_SIZE +
       LDM_compress(src, statbuf.st_size,
                    dst + LDM_HEADER_SIZE, maxCompressedSize);
+
   gettimeofday(&tv2, NULL);
 
   // Write compress and decompress size to header
   // TODO: should depend on LDM_DECOMPRESS_SIZE write32
-  memcpy(dst, &compressedSize, 8);
-  memcpy(dst + 8, &(statbuf.st_size), 8);
-
-#ifdef DEBUG
-  printf("Compressed size: %zu\n", compressedSize);
-  printf("Decompressed size: %zu\n", (size_t)statbuf.st_size);
-#endif
+  LDM_writeHeader(dst, compressedSize, statbuf.st_size);
 
   // Truncate file to compressedSize.
   ftruncate(fdout, compressedSize);
 
+
   printf("%25s : %10lu -> %10lu - %s (%.2fx --- %.1f%%)\n", fname,
          (size_t)statbuf.st_size, (size_t)compressedSize, oname,
          (statbuf.st_size) / (double)compressedSize,
@@ -102,7 +99,7 @@ static int compress(const char *fname, const char *oname) {
   timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
               (double) (tv2.tv_sec - tv1.tv_sec),
 
-  printf("Total compress time = %.3f seconds, Average compression speed: %.3f MB/s\n",
+  printf("Total compress time = %.3f seconds, Average scanning speed: %.3f MB/s\n",
          timeTaken,
          ((double)statbuf.st_size / (double) (1 << 20)) / timeTaken);
 

From 6eefa3291195e3fc7f592484125ff3bc44ba3a50 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 24 Jul 2017 12:40:59 -0700
Subject: [PATCH 089/100] Deduplicate code

---
 contrib/long_distance_matching/Makefile       |   6 +-
 contrib/long_distance_matching/ldm.c          | 156 --------------
 contrib/long_distance_matching/ldm.h          |  16 +-
 contrib/long_distance_matching/ldm_64_hash.c  | 198 ++----------------
 contrib/long_distance_matching/ldm_common.c   | 113 ++++++++++
 .../long_distance_matching/ldm_integrated.c   | 116 ----------
 6 files changed, 153 insertions(+), 452 deletions(-)
 create mode 100644 contrib/long_distance_matching/ldm_common.c

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index c8129f67..e1c31112 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -27,13 +27,13 @@ default: all
 
 all: main-circular-buffer main-integrated main-64 
 
-main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
+main-circular-buffer: ldm_common.c circular_buffer_table.c ldm.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-64: ldm_64_hash.c main-ldm.c
+main-64: ldm_common.c ldm_64_hash.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-integrated: ldm_integrated.c main-ldm.c
+main-integrated: ldm_common.c ldm_integrated.c main-ldm.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index a5594ff6..9d3eda32 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -8,28 +8,16 @@
 #include "ldm_hashtable.h"
 
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
-// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
-#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
 #define COMPUTE_STATS
 #define OUTPUT_CONFIGURATION
 #define CHECKSUM_CHAR_OFFSET 10
 
 //#define RUN_CHECKS
-//#define TMP_RECOMPUTE_LENGTHS
 
 typedef U32 checksum_t;
 
-// TODO: Memory usage
 struct LDM_compressStats {
   U32 windowSizeLog, hashTableSizeLog;
   U32 numMatches;
@@ -130,27 +118,6 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("=====================\n");
 }
 
-/*
-static int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
-  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
-  const BYTE *curIn = pIn;
-  const BYTE *curMatch = pMatch;
-
-  if (pIn - pMatch > LDM_WINDOW_SIZE) {
-    return 0;
-  }
-
-  for (; lengthLeft >= 4; lengthLeft -= 4) {
-    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
-      return 0;
-    }
-    curIn += 4;
-    curMatch += 4;
-  }
-  return 1;
-}
-*/
-
 /**
  * Convert a sum computed from getChecksum to a hash value in the range
  * of the hash table.
@@ -312,50 +279,6 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   putHashOfCurrentPositionFromHash(cctx, hash, sum);
 }
 
-/*
-U32 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = (*pMatch) ^ *(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (U32)(pIn - pStart);
-  }
-  return (U32)(pIn - pStart);
-}
-*/
-
-void LDM_outputConfiguration(void) {
-  printf("=====================\n");
-  printf("Configuration\n");
-  printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG);
-  printf("Min match, hash length: %d, %d\n",
-         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
-  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
-  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
-  printf("LDM_LAG %d\n", LDM_LAG);
-  printf("=====================\n");
-}
-
-void LDM_readHeader(const void *src, U64 *compressedSize,
-                    U64 *decompressedSize) {
-  const BYTE *ip = (const BYTE *)src;
-  *compressedSize = MEM_readLE64(ip);
-  ip += sizeof(U64);
-  *decompressedSize = MEM_readLE64(ip);
-  // ip += sizeof(U64);
-}
-
-void LDM_writeHeader(void *memPtr, U64 compressedSize,
-                     U64 decompressedSize) {
-  MEM_write64(memPtr, compressedSize);
-  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
-}
-
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
@@ -587,85 +510,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
   }
 }
 
-struct LDM_DCtx {
-  size_t compressedSize;
-  size_t maxDecompressedSize;
-
-  const BYTE *ibase;   /* Base of input */
-  const BYTE *ip;      /* Current input position */
-  const BYTE *iend;    /* End of source */
-
-  const BYTE *obase;   /* Base of output */
-  BYTE *op;            /* Current output position */
-  const BYTE *oend;    /* End of output */
-};
-
-void LDM_initializeDCtx(LDM_DCtx *dctx,
-                        const void *src, size_t compressedSize,
-                        void *dst, size_t maxDecompressedSize) {
-  dctx->compressedSize = compressedSize;
-  dctx->maxDecompressedSize = maxDecompressedSize;
-
-  dctx->ibase = src;
-  dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressedSize;
-  dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressedSize;
-}
-
-size_t LDM_decompress(const void *src, size_t compressedSize,
-                      void *dst, size_t maxDecompressedSize) {
-  LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
-
-  while (dctx.ip < dctx.iend) {
-    BYTE *cpy;
-    const BYTE *match;
-    size_t length, offset;
-
-    /* Get the literal length. */
-    const unsigned token = *(dctx.ip)++;
-    if ((length = (token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-
-    /* Copy the literals. */
-    cpy = dctx.op + length;
-    memcpy(dctx.op, dctx.ip, length);
-    dctx.ip += length;
-    dctx.op = cpy;
-
-    //TODO : dynamic offset size
-    offset = MEM_read32(dctx.ip);
-    dctx.ip += LDM_OFFSET_SIZE;
-    match = dctx.op - offset;
-
-    /* Get the match length. */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-    length += LDM_MIN_MATCH_LENGTH;
-
-    /* Copy match. */
-    cpy = dctx.op + length;
-
-    // Inefficient for now.
-    while (match < cpy - offset && dctx.op < dctx.oend) {
-      *(dctx.op)++ = *match++;
-    }
-  }
-  return dctx.op - (BYTE *)dst;
-}
-
 void LDM_test(const BYTE *src) {
   (void)src;
 }
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index adbe35bf..3078fb8c 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -9,6 +9,11 @@
 #define LDM_DECOMPRESSED_SIZE 8
 #define LDM_HEADER_SIZE ((LDM_COMPRESSED_SIZE)+(LDM_DECOMPRESSED_SIZE))
 
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
 // THe number of bytes storing the offset.
 #define LDM_OFFSET_SIZE 4
 
@@ -33,9 +38,18 @@
 // Experimental.
 //#define TMP_EVICTION
 #define TMP_TAG_INSERT
-//#define TMP_SIMPLE_LOWER
 //#define TMP_FORCE_HASH_ONLY
 
+#define LDM_HASH_ENTRY_SIZE_LOG 3
+
+// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
+#ifdef TMP_FORCE_HASH_ONLY
+  #define HASH_ONLY_EVERY_LOG 7
+#else
+  #define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
+#endif
+#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
+
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
 typedef struct LDM_DCtx LDM_DCtx;
diff --git a/contrib/long_distance_matching/ldm_64_hash.c b/contrib/long_distance_matching/ldm_64_hash.c
index d0080efd..95865f70 100644
--- a/contrib/long_distance_matching/ldm_64_hash.c
+++ b/contrib/long_distance_matching/ldm_64_hash.c
@@ -7,27 +7,12 @@
 #include "ldm.h"
 
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASH_ENTRY_SIZE_LOG 3
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
-// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#ifdef TMP_FORCE_HASH_ONLY
-  #define HASH_ONLY_EVERY_LOG 7
-#else
-  #define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
-#endif
-#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
-
 /* Hash table stuff. */
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
 #define COMPUTE_STATS
 #define OUTPUT_CONFIGURATION
 #define CHECKSUM_CHAR_OFFSET 1
@@ -247,7 +232,7 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
  */
 size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
                         const BYTE *pMatch, const BYTE *pBase) {
-  size_T matchLength = 0;
+  size_t matchLength = 0;
   while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
     pIn--;
     pMatch--;
@@ -319,7 +304,6 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
 }
 
 #ifdef TMP_EVICTION
-
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry,
                  LDM_CCtx *cctx) {
@@ -381,9 +365,6 @@ void HASH_insert(LDM_hashTable *table,
    *
    * Insert forwards. If > tag, keep. Else evict.
    *
-   *
-   *
-   *
    */
 
 
@@ -428,7 +409,7 @@ void HASH_outputTableOccupancy(const LDM_hashTable *table) {
     }
   }
 
-  // TODO: repeat numBuckets as a check for now.
+  // The number of buckets is repeated as a check for now.
   printf("Num buckets, bucket size: %d (2^%d), %d\n",
          table->numBuckets, LDM_HASHLOG, HASH_BUCKET_SIZE);
   printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
@@ -498,31 +479,16 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("=====================\n");
 }
 
-int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
-  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
-  const BYTE *curIn = pIn;
-  const BYTE *curMatch = pMatch;
-
-  if (pIn - pMatch > LDM_WINDOW_SIZE) {
-    return 0;
-  }
-
-  for (; lengthLeft >= 4; lengthLeft -= 4) {
-    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
-      return 0;
-    }
-    curIn += 4;
-    curMatch += 4;
-  }
-  return 1;
-}
-
-// Upper LDM_HASHLOG bits.
+/**
+ * Return the upper (most significant) LDM_HASHLOG bits.
+ */
 static hash_t checksumToHash(U64 sum) {
   return sum >> (64 - LDM_HASHLOG);
 }
 
-// 32 bits after LDM_HASH_LOG bits.
+/**
+ * Return the 32 bits after the upper LDM_HASHLOG bits.
+ */
 static U32 checksumFromHfHash(U64 hfHash) {
   return (hfHash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
 }
@@ -534,9 +500,6 @@ static U32 lowerBitsFromHfHash(U64 hfHash) {
   // Occasional hashing requires HASH_ONLY_EVERY_LOG bits.
   // So if 32 - LDMHASHLOG < HASH_ONLY_EVERY_LOG, just return lower bits
   // allowing for reuse of bits.
-#ifdef TMP_SIMPLE_LOWER
-  return hfHash & HASH_ONLY_EVERY;
-#else
   if (32 - LDM_HASHLOG < HASH_ONLY_EVERY_LOG) {
     return hfHash & HASH_ONLY_EVERY;
   } else {
@@ -544,12 +507,20 @@ static U32 lowerBitsFromHfHash(U64 hfHash) {
     return (hfHash >> (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG)) &
            HASH_ONLY_EVERY;
   }
-#endif
 }
 #endif
 
-
-
+/**
+ * Get a 64-bit hash using the first len bytes from buf.
+ *
+ * Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be
+ * H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0)
+ *
+ * where the constant a is defined to be prime8bytes.
+ *
+ * The implementation adds an offset to each byte, so
+ * H(s) = (s_1 + CHECKSUM_CHAR_OFFSET)*(a^(k-1)) + ...
+ */
 static U64 getChecksum(const BYTE *buf, U32 len) {
   U64 ret = 0;
   U32 i;
@@ -575,8 +546,8 @@ static U64 ipow(U64 base, U64 exp) {
 
 static U64 updateChecksum(U64 sum, U32 len,
                           BYTE toRemove, BYTE toAdd) {
-  // TODO: relying on compiler optimization here.
-  // The exponential can (should?) be calculated explicitly.
+  // TODO: this relies on compiler optimization.
+  // The exponential can be calculated explicitly as len is constant.
   sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
           ipow(prime8bytes, len - 1));
   sum *= prime8bytes;
@@ -618,7 +589,6 @@ static void setNextHash(LDM_CCtx *cctx) {
 #endif
 
 #if LDM_LAG
-//  printf("LDM_LAG %zu\n", cctx->ip - cctx->lagIp);
   if (cctx->ip - cctx->ibase > LDM_LAG) {
     cctx->lagSum = updateChecksum(
       cctx->lagSum, LDM_HASH_LENGTH,
@@ -647,7 +617,7 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hfHash) {
   // Note: this works only when cctx->step is 1.
 #if LDM_LAG
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    // TODO: off by 1, but whatever.
+    // TODO: Off by one, but not important.
     if (cctx->lagIp - cctx->ibase > 0) {
       U32 hash = checksumToHash(cctx->lagSum);
       U32 sum = checksumFromHfHash(cctx->lagSum);
@@ -724,50 +694,6 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   putHashOfCurrentPositionFromHash(cctx, sum);
 }
 
-U64 LDM_countMatchLength(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *pInLimit) {
-  const BYTE * const pStart = pIn;
-  while (pIn < pInLimit - 1) {
-    BYTE const diff = (*pMatch) ^ *(pIn);
-    if (!diff) {
-      pIn++;
-      pMatch++;
-      continue;
-    }
-    return (U64)(pIn - pStart);
-  }
-  return (U64)(pIn - pStart);
-}
-
-void LDM_outputConfiguration(void) {
-  printf("=====================\n");
-  printf("Configuration\n");
-  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
-  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
-         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
-  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
-  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
-  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
-  printf("LDM_LAG %d\n", LDM_LAG);
-  printf("=====================\n");
-}
-
-void LDM_readHeader(const void *src, U64 *compressedSize,
-                    U64 *decompressedSize) {
-  const BYTE *ip = (const BYTE *)src;
-  *compressedSize = MEM_readLE64(ip);
-  ip += sizeof(U64);
-  *decompressedSize = MEM_readLE64(ip);
-  // ip += sizeof(U64);
-}
-
-void LDM_writeHeader(void *memPtr, U64 compressedSize,
-                     U64 decompressedSize) {
-  MEM_write64(memPtr, compressedSize);
-  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
-}
-
-
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
@@ -1013,86 +939,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
   }
 }
 
-struct LDM_DCtx {
-  size_t compressedSize;
-  size_t maxDecompressedSize;
-
-  const BYTE *ibase;   /* Base of input */
-  const BYTE *ip;      /* Current input position */
-  const BYTE *iend;    /* End of source */
-
-  const BYTE *obase;   /* Base of output */
-  BYTE *op;            /* Current output position */
-  const BYTE *oend;    /* End of output */
-};
-
-void LDM_initializeDCtx(LDM_DCtx *dctx,
-                        const void *src, size_t compressedSize,
-                        void *dst, size_t maxDecompressedSize) {
-  dctx->compressedSize = compressedSize;
-  dctx->maxDecompressedSize = maxDecompressedSize;
-
-  dctx->ibase = src;
-  dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressedSize;
-  dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressedSize;
-}
-
-size_t LDM_decompress(const void *src, size_t compressedSize,
-                      void *dst, size_t maxDecompressedSize) {
-
-  LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
-
-  while (dctx.ip < dctx.iend) {
-    BYTE *cpy;
-    const BYTE *match;
-    size_t length, offset;
-
-    /* Get the literal length. */
-    const unsigned token = *(dctx.ip)++;
-    if ((length = (token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-
-    /* Copy the literals. */
-    cpy = dctx.op + length;
-    memcpy(dctx.op, dctx.ip, length);
-    dctx.ip += length;
-    dctx.op = cpy;
-
-    //TODO : dynamic offset size
-    offset = MEM_read32(dctx.ip);
-    dctx.ip += LDM_OFFSET_SIZE;
-    match = dctx.op - offset;
-
-    /* Get the match length. */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-    length += LDM_MIN_MATCH_LENGTH;
-
-    /* Copy match. */
-    cpy = dctx.op + length;
-
-    // Inefficient for now.
-    while (match < cpy - offset && dctx.op < dctx.oend) {
-      *(dctx.op)++ = *match++;
-    }
-  }
-  return dctx.op - (BYTE *)dst;
-}
-
 // TODO: implement and test hash function
 void LDM_test(const BYTE *src) {
   const U32 diff = 100;
diff --git a/contrib/long_distance_matching/ldm_common.c b/contrib/long_distance_matching/ldm_common.c
new file mode 100644
index 00000000..673959db
--- /dev/null
+++ b/contrib/long_distance_matching/ldm_common.c
@@ -0,0 +1,113 @@
+#include <stdio.h>
+
+#include "ldm.h"
+
+void LDM_outputConfiguration(void) {
+  printf("=====================\n");
+  printf("Configuration\n");
+  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
+         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
+  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
+  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
+  printf("LDM_LAG %d\n", LDM_LAG);
+  printf("=====================\n");
+}
+
+void LDM_readHeader(const void *src, U64 *compressedSize,
+                    U64 *decompressedSize) {
+  const BYTE *ip = (const BYTE *)src;
+  *compressedSize = MEM_readLE64(ip);
+  ip += sizeof(U64);
+  *decompressedSize = MEM_readLE64(ip);
+  // ip += sizeof(U64);
+}
+
+void LDM_writeHeader(void *memPtr, U64 compressedSize,
+                     U64 decompressedSize) {
+  MEM_write64(memPtr, compressedSize);
+  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
+}
+
+struct LDM_DCtx {
+  size_t compressedSize;
+  size_t maxDecompressedSize;
+
+  const BYTE *ibase;   /* Base of input */
+  const BYTE *ip;      /* Current input position */
+  const BYTE *iend;    /* End of source */
+
+  const BYTE *obase;   /* Base of output */
+  BYTE *op;            /* Current output position */
+  const BYTE *oend;    /* End of output */
+};
+
+void LDM_initializeDCtx(LDM_DCtx *dctx,
+                        const void *src, size_t compressedSize,
+                        void *dst, size_t maxDecompressedSize) {
+  dctx->compressedSize = compressedSize;
+  dctx->maxDecompressedSize = maxDecompressedSize;
+
+  dctx->ibase = src;
+  dctx->ip = (const BYTE *)src;
+  dctx->iend = dctx->ip + dctx->compressedSize;
+  dctx->op = dst;
+  dctx->oend = dctx->op + dctx->maxDecompressedSize;
+}
+
+size_t LDM_decompress(const void *src, size_t compressedSize,
+                      void *dst, size_t maxDecompressedSize) {
+
+  LDM_DCtx dctx;
+  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
+
+  while (dctx.ip < dctx.iend) {
+    BYTE *cpy;
+    const BYTE *match;
+    size_t length, offset;
+
+    /* Get the literal length. */
+    const unsigned token = *(dctx.ip)++;
+    if ((length = (token >> ML_BITS)) == RUN_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+
+    /* Copy the literals. */
+    cpy = dctx.op + length;
+    memcpy(dctx.op, dctx.ip, length);
+    dctx.ip += length;
+    dctx.op = cpy;
+
+    //TODO : dynamic offset size
+    offset = MEM_read32(dctx.ip);
+    dctx.ip += LDM_OFFSET_SIZE;
+    match = dctx.op - offset;
+
+    /* Get the match length. */
+    length = token & ML_MASK;
+    if (length == ML_MASK) {
+      unsigned s;
+      do {
+        s = *(dctx.ip)++;
+        length += s;
+      } while (s == 255);
+    }
+    length += LDM_MIN_MATCH_LENGTH;
+
+    /* Copy match. */
+    cpy = dctx.op + length;
+
+    // Inefficient for now.
+    while (match < cpy - offset && dctx.op < dctx.oend) {
+      *(dctx.op)++ = *match++;
+    }
+  }
+  return dctx.op - (BYTE *)dst;
+}
+
+
diff --git a/contrib/long_distance_matching/ldm_integrated.c b/contrib/long_distance_matching/ldm_integrated.c
index 7733d4e9..d51c1e9d 100644
--- a/contrib/long_distance_matching/ldm_integrated.c
+++ b/contrib/long_distance_matching/ldm_integrated.c
@@ -11,19 +11,10 @@
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
-// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE) - (LDM_HASH_ENTRY_SIZE_LOG)))
-#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
-
 /* Hash table stuff. */
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
 #define COMPUTE_STATS
 #define OUTPUT_CONFIGURATION
 #define CHECKSUM_CHAR_OFFSET 10
@@ -563,34 +554,6 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   putHashOfCurrentPositionFromHash(cctx, hash, sum);
 }
 
-void LDM_outputConfiguration(void) {
-  printf("=====================\n");
-  printf("Configuration\n");
-  printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG);
-  printf("Min match, hash length: %d, %d\n",
-         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
-  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
-  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
-  printf("LDM_LAG %d\n", LDM_LAG);
-  printf("=====================\n");
-}
-
-void LDM_readHeader(const void *src, U64 *compressedSize,
-                    U64 *decompressedSize) {
-  const BYTE *ip = (const BYTE *)src;
-  *compressedSize = MEM_readLE64(ip);
-  ip += sizeof(U64);
-  *decompressedSize = MEM_readLE64(ip);
-  // ip += sizeof(U64);
-}
-
-void LDM_writeHeader(void *memPtr, U64 compressedSize,
-                     U64 decompressedSize) {
-  MEM_write64(memPtr, compressedSize);
-  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
-}
-
-
 void LDM_initializeCCtx(LDM_CCtx *cctx,
                         const void *src, size_t srcSize,
                         void *dst, size_t maxDstSize) {
@@ -821,85 +784,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
   }
 }
 
-struct LDM_DCtx {
-  size_t compressedSize;
-  size_t maxDecompressedSize;
-
-  const BYTE *ibase;   /* Base of input */
-  const BYTE *ip;      /* Current input position */
-  const BYTE *iend;    /* End of source */
-
-  const BYTE *obase;   /* Base of output */
-  BYTE *op;            /* Current output position */
-  const BYTE *oend;    /* End of output */
-};
-
-void LDM_initializeDCtx(LDM_DCtx *dctx,
-                        const void *src, size_t compressedSize,
-                        void *dst, size_t maxDecompressedSize) {
-  dctx->compressedSize = compressedSize;
-  dctx->maxDecompressedSize = maxDecompressedSize;
-
-  dctx->ibase = src;
-  dctx->ip = (const BYTE *)src;
-  dctx->iend = dctx->ip + dctx->compressedSize;
-  dctx->op = dst;
-  dctx->oend = dctx->op + dctx->maxDecompressedSize;
-}
-
-size_t LDM_decompress(const void *src, size_t compressedSize,
-                      void *dst, size_t maxDecompressedSize) {
-  LDM_DCtx dctx;
-  LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
-
-  while (dctx.ip < dctx.iend) {
-    BYTE *cpy;
-    const BYTE *match;
-    size_t length, offset;
-
-    /* Get the literal length. */
-    const unsigned token = *(dctx.ip)++;
-    if ((length = (token >> ML_BITS)) == RUN_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-
-    /* Copy the literals. */
-    cpy = dctx.op + length;
-    memcpy(dctx.op, dctx.ip, length);
-    dctx.ip += length;
-    dctx.op = cpy;
-
-    //TODO : dynamic offset size
-    offset = MEM_read32(dctx.ip);
-    dctx.ip += LDM_OFFSET_SIZE;
-    match = dctx.op - offset;
-
-    /* Get the match length. */
-    length = token & ML_MASK;
-    if (length == ML_MASK) {
-      unsigned s;
-      do {
-        s = *(dctx.ip)++;
-        length += s;
-      } while (s == 255);
-    }
-    length += LDM_MIN_MATCH_LENGTH;
-
-    /* Copy match. */
-    cpy = dctx.op + length;
-
-    // Inefficient for now.
-    while (match < cpy - offset && dctx.op < dctx.oend) {
-      *(dctx.op)++ = *match++;
-    }
-  }
-  return dctx.op - (BYTE *)dst;
-}
-
 // TODO: implement and test hash function
 void LDM_test(const BYTE *src) {
   (void)src;

From 08a6e9a141025f4d41177fc77495c32b138d6e54 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 24 Jul 2017 13:22:00 -0700
Subject: [PATCH 090/100] Minor code cleanup

---
 contrib/long_distance_matching/ldm_64_hash.c  | 291 +++++++++---------
 contrib/long_distance_matching/ldm_common.c   |   2 -
 .../long_distance_matching/ldm_integrated.c   |   2 +-
 3 files changed, 146 insertions(+), 149 deletions(-)

diff --git a/contrib/long_distance_matching/ldm_64_hash.c b/contrib/long_distance_matching/ldm_64_hash.c
index 95865f70..06ddf520 100644
--- a/contrib/long_distance_matching/ldm_64_hash.c
+++ b/contrib/long_distance_matching/ldm_64_hash.c
@@ -15,7 +15,7 @@
 
 #define COMPUTE_STATS
 #define OUTPUT_CONFIGURATION
-#define CHECKSUM_CHAR_OFFSET 1
+#define HASH_CHAR_OFFSET 10
 
 // Take first match only.
 //#define ZSTD_SKIP
@@ -24,8 +24,7 @@
 
 static const U64 prime8bytes = 11400714785074694791ULL;
 
-/* Hash table stuff */
-
+// Type of the small hash used to index into the hash table.
 typedef U32 hash_t;
 
 typedef struct LDM_hashEntry {
@@ -41,7 +40,6 @@ struct LDM_compressStats {
   U64 totalOffset;
 
   U32 minOffset, maxOffset;
-
   U32 offsetHistogram[32];
 
   U64 TMP_hashCount[1 << HASH_ONLY_EVERY_LOG];
@@ -56,8 +54,8 @@ struct LDM_compressStats {
 typedef struct LDM_hashTable LDM_hashTable;
 
 struct LDM_CCtx {
-  U64 isize;             /* Input size */
-  U64 maxOSize;          /* Maximum output size */
+  size_t isize;             /* Input size */
+  size_t maxOSize;          /* Maximum output size */
 
   const BYTE *ibase;        /* Base of input */
   const BYTE *ip;           /* Current input position */
@@ -80,23 +78,21 @@ struct LDM_CCtx {
 
   LDM_hashTable *hashTable;
 
-//  LDM_hashEntry hashTable[LDM_HASHTABLESIZE_U32];
-
   const BYTE *lastPosHashed;          /* Last position hashed */
-  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  U64 lastSum;
+  U64 lastHash;
 
-  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
+  const BYTE *nextIp;                 // TODO: this is redundant (ip + step)
   const BYTE *nextPosHashed;
-  U64 nextSum;
+  U64 nextHash;
 
   unsigned step;                      // ip step, should be 1.
 
   const BYTE *lagIp;
-  U64 lagSum;
+  U64 lagHash;
 
-  // DEBUG
+#ifdef RUN_CHECKS
   const BYTE *DEBUG_setNextHash;
+#endif
 };
 
 struct LDM_hashTable {
@@ -107,7 +103,6 @@ struct LDM_hashTable {
   BYTE *bucketOffsets;     // A pointer (per bucket) to the next insert position.
 };
 
-
 /**
  * Create a hash table that can contain size elements.
  * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
@@ -126,70 +121,74 @@ static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
 }
 
 static unsigned ZSTD_NbCommonBytes (register size_t val) {
-    if (MEM_isLittleEndian()) {
-        if (MEM_64bits()) {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanForward64( &r, (U64)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctzll((U64)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
-                                                     0, 3, 1, 3, 1, 4, 2, 7,
-                                                     0, 2, 3, 6, 1, 5, 3, 5,
-                                                     1, 3, 4, 4, 2, 5, 6, 7,
-                                                     7, 0, 1, 2, 3, 3, 4, 6,
-                                                     2, 6, 5, 5, 3, 4, 5, 6,
-                                                     7, 1, 2, 4, 6, 4, 4, 5,
-                                                     7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+  if (MEM_isLittleEndian()) {
+    if (MEM_64bits()) {
+#    if defined(_MSC_VER) && defined(_WIN64)
+      unsigned long r = 0;
+      _BitScanForward64( &r, (U64)val );
+      return (unsigned)(r>>3);
+#     elif defined(__GNUC__) && (__GNUC__ >= 3)
+      return (__builtin_ctzll((U64)val) >> 3);
+#     else
+      static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+                                               0, 3, 1, 3, 1, 4, 2, 7,
+                                               0, 2, 3, 6, 1, 5, 3, 5,
+                                               1, 3, 4, 4, 2, 5, 6, 7,
+                                               7, 0, 1, 2, 3, 3, 4, 6,
+                                               2, 6, 5, 5, 3, 4, 5, 6,
+                                               7, 1, 2, 4, 6, 4, 4, 5,
+                                               7, 2, 6, 5, 7, 6, 7, 7 };
+      return DeBruijnBytePos[
+          ((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+#     endif
+  } else { /* 32 bits */
+#     if defined(_MSC_VER)
+      unsigned long r=0;
+      _BitScanForward( &r, (U32)val );
+      return (unsigned)(r>>3);
+#     elif defined(__GNUC__) && (__GNUC__ >= 3)
+      return (__builtin_ctz((U32)val) >> 3);
+#     else
+      static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+                                               3, 2, 2, 1, 3, 2, 0, 1,
+                                               3, 3, 1, 2, 2, 2, 2, 0,
+                                               3, 1, 2, 0, 1, 0, 1, 1 };
+      return DeBruijnBytePos[
+          ((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+#     endif
+    }
+  } else {  /* Big Endian CPU */
+    if (MEM_64bits()) {
+#     if defined(_MSC_VER) && defined(_WIN64)
+      unsigned long r = 0;
+      _BitScanReverse64( &r, val );
+      return (unsigned)(r>>3);
+#     elif defined(__GNUC__) && (__GNUC__ >= 3)
+      return (__builtin_clzll(val) >> 3);
+#     else
+      unsigned r;
+      /* calculate this way due to compiler complaining in 32-bits mode */
+      const unsigned n32 = sizeof(size_t)*4;
+      if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+      if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+      r += (!val);
+      return r;
 #       endif
-        } else { /* 32 bits */
-#       if defined(_MSC_VER)
-            unsigned long r=0;
-            _BitScanForward( &r, (U32)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctz((U32)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
-                                                     3, 2, 2, 1, 3, 2, 0, 1,
-                                                     3, 3, 1, 2, 2, 2, 2, 0,
-                                                     3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-        }
-    } else {  /* Big Endian CPU */
-        if (MEM_64bits()) {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanReverse64( &r, val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clzll(val) >> 3);
-#       else
-            unsigned r;
-            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
-            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
-            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        } else { /* 32 bits */
-#       if defined(_MSC_VER)
-            unsigned long r = 0;
-            _BitScanReverse( &r, (unsigned long)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clz((U32)val) >> 3);
-#       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-    }   }
+    } else { /* 32 bits */
+#     if defined(_MSC_VER)
+      unsigned long r = 0;
+      _BitScanReverse( &r, (unsigned long)val );
+      return (unsigned)(r>>3);
+#     elif defined(__GNUC__) && (__GNUC__ >= 3)
+      return (__builtin_clz((U32)val) >> 3);
+#     else
+      unsigned r;
+      if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+      r += (!val);
+      return r;
+#     endif
+    }
+  }
 }
 
 // From lib/compress/zstd_compress.c
@@ -230,8 +229,8 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
  *
  * We count only bytes where pMatch > pBaes and pIn > pAnchor.
  */
-size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
-                        const BYTE *pMatch, const BYTE *pBase) {
+static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+                                  const BYTE *pMatch, const BYTE *pBase) {
   size_t matchLength = 0;
   while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
     pIn--;
@@ -482,29 +481,29 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
 /**
  * Return the upper (most significant) LDM_HASHLOG bits.
  */
-static hash_t checksumToHash(U64 sum) {
-  return sum >> (64 - LDM_HASHLOG);
+static hash_t getSmallHash(U64 hash) {
+  return hash >> (64 - LDM_HASHLOG);
 }
 
 /**
  * Return the 32 bits after the upper LDM_HASHLOG bits.
  */
-static U32 checksumFromHfHash(U64 hfHash) {
-  return (hfHash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
+static U32 getChecksum(U64 hash) {
+  return (hash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
 }
 
 #ifdef TMP_TAG_INSERT
-static U32 lowerBitsFromHfHash(U64 hfHash) {
+static U32 lowerBitsFromHfHash(U64 hash) {
   // The number of bits used so far is LDM_HASHLOG + 32.
   // So there are 32 - LDM_HASHLOG bits left.
   // Occasional hashing requires HASH_ONLY_EVERY_LOG bits.
   // So if 32 - LDMHASHLOG < HASH_ONLY_EVERY_LOG, just return lower bits
   // allowing for reuse of bits.
   if (32 - LDM_HASHLOG < HASH_ONLY_EVERY_LOG) {
-    return hfHash & HASH_ONLY_EVERY;
+    return hash & HASH_ONLY_EVERY;
   } else {
     // Otherwise shift by (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG) bits first.
-    return (hfHash >> (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG)) &
+    return (hash >> (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG)) &
            HASH_ONLY_EVERY;
   }
 }
@@ -519,14 +518,14 @@ static U32 lowerBitsFromHfHash(U64 hfHash) {
  * where the constant a is defined to be prime8bytes.
  *
  * The implementation adds an offset to each byte, so
- * H(s) = (s_1 + CHECKSUM_CHAR_OFFSET)*(a^(k-1)) + ...
+ * H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ...
  */
-static U64 getChecksum(const BYTE *buf, U32 len) {
+static U64 getHash(const BYTE *buf, U32 len) {
   U64 ret = 0;
   U32 i;
   for (i = 0; i < len; i++) {
     ret *= prime8bytes;
-    ret += buf[i] + CHECKSUM_CHAR_OFFSET;
+    ret += buf[i] + HASH_CHAR_OFFSET;
   }
   return ret;
 
@@ -544,20 +543,20 @@ static U64 ipow(U64 base, U64 exp) {
   return ret;
 }
 
-static U64 updateChecksum(U64 sum, U32 len,
-                          BYTE toRemove, BYTE toAdd) {
+static U64 updateHash(U64 hash, U32 len,
+                      BYTE toRemove, BYTE toAdd) {
   // TODO: this relies on compiler optimization.
   // The exponential can be calculated explicitly as len is constant.
-  sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
+  hash -= ((toRemove + HASH_CHAR_OFFSET) *
           ipow(prime8bytes, len - 1));
-  sum *= prime8bytes;
-  sum += toAdd + CHECKSUM_CHAR_OFFSET;
-  return sum;
+  hash *= prime8bytes;
+  hash += toAdd + HASH_CHAR_OFFSET;
+  return hash;
 }
 
 /**
- * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
- * based on cctx->lastSum and cctx->lastPosHashed.
+ * Update cctx->nextHash and cctx->nextPosHashed
+ * based on cctx->lastHash and cctx->lastPosHashed.
  *
  * This uses a rolling hash and requires that the last position hashed
  * corresponds to cctx->nextIp - step.
@@ -574,15 +573,15 @@ static void setNextHash(LDM_CCtx *cctx) {
   cctx->DEBUG_setNextHash = cctx->nextIp;
 #endif
 
-  cctx->nextSum = updateChecksum(
-      cctx->lastSum, LDM_HASH_LENGTH,
+  cctx->nextHash = updateHash(
+      cctx->lastHash, LDM_HASH_LENGTH,
       cctx->lastPosHashed[0],
       cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
 
 #ifdef TMP_TAG_INSERT
   {
-    U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextSum);
+    U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextHash);
     cctx->stats.TMP_totalHashCount++;
     cctx->stats.TMP_hashCount[hashEveryMask]++;
   }
@@ -590,18 +589,18 @@ static void setNextHash(LDM_CCtx *cctx) {
 
 #if LDM_LAG
   if (cctx->ip - cctx->ibase > LDM_LAG) {
-    cctx->lagSum = updateChecksum(
-      cctx->lagSum, LDM_HASH_LENGTH,
+    cctx->lagHash = updateHash(
+      cctx->lagHash, LDM_HASH_LENGTH,
       cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
     cctx->lagIp++;
   }
 #endif
 
 #ifdef RUN_CHECKS
-  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
+  check = getHash(cctx->nextIp, LDM_HASH_LENGTH);
 
-  if (check != cctx->nextSum) {
-    printf("CHECK: setNextHash failed %llu %llu\n", check, cctx->nextSum);
+  if (check != cctx->nextHash) {
+    printf("CHECK: setNextHash failed %llu %llu\n", check, cctx->nextHash);
   }
 
   if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
@@ -612,58 +611,57 @@ static void setNextHash(LDM_CCtx *cctx) {
 #endif
 }
 
-static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hfHash) {
+static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
 #if LDM_LAG
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
     // TODO: Off by one, but not important.
     if (cctx->lagIp - cctx->ibase > 0) {
-      U32 hash = checksumToHash(cctx->lagSum);
-      U32 sum = checksumFromHfHash(cctx->lagSum);
-      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, sum };
+      U32 smallHash = getSmallHash(cctx->lagHash);
+      U32 checksum = getChecksum(cctx->lagHash);
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, checksum };
 #ifdef TMP_EVICTION
-      HASH_insert(cctx->hashTable, hash, entry, cctx);
+      HASH_insert(cctx->hashTable, smallHash, entry, cctx);
 #else
-      HASH_insert(cctx->hashTable, hash, entry);
+      HASH_insert(cctx->hashTable, smallHash, entry);
 #endif
     } else {
-      U32 hash = checksumToHash(hfHash);
-      U32 sum = checksumFromHfHash(hfHash);
+      U32 smallHash = getSmallHash(hash);
+      U32 checksum = getChecksum(hash);
 
-      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
 #ifdef TMP_EVICTION
-      HASH_insert(cctx->hashTable, hash, entry, cctx);
+      HASH_insert(cctx->hashTable, smallHash, entry, cctx);
 #else
-      HASH_insert(cctx->hashTable, hash, entry);
+      HASH_insert(cctx->hashTable, smallHash, entry);
 #endif
     }
   }
 #else
 #ifdef TMP_TAG_INSERT
-  U32 hashEveryMask = lowerBitsFromHfHash(hfHash);
-  // TODO: look at stats.
+  U32 hashEveryMask = lowerBitsFromHfHash(hash);
   if (hashEveryMask == HASH_ONLY_EVERY) {
 #else
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
 #endif
-    U32 hash = checksumToHash(hfHash);
-    U32 sum = checksumFromHfHash(hfHash);
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
+    U32 smallHash = getSmallHash(hash);
+    U32 checksum = getChecksum(hash);
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
 #ifdef TMP_EVICTION
-    HASH_insert(cctx->hashTable, hash, entry, cctx);
+    HASH_insert(cctx->hashTable, smallHash, entry, cctx);
 #else
-    HASH_insert(cctx->hashTable, hash, entry);
+    HASH_insert(cctx->hashTable, smallHash, entry);
 #endif
   }
 #endif
 
   cctx->lastPosHashed = cctx->ip;
-  cctx->lastSum = hfHash;
+  cctx->lastHash = hash;
 }
 
 /**
- * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
+ * Copy over the cctx->lastHash, and cctx->lastPosHashed
  * fields from the "next" fields.
  *
  * This requires that cctx->ip == cctx->nextPosHashed.
@@ -675,14 +673,14 @@ static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
            cctx->ip - cctx->ibase);
   }
 #endif
-  putHashOfCurrentPositionFromHash(cctx, cctx->nextSum);
+  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash);
 }
 
 /**
  * Insert hash of the current position into the hash table.
  */
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U64 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
+  U64 hash = getHash(cctx->ip, LDM_HASH_LENGTH);
 
 #ifdef RUN_CHECKS
   if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
@@ -691,7 +689,7 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   }
 #endif
 
-  putHashOfCurrentPositionFromHash(cctx, sum);
+  putHashOfCurrentPositionFromHash(cctx, hash);
 }
 
 void LDM_initializeCCtx(LDM_CCtx *cctx,
@@ -726,7 +724,9 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->nextIp = cctx->ip + cctx->step;
   cctx->nextPosHashed = 0;
 
+#ifdef RUN_CHECKS
   cctx->DEBUG_setNextHash = 0;
+#endif
 }
 
 void LDM_destroyCCtx(LDM_CCtx *cctx) {
@@ -748,16 +748,16 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
   cctx->nextIp = cctx->ip + cctx->step;
 
   while (entry == NULL) {
-    hash_t h;
     U64 hash;
-    U32 sum;
+    hash_t smallHash;
+    U32 checksum;
 #ifdef TMP_TAG_INSERT
     U32 hashEveryMask;
 #endif
     setNextHash(cctx);
-    hash = cctx->nextSum;
-    h = checksumToHash(hash);
-    sum = checksumFromHfHash(hash);
+    hash = cctx->nextHash;
+    smallHash = getSmallHash(hash);
+    checksum = getChecksum(hash);
 #ifdef TMP_TAG_INSERT
     hashEveryMask = lowerBitsFromHfHash(hash);
 #endif
@@ -770,11 +770,11 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     }
 #ifdef TMP_TAG_INSERT
     if (hashEveryMask == HASH_ONLY_EVERY) {
-      entry = HASH_getBestEntry(cctx, h, sum,
+      entry = HASH_getBestEntry(cctx, smallHash, checksum,
                                 forwardMatchLength, backwardMatchLength);
     }
 #else
-    entry = HASH_getBestEntry(cctx, h, sum,
+    entry = HASH_getBestEntry(cctx, smallHash, checksum,
                               forwardMatchLength, backwardMatchLength);
 #endif
 
@@ -850,15 +850,16 @@ size_t LDM_compress(const void *src, size_t srcSize,
   U64 backwardsMatchLength = 0;
 
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+#ifdef OUTPUT_CONFIGURATION
   LDM_outputConfiguration();
+#endif
 
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
 
 #if LDM_LAG
   cctx.lagIp = cctx.ip;
-//  cctx.lagHash = cctx.lastHash;
-  cctx.lagSum = cctx.lastSum;
+  cctx.lagHash = cctx.lastHash;
 #endif
   /**
    * Find a match.
@@ -918,8 +919,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
     LDM_updateLastHashFromNextHash(&cctx);
   }
 
-  // HASH_outputTableOffsetHistogram(&cctx);
-
   /* Encode the last literals (no more matches). */
   {
     const U64 lastRun = cctx.iend - cctx.anchor;
@@ -943,14 +942,14 @@ size_t LDM_compress(const void *src, size_t srcSize,
 void LDM_test(const BYTE *src) {
   const U32 diff = 100;
   const BYTE *pCur = src + diff;
-  U64 checksum = getChecksum(pCur, LDM_HASH_LENGTH);
+  U64 hash = getHash(pCur, LDM_HASH_LENGTH);
 
   for (; pCur < src + diff + 60; ++pCur) {
-    U64 nextSum = getChecksum(pCur + 1, LDM_HASH_LENGTH);
-    U64 updateSum = updateChecksum(checksum, LDM_HASH_LENGTH,
-                                   pCur[0], pCur[LDM_HASH_LENGTH]);
-    checksum = nextSum;
-    printf("%llu %llu\n", nextSum, updateSum);
+    U64 nextHash = getHash(pCur + 1, LDM_HASH_LENGTH);
+    U64 updatedHash = updateHash(hash, LDM_HASH_LENGTH,
+                                 pCur[0], pCur[LDM_HASH_LENGTH]);
+    hash = nextHash;
+    printf("%llu %llu\n", nextHash, updatedHash);
   }
 }
 
diff --git a/contrib/long_distance_matching/ldm_common.c b/contrib/long_distance_matching/ldm_common.c
index 673959db..1aa664f0 100644
--- a/contrib/long_distance_matching/ldm_common.c
+++ b/contrib/long_distance_matching/ldm_common.c
@@ -109,5 +109,3 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
   }
   return dctx.op - (BYTE *)dst;
 }
-
-
diff --git a/contrib/long_distance_matching/ldm_integrated.c b/contrib/long_distance_matching/ldm_integrated.c
index d51c1e9d..6440c009 100644
--- a/contrib/long_distance_matching/ldm_integrated.c
+++ b/contrib/long_distance_matching/ldm_integrated.c
@@ -17,7 +17,7 @@
 
 #define COMPUTE_STATS
 #define OUTPUT_CONFIGURATION
-#define CHECKSUM_CHAR_OFFSET 10
+#define CHECKSUM_CHAR_OFFSET 1
 
 // Take first match only.
 //#define ZSTD_SKIP

From 0295a27133e68f61ce398006a080353453cb2b40 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Mon, 24 Jul 2017 15:26:44 -0700
Subject: [PATCH 091/100] Experiment with not using a checksum

---
 contrib/long_distance_matching/ldm.c          |  23 ++++
 contrib/long_distance_matching/ldm.h          |  20 +---
 contrib/long_distance_matching/ldm_64_hash.c  | 104 +++++++++++++++---
 contrib/long_distance_matching/ldm_common.c   |  13 ---
 .../long_distance_matching/ldm_integrated.c   |  37 ++++++-
 5 files changed, 151 insertions(+), 46 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 9d3eda32..fae35f9e 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -10,6 +10,14 @@
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
+#define LDM_HASH_ENTRY_SIZE_LOG 3
+
+//#define HASH_ONLY_EVERY_LOG 7
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
+
+#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
+
+
 #define COMPUTE_STATS
 #define OUTPUT_CONFIGURATION
 #define CHECKSUM_CHAR_OFFSET 10
@@ -510,6 +518,21 @@ size_t LDM_compress(const void *src, size_t srcSize,
   }
 }
 
+void LDM_outputConfiguration(void) {
+  printf("=====================\n");
+  printf("Configuration\n");
+  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
+         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
+  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
+  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
+  printf("LDM_LAG %d\n", LDM_LAG);
+  printf("=====================\n");
+}
+
+
+
 void LDM_test(const BYTE *src) {
   (void)src;
 }
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 3078fb8c..e1a005e3 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -32,23 +32,15 @@
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
 //These should be multiples of four (and perhaps set to the same value?).
-#define LDM_MIN_MATCH_LENGTH 16
-#define LDM_HASH_LENGTH 16
+#define LDM_MIN_MATCH_LENGTH 64
+#define LDM_HASH_LENGTH 64
 
 // Experimental.
-//#define TMP_EVICTION
-#define TMP_TAG_INSERT
-//#define TMP_FORCE_HASH_ONLY
+//#define TMP_EVICTION    // Experiment with eviction policies.
+#define TMP_TAG_INSERT    // Insertion policy based on hash.
 
-#define LDM_HASH_ENTRY_SIZE_LOG 3
-
-// Insert every (HASH_ONLY_EVERY + 1) into the hash table.
-#ifdef TMP_FORCE_HASH_ONLY
-  #define HASH_ONLY_EVERY_LOG 7
-#else
-  #define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
-#endif
-#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
+#define USE_CHECKSUM 1
+//#define USE_CHECKSUM (HASH_BUCKET_SIZE_LOG)
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
diff --git a/contrib/long_distance_matching/ldm_64_hash.c b/contrib/long_distance_matching/ldm_64_hash.c
index 06ddf520..c74d71ff 100644
--- a/contrib/long_distance_matching/ldm_64_hash.c
+++ b/contrib/long_distance_matching/ldm_64_hash.c
@@ -7,9 +7,20 @@
 #include "ldm.h"
 
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
+#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
-/* Hash table stuff. */
+#if USE_CHECKSUM
+  #define LDM_HASH_ENTRY_SIZE_LOG 3
+#else
+  #define LDM_HASH_ENTRY_SIZE_LOG 2
+#endif
+
+//#define HASH_ONLY_EVERY_LOG 7
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
+
+#define HASH_ONLY_EVERY ((1 << (HASH_ONLY_EVERY_LOG)) - 1)
+
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
@@ -27,10 +38,16 @@ static const U64 prime8bytes = 11400714785074694791ULL;
 // Type of the small hash used to index into the hash table.
 typedef U32 hash_t;
 
+#if USE_CHECKSUM
 typedef struct LDM_hashEntry {
   U32 offset;
   U32 checksum;
 } LDM_hashEntry;
+#else
+typedef struct LDM_hashEntry {
+  U32 offset;
+} LDM_hashEntry;
+#endif
 
 struct LDM_compressStats {
   U32 windowSizeLog, hashTableSizeLog;
@@ -39,6 +56,8 @@ struct LDM_compressStats {
   U64 totalLiteralLength;
   U64 totalOffset;
 
+  U32 matchLengthHistogram[32];
+
   U32 minOffset, maxOffset;
   U32 offsetHistogram[32];
 
@@ -262,12 +281,19 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
   LDM_hashEntry *cur = bucket;
   LDM_hashEntry *bestEntry = NULL;
   U64 bestMatchLength = 0;
+#if !(USE_CHECKSUM)
+  (void)checksum;
+#endif
   for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
     const BYTE *pMatch = cur->offset + cctx->ibase;
 
     // Check checksum for faster check.
+#if USE_CHECKSUM
     if (cur->checksum == checksum &&
         cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
+#else
+    if (cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
+#endif
       U64 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
       U64 backwardMatchLength, totalMatchLength;
 
@@ -448,12 +474,18 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
          stats->minOffset, stats->maxOffset);
 
   printf("\n");
-  printf("offset histogram: offset, num matches, %% of matches\n");
+  printf("offset histogram | match length histogram\n");
+  printf("offset/ML, num matches, %% of matches | num matches, %% of matches\n");
 
   for (; i <= intLog2(stats->maxOffset); i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+    printf("2^%*d: %10u    %6.3f%% |2^%*d:  %10u    %6.3f \n",
+           2, i,
            stats->offsetHistogram[i],
            100.0 * (double) stats->offsetHistogram[i] /
+                   (double) stats->numMatches,
+           2, i,
+           stats->matchLengthHistogram[i],
+           100.0 * (double) stats->matchLengthHistogram[i] /
                    (double) stats->numMatches);
   }
   printf("\n");
@@ -619,23 +651,32 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
     // TODO: Off by one, but not important.
     if (cctx->lagIp - cctx->ibase > 0) {
       U32 smallHash = getSmallHash(cctx->lagHash);
+
+#   if USE_CHECKSUM
       U32 checksum = getChecksum(cctx->lagHash);
       const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, checksum };
-#ifdef TMP_EVICTION
-      HASH_insert(cctx->hashTable, smallHash, entry, cctx);
-#else
-      HASH_insert(cctx->hashTable, smallHash, entry);
-#endif
-    } else {
-      U32 smallHash = getSmallHash(hash);
-      U32 checksum = getChecksum(hash);
+#   else
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase };
+#   endif
 
-      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
-#ifdef TMP_EVICTION
+#   ifdef TMP_EVICTION
       HASH_insert(cctx->hashTable, smallHash, entry, cctx);
-#else
+#   else
       HASH_insert(cctx->hashTable, smallHash, entry);
-#endif
+#   endif
+    } else {
+#   if USE_CHECKSUM
+      U32 checksum = getChecksum(hash);
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, checksum };
+#   else
+      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase };
+#   endif
+
+#   ifdef TMP_EVICTION
+      HASH_insert(cctx->hashTable, smallHash, entry, cctx);
+#   else
+      HASH_insert(cctx->hashTable, smallHash, entry);
+#    endif
     }
   }
 #else
@@ -646,8 +687,12 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
   if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
 #endif
     U32 smallHash = getSmallHash(hash);
+#if USE_CHECKSUM
     U32 checksum = getChecksum(hash);
     const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
+#else
+    const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
+#endif
 #ifdef TMP_EVICTION
     HASH_insert(cctx->hashTable, smallHash, entry, cctx);
 #else
@@ -711,8 +756,11 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->anchor = cctx->ibase;
 
   memset(&(cctx->stats), 0, sizeof(cctx->stats));
+#if USE_CHECKSUM
   cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64);
-
+#else
+  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U32);
+#endif
   cctx->stats.minOffset = UINT_MAX;
   cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
   cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
@@ -755,6 +803,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     U32 hashEveryMask;
 #endif
     setNextHash(cctx);
+
     hash = cctx->nextHash;
     smallHash = getSmallHash(hash);
     checksum = getChecksum(hash);
@@ -770,6 +819,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     }
 #ifdef TMP_TAG_INSERT
     if (hashEveryMask == HASH_ONLY_EVERY) {
+
       entry = HASH_getBestEntry(cctx, smallHash, checksum,
                                 forwardMatchLength, backwardMatchLength);
     }
@@ -781,7 +831,9 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     if (entry != NULL) {
       *match = entry->offset + cctx->ibase;
     }
+
     putHashOfCurrentPositionFromHash(cctx, hash);
+
   }
   setNextHash(cctx);
   return 0;
@@ -850,6 +902,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
   U64 backwardsMatchLength = 0;
 
   LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+
 #ifdef OUTPUT_CONFIGURATION
   LDM_outputConfiguration();
 #endif
@@ -869,6 +922,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
    */
   while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
          &backwardsMatchLength) == 0) {
+
 #ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
 #endif
@@ -898,6 +952,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
       cctx.stats.maxOffset =
           offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
       cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
+      cctx.stats.matchLengthHistogram[
+          (U32)intLog2(matchLength + LDM_MIN_MATCH_LENGTH)]++;
 #endif
 
       // Move ip to end of block, inserting hashes at each position.
@@ -938,6 +994,22 @@ size_t LDM_compress(const void *src, size_t srcSize,
   }
 }
 
+void LDM_outputConfiguration(void) {
+  printf("=====================\n");
+  printf("Configuration\n");
+  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
+         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
+  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
+  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
+  printf("LDM_LAG %d\n", LDM_LAG);
+  printf("USE_CHECKSUM %d\n", USE_CHECKSUM);
+  printf("=====================\n");
+}
+
+
+
 // TODO: implement and test hash function
 void LDM_test(const BYTE *src) {
   const U32 diff = 100;
diff --git a/contrib/long_distance_matching/ldm_common.c b/contrib/long_distance_matching/ldm_common.c
index 1aa664f0..1953656e 100644
--- a/contrib/long_distance_matching/ldm_common.c
+++ b/contrib/long_distance_matching/ldm_common.c
@@ -2,19 +2,6 @@
 
 #include "ldm.h"
 
-void LDM_outputConfiguration(void) {
-  printf("=====================\n");
-  printf("Configuration\n");
-  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
-  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
-         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
-  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
-  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
-  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
-  printf("LDM_LAG %d\n", LDM_LAG);
-  printf("=====================\n");
-}
-
 void LDM_readHeader(const void *src, U64 *compressedSize,
                     U64 *decompressedSize) {
   const BYTE *ip = (const BYTE *)src;
diff --git a/contrib/long_distance_matching/ldm_integrated.c b/contrib/long_distance_matching/ldm_integrated.c
index 6440c009..b80a5c01 100644
--- a/contrib/long_distance_matching/ldm_integrated.c
+++ b/contrib/long_distance_matching/ldm_integrated.c
@@ -7,10 +7,16 @@
 #include "ldm.h"
 
 #define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASH_ENTRY_SIZE_LOG 3
 #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
 #define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
 
+#define LDM_HASH_ENTRY_SIZE_LOG 3
+//#define HASH_ONLY_EVERY_LOG 7
+#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
+
+#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
+
+
 /* Hash table stuff. */
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
@@ -38,6 +44,8 @@ struct LDM_compressStats {
   U64 totalLiteralLength;
   U64 totalOffset;
 
+  U32 matchLengthHistogram[32];
+
   U32 minOffset, maxOffset;
 
   U32 offsetHistogram[32];
@@ -358,12 +366,18 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
          stats->minOffset, stats->maxOffset);
 
   printf("\n");
-  printf("offset histogram: offset, num matches, %% of matches\n");
+  printf("offset histogram | match length histogram\n");
+  printf("offset/ML, num matches, %% of matches | num matches, %% of matches\n");
 
   for (; i <= intLog2(stats->maxOffset); i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
+    printf("2^%*d: %10u    %6.3f%% |2^%*d:  %10u    %6.3f \n",
+           2, i,
            stats->offsetHistogram[i],
            100.0 * (double) stats->offsetHistogram[i] /
+                   (double) stats->numMatches,
+           2, i,
+           stats->matchLengthHistogram[i],
+           100.0 * (double) stats->matchLengthHistogram[i] /
                    (double) stats->numMatches);
   }
   printf("\n");
@@ -742,6 +756,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
       cctx.stats.maxOffset =
           offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
       cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
+      cctx.stats.matchLengthHistogram[
+          (U32)intLog2(matchLength + LDM_MIN_MATCH_LENGTH)]++;
 #endif
 
       // Move ip to end of block, inserting hashes at each position.
@@ -784,6 +800,21 @@ size_t LDM_compress(const void *src, size_t srcSize,
   }
 }
 
+void LDM_outputConfiguration(void) {
+  printf("=====================\n");
+  printf("Configuration\n");
+  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
+  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
+         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
+  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
+  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
+  printf("LDM_LAG %d\n", LDM_LAG);
+  printf("=====================\n");
+}
+
+
+
 // TODO: implement and test hash function
 void LDM_test(const BYTE *src) {
   (void)src;

From ae20d413daf35cac05fdb76cb20bbb3b1bf053a1 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Tue, 25 Jul 2017 12:52:01 -0700
Subject: [PATCH 092/100] [libzstd] Fix CHECK_V_F macros

---
 lib/compress/fse_compress.c | 2 +-
 lib/compress/huf_compress.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 26e8052d..3a03627c 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -781,7 +781,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
 
 size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
 
-#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
+#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
 #define CHECK_F(f)   { CHECK_V_F(_var_err__, f); }
 
 /* FSE_compress_wksp() :
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index beb4fdb6..953cb5f2 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -56,7 +56,7 @@
 *  Error Management
 ****************************************************************/
 #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
-#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
+#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
 #define CHECK_F(f)   { CHECK_V_F(_var_err__, f); }
 
 

From 629c30011880ed0f21d454993d53db7aae0cfdbc Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Tue, 25 Jul 2017 15:17:36 -0700
Subject: [PATCH 093/100] Rename and remove unneeded files

---
 contrib/long_distance_matching/Makefile       |  13 +-
 .../circular_buffer_table.c                   | 256 ------------------
 contrib/long_distance_matching/ldm.h          |  14 +-
 .../{ldm_integrated.c => ldm_hash32.c}        |   0
 .../{ldm_64_hash.c => ldm_hash64.c}           |  65 +++--
 .../long_distance_matching/ldm_hashtable.h    |  91 -------
 .../{main-ldm.c => main.c}                    |  15 +-
 7 files changed, 65 insertions(+), 389 deletions(-)
 delete mode 100644 contrib/long_distance_matching/circular_buffer_table.c
 rename contrib/long_distance_matching/{ldm_integrated.c => ldm_hash32.c} (100%)
 rename contrib/long_distance_matching/{ldm_64_hash.c => ldm_hash64.c} (96%)
 delete mode 100644 contrib/long_distance_matching/ldm_hashtable.h
 rename contrib/long_distance_matching/{main-ldm.c => main.c} (95%)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index e1c31112..292ce851 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -1,5 +1,5 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-present, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under the BSD-style license found in the
@@ -25,19 +25,16 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-circular-buffer main-integrated main-64 
+all: main-hash32 main-hash64 
 
-main-circular-buffer: ldm_common.c circular_buffer_table.c ldm.c main-ldm.c
+main-hash64: ldm_common.c ldm_hash64.c main.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-64: ldm_common.c ldm_64_hash.c main-ldm.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-main-integrated: ldm_common.c ldm_integrated.c main-ldm.c
+main-hash32: ldm_common.c ldm_hash32.c main.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-circular-buffer main-64 main-integrated 
+	main-hash64 main-hash32
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/circular_buffer_table.c b/contrib/long_distance_matching/circular_buffer_table.c
deleted file mode 100644
index 92ffc55b..00000000
--- a/contrib/long_distance_matching/circular_buffer_table.c
+++ /dev/null
@@ -1,256 +0,0 @@
-#include <stdlib.h>
-#include <stdio.h>
-
-#include "ldm.h"
-#include "ldm_hashtable.h"
-#include "mem.h"
-
-// THe number of elements per hash bucket.
-// HASH_BUCKET_SIZE_LOG is defined in ldm.h.
-#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
-
-// The number of hash buckets.
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
-
-// If ZSTD_SKIP is defined, then the first entry is returned in HASH_getBestEntry
-// (without looking at other entries in the bucket).
-//#define ZSTD_SKIP
-
-struct LDM_hashTable {
-  U32 numBuckets;          // The number of buckets.
-  U32 numEntries;          // numBuckets * HASH_BUCKET_SIZE.
-  LDM_hashEntry *entries;
-  BYTE *bucketOffsets;     // A pointer (per bucket) to the next insert position.
-
-  const BYTE *offsetBase;  // Corresponds to offset=0 in LDM_hashEntry.
-  U32 minMatchLength;
-  U32 maxWindowSize;
-};
-
-LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
-                                U32 minMatchLength, U32 maxWindowSize) {
-  LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
-  table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
-  table->numEntries = size;
-  table->entries = calloc(size, sizeof(LDM_hashEntry));
-  table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
-  table->offsetBase = offsetBase;
-  table->minMatchLength = minMatchLength;
-  table->maxWindowSize = maxWindowSize;
-  return table;
-}
-
-static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
-  return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
-}
-
-// From lib/compress/zstd_compress.c
-static unsigned ZSTD_NbCommonBytes (register size_t val)
-{
-    if (MEM_isLittleEndian()) {
-        if (MEM_64bits()) {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanForward64( &r, (U64)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctzll((U64)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
-                                                     0, 3, 1, 3, 1, 4, 2, 7,
-                                                     0, 2, 3, 6, 1, 5, 3, 5,
-                                                     1, 3, 4, 4, 2, 5, 6, 7,
-                                                     7, 0, 1, 2, 3, 3, 4, 6,
-                                                     2, 6, 5, 5, 3, 4, 5, 6,
-                                                     7, 1, 2, 4, 6, 4, 4, 5,
-                                                     7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-#       endif
-        } else { /* 32 bits */
-#       if defined(_MSC_VER)
-            unsigned long r=0;
-            _BitScanForward( &r, (U32)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctz((U32)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
-                                                     3, 2, 2, 1, 3, 2, 0, 1,
-                                                     3, 3, 1, 2, 2, 2, 2, 0,
-                                                     3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-        }
-    } else {  /* Big Endian CPU */
-        if (MEM_64bits()) {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanReverse64( &r, val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clzll(val) >> 3);
-#       else
-            unsigned r;
-            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
-            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
-            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        } else { /* 32 bits */
-#       if defined(_MSC_VER)
-            unsigned long r = 0;
-            _BitScanReverse( &r, (unsigned long)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clz((U32)val) >> 3);
-#       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-    }   }
-}
-
-/**
- * From lib/compress/zstd_compress.c
- * Returns the number of bytes (consecutively) in common between pIn and pMatch
- * up to pInLimit.
- */
-static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *const pInLimit) {
-    const BYTE * const pStart = pIn;
-    const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
-
-    while (pIn < pInLoopLimit) {
-        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
-        if (!diff) {
-          pIn += sizeof(size_t);
-          pMatch += sizeof(size_t);
-          continue;
-        }
-        pIn += ZSTD_NbCommonBytes(diff);
-        return (size_t)(pIn - pStart);
-    }
-
-    if (MEM_64bits()) {
-      if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
-        pIn += 4;
-        pMatch += 4;
-      }
-    }
-    if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
-      pIn += 2;
-      pMatch += 2;
-    }
-    if ((pIn < pInLimit) && (*pMatch == *pIn)) {
-      pIn++;
-    }
-    return (size_t)(pIn - pStart);
-}
-
-/**
- * Returns the number of bytes in common between pIn and pMatch,
- * counting backwards, with pIn having a lower limit of pAnchor and
- * pMatch having a lower limit of pBase.
- */
-static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
-                                  const BYTE *pMatch, const BYTE *pBase) {
-  size_t matchLength = 0;
-  while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
-    pIn--;
-    pMatch--;
-    matchLength++;
-  }
-  return matchLength;
-}
-
-LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
-                                 const hash_t hash,
-                                 const U32 checksum,
-                                 const BYTE *pIn,
-                                 const BYTE *pEnd,
-                                 const BYTE *pAnchor,
-                                 U64 *pForwardMatchLength,
-                                 U64 *pBackwardMatchLength) {
-  LDM_hashEntry *bucket = getBucket(table, hash);
-  LDM_hashEntry *cur = bucket;
-  LDM_hashEntry *bestEntry = NULL;
-  U64 bestMatchLength = 0;
-  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    const BYTE *pMatch = cur->offset + table->offsetBase;
-
-    // Check checksum for faster check.
-    if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
-      U64 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
-      U64 backwardMatchLength, totalMatchLength;
-
-      // Only take matches where the forwardMatchLength is large enough
-      // for speed.
-      if (forwardMatchLength < table->minMatchLength) {
-        continue;
-      }
-      backwardMatchLength =
-          countBackwardsMatch(pIn, pAnchor, cur->offset + table->offsetBase,
-                              table->offsetBase);
-
-      totalMatchLength = forwardMatchLength + backwardMatchLength;
-
-      if (totalMatchLength >= bestMatchLength) {
-        bestMatchLength = totalMatchLength;
-        *pForwardMatchLength = forwardMatchLength;
-        *pBackwardMatchLength = backwardMatchLength;
-
-        bestEntry = cur;
-
-#ifdef ZSTD_SKIP
-        return cur;
-#endif
-      }
-    }
-  }
-  if (bestEntry != NULL) {
-    return bestEntry;
-  }
-  return NULL;
-}
-
-hash_t HASH_hashU32(U32 value) {
-  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
-}
-
-void HASH_insert(LDM_hashTable *table,
-                 const hash_t hash, const LDM_hashEntry entry) {
-  // Circular buffer.
-  *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
-  table->bucketOffsets[hash]++;
-  table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
-}
-
-U32 HASH_getSize(const LDM_hashTable *table) {
-  return table->numBuckets;
-}
-
-void HASH_destroyTable(LDM_hashTable *table) {
-  free(table->entries);
-  free(table->bucketOffsets);
-  free(table);
-}
-
-void HASH_outputTableOccupancy(const LDM_hashTable *table) {
-  U32 ctr = 0;
-  LDM_hashEntry *cur = table->entries;
-  LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
-  for (; cur < end; ++cur) {
-    if (cur->offset == 0) {
-      ctr++;
-    }
-  }
-
-  printf("Num buckets, bucket size: %d, %d\n",
-         table->numBuckets, HASH_BUCKET_SIZE);
-  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         table->numEntries, ctr,
-         100.0 * (double)(ctr) / table->numEntries);
-}
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index e1a005e3..f9ad383e 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -17,17 +17,22 @@
 // THe number of bytes storing the offset.
 #define LDM_OFFSET_SIZE 4
 
+// =============================================================================
+// User parameters.
+// =============================================================================
+
 // Defines the size of the hash table.
 // Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
-#define LDM_MEMORY_USAGE 23
+#define LDM_MEMORY_USAGE 25
 
 // The number of entries in a hash bucket.
-#define HASH_BUCKET_SIZE_LOG 0 // The maximum is 4 for now.
+#define HASH_BUCKET_SIZE_LOG 3 // The maximum is 4 for now.
 
 // Defines the lag in inserting elements into the hash table.
 #define LDM_LAG 0
 
+// The maximum window size.
 #define LDM_WINDOW_SIZE_LOG 28 // Max value is 30
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 
@@ -37,10 +42,11 @@
 
 // Experimental.
 //#define TMP_EVICTION    // Experiment with eviction policies.
-#define TMP_TAG_INSERT    // Insertion policy based on hash.
+#define INSERT_BY_TAG // Insertion policy based on hash.
 
 #define USE_CHECKSUM 1
-//#define USE_CHECKSUM (HASH_BUCKET_SIZE_LOG)
+
+// =============================================================================
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
diff --git a/contrib/long_distance_matching/ldm_integrated.c b/contrib/long_distance_matching/ldm_hash32.c
similarity index 100%
rename from contrib/long_distance_matching/ldm_integrated.c
rename to contrib/long_distance_matching/ldm_hash32.c
diff --git a/contrib/long_distance_matching/ldm_64_hash.c b/contrib/long_distance_matching/ldm_hash64.c
similarity index 96%
rename from contrib/long_distance_matching/ldm_64_hash.c
rename to contrib/long_distance_matching/ldm_hash64.c
index c74d71ff..e51ac57d 100644
--- a/contrib/long_distance_matching/ldm_64_hash.c
+++ b/contrib/long_distance_matching/ldm_hash64.c
@@ -489,7 +489,7 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
                    (double) stats->numMatches);
   }
   printf("\n");
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
 /*
   printf("Lower bit distribution\n");
   for (i = 0; i < (1 << HASH_ONLY_EVERY_LOG); i++) {
@@ -524,7 +524,7 @@ static U32 getChecksum(U64 hash) {
   return (hash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
 }
 
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
 static U32 lowerBitsFromHfHash(U64 hash) {
   // The number of bits used so far is LDM_HASHLOG + 32.
   // So there are 32 - LDM_HASHLOG bits left.
@@ -611,7 +611,7 @@ static void setNextHash(LDM_CCtx *cctx) {
       cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
 
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
   {
     U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextHash);
     cctx->stats.TMP_totalHashCount++;
@@ -647,9 +647,13 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
   // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
   // Note: this works only when cctx->step is 1.
 #if LDM_LAG
-  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-    // TODO: Off by one, but not important.
-    if (cctx->lagIp - cctx->ibase > 0) {
+  if (cctx -> lagIp - cctx->ibase > 0) {
+#ifdef INSERT_BY_TAG
+    U32 hashEveryMask = lowerBitsFromHfHash(cctx->lagHash);
+    if (hashEveryMask == HASH_ONLY_EVERY) {
+#else
+    if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+#endif
       U32 smallHash = getSmallHash(cctx->lagHash);
 
 #   if USE_CHECKSUM
@@ -664,23 +668,32 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
 #   else
       HASH_insert(cctx->hashTable, smallHash, entry);
 #   endif
-    } else {
-#   if USE_CHECKSUM
-      U32 checksum = getChecksum(hash);
-      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, checksum };
-#   else
-      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase };
-#   endif
+    }
+  } else {
+#ifdef INSERT_BY_TAG
+    U32 hashEveryMask = lowerBitsFromHfHash(hash);
+    if (hashEveryMask == HASH_ONLY_EVERY) {
+#else
+    if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+#endif
+      U32 smallHash = getSmallHash(hash);
 
-#   ifdef TMP_EVICTION
+#if USE_CHECKSUM
+      U32 checksum = getChecksum(hash);
+      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
+#else
+      const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
+#endif
+
+#ifdef TMP_EVICTION
       HASH_insert(cctx->hashTable, smallHash, entry, cctx);
-#   else
+#else
       HASH_insert(cctx->hashTable, smallHash, entry);
-#    endif
+#endif
     }
   }
 #else
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
   U32 hashEveryMask = lowerBitsFromHfHash(hash);
   if (hashEveryMask == HASH_ONLY_EVERY) {
 #else
@@ -799,7 +812,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     U64 hash;
     hash_t smallHash;
     U32 checksum;
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
     U32 hashEveryMask;
 #endif
     setNextHash(cctx);
@@ -807,7 +820,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     hash = cctx->nextHash;
     smallHash = getSmallHash(hash);
     checksum = getChecksum(hash);
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
     hashEveryMask = lowerBitsFromHfHash(hash);
 #endif
 
@@ -817,7 +830,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     if (cctx->ip > cctx->imatchLimit) {
       return 1;
     }
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
     if (hashEveryMask == HASH_ONLY_EVERY) {
 
       entry = HASH_getBestEntry(cctx, smallHash, checksum,
@@ -1003,13 +1016,17 @@ void LDM_outputConfiguration(void) {
   printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
   printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
   printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
-  printf("LDM_LAG %d\n", LDM_LAG);
-  printf("USE_CHECKSUM %d\n", USE_CHECKSUM);
+  printf("LDM_LAG: %d\n", LDM_LAG);
+  printf("USE_CHECKSUM: %d\n", USE_CHECKSUM);
+#ifdef INSERT_BY_TAG
+  printf("INSERT_BY_TAG: %d\n", 1);
+#else
+  printf("INSERT_BY_TAG: %d\n", 0);
+#endif
+  printf("HASH_CHAR_OFFSET: %d\n", HASH_CHAR_OFFSET);
   printf("=====================\n");
 }
 
-
-
 // TODO: implement and test hash function
 void LDM_test(const BYTE *src) {
   const U32 diff = 100;
diff --git a/contrib/long_distance_matching/ldm_hashtable.h b/contrib/long_distance_matching/ldm_hashtable.h
deleted file mode 100644
index 6093197d..00000000
--- a/contrib/long_distance_matching/ldm_hashtable.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * A "hash" table used in LDM compression.
- *
- * This is not exactly a hash table in the sense that inserted entries
- * are not guaranteed to remain in the hash table.
- */
-
-#ifndef LDM_HASHTABLE_H
-#define LDM_HASHTABLE_H
-
-#include "mem.h"
-
-// The log size of LDM_hashEntry in bytes.
-#define LDM_HASH_ENTRY_SIZE_LOG 3
-
-typedef U32 hash_t;
-
-typedef struct LDM_hashEntry {
-  U32 offset;       // Represents the offset of the entry from offsetBase.
-  U32 checksum;     // A checksum to select entries with the same hash value.
-} LDM_hashEntry;
-
-typedef struct LDM_hashTable LDM_hashTable;
-
-/**
- * Create a table that can contain size elements. This does not necessarily
- * correspond to the number of hash buckets. The number of hash buckets
- * is size / (1 << HASH_BUCKET_SIZE_LOG)
- *
- * minMatchLength is the minimum match length required in HASH_getBestEntry.
- *
- * maxWindowSize is the maximum distance from pIn in HASH_getBestEntry.
- * The window is defined to be (pIn - offsetBase - offset).
- */
-LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
-                                U32 minMatchLength, U32 maxWindowSize);
-
-/**
- * Return the "best" entry from the table with the same hash and checksum.
- *
- * pIn: a pointer to the current input position.
- * pEnd: a pointer to the maximum input position.
- * pAnchor: a pointer to the minimum input position.
- *
- * This function computes the forward and backward match length from pIn
- * and writes it to forwardMatchLength and backwardsMatchLength.
- *
- * E.g. for the two strings "aaabbbb" "aaabbbb" with pIn and the
- * entry pointing at the first "b", the forward match length would be
- * four (representing the "b" matches) and the backward match length would
- * three (representing the "a" matches before the pointer).
- */
-LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
-                                 const hash_t hash,
-                                 const U32 checksum,
-                                 const BYTE *pIn,
-                                 const BYTE *pEnd,
-                                 const BYTE *pAnchor,
-                                 U64 *forwardMatchLength,
-                                 U64 *backwardsMatchLength);
-
-/**
- * Return a hash of the value.
- */
-hash_t HASH_hashU32(U32 value);
-
-/**
- * Insert an LDM_hashEntry into the bucket corresponding to hash.
- *
- * An entry may be evicted in the process.
- */
-void HASH_insert(LDM_hashTable *table, const hash_t hash,
-                 const LDM_hashEntry entry);
-
-/**
- * Return the number of distinct hash buckets.
- */
-U32 HASH_getSize(const LDM_hashTable *table);
-
-/**
- * Destroy the table.
- */
-void HASH_destroyTable(LDM_hashTable *table);
-
-/**
- * Prints the percentage of the hash table occupied (where occupied is defined
- * as the entry being non-zero).
- */
-void HASH_outputTableOccupancy(const LDM_hashTable *hashTable);
-
-#endif /* LDM_HASHTABLE_H */
diff --git a/contrib/long_distance_matching/main-ldm.c b/contrib/long_distance_matching/main.c
similarity index 95%
rename from contrib/long_distance_matching/main-ldm.c
rename to contrib/long_distance_matching/main.c
index 232c14a2..cee5edba 100644
--- a/contrib/long_distance_matching/main-ldm.c
+++ b/contrib/long_distance_matching/main.c
@@ -12,7 +12,7 @@
 #include "ldm.h"
 #include "zstd.h"
 
-//#define TEST
+//#define DECOMPRESS_AND_VERIFY
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
@@ -91,9 +91,10 @@ static int compress(const char *fname, const char *oname) {
   ftruncate(fdout, compressedSize);
 
 
-  printf("%25s : %10lu -> %10lu - %s (%.2fx --- %.1f%%)\n", fname,
-         (size_t)statbuf.st_size, (size_t)compressedSize, oname,
-         (statbuf.st_size) / (double)compressedSize,
+  printf("%25s : %10lu -> %10lu - %s \n", fname,
+         (size_t)statbuf.st_size, (size_t)compressedSize, oname);
+  printf("Compression ratio: %.2fx --- %.1f%%\n",
+         (double)statbuf.st_size / (double)compressedSize,
          (double)compressedSize / (double)(statbuf.st_size) * 100.0);
 
   timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
@@ -110,6 +111,7 @@ static int compress(const char *fname, const char *oname) {
   return 0;
 }
 
+#ifdef DECOMPRESS
 /* Decompress file compressed using LDM_compress.
  * The input file should have the LDM_HEADER followed by payload.
  * Returns 0 if succesful, and an error code otherwise.
@@ -162,7 +164,6 @@ static int decompress(const char *fname, const char *oname) {
       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
       dst, decompressedSize);
   printf("Ret size out: %zu\n", outSize);
-//  ftruncate(fdout, decompressedSize);
 
   close(fdin);
   close(fdout);
@@ -207,6 +208,7 @@ static void verify(const char *inpFilename, const char *decFilename) {
 	fclose(decFp);
 	fclose(inpFp);
 }
+#endif
 
 int main(int argc, const char *argv[]) {
   const char * const exeName = argv[0];
@@ -237,6 +239,7 @@ int main(int argc, const char *argv[]) {
     }
   }
 
+#ifdef DECOMPRESS_AND_VERIFY
   /* Decompress */
   {
     struct timeval tv1, tv2;
@@ -252,6 +255,6 @@ int main(int argc, const char *argv[]) {
   }
   /* verify */
   verify(inpFilename, decFilename);
-
+#endif
   return 0;
 }

From e9161637b28f3b9fb26398d5e85d74e8959ca2c8 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Tue, 25 Jul 2017 18:13:27 -0700
Subject: [PATCH 094/100] Allow parameters to be modified from a separate file

---
 contrib/long_distance_matching/Makefile     |   8 +-
 contrib/long_distance_matching/ldm.c        | 539 --------------------
 contrib/long_distance_matching/ldm.h        |  30 +-
 contrib/long_distance_matching/ldm_common.c |   5 +-
 contrib/long_distance_matching/ldm_hash32.c |  12 +-
 contrib/long_distance_matching/ldm_hash64.c | 164 +-----
 contrib/long_distance_matching/ldm_params.h |  10 +
 contrib/long_distance_matching/main.c       |   8 +-
 8 files changed, 47 insertions(+), 729 deletions(-)
 delete mode 100644 contrib/long_distance_matching/ldm.c
 create mode 100644 contrib/long_distance_matching/ldm_params.h

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 292ce851..b1fd3a1e 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,16 +25,16 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-hash32 main-hash64 
+all: main-64 main-integrated 
 
-main-hash64: ldm_common.c ldm_hash64.c main.c
+main-64: ldm_common.c ldm_hash64.c main.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
-main-hash32: ldm_common.c ldm_hash32.c main.c
+main-integrated: ldm_common.c ldm_hash32.c main.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-hash64 main-hash32
+	main-hash64 main-hash32 main-64 main-integrated
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
deleted file mode 100644
index fae35f9e..00000000
--- a/contrib/long_distance_matching/ldm.c
+++ /dev/null
@@ -1,539 +0,0 @@
-#include <limits.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "ldm.h"
-#include "ldm_hashtable.h"
-
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
-
-#define LDM_HASH_ENTRY_SIZE_LOG 3
-
-//#define HASH_ONLY_EVERY_LOG 7
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
-
-#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
-
-
-#define COMPUTE_STATS
-#define OUTPUT_CONFIGURATION
-#define CHECKSUM_CHAR_OFFSET 10
-
-//#define RUN_CHECKS
-
-typedef U32 checksum_t;
-
-struct LDM_compressStats {
-  U32 windowSizeLog, hashTableSizeLog;
-  U32 numMatches;
-  U64 totalMatchLength;
-  U64 totalLiteralLength;
-  U64 totalOffset;
-
-  U32 minOffset, maxOffset;
-
-  U32 offsetHistogram[32];
-};
-
-struct LDM_CCtx {
-  U64 isize;             /* Input size */
-  U64 maxOSize;          /* Maximum output size */
-
-  const BYTE *ibase;        /* Base of input */
-  const BYTE *ip;           /* Current input position */
-  const BYTE *iend;         /* End of input */
-
-  // Maximum input position such that hashing at the position does not exceed
-  // end of input.
-  const BYTE *ihashLimit;
-
-  // Maximum input position such that finding a match of at least the minimum
-  // match length does not exceed end of input.
-  const BYTE *imatchLimit;
-
-  const BYTE *obase;        /* Base of output */
-  BYTE *op;                 /* Output */
-
-  const BYTE *anchor;       /* Anchor to start of current (match) block */
-
-  LDM_compressStats stats;            /* Compression statistics */
-
-  LDM_hashTable *hashTable;
-
-  const BYTE *lastPosHashed;          /* Last position hashed */
-  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  checksum_t lastSum;
-
-  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
-  const BYTE *nextPosHashed;
-  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
-  checksum_t nextSum;
-
-  unsigned step;                      // ip step, should be 1.
-
-  const BYTE *lagIp;
-  hash_t lagHash;
-  checksum_t lagSum;
-
-  // DEBUG
-  const BYTE *DEBUG_setNextHash;
-};
-
-// TODO: This can be done more efficiently (but it is not that important as it
-// is only used for computing stats).
-static int intLog2(U32 x) {
-  int ret = 0;
-  while (x >>= 1) {
-    ret++;
-  }
-  return ret;
-}
-
-void LDM_printCompressStats(const LDM_compressStats *stats) {
-  int i = 0;
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
-          stats->windowSizeLog, stats->hashTableSizeLog);
-  printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
-          stats->numMatches,
-          stats->totalMatchLength,
-          100.0 * (double)stats->totalMatchLength /
-              (double)(stats->totalMatchLength + stats->totalLiteralLength));
-  printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
-                                         (double)stats->numMatches);
-  printf("avg literal length, total literalLength: %.1f, %llu\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches,
-         stats->totalLiteralLength);
-  printf("avg offset length: %.1f\n",
-         ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("min offset, max offset: %u, %u\n",
-         stats->minOffset, stats->maxOffset);
-
-  printf("\n");
-  printf("offset histogram: offset, num matches, %% of matches\n");
-
-  for (; i <= intLog2(stats->maxOffset); i++) {
-    printf("2^%*d: %10u    %6.3f%%\n", 2, i,
-           stats->offsetHistogram[i],
-           100.0 * (double) stats->offsetHistogram[i] /
-                   (double) stats->numMatches);
-  }
-  printf("\n");
-  printf("=====================\n");
-}
-
-/**
- * Convert a sum computed from getChecksum to a hash value in the range
- * of the hash table.
- */
-static hash_t checksumToHash(U32 sum) {
-  return HASH_hashU32(sum);
-}
-
-/**
- * Computes a 32-bit checksum based on rsync's checksum.
- *
- * a(k,l) = \sum_{i = k}^l x_i (mod M)
- * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
- * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
-  */
-static checksum_t getChecksum(const BYTE *buf, U32 len) {
-  U32 i;
-  checksum_t s1, s2;
-
-  s1 = s2 = 0;
-  for (i = 0; i < (len - 4); i += 4) {
-    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]) +
-          (10 * CHECKSUM_CHAR_OFFSET);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
-          + (4 * CHECKSUM_CHAR_OFFSET);
-
-  }
-  for(; i < len; i++) {
-    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
-    s2 += s1;
-  }
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-/**
- * Update a checksum computed from getChecksum(data, len).
- *
- * The checksum can be updated along its ends as follows:
- * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
- * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
- *
- * Thus toRemove should correspond to data[0].
- */
-static checksum_t updateChecksum(checksum_t sum, U32 len,
-                                 BYTE toRemove, BYTE toAdd) {
-  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
-  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
-
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-/**
- * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
- * based on cctx->lastSum and cctx->lastPosHashed.
- *
- * This uses a rolling hash and requires that the last position hashed
- * corresponds to cctx->nextIp - step.
- */
-static void setNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  U32 check;
-  if ((cctx->nextIp - cctx->ibase != 1) &&
-      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
-    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
-            cctx->DEBUG_setNextHash - cctx->ibase);
-  }
-
-  cctx->DEBUG_setNextHash = cctx->nextIp;
-#endif
-
-  cctx->nextSum = updateChecksum(
-      cctx->lastSum, LDM_HASH_LENGTH,
-      cctx->lastPosHashed[0],
-      cctx->lastPosHashed[LDM_HASH_LENGTH]);
-  cctx->nextPosHashed = cctx->nextIp;
-  cctx->nextHash = checksumToHash(cctx->nextSum);
-
-#if LDM_LAG
-  if (cctx->ip - cctx->ibase > LDM_LAG) {
-    cctx->lagSum = updateChecksum(
-      cctx->lagSum, LDM_HASH_LENGTH,
-      cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
-    cctx->lagIp++;
-    cctx->lagHash = checksumToHash(cctx->lagSum);
-  }
-#endif
-
-#ifdef RUN_CHECKS
-  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
-
-  if (check != cctx->nextSum) {
-    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
-  }
-
-  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
-    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
-            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
-            cctx->ip - cctx->ibase);
-  }
-#endif
-}
-
-static void putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash, U32 checksum) {
-  // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
-  // Note: this works only when cctx->step is 1.
-  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-#if LDM_LAG
-    // Off by 1, but whatever
-    if (cctx->lagIp - cctx->ibase > 0) {
-      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
-      HASH_insert(cctx->hashTable, cctx->lagHash, entry);
-    } else {
-      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
-      HASH_insert(cctx->hashTable, hash, entry);
-    }
-#else
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
-    HASH_insert(cctx->hashTable, hash, entry);
-#endif
-  }
-
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-  cctx->lastSum = checksum;
-}
-
-/**
- * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
- * fields from the "next" fields.
- *
- * This requires that cctx->ip == cctx->nextPosHashed.
- */
-static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  if (cctx->ip != cctx->nextPosHashed) {
-    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
-  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
-}
-
-/**
- * Insert hash of the current position into the hash table.
- */
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  checksum_t sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
-  hash_t hash = checksumToHash(sum);
-
-#ifdef RUN_CHECKS
-  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
-    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
-
-  putHashOfCurrentPositionFromHash(cctx, hash, sum);
-}
-
-void LDM_initializeCCtx(LDM_CCtx *cctx,
-                        const void *src, size_t srcSize,
-                        void *dst, size_t maxDstSize) {
-  cctx->isize = srcSize;
-  cctx->maxOSize = maxDstSize;
-
-  cctx->ibase = (const BYTE *)src;
-  cctx->ip = cctx->ibase;
-  cctx->iend = cctx->ibase + srcSize;
-
-  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
-  cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
-
-  cctx->obase = (BYTE *)dst;
-  cctx->op = (BYTE *)dst;
-
-  cctx->anchor = cctx->ibase;
-
-  memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64, cctx->ibase,
-                                     LDM_MIN_MATCH_LENGTH, LDM_WINDOW_SIZE);
-
-  cctx->stats.minOffset = UINT_MAX;
-  cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
-  cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
-
-
-  cctx->lastPosHashed = NULL;
-
-  cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
-  cctx->nextIp = cctx->ip + cctx->step;
-  cctx->nextPosHashed = 0;
-
-  cctx->DEBUG_setNextHash = 0;
-}
-
-void LDM_destroyCCtx(LDM_CCtx *cctx) {
-  HASH_destroyTable(cctx->hashTable);
-}
-
-/**
- * Finds the "best" match.
- *
- * Returns 0 if successful and 1 otherwise (i.e. no match can be found
- * in the remaining input that is long enough).
- *
- * matchLength contains the forward length of the match.
- */
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
-                             U64 *matchLength, U64 *backwardMatchLength) {
-
-  LDM_hashEntry *entry = NULL;
-  cctx->nextIp = cctx->ip + cctx->step;
-
-  while (entry == NULL) {
-    hash_t h;
-    checksum_t sum;
-    setNextHash(cctx);
-    h = cctx->nextHash;
-    sum = cctx->nextSum;
-    cctx->ip = cctx->nextIp;
-    cctx->nextIp += cctx->step;
-
-    if (cctx->ip > cctx->imatchLimit) {
-      return 1;
-    }
-
-    entry = HASH_getBestEntry(cctx->hashTable, h, sum,
-                              cctx->ip, cctx->iend,
-                              cctx->anchor,
-                              matchLength, backwardMatchLength);
-
-    if (entry != NULL) {
-      *match = entry->offset + cctx->ibase;
-    }
-    putHashOfCurrentPositionFromHash(cctx, h, sum);
-  }
-  setNextHash(cctx);
-  return 0;
-}
-
-void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
-  /* Encode the literal length. */
-  if (literalLength >= RUN_MASK) {
-    int len = (int)literalLength - RUN_MASK;
-    *pToken = (RUN_MASK << ML_BITS);
-    for (; len >= 255; len -= 255) {
-      *(cctx->op)++ = 255;
-    }
-    *(cctx->op)++ = (BYTE)len;
-  } else {
-    *pToken = (BYTE)(literalLength << ML_BITS);
-  }
-
-  /* Encode the literals. */
-  memcpy(cctx->op, cctx->anchor, literalLength);
-  cctx->op += literalLength;
-}
-
-void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U64 literalLength,
-                     const U32 offset,
-                     const U64 matchLength) {
-  BYTE *pToken = cctx->op++;
-
-  /* Encode the literal length and literals. */
-  LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
-
-  /* Encode the offset. */
-  MEM_write32(cctx->op, offset);
-  cctx->op += LDM_OFFSET_SIZE;
-
-  /* Encode the match length. */
-  if (matchLength >= ML_MASK) {
-    U64 matchLengthRemaining = matchLength;
-    *pToken += ML_MASK;
-    matchLengthRemaining -= ML_MASK;
-    MEM_write32(cctx->op, 0xFFFFFFFF);
-    while (matchLengthRemaining >= 4*0xFF) {
-      cctx->op += 4;
-      MEM_write32(cctx->op, 0xffffffff);
-      matchLengthRemaining -= 4*0xFF;
-    }
-    cctx->op += matchLengthRemaining / 255;
-    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
-  } else {
-    *pToken += (BYTE)(matchLength);
-  }
-}
-
-// TODO: maxDstSize is unused. This function may seg fault when writing
-// beyond the size of dst, as it does not check maxDstSize. Writing to
-// a buffer and performing checks is a possible solution.
-//
-// This is based upon lz4.
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize) {
-  LDM_CCtx cctx;
-  const BYTE *match = NULL;
-  U64 forwardMatchLength = 0;
-  U64 backwardsMatchLength = 0;
-
-  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
-  LDM_outputConfiguration();
-
-  /* Hash the first position and put it into the hash table. */
-  LDM_putHashOfCurrentPosition(&cctx);
-
-#if LDM_LAG
-  cctx.lagIp = cctx.ip;
-  cctx.lagHash = cctx.lastHash;
-  cctx.lagSum = cctx.lastSum;
-#endif
-  /**
-   * Find a match.
-   * If no more matches can be found (i.e. the length of the remaining input
-   * is less than the minimum match length), then stop searching for matches
-   * and encode the final literals.
-   */
-  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
-         &backwardsMatchLength) == 0) {
-#ifdef COMPUTE_STATS
-    cctx.stats.numMatches++;
-#endif
-
-     cctx.ip -= backwardsMatchLength;
-     match -= backwardsMatchLength;
-
-    /**
-     * Write current block (literals, literal length, match offset, match
-     * length) and update pointers and hashes.
-     */
-    {
-      const U32 literalLength = cctx.ip - cctx.anchor;
-      const U32 offset = cctx.ip - match;
-      const U32 matchLength = forwardMatchLength +
-                              backwardsMatchLength -
-                              LDM_MIN_MATCH_LENGTH;
-
-      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
-
-#ifdef COMPUTE_STATS
-      cctx.stats.totalLiteralLength += literalLength;
-      cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
-      cctx.stats.minOffset =
-          offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
-      cctx.stats.maxOffset =
-          offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
-      cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
-#endif
-
-      // Move ip to end of block, inserting hashes at each position.
-      cctx.nextIp = cctx.ip + cctx.step;
-      while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
-                       matchLength + literalLength) {
-        if (cctx.ip > cctx.lastPosHashed) {
-          // TODO: Simplify.
-          LDM_updateLastHashFromNextHash(&cctx);
-          setNextHash(&cctx);
-        }
-        cctx.ip++;
-        cctx.nextIp++;
-      }
-    }
-
-    // Set start of next block to current input pointer.
-    cctx.anchor = cctx.ip;
-    LDM_updateLastHashFromNextHash(&cctx);
-  }
-
-  /* Encode the last literals (no more matches). */
-  {
-    const U32 lastRun = cctx.iend - cctx.anchor;
-    BYTE *pToken = cctx.op++;
-    LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
-  }
-
-#ifdef COMPUTE_STATS
-  LDM_printCompressStats(&cctx.stats);
-  HASH_outputTableOccupancy(cctx.hashTable);
-#endif
-
-  {
-    const size_t ret = cctx.op - cctx.obase;
-    LDM_destroyCCtx(&cctx);
-    return ret;
-  }
-}
-
-void LDM_outputConfiguration(void) {
-  printf("=====================\n");
-  printf("Configuration\n");
-  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
-  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
-         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
-  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
-  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
-  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
-  printf("LDM_LAG %d\n", LDM_LAG);
-  printf("=====================\n");
-}
-
-
-
-void LDM_test(const BYTE *src) {
-  (void)src;
-}
-
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index f9ad383e..b87a57bc 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -2,6 +2,7 @@
 #define LDM_H
 
 #include "mem.h"    // from /lib/common/mem.h
+#include "ldm_params.h"
 
 // The number of bytes storing the compressed and decompressed size
 // in the header.
@@ -18,35 +19,38 @@
 #define LDM_OFFSET_SIZE 4
 
 // =============================================================================
-// User parameters.
+// Modify parameters in ldm_params.h if "ldm_params.h" is included.
 // =============================================================================
 
+#ifndef LDM_PARAMS_H
 // Defines the size of the hash table.
 // Note that this is not the number of buckets.
 // Currently this should be less than WINDOW_SIZE_LOG + 4?
-#define LDM_MEMORY_USAGE 25
+  #define LDM_MEMORY_USAGE 25
 
 // The number of entries in a hash bucket.
-#define HASH_BUCKET_SIZE_LOG 3 // The maximum is 4 for now.
+  #define HASH_BUCKET_SIZE_LOG 3 // The maximum is 4 for now.
 
 // Defines the lag in inserting elements into the hash table.
-#define LDM_LAG 0
+  #define LDM_LAG 0
 
 // The maximum window size.
-#define LDM_WINDOW_SIZE_LOG 28 // Max value is 30
-#define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
+  #define LDM_WINDOW_SIZE_LOG 28 // Max value is 30
 
 //These should be multiples of four (and perhaps set to the same value?).
-#define LDM_MIN_MATCH_LENGTH 64
-#define LDM_HASH_LENGTH 64
+  #define LDM_MIN_MATCH_LENGTH 64
 
-// Experimental.
-//#define TMP_EVICTION    // Experiment with eviction policies.
-#define INSERT_BY_TAG // Insertion policy based on hash.
+  #define INSERT_BY_TAG 1 // Insertion policy based on hash.
 
-#define USE_CHECKSUM 1
+  #define USE_CHECKSUM 1
+#endif
 
 // =============================================================================
+#define COMPUTE_STATS
+#define OUTPUT_CONFIGURATION
+
+#define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
+#define LDM_HASH_LENGTH LDM_MIN_MATCH_LENGTH
 
 typedef struct LDM_compressStats LDM_compressStats;
 typedef struct LDM_CCtx LDM_CCtx;
@@ -164,6 +168,4 @@ void LDM_writeHeader(void *memPtr, U64 compressedSize,
  */
 void LDM_outputConfiguration(void);
 
-void LDM_test(const BYTE *src);
-
 #endif /* LDM_H */
diff --git a/contrib/long_distance_matching/ldm_common.c b/contrib/long_distance_matching/ldm_common.c
index 1953656e..26b716a1 100644
--- a/contrib/long_distance_matching/ldm_common.c
+++ b/contrib/long_distance_matching/ldm_common.c
@@ -70,7 +70,8 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
     dctx.ip += length;
     dctx.op = cpy;
 
-    //TODO : dynamic offset size
+    //TODO: dynamic offset size?
+    /* Encode the offset. */
     offset = MEM_read32(dctx.ip);
     dctx.ip += LDM_OFFSET_SIZE;
     match = dctx.op - offset;
@@ -89,7 +90,7 @@ size_t LDM_decompress(const void *src, size_t compressedSize,
     /* Copy match. */
     cpy = dctx.op + length;
 
-    // Inefficient for now.
+    // TODO: this can be made more efficient.
     while (match < cpy - offset && dctx.op < dctx.oend) {
       *(dctx.op)++ = *match++;
     }
diff --git a/contrib/long_distance_matching/ldm_hash32.c b/contrib/long_distance_matching/ldm_hash32.c
index b80a5c01..94fa5e92 100644
--- a/contrib/long_distance_matching/ldm_hash32.c
+++ b/contrib/long_distance_matching/ldm_hash32.c
@@ -21,9 +21,7 @@
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
-#define COMPUTE_STATS
-#define OUTPUT_CONFIGURATION
-#define CHECKSUM_CHAR_OFFSET 1
+#define CHECKSUM_CHAR_OFFSET 10
 
 // Take first match only.
 //#define ZSTD_SKIP
@@ -779,8 +777,6 @@ size_t LDM_compress(const void *src, size_t srcSize,
     LDM_updateLastHashFromNextHash(&cctx);
   }
 
-  // HASH_outputTableOffsetHistogram(&cctx);
-
   /* Encode the last literals (no more matches). */
   {
     const U32 lastRun = cctx.iend - cctx.anchor;
@@ -815,9 +811,3 @@ void LDM_outputConfiguration(void) {
 
 
 
-// TODO: implement and test hash function
-void LDM_test(const BYTE *src) {
-  (void)src;
-}
-
-
diff --git a/contrib/long_distance_matching/ldm_hash64.c b/contrib/long_distance_matching/ldm_hash64.c
index e51ac57d..884f7b72 100644
--- a/contrib/long_distance_matching/ldm_hash64.c
+++ b/contrib/long_distance_matching/ldm_hash64.c
@@ -24,8 +24,6 @@
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
 #define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
-#define COMPUTE_STATS
-#define OUTPUT_CONFIGURATION
 #define HASH_CHAR_OFFSET 10
 
 // Take first match only.
@@ -63,11 +61,6 @@ struct LDM_compressStats {
 
   U64 TMP_hashCount[1 << HASH_ONLY_EVERY_LOG];
   U64 TMP_totalHashCount;
-
-  U64 TMP_totalInWindow;
-  U64 TMP_totalInserts;
-
-  U64 TMP_matchCount;
 };
 
 typedef struct LDM_hashTable LDM_hashTable;
@@ -328,91 +321,12 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
   return NULL;
 }
 
-#ifdef TMP_EVICTION
-void HASH_insert(LDM_hashTable *table,
-                 const hash_t hash, const LDM_hashEntry entry,
-                 LDM_CCtx *cctx) {
-  // Overwrite based on part of checksum.
-  /*
-  LDM_hashEntry *toOverwrite =
-    getBucket(table, hash) + table->bucketOffsets[hash];
-  const BYTE *pMatch = toOverwrite->offset + cctx->ibase;
-  if (toOverwrite->offset != 0 &&
-      cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
-    cctx->stats.TMP_totalInWindow++;
-  }
-
-  cctx->stats.TMP_totalInserts++;
-  *(toOverwrite) = entry;
-  */
-
-  /*
-  int i;
-  LDM_hashEntry *bucket = getBucket(table, hash);
-  for (i = 0; i < HASH_BUCKET_SIZE; i++) {
-    if (bucket[i].checksum == entry.checksum) {
-      bucket[i] = entry;
-      cctx->stats.TMP_matchCount++;
-      return;
-    }
-  }
-  */
-
-  // Find entry beyond window size, replace. Else, random.
-  int i;
-  LDM_hashEntry *bucket = getBucket(table, hash);
-  for (i = 0; i < HASH_BUCKET_SIZE; i++) {
-    if (cctx->ip - cctx->ibase - bucket[i].offset > LDM_WINDOW_SIZE) {
-      bucket[i] = entry;
-      return;
-    }
-  }
-
-  i = rand() & (HASH_BUCKET_SIZE - 1);
-  *(bucket + i) = entry;
-
-
-  /**
-   * Sliding buffer style pointer
-   * Keep old entry as temporary. If the old entry is outside the window,
-   * overwrite and we are done.
-   *
-   * Backwards (insert at x):
-   * x, a, b b, c c c c, d d d d d d d d
-   * x, d d d d d d d d, c c c c, b b, a
-   *
-   * Else, find something to evict.
-   * If old entry has more ones, it takes
-   * the next spot. <-- reversed order?
-   *
-   * If window size > LDM_WINDOW_SIZE,
-   * overwrite,
-   *
-   * Insert forwards. If > tag, keep. Else evict.
-   *
-   */
-
-
-  /*
-  *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
-  table->bucketOffsets[hash]++;
-  table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
-  */
-
-//  U16 mask = entry.checksum & (HASH_BUCKET_SIZE - 1);
-//  *(getBucket(table, hash) + mask) = entry;
-}
-
-#else
-
 void HASH_insert(LDM_hashTable *table,
                  const hash_t hash, const LDM_hashEntry entry) {
   *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
   table->bucketOffsets[hash]++;
   table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
 }
-#endif // TMP_EVICTION
-
 
 U32 HASH_getSize(const LDM_hashTable *table) {
   return table->numBuckets;
@@ -489,7 +403,7 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
                    (double) stats->numMatches);
   }
   printf("\n");
-#ifdef INSERT_BY_TAG
+#if INSERT_BY_TAG
 /*
   printf("Lower bit distribution\n");
   for (i = 0; i < (1 << HASH_ONLY_EVERY_LOG); i++) {
@@ -500,13 +414,6 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
 */
 #endif
 
-#ifdef TMP_EVICTION
-  printf("Evicted something in window: %llu %6.3f\n",
-         stats->TMP_totalInWindow,
-         100.0 * (double)stats->TMP_totalInWindow /
-                 (double)stats->TMP_totalInserts);
-  printf("Match count: %llu\n", stats->TMP_matchCount);
-#endif
   printf("=====================\n");
 }
 
@@ -524,7 +431,7 @@ static U32 getChecksum(U64 hash) {
   return (hash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
 }
 
-#ifdef INSERT_BY_TAG
+#if INSERT_BY_TAG
 static U32 lowerBitsFromHfHash(U64 hash) {
   // The number of bits used so far is LDM_HASHLOG + 32.
   // So there are 32 - LDM_HASHLOG bits left.
@@ -611,7 +518,7 @@ static void setNextHash(LDM_CCtx *cctx) {
       cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
 
-#ifdef INSERT_BY_TAG
+#if INSERT_BY_TAG
   {
     U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextHash);
     cctx->stats.TMP_totalHashCount++;
@@ -648,7 +555,7 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
   // Note: this works only when cctx->step is 1.
 #if LDM_LAG
   if (cctx -> lagIp - cctx->ibase > 0) {
-#ifdef INSERT_BY_TAG
+#if INSERT_BY_TAG
     U32 hashEveryMask = lowerBitsFromHfHash(cctx->lagHash);
     if (hashEveryMask == HASH_ONLY_EVERY) {
 #else
@@ -663,14 +570,11 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
       const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase };
 #   endif
 
-#   ifdef TMP_EVICTION
-      HASH_insert(cctx->hashTable, smallHash, entry, cctx);
-#   else
       HASH_insert(cctx->hashTable, smallHash, entry);
-#   endif
     }
   } else {
-#ifdef INSERT_BY_TAG
+#endif // LDM_LAG
+#if INSERT_BY_TAG
     U32 hashEveryMask = lowerBitsFromHfHash(hash);
     if (hashEveryMask == HASH_ONLY_EVERY) {
 #else
@@ -684,33 +588,9 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
 #else
       const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
 #endif
-
-#ifdef TMP_EVICTION
-      HASH_insert(cctx->hashTable, smallHash, entry, cctx);
-#else
       HASH_insert(cctx->hashTable, smallHash, entry);
-#endif
     }
-  }
-#else
-#ifdef INSERT_BY_TAG
-  U32 hashEveryMask = lowerBitsFromHfHash(hash);
-  if (hashEveryMask == HASH_ONLY_EVERY) {
-#else
-  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-#endif
-    U32 smallHash = getSmallHash(hash);
-#if USE_CHECKSUM
-    U32 checksum = getChecksum(hash);
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
-#else
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
-#endif
-#ifdef TMP_EVICTION
-    HASH_insert(cctx->hashTable, smallHash, entry, cctx);
-#else
-    HASH_insert(cctx->hashTable, smallHash, entry);
-#endif
+#if LDM_LAG
   }
 #endif
 
@@ -812,7 +692,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     U64 hash;
     hash_t smallHash;
     U32 checksum;
-#ifdef INSERT_BY_TAG
+#if INSERT_BY_TAG
     U32 hashEveryMask;
 #endif
     setNextHash(cctx);
@@ -820,7 +700,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     hash = cctx->nextHash;
     smallHash = getSmallHash(hash);
     checksum = getChecksum(hash);
-#ifdef INSERT_BY_TAG
+#if INSERT_BY_TAG
     hashEveryMask = lowerBitsFromHfHash(hash);
 #endif
 
@@ -830,7 +710,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
     if (cctx->ip > cctx->imatchLimit) {
       return 1;
     }
-#ifdef INSERT_BY_TAG
+#if INSERT_BY_TAG
     if (hashEveryMask == HASH_ONLY_EVERY) {
 
       entry = HASH_getBestEntry(cctx, smallHash, checksum,
@@ -923,10 +803,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
   /* Hash the first position and put it into the hash table. */
   LDM_putHashOfCurrentPosition(&cctx);
 
-#if LDM_LAG
   cctx.lagIp = cctx.ip;
   cctx.lagHash = cctx.lastHash;
-#endif
   /**
    * Find a match.
    * If no more matches can be found (i.e. the length of the remaining input
@@ -1018,28 +896,8 @@ void LDM_outputConfiguration(void) {
   printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
   printf("LDM_LAG: %d\n", LDM_LAG);
   printf("USE_CHECKSUM: %d\n", USE_CHECKSUM);
-#ifdef INSERT_BY_TAG
-  printf("INSERT_BY_TAG: %d\n", 1);
-#else
-  printf("INSERT_BY_TAG: %d\n", 0);
-#endif
+  printf("INSERT_BY_TAG: %d\n", INSERT_BY_TAG);
   printf("HASH_CHAR_OFFSET: %d\n", HASH_CHAR_OFFSET);
   printf("=====================\n");
 }
 
-// TODO: implement and test hash function
-void LDM_test(const BYTE *src) {
-  const U32 diff = 100;
-  const BYTE *pCur = src + diff;
-  U64 hash = getHash(pCur, LDM_HASH_LENGTH);
-
-  for (; pCur < src + diff + 60; ++pCur) {
-    U64 nextHash = getHash(pCur + 1, LDM_HASH_LENGTH);
-    U64 updatedHash = updateHash(hash, LDM_HASH_LENGTH,
-                                 pCur[0], pCur[LDM_HASH_LENGTH]);
-    hash = nextHash;
-    printf("%llu %llu\n", nextHash, updatedHash);
-  }
-}
-
-
diff --git a/contrib/long_distance_matching/ldm_params.h b/contrib/long_distance_matching/ldm_params.h
new file mode 100644
index 00000000..0fcd30bd
--- /dev/null
+++ b/contrib/long_distance_matching/ldm_params.h
@@ -0,0 +1,10 @@
+#ifndef LDM_PARAMS_H
+#define LDM_PARAMS_H
+#define LDM_MEMORY_USAGE 23
+#define HASH_BUCKET_SIZE_LOG 3
+#define LDM_LAG 0
+#define LDM_WINDOW_SIZE_LOG 28
+#define LDM_MIN_MATCH_LENGTH 64
+#define INSERT_BY_TAG 1
+#define USE_CHECKSUM 1
+#endif
diff --git a/contrib/long_distance_matching/main.c b/contrib/long_distance_matching/main.c
index cee5edba..bdd385ce 100644
--- a/contrib/long_distance_matching/main.c
+++ b/contrib/long_distance_matching/main.c
@@ -12,7 +12,7 @@
 #include "ldm.h"
 #include "zstd.h"
 
-//#define DECOMPRESS_AND_VERIFY
+#define DECOMPRESS_AND_VERIFY
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
@@ -71,10 +71,6 @@ static int compress(const char *fname, const char *oname) {
       return 1;
   }
 
-#ifdef TEST
-  LDM_test((const BYTE *)src);
-#endif
-
   gettimeofday(&tv1, NULL);
 
   compressedSize = LDM_HEADER_SIZE +
@@ -111,7 +107,7 @@ static int compress(const char *fname, const char *oname) {
   return 0;
 }
 
-#ifdef DECOMPRESS
+#ifdef DECOMPRESS_AND_VERIFY
 /* Decompress file compressed using LDM_compress.
  * The input file should have the LDM_HEADER followed by payload.
  * Returns 0 if succesful, and an error code otherwise.

From 40759bade90d1ecb1bb52b55e2c7a3399e625998 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 26 Jul 2017 13:18:53 -0700
Subject: [PATCH 095/100] Add README and clean up code

---
 contrib/long_distance_matching/Makefile       |  11 +-
 contrib/long_distance_matching/README.md      |  39 +
 .../{ldm_hash64.c => ldm.c}                   | 109 +--
 contrib/long_distance_matching/ldm.h          |  87 +-
 contrib/long_distance_matching/ldm_hash32.c   | 813 ------------------
 contrib/long_distance_matching/ldm_params.h   |   4 +-
 contrib/long_distance_matching/main.c         |  13 +-
 7 files changed, 134 insertions(+), 942 deletions(-)
 create mode 100644 contrib/long_distance_matching/README.md
 rename contrib/long_distance_matching/{ldm_hash64.c => ldm.c} (90%)
 delete mode 100644 contrib/long_distance_matching/ldm_hash32.c

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index b1fd3a1e..8bc7ac47 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -25,16 +25,13 @@ LDFLAGS += -lzstd
 
 default: all
 
-all: main-64 main-integrated 
-
-main-64: ldm_common.c ldm_hash64.c main.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-main-integrated: ldm_common.c ldm_hash32.c main.c
+all: ldm 
+	
+ldm: ldm_common.c ldm.c main.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	main-hash64 main-hash32 main-64 main-integrated
+	ldm	
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/README.md b/contrib/long_distance_matching/README.md
new file mode 100644
index 00000000..d9cb0895
--- /dev/null
+++ b/contrib/long_distance_matching/README.md
@@ -0,0 +1,39 @@
+This is a compression algorithm focused on finding long distance matches.
+
+It is based upon lz4 and uses nearly the same block format (github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md). The number of bytes to encode the offset is four instead of two in lz4 to reflect the longer distance matching. The block format is descriped in `ldm.h`.
+
+### Build
+
+Run `make`.
+
+### Compressing a file
+
+`ldm <filename>`
+
+Decompression and verification can be enabled by defining `DECOMPRESS_AND_VERIFY` in `main.c`.
+The output file names are as follows:
+- `<filename>.ldm` : compressed file
+- `<filename>.ldm.dec` : decompressed file
+
+### Parameters
+
+There are various parameters that can be tuned. These parameters can be tuned in `ldm.h` or, alternatively if `ldm_params.h` is included, in `ldm_params.h` (for easier configuration).
+
+The parameters are as follows and must all be defined:
+- `LDM_MEMORY_USAGE` : the memory usage of the underlying hash table in bytes.
+- `HASH_BUCKET_SIZE_LOG` : the log size of each bucket in the hash table (used in collision resolution).
+- `LDM_LAG` : the lag (in bytes) in inserting entries into the hash table.
+- `LDM_WINDOW_SIZE_LOG` : the log maximum window size when searching for matches.
+- `LDM_MIN_MATCH_LENGTH` : the minimum match length.
+- `INSERT_BY_TAG` : insert entries into the hash table as a function of the hash. This increases speed by reducing the number of hash table lookups and match comparisons. Certain hashes will never be inserted.
+- `USE_CHECKSUM`  : store a checksum with the hash table entries for faster comparison. This halves the number of entries the hash table can contain.
+
+### Compression statistics
+
+Compression statistics (and the configuration) can be enabled/disabled via `COMPUTE_STATS` and `OUTPUT_CONFIGURATION` in `ldm.h`.
+
+
+
+
+
+
diff --git a/contrib/long_distance_matching/ldm_hash64.c b/contrib/long_distance_matching/ldm.c
similarity index 90%
rename from contrib/long_distance_matching/ldm_hash64.c
rename to contrib/long_distance_matching/ldm.c
index 884f7b72..9a843838 100644
--- a/contrib/long_distance_matching/ldm_hash64.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -16,21 +16,21 @@
   #define LDM_HASH_ENTRY_SIZE_LOG 2
 #endif
 
+// Force the "probability" of insertion to be some value.
+// Entries are inserted into the table HASH_ONLY_EVERY + 1 times "on average".
+
 //#define HASH_ONLY_EVERY_LOG 7
 #define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
-
 #define HASH_ONLY_EVERY ((1 << (HASH_ONLY_EVERY_LOG)) - 1)
 
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
+#define NUM_HASH_BUCKETS_LOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
 
 #define HASH_CHAR_OFFSET 10
 
-// Take first match only.
+// Take the first match in the hash bucket only.
 //#define ZSTD_SKIP
 
-//#define RUN_CHECKS
-
 static const U64 prime8bytes = 11400714785074694791ULL;
 
 // Type of the small hash used to index into the hash table.
@@ -101,10 +101,6 @@ struct LDM_CCtx {
 
   const BYTE *lagIp;
   U64 lagHash;
-
-#ifdef RUN_CHECKS
-  const BYTE *DEBUG_setNextHash;
-#endif
 };
 
 struct LDM_hashTable {
@@ -119,7 +115,7 @@ struct LDM_hashTable {
  * Create a hash table that can contain size elements.
  * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
  */
-LDM_hashTable *HASH_createTable(U32 size) {
+static LDM_hashTable *HASH_createTable(U32 size) {
   LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
   table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
   table->numEntries = size;
@@ -239,7 +235,7 @@ static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
 /**
  * Count number of bytes that match backwards before pIn and pMatch.
  *
- * We count only bytes where pMatch > pBaes and pIn > pAnchor.
+ * We count only bytes where pMatch > pBase and pIn > pAnchor.
  */
 static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
                                   const BYTE *pMatch, const BYTE *pBase) {
@@ -262,13 +258,12 @@ static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
  * The forward match is computed from cctx->ip and entry->offset + cctx->ibase.
  * The backward match is computed backwards from cctx->ip and
  * cctx->ibase only if the forward match is longer than LDM_MIN_MATCH_LENGTH.
- *
  */
-LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
-                                 const hash_t hash,
-                                 const U32 checksum,
-                                 U64 *pForwardMatchLength,
-                                 U64 *pBackwardMatchLength) {
+static LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
+                                        const hash_t hash,
+                                        const U32 checksum,
+                                        U64 *pForwardMatchLength,
+                                        U64 *pBackwardMatchLength) {
   LDM_hashTable *table = cctx->hashTable;
   LDM_hashEntry *bucket = getBucket(table, hash);
   LDM_hashEntry *cur = bucket;
@@ -321,24 +316,24 @@ LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
   return NULL;
 }
 
-void HASH_insert(LDM_hashTable *table,
-                 const hash_t hash, const LDM_hashEntry entry) {
+/**
+ * Insert an entry into the hash table. The table uses a "circular buffer",
+ * with the oldest entry overwritten.
+ */
+static void HASH_insert(LDM_hashTable *table,
+                        const hash_t hash, const LDM_hashEntry entry) {
   *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
   table->bucketOffsets[hash]++;
   table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
 }
 
-U32 HASH_getSize(const LDM_hashTable *table) {
-  return table->numBuckets;
-}
-
-void HASH_destroyTable(LDM_hashTable *table) {
+static void HASH_destroyTable(LDM_hashTable *table) {
   free(table->entries);
   free(table->bucketOffsets);
   free(table);
 }
 
-void HASH_outputTableOccupancy(const LDM_hashTable *table) {
+static void HASH_outputTableOccupancy(const LDM_hashTable *table) {
   U32 ctr = 0;
   LDM_hashEntry *cur = table->entries;
   LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
@@ -350,7 +345,7 @@ void HASH_outputTableOccupancy(const LDM_hashTable *table) {
 
   // The number of buckets is repeated as a check for now.
   printf("Num buckets, bucket size: %d (2^%d), %d\n",
-         table->numBuckets, LDM_HASHLOG, HASH_BUCKET_SIZE);
+         table->numBuckets, NUM_HASH_BUCKETS_LOG, HASH_BUCKET_SIZE);
   printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
          table->numEntries, ctr,
          100.0 * (double)(ctr) / table->numEntries);
@@ -418,31 +413,32 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
 }
 
 /**
- * Return the upper (most significant) LDM_HASHLOG bits.
+ * Return the upper (most significant) NUM_HASH_BUCKETS_LOG bits.
  */
 static hash_t getSmallHash(U64 hash) {
-  return hash >> (64 - LDM_HASHLOG);
+  return hash >> (64 - NUM_HASH_BUCKETS_LOG);
 }
 
 /**
- * Return the 32 bits after the upper LDM_HASHLOG bits.
+ * Return the 32 bits after the upper NUM_HASH_BUCKETS_LOG bits.
  */
 static U32 getChecksum(U64 hash) {
-  return (hash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
+  return (hash >> (64 - 32 - NUM_HASH_BUCKETS_LOG)) & 0xFFFFFFFF;
 }
 
 #if INSERT_BY_TAG
 static U32 lowerBitsFromHfHash(U64 hash) {
-  // The number of bits used so far is LDM_HASHLOG + 32.
-  // So there are 32 - LDM_HASHLOG bits left.
+  // The number of bits used so far is NUM_HASH_BUCKETS_LOG + 32.
+  // So there are 32 - NUM_HASH_BUCKETS_LOG bits left.
   // Occasional hashing requires HASH_ONLY_EVERY_LOG bits.
   // So if 32 - LDMHASHLOG < HASH_ONLY_EVERY_LOG, just return lower bits
   // allowing for reuse of bits.
-  if (32 - LDM_HASHLOG < HASH_ONLY_EVERY_LOG) {
+  if (32 - NUM_HASH_BUCKETS_LOG < HASH_ONLY_EVERY_LOG) {
     return hash & HASH_ONLY_EVERY;
   } else {
-    // Otherwise shift by (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG) bits first.
-    return (hash >> (32 - LDM_HASHLOG - HASH_ONLY_EVERY_LOG)) &
+    // Otherwise shift by
+    // (32 - NUM_HASH_BUCKETS_LOG - HASH_ONLY_EVERY_LOG) bits first.
+    return (hash >> (32 - NUM_HASH_BUCKETS_LOG - HASH_ONLY_EVERY_LOG)) &
            HASH_ONLY_EVERY;
   }
 }
@@ -501,17 +497,6 @@ static U64 updateHash(U64 hash, U32 len,
  * corresponds to cctx->nextIp - step.
  */
 static void setNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  U64 check;
-  if ((cctx->nextIp - cctx->ibase != 1) &&
-      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
-    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
-            cctx->DEBUG_setNextHash - cctx->ibase);
-  }
-
-  cctx->DEBUG_setNextHash = cctx->nextIp;
-#endif
-
   cctx->nextHash = updateHash(
       cctx->lastHash, LDM_HASH_LENGTH,
       cctx->lastPosHashed[0],
@@ -534,20 +519,6 @@ static void setNextHash(LDM_CCtx *cctx) {
     cctx->lagIp++;
   }
 #endif
-
-#ifdef RUN_CHECKS
-  check = getHash(cctx->nextIp, LDM_HASH_LENGTH);
-
-  if (check != cctx->nextHash) {
-    printf("CHECK: setNextHash failed %llu %llu\n", check, cctx->nextHash);
-  }
-
-  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
-    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
-            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
-            cctx->ip - cctx->ibase);
-  }
-#endif
 }
 
 static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
@@ -605,12 +576,6 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
  * This requires that cctx->ip == cctx->nextPosHashed.
  */
 static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  if (cctx->ip != cctx->nextPosHashed) {
-    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
   putHashOfCurrentPositionFromHash(cctx, cctx->nextHash);
 }
 
@@ -620,13 +585,6 @@ static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
 static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   U64 hash = getHash(cctx->ip, LDM_HASH_LENGTH);
 
-#ifdef RUN_CHECKS
-  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
-    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
-
   putHashOfCurrentPositionFromHash(cctx, hash);
 }
 
@@ -664,10 +622,6 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
   cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
   cctx->nextIp = cctx->ip + cctx->step;
   cctx->nextPosHashed = 0;
-
-#ifdef RUN_CHECKS
-  cctx->DEBUG_setNextHash = 0;
-#endif
 }
 
 void LDM_destroyCCtx(LDM_CCtx *cctx) {
@@ -805,6 +759,7 @@ size_t LDM_compress(const void *src, size_t srcSize,
 
   cctx.lagIp = cctx.ip;
   cctx.lagHash = cctx.lastHash;
+
   /**
    * Find a match.
    * If no more matches can be found (i.e. the length of the remaining input
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index b87a57bc..38d24015 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -2,7 +2,52 @@
 #define LDM_H
 
 #include "mem.h"    // from /lib/common/mem.h
-#include "ldm_params.h"
+
+// #include "ldm_params.h"
+
+// =============================================================================
+// Modify the parameters in ldm_params.h if "ldm_params.h" is included.
+// Otherwise, modify the parameters here.
+// =============================================================================
+
+#ifndef LDM_PARAMS_H
+  // Defines the size of the hash table.
+  // Note that this is not the number of buckets.
+  // Currently this should be less than WINDOW_SIZE_LOG + 4.
+  #define LDM_MEMORY_USAGE 23
+
+  // The number of entries in a hash bucket.
+  #define HASH_BUCKET_SIZE_LOG 3 // The maximum is 4 for now.
+
+  // Defines the lag in inserting elements into the hash table.
+  #define LDM_LAG 0
+
+  // The maximum window size when searching for matches.
+  // The maximum value is 30.
+  #define LDM_WINDOW_SIZE_LOG 28
+
+  // The minimum match length.
+  // This should be a multiple of four.
+  #define LDM_MIN_MATCH_LENGTH 64
+
+  // If INSERT_BY_TAG, insert entries into the hash table as a function of the
+  // hash. Certain hashes will not be inserted.
+  //
+  // Otherwise, insert as a function of the position.
+  #define INSERT_BY_TAG 1
+
+  // Store a checksum with the hash table entries for faster comparison.
+  // This halves the number of entries the hash table can contain.
+  #define USE_CHECKSUM 1
+#endif
+
+// Output compression statistics.
+#define COMPUTE_STATS
+
+// Output the configuration.
+#define OUTPUT_CONFIGURATION
+
+// =============================================================================
 
 // The number of bytes storing the compressed and decompressed size
 // in the header.
@@ -15,40 +60,9 @@
 #define RUN_BITS (8-ML_BITS)
 #define RUN_MASK ((1U<<RUN_BITS)-1)
 
-// THe number of bytes storing the offset.
+// The number of bytes storing the offset.
 #define LDM_OFFSET_SIZE 4
 
-// =============================================================================
-// Modify parameters in ldm_params.h if "ldm_params.h" is included.
-// =============================================================================
-
-#ifndef LDM_PARAMS_H
-// Defines the size of the hash table.
-// Note that this is not the number of buckets.
-// Currently this should be less than WINDOW_SIZE_LOG + 4?
-  #define LDM_MEMORY_USAGE 25
-
-// The number of entries in a hash bucket.
-  #define HASH_BUCKET_SIZE_LOG 3 // The maximum is 4 for now.
-
-// Defines the lag in inserting elements into the hash table.
-  #define LDM_LAG 0
-
-// The maximum window size.
-  #define LDM_WINDOW_SIZE_LOG 28 // Max value is 30
-
-//These should be multiples of four (and perhaps set to the same value?).
-  #define LDM_MIN_MATCH_LENGTH 64
-
-  #define INSERT_BY_TAG 1 // Insertion policy based on hash.
-
-  #define USE_CHECKSUM 1
-#endif
-
-// =============================================================================
-#define COMPUTE_STATS
-#define OUTPUT_CONFIGURATION
-
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
 #define LDM_HASH_LENGTH LDM_MIN_MATCH_LENGTH
 
@@ -81,7 +95,6 @@ typedef struct LDM_DCtx LDM_DCtx;
  *  bytes added similarly to the additional literal length bytes after the offset.
  *
  *  The last sequence is incomplete and stops right after the literals.
- *
  */
 size_t LDM_compress(const void *src, size_t srcSize,
                     void *dst, size_t maxDstSize);
@@ -96,7 +109,7 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
                         void *dst, size_t maxDstSize);
 
 /**
- * Frees up memory allocating in initializeCCtx
+ * Frees up memory allocated in LDM_initializeCCtx().
  */
 void LDM_destroyCCtx(LDM_CCtx *cctx);
 
@@ -121,8 +134,8 @@ void LDM_printCompressStats(const LDM_compressStats *stats);
  *
  * This is followed by literalLength bytes corresponding to the literals.
  */
-void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength);
+void LDM_encodeLiteralLengthAndLiterals(LDM_CCtx *cctx, BYTE *pToken,
+                                        const U64 literalLength);
 
 /**
  * Write current block (literals, literal length, match offset,
diff --git a/contrib/long_distance_matching/ldm_hash32.c b/contrib/long_distance_matching/ldm_hash32.c
deleted file mode 100644
index 94fa5e92..00000000
--- a/contrib/long_distance_matching/ldm_hash32.c
+++ /dev/null
@@ -1,813 +0,0 @@
-#include <limits.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "ldm.h"
-
-#define LDM_HASHTABLESIZE (1 << (LDM_MEMORY_USAGE))
-#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
-#define LDM_HASHTABLESIZE_U64 ((LDM_HASHTABLESIZE) >> 3)
-
-#define LDM_HASH_ENTRY_SIZE_LOG 3
-//#define HASH_ONLY_EVERY_LOG 7
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
-
-#define HASH_ONLY_EVERY ((1 << HASH_ONLY_EVERY_LOG) - 1)
-
-
-/* Hash table stuff. */
-#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
-
-#define CHECKSUM_CHAR_OFFSET 10
-
-// Take first match only.
-//#define ZSTD_SKIP
-
-//#define RUN_CHECKS
-
-typedef U32 hash_t;
-
-typedef struct LDM_hashEntry {
-  U32 offset;
-  U32 checksum;
-} LDM_hashEntry;
-
-struct LDM_compressStats {
-  U32 windowSizeLog, hashTableSizeLog;
-  U32 numMatches;
-  U64 totalMatchLength;
-  U64 totalLiteralLength;
-  U64 totalOffset;
-
-  U32 matchLengthHistogram[32];
-
-  U32 minOffset, maxOffset;
-
-  U32 offsetHistogram[32];
-};
-
-typedef struct LDM_hashTable LDM_hashTable;
-
-struct LDM_CCtx {
-  U64 isize;             /* Input size */
-  U64 maxOSize;          /* Maximum output size */
-
-  const BYTE *ibase;        /* Base of input */
-  const BYTE *ip;           /* Current input position */
-  const BYTE *iend;         /* End of input */
-
-  // Maximum input position such that hashing at the position does not exceed
-  // end of input.
-  const BYTE *ihashLimit;
-
-  // Maximum input position such that finding a match of at least the minimum
-  // match length does not exceed end of input.
-  const BYTE *imatchLimit;
-
-  const BYTE *obase;        /* Base of output */
-  BYTE *op;                 /* Output */
-
-  const BYTE *anchor;       /* Anchor to start of current (match) block */
-
-  LDM_compressStats stats;            /* Compression statistics */
-
-  LDM_hashTable *hashTable;
-
-  const BYTE *lastPosHashed;          /* Last position hashed */
-  hash_t lastHash;                    /* Hash corresponding to lastPosHashed */
-  U32 lastSum;
-
-  const BYTE *nextIp;                 // TODO: this is  redundant (ip + step)
-  const BYTE *nextPosHashed;
-  hash_t nextHash;                    /* Hash corresponding to nextPosHashed */
-  U32 nextSum;
-
-  unsigned step;                      // ip step, should be 1.
-
-  const BYTE *lagIp;
-  hash_t lagHash;
-  U32 lagSum;
-
-  U64 numHashInserts;
-  // DEBUG
-  const BYTE *DEBUG_setNextHash;
-};
-
-struct LDM_hashTable {
-  U32 numBuckets;  // Number of buckets
-  U32 numEntries;
-  LDM_hashEntry *entries;
-
-  BYTE *bucketOffsets;
-};
-
-/**
- * Create a hash table that can contain size elements.
- * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
- */
-LDM_hashTable *HASH_createTable(U32 size) {
-  LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
-  table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
-  table->numEntries = size;
-  table->entries = calloc(size, sizeof(LDM_hashEntry));
-  table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
-  return table;
-}
-
-static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
-  return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
-}
-
-static unsigned ZSTD_NbCommonBytes (register size_t val) {
-    if (MEM_isLittleEndian()) {
-        if (MEM_64bits()) {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanForward64( &r, (U64)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctzll((U64)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
-                                                     0, 3, 1, 3, 1, 4, 2, 7,
-                                                     0, 2, 3, 6, 1, 5, 3, 5,
-                                                     1, 3, 4, 4, 2, 5, 6, 7,
-                                                     7, 0, 1, 2, 3, 3, 4, 6,
-                                                     2, 6, 5, 5, 3, 4, 5, 6,
-                                                     7, 1, 2, 4, 6, 4, 4, 5,
-                                                     7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-#       endif
-        } else { /* 32 bits */
-#       if defined(_MSC_VER)
-            unsigned long r=0;
-            _BitScanForward( &r, (U32)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_ctz((U32)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
-                                                     3, 2, 2, 1, 3, 2, 0, 1,
-                                                     3, 3, 1, 2, 2, 2, 2, 0,
-                                                     3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-        }
-    } else {  /* Big Endian CPU */
-        if (MEM_64bits()) {
-#       if defined(_MSC_VER) && defined(_WIN64)
-            unsigned long r = 0;
-            _BitScanReverse64( &r, val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clzll(val) >> 3);
-#       else
-            unsigned r;
-            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
-            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
-            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        } else { /* 32 bits */
-#       if defined(_MSC_VER)
-            unsigned long r = 0;
-            _BitScanReverse( &r, (unsigned long)val );
-            return (unsigned)(r>>3);
-#       elif defined(__GNUC__) && (__GNUC__ >= 3)
-            return (__builtin_clz((U32)val) >> 3);
-#       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-    }   }
-}
-
-// From lib/compress/zstd_compress.c
-static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
-                         const BYTE *const pInLimit) {
-    const BYTE * const pStart = pIn;
-    const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
-
-    while (pIn < pInLoopLimit) {
-        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
-        if (!diff) {
-          pIn += sizeof(size_t);
-          pMatch += sizeof(size_t);
-          continue;
-        }
-        pIn += ZSTD_NbCommonBytes(diff);
-        return (size_t)(pIn - pStart);
-    }
-
-    if (MEM_64bits()) {
-      if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
-        pIn += 4;
-        pMatch += 4;
-      }
-    }
-    if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
-      pIn += 2;
-      pMatch += 2;
-    }
-    if ((pIn < pInLimit) && (*pMatch == *pIn)) {
-      pIn++;
-    }
-    return (size_t)(pIn - pStart);
-}
-
-/**
- * Count number of bytes that match backwards before pIn and pMatch.
- *
- * We count only bytes where pMatch > pBaes and pIn > pAnchor.
- */
-U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
-                        const BYTE *pMatch, const BYTE *pBase) {
-  U32 matchLength = 0;
-  while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
-    pIn--;
-    pMatch--;
-    matchLength++;
-  }
-  return matchLength;
-}
-
-/**
- * Returns a pointer to the entry in the hash table matching the hash and
- * checksum with the "longest match length" as defined below. The forward and
- * backward match lengths are written to *pForwardMatchLength and
- * *pBackwardMatchLength.
- *
- * The match length is defined based on cctx->ip and the entry's offset.
- * The forward match is computed from cctx->ip and entry->offset + cctx->ibase.
- * The backward match is computed backwards from cctx->ip and
- * cctx->ibase only if the forward match is longer than LDM_MIN_MATCH_LENGTH.
- *
- */
-LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
-                                 const hash_t hash,
-                                 const U32 checksum,
-                                 U32 *pForwardMatchLength,
-                                 U32 *pBackwardMatchLength) {
-  LDM_hashTable *table = cctx->hashTable;
-  LDM_hashEntry *bucket = getBucket(table, hash);
-  LDM_hashEntry *cur = bucket;
-  LDM_hashEntry *bestEntry = NULL;
-  U32 bestMatchLength = 0;
-  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
-    const BYTE *pMatch = cur->offset + cctx->ibase;
-
-    // Check checksum for faster check.
-    if (cur->checksum == checksum &&
-        cctx->ip - pMatch <= LDM_WINDOW_SIZE) {
-      U32 forwardMatchLength = ZSTD_count(cctx->ip, pMatch, cctx->iend);
-      U32 backwardMatchLength, totalMatchLength;
-
-      // For speed.
-      if (forwardMatchLength < LDM_MIN_MATCH_LENGTH) {
-        continue;
-      }
-
-      backwardMatchLength =
-          countBackwardsMatch(cctx->ip, cctx->anchor,
-                              cur->offset + cctx->ibase,
-                              cctx->ibase);
-
-      totalMatchLength = forwardMatchLength + backwardMatchLength;
-
-      if (totalMatchLength >= bestMatchLength) {
-        bestMatchLength = totalMatchLength;
-        *pForwardMatchLength = forwardMatchLength;
-        *pBackwardMatchLength = backwardMatchLength;
-
-        bestEntry = cur;
-#ifdef ZSTD_SKIP
-        return cur;
-#endif
-      }
-    }
-  }
-  if (bestEntry != NULL) {
-    return bestEntry;
-  }
-  return NULL;
-}
-
-void HASH_insert(LDM_hashTable *table,
-                 const hash_t hash, const LDM_hashEntry entry) {
-  *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
-  table->bucketOffsets[hash]++;
-  table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
-}
-
-U32 HASH_getSize(const LDM_hashTable *table) {
-  return table->numBuckets;
-}
-
-void HASH_destroyTable(LDM_hashTable *table) {
-  free(table->entries);
-  free(table->bucketOffsets);
-  free(table);
-}
-
-void HASH_outputTableOccupancy(const LDM_hashTable *table) {
-  U32 ctr = 0;
-  LDM_hashEntry *cur = table->entries;
-  LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
-  for (; cur < end; ++cur) {
-    if (cur->offset == 0) {
-      ctr++;
-    }
-  }
-
-  printf("Num buckets, bucket size: %d, %d\n",
-         table->numBuckets, HASH_BUCKET_SIZE);
-  printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
-         table->numEntries, ctr,
-         100.0 * (double)(ctr) / table->numEntries);
-}
-
-// TODO: This can be done more efficiently (but it is not that important as it
-// is only used for computing stats).
-static int intLog2(U32 x) {
-  int ret = 0;
-  while (x >>= 1) {
-    ret++;
-  }
-  return ret;
-}
-
-void LDM_printCompressStats(const LDM_compressStats *stats) {
-  int i = 0;
-  printf("=====================\n");
-  printf("Compression statistics\n");
-  printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
-          stats->windowSizeLog, stats->hashTableSizeLog);
-  printf("num matches, total match length, %% matched: %u, %llu, %.3f\n",
-          stats->numMatches,
-          stats->totalMatchLength,
-          100.0 * (double)stats->totalMatchLength /
-              (double)(stats->totalMatchLength + stats->totalLiteralLength));
-  printf("avg match length: %.1f\n", ((double)stats->totalMatchLength) /
-                                         (double)stats->numMatches);
-  printf("avg literal length, total literalLength: %.1f, %llu\n",
-         ((double)stats->totalLiteralLength) / (double)stats->numMatches,
-         stats->totalLiteralLength);
-  printf("avg offset length: %.1f\n",
-         ((double)stats->totalOffset) / (double)stats->numMatches);
-  printf("min offset, max offset: %u, %u\n",
-         stats->minOffset, stats->maxOffset);
-
-  printf("\n");
-  printf("offset histogram | match length histogram\n");
-  printf("offset/ML, num matches, %% of matches | num matches, %% of matches\n");
-
-  for (; i <= intLog2(stats->maxOffset); i++) {
-    printf("2^%*d: %10u    %6.3f%% |2^%*d:  %10u    %6.3f \n",
-           2, i,
-           stats->offsetHistogram[i],
-           100.0 * (double) stats->offsetHistogram[i] /
-                   (double) stats->numMatches,
-           2, i,
-           stats->matchLengthHistogram[i],
-           100.0 * (double) stats->matchLengthHistogram[i] /
-                   (double) stats->numMatches);
-  }
-  printf("\n");
-  printf("=====================\n");
-}
-
-int LDM_isValidMatch(const BYTE *pIn, const BYTE *pMatch) {
-  U32 lengthLeft = LDM_MIN_MATCH_LENGTH;
-  const BYTE *curIn = pIn;
-  const BYTE *curMatch = pMatch;
-
-  if (pIn - pMatch > LDM_WINDOW_SIZE) {
-    return 0;
-  }
-
-  for (; lengthLeft >= 4; lengthLeft -= 4) {
-    if (MEM_read32(curIn) != MEM_read32(curMatch)) {
-      return 0;
-    }
-    curIn += 4;
-    curMatch += 4;
-  }
-  return 1;
-}
-
-hash_t HASH_hashU32(U32 value) {
-  return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
-}
-
-/**
- * Convert a sum computed from getChecksum to a hash value in the range
- * of the hash table.
- */
-static hash_t checksumToHash(U32 sum) {
-  return HASH_hashU32(sum);
-}
-
-/**
- * Computes a checksum based on rsync's checksum.
- *
- * a(k,l) = \sum_{i = k}^l x_i (mod M)
- * b(k,l) = \sum_{i = k}^l ((l - i + 1) * x_i) (mod M)
- * checksum(k,l) = a(k,l) + 2^{16} * b(k,l)
-  */
-static U32 getChecksum(const BYTE *buf, U32 len) {
-  U32 i;
-  U32 s1, s2;
-
-  s1 = s2 = 0;
-  for (i = 0; i < (len - 4); i += 4) {
-    s2 += (4 * (s1 + buf[i])) + (3 * buf[i + 1]) +
-          (2 * buf[i + 2]) + (buf[i + 3]) +
-          (10 * CHECKSUM_CHAR_OFFSET);
-    s1 += buf[i] + buf[i + 1] + buf[i + 2] + buf[i + 3] +
-          + (4 * CHECKSUM_CHAR_OFFSET);
-
-  }
-  for(; i < len; i++) {
-    s1 += buf[i] + CHECKSUM_CHAR_OFFSET;
-    s2 += s1;
-  }
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-/**
- * Update a checksum computed from getChecksum(data, len).
- *
- * The checksum can be updated along its ends as follows:
- * a(k+1, l+1) = (a(k,l) - x_k + x_{l+1}) (mod M)
- * b(k+1, l+1) = (b(k,l) - (l-k+1)*x_k + (a(k+1,l+1)) (mod M)
- *
- * Thus toRemove should correspond to data[0].
- */
-static U32 updateChecksum(U32 sum, U32 len,
-                          BYTE toRemove, BYTE toAdd) {
-  U32 s1 = (sum & 0xffff) - toRemove + toAdd;
-  U32 s2 = (sum >> 16) - ((toRemove + CHECKSUM_CHAR_OFFSET) * len) + s1;
-
-  return (s1 & 0xffff) + (s2 << 16);
-}
-
-/**
- * Update cctx->nextSum, cctx->nextHash, and cctx->nextPosHashed
- * based on cctx->lastSum and cctx->lastPosHashed.
- *
- * This uses a rolling hash and requires that the last position hashed
- * corresponds to cctx->nextIp - step.
- */
-static void setNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  U32 check;
-  if ((cctx->nextIp - cctx->ibase != 1) &&
-      (cctx->nextIp - cctx->DEBUG_setNextHash != 1)) {
-    printf("CHECK debug fail: %zu %zu\n", cctx->nextIp - cctx->ibase,
-            cctx->DEBUG_setNextHash - cctx->ibase);
-  }
-
-  cctx->DEBUG_setNextHash = cctx->nextIp;
-#endif
-
-  cctx->nextSum = updateChecksum(
-      cctx->lastSum, LDM_HASH_LENGTH,
-      cctx->lastPosHashed[0],
-      cctx->lastPosHashed[LDM_HASH_LENGTH]);
-  cctx->nextPosHashed = cctx->nextIp;
-  cctx->nextHash = checksumToHash(cctx->nextSum);
-
-#if LDM_LAG
-  if (cctx->ip - cctx->ibase > LDM_LAG) {
-    cctx->lagSum = updateChecksum(
-      cctx->lagSum, LDM_HASH_LENGTH,
-      cctx->lagIp[0], cctx->lagIp[LDM_HASH_LENGTH]);
-    cctx->lagIp++;
-    cctx->lagHash = checksumToHash(cctx->lagSum);
-  }
-#endif
-
-#ifdef RUN_CHECKS
-  check = getChecksum(cctx->nextIp, LDM_HASH_LENGTH);
-
-  if (check != cctx->nextSum) {
-    printf("CHECK: setNextHash failed %u %u\n", check, cctx->nextSum);
-  }
-
-  if ((cctx->nextIp - cctx->lastPosHashed) != 1) {
-    printf("setNextHash: nextIp != lastPosHashed + 1. %zu %zu %zu\n",
-            cctx->nextIp - cctx->ibase, cctx->lastPosHashed - cctx->ibase,
-            cctx->ip - cctx->ibase);
-  }
-#endif
-}
-
-static void putHashOfCurrentPositionFromHash(
-    LDM_CCtx *cctx, hash_t hash, U32 sum) {
-  // Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
-  // Note: this works only when cctx->step is 1.
-  if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
-#if LDM_LAG
-    // TODO: off by 1, but whatever
-    if (cctx->lagIp - cctx->ibase > 0) {
-      const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, cctx->lagSum };
-      HASH_insert(cctx->hashTable, cctx->lagHash, entry);
-    } else {
-      const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
-      HASH_insert(cctx->hashTable, hash, entry);
-    }
-#else
-    const LDM_hashEntry entry = { cctx->ip - cctx->ibase, sum };
-    HASH_insert(cctx->hashTable, hash, entry);
-#endif
-  }
-
-  cctx->lastPosHashed = cctx->ip;
-  cctx->lastHash = hash;
-  cctx->lastSum = sum;
-}
-
-/**
- * Copy over the cctx->lastHash, cctx->lastSum, and cctx->lastPosHashed
- * fields from the "next" fields.
- *
- * This requires that cctx->ip == cctx->nextPosHashed.
- */
-static void LDM_updateLastHashFromNextHash(LDM_CCtx *cctx) {
-#ifdef RUN_CHECKS
-  if (cctx->ip != cctx->nextPosHashed) {
-    printf("CHECK failed: updateLastHashFromNextHash %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
-  putHashOfCurrentPositionFromHash(cctx, cctx->nextHash, cctx->nextSum);
-}
-
-/**
- * Insert hash of the current position into the hash table.
- */
-static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
-  U32 sum = getChecksum(cctx->ip, LDM_HASH_LENGTH);
-  hash_t hash = checksumToHash(sum);
-
-#ifdef RUN_CHECKS
-  if (cctx->nextPosHashed != cctx->ip && (cctx->ip != cctx->ibase)) {
-    printf("CHECK failed: putHashOfCurrentPosition %zu\n",
-           cctx->ip - cctx->ibase);
-  }
-#endif
-
-  putHashOfCurrentPositionFromHash(cctx, hash, sum);
-}
-
-void LDM_initializeCCtx(LDM_CCtx *cctx,
-                        const void *src, size_t srcSize,
-                        void *dst, size_t maxDstSize) {
-  cctx->isize = srcSize;
-  cctx->maxOSize = maxDstSize;
-
-  cctx->ibase = (const BYTE *)src;
-  cctx->ip = cctx->ibase;
-  cctx->iend = cctx->ibase + srcSize;
-
-  cctx->ihashLimit = cctx->iend - LDM_HASH_LENGTH;
-  cctx->imatchLimit = cctx->iend - LDM_MIN_MATCH_LENGTH;
-
-  cctx->obase = (BYTE *)dst;
-  cctx->op = (BYTE *)dst;
-
-  cctx->anchor = cctx->ibase;
-
-  memset(&(cctx->stats), 0, sizeof(cctx->stats));
-  cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U64);
-
-  cctx->stats.minOffset = UINT_MAX;
-  cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
-  cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
-
-
-  cctx->lastPosHashed = NULL;
-
-  cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
-  cctx->nextIp = cctx->ip + cctx->step;
-  cctx->nextPosHashed = 0;
-
-  cctx->DEBUG_setNextHash = 0;
-}
-
-void LDM_destroyCCtx(LDM_CCtx *cctx) {
-  HASH_destroyTable(cctx->hashTable);
-}
-
-/**
- * Finds the "best" match.
- *
- * Returns 0 if successful and 1 otherwise (i.e. no match can be found
- * in the remaining input that is long enough).
- *
- * forwardMatchLength contains the forward length of the match.
- */
-static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
-                             U32 *forwardMatchLength, U32 *backwardMatchLength) {
-
-  LDM_hashEntry *entry = NULL;
-  cctx->nextIp = cctx->ip + cctx->step;
-
-  while (entry == NULL) {
-    hash_t h;
-    U32 sum;
-    setNextHash(cctx);
-    h = cctx->nextHash;
-    sum = cctx->nextSum;
-    cctx->ip = cctx->nextIp;
-    cctx->nextIp += cctx->step;
-
-    if (cctx->ip > cctx->imatchLimit) {
-      return 1;
-    }
-
-    entry = HASH_getBestEntry(cctx, h, sum,
-                              forwardMatchLength, backwardMatchLength);
-
-    if (entry != NULL) {
-      *match = entry->offset + cctx->ibase;
-    }
-    putHashOfCurrentPositionFromHash(cctx, h, sum);
-  }
-  setNextHash(cctx);
-  return 0;
-}
-
-void LDM_encodeLiteralLengthAndLiterals(
-    LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
-  /* Encode the literal length. */
-  if (literalLength >= RUN_MASK) {
-    U64 len = (U64)literalLength - RUN_MASK;
-    *pToken = (RUN_MASK << ML_BITS);
-    for (; len >= 255; len -= 255) {
-      *(cctx->op)++ = 255;
-    }
-    *(cctx->op)++ = (BYTE)len;
-  } else {
-    *pToken = (BYTE)(literalLength << ML_BITS);
-  }
-
-  /* Encode the literals. */
-  memcpy(cctx->op, cctx->anchor, literalLength);
-  cctx->op += literalLength;
-}
-
-void LDM_outputBlock(LDM_CCtx *cctx,
-                     const U64 literalLength,
-                     const U32 offset,
-                     const U64 matchLength) {
-  BYTE *pToken = cctx->op++;
-
-  /* Encode the literal length and literals. */
-  LDM_encodeLiteralLengthAndLiterals(cctx, pToken, literalLength);
-
-  /* Encode the offset. */
-  MEM_write32(cctx->op, offset);
-  cctx->op += LDM_OFFSET_SIZE;
-
-  /* Encode the match length. */
-  if (matchLength >= ML_MASK) {
-    unsigned matchLengthRemaining = matchLength;
-    *pToken += ML_MASK;
-    matchLengthRemaining -= ML_MASK;
-    MEM_write32(cctx->op, 0xFFFFFFFF);
-    while (matchLengthRemaining >= 4*0xFF) {
-      cctx->op += 4;
-      MEM_write32(cctx->op, 0xffffffff);
-      matchLengthRemaining -= 4*0xFF;
-    }
-    cctx->op += matchLengthRemaining / 255;
-    *(cctx->op)++ = (BYTE)(matchLengthRemaining % 255);
-  } else {
-    *pToken += (BYTE)(matchLength);
-  }
-}
-
-// TODO: maxDstSize is unused. This function may seg fault when writing
-// beyond the size of dst, as it does not check maxDstSize. Writing to
-// a buffer and performing checks is a possible solution.
-//
-// This is based upon lz4.
-size_t LDM_compress(const void *src, size_t srcSize,
-                    void *dst, size_t maxDstSize) {
-  LDM_CCtx cctx;
-  const BYTE *match = NULL;
-  U32 forwardMatchLength = 0;
-  U32 backwardsMatchLength = 0;
-
-  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
-  LDM_outputConfiguration();
-
-  /* Hash the first position and put it into the hash table. */
-  LDM_putHashOfCurrentPosition(&cctx);
-
-#if LDM_LAG
-  cctx.lagIp = cctx.ip;
-  cctx.lagHash = cctx.lastHash;
-  cctx.lagSum = cctx.lastSum;
-#endif
-  /**
-   * Find a match.
-   * If no more matches can be found (i.e. the length of the remaining input
-   * is less than the minimum match length), then stop searching for matches
-   * and encode the final literals.
-   */
-  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
-         &backwardsMatchLength) == 0) {
-#ifdef COMPUTE_STATS
-    cctx.stats.numMatches++;
-#endif
-
-     cctx.ip -= backwardsMatchLength;
-     match -= backwardsMatchLength;
-
-    /**
-     * Write current block (literals, literal length, match offset, match
-     * length) and update pointers and hashes.
-     */
-    {
-      const U64 literalLength = cctx.ip - cctx.anchor;
-      const U32 offset = cctx.ip - match;
-      const U64 matchLength = forwardMatchLength +
-                              backwardsMatchLength -
-                              LDM_MIN_MATCH_LENGTH;
-
-      LDM_outputBlock(&cctx, literalLength, offset, matchLength);
-
-#ifdef COMPUTE_STATS
-      cctx.stats.totalLiteralLength += literalLength;
-      cctx.stats.totalOffset += offset;
-      cctx.stats.totalMatchLength += matchLength + LDM_MIN_MATCH_LENGTH;
-      cctx.stats.minOffset =
-          offset < cctx.stats.minOffset ? offset : cctx.stats.minOffset;
-      cctx.stats.maxOffset =
-          offset > cctx.stats.maxOffset ? offset : cctx.stats.maxOffset;
-      cctx.stats.offsetHistogram[(U32)intLog2(offset)]++;
-      cctx.stats.matchLengthHistogram[
-          (U32)intLog2(matchLength + LDM_MIN_MATCH_LENGTH)]++;
-#endif
-
-      // Move ip to end of block, inserting hashes at each position.
-      cctx.nextIp = cctx.ip + cctx.step;
-      while (cctx.ip < cctx.anchor + LDM_MIN_MATCH_LENGTH +
-                       matchLength + literalLength) {
-        if (cctx.ip > cctx.lastPosHashed) {
-          // TODO: Simplify.
-          LDM_updateLastHashFromNextHash(&cctx);
-          setNextHash(&cctx);
-        }
-        cctx.ip++;
-        cctx.nextIp++;
-      }
-    }
-
-    // Set start of next block to current input pointer.
-    cctx.anchor = cctx.ip;
-    LDM_updateLastHashFromNextHash(&cctx);
-  }
-
-  /* Encode the last literals (no more matches). */
-  {
-    const U32 lastRun = cctx.iend - cctx.anchor;
-    BYTE *pToken = cctx.op++;
-    LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
-  }
-
-#ifdef COMPUTE_STATS
-  LDM_printCompressStats(&cctx.stats);
-  HASH_outputTableOccupancy(cctx.hashTable);
-#endif
-
-  {
-    const size_t ret = cctx.op - cctx.obase;
-    LDM_destroyCCtx(&cctx);
-    return ret;
-  }
-}
-
-void LDM_outputConfiguration(void) {
-  printf("=====================\n");
-  printf("Configuration\n");
-  printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
-  printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
-         LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
-  printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
-  printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
-  printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
-  printf("LDM_LAG %d\n", LDM_LAG);
-  printf("=====================\n");
-}
-
-
-
diff --git a/contrib/long_distance_matching/ldm_params.h b/contrib/long_distance_matching/ldm_params.h
index 0fcd30bd..a541581b 100644
--- a/contrib/long_distance_matching/ldm_params.h
+++ b/contrib/long_distance_matching/ldm_params.h
@@ -1,5 +1,6 @@
 #ifndef LDM_PARAMS_H
 #define LDM_PARAMS_H
+
 #define LDM_MEMORY_USAGE 23
 #define HASH_BUCKET_SIZE_LOG 3
 #define LDM_LAG 0
@@ -7,4 +8,5 @@
 #define LDM_MIN_MATCH_LENGTH 64
 #define INSERT_BY_TAG 1
 #define USE_CHECKSUM 1
-#endif
+
+#endif  // LDM_PARAMS_H
diff --git a/contrib/long_distance_matching/main.c b/contrib/long_distance_matching/main.c
index bdd385ce..d55e01d3 100644
--- a/contrib/long_distance_matching/main.c
+++ b/contrib/long_distance_matching/main.c
@@ -12,11 +12,13 @@
 #include "ldm.h"
 #include "zstd.h"
 
-#define DECOMPRESS_AND_VERIFY
+// #define DECOMPRESS_AND_VERIFY
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
  *
+ * This adds a header from LDM_writeHeader to the beginning of the output.
+ *
  * This might seg fault if the compressed size is > the decompress
  * size due to the mmapping and output file size allocated to be the input size
  * The compress function should check before writing or buffer writes.
@@ -52,7 +54,7 @@ static int compress(const char *fname, const char *oname) {
   maxCompressedSize = (statbuf.st_size + LDM_HEADER_SIZE);
 
   // Handle case where compressed size is > decompressed size.
-  // The compress function should check before writing or buffer writes.
+  // TODO: The compress function should check before writing or buffer writes.
   maxCompressedSize += statbuf.st_size / 255;
 
   ftruncate(fdout, maxCompressedSize);
@@ -64,7 +66,7 @@ static int compress(const char *fname, const char *oname) {
       return 1;
   }
 
-  /* mmap the output file */
+  /* mmap the output file. */
   if ((dst = mmap(0, maxCompressedSize, PROT_READ | PROT_WRITE,
                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
       perror("mmap error for output");
@@ -79,14 +81,12 @@ static int compress(const char *fname, const char *oname) {
 
   gettimeofday(&tv2, NULL);
 
-  // Write compress and decompress size to header
-  // TODO: should depend on LDM_DECOMPRESS_SIZE write32
+  // Write the header.
   LDM_writeHeader(dst, compressedSize, statbuf.st_size);
 
   // Truncate file to compressedSize.
   ftruncate(fdout, compressedSize);
 
-
   printf("%25s : %10lu -> %10lu - %s \n", fname,
          (size_t)statbuf.st_size, (size_t)compressedSize, oname);
   printf("Compression ratio: %.2fx --- %.1f%%\n",
@@ -100,7 +100,6 @@ static int compress(const char *fname, const char *oname) {
          timeTaken,
          ((double)statbuf.st_size / (double) (1 << 20)) / timeTaken);
 
-
   // Close files.
   close(fdin);
   close(fdout);

From 9eaf3d22d0818fc977b7df6af4ce78654613a97e Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Wed, 26 Jul 2017 16:43:25 -0700
Subject: [PATCH 096/100] Allow HASH_ONLY_EVERY_LOG to be configured in ldm.h

---
 contrib/long_distance_matching/Makefile |  2 +-
 contrib/long_distance_matching/ldm.c    |  6 +++---
 contrib/long_distance_matching/ldm.h    | 11 +++++++++--
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/contrib/long_distance_matching/Makefile b/contrib/long_distance_matching/Makefile
index 8bc7ac47..4193cb32 100644
--- a/contrib/long_distance_matching/Makefile
+++ b/contrib/long_distance_matching/Makefile
@@ -32,6 +32,6 @@ ldm: ldm_common.c ldm.c main.c
 
 clean:
 	@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
-	ldm	
+	ldm
 	@echo Cleaning completed
 
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 9a843838..25bf5c83 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -16,11 +16,11 @@
   #define LDM_HASH_ENTRY_SIZE_LOG 2
 #endif
 
-// Force the "probability" of insertion to be some value.
 // Entries are inserted into the table HASH_ONLY_EVERY + 1 times "on average".
+#ifndef HASH_ONLY_EVERY_LOG
+  #define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
+#endif
 
-//#define HASH_ONLY_EVERY_LOG 7
-#define HASH_ONLY_EVERY_LOG (LDM_WINDOW_SIZE_LOG-((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)))
 #define HASH_ONLY_EVERY ((1 << (HASH_ONLY_EVERY_LOG)) - 1)
 
 #define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 38d24015..af35130e 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -3,7 +3,7 @@
 
 #include "mem.h"    // from /lib/common/mem.h
 
-// #include "ldm_params.h"
+//#include "ldm_params.h"
 
 // =============================================================================
 // Modify the parameters in ldm_params.h if "ldm_params.h" is included.
@@ -23,7 +23,7 @@
   #define LDM_LAG 0
 
   // The maximum window size when searching for matches.
-  // The maximum value is 30.
+  // The maximum value is 30
   #define LDM_WINDOW_SIZE_LOG 28
 
   // The minimum match length.
@@ -47,6 +47,13 @@
 // Output the configuration.
 #define OUTPUT_CONFIGURATION
 
+// If defined, forces the probability of insertion to be approximately
+// one per (1 << HASH_ONLY_EVERY_LOG). If not defined, the probability will be
+// calculated based on the memory usage and window size for "even" insertion
+// throughout the window.
+
+// #define HASH_ONLY_EVERY_LOG 8
+
 // =============================================================================
 
 // The number of bytes storing the compressed and decompressed size

From c105f605e66bd3bd985ceeaa660e7159e4d16298 Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 27 Jul 2017 11:11:35 -0700
Subject: [PATCH 097/100] Update README

---
 contrib/long_distance_matching/README.md | 75 ++++++++++++++++++++++--
 contrib/long_distance_matching/ldm.c     | 22 -------
 contrib/long_distance_matching/ldm.h     |  3 +
 3 files changed, 72 insertions(+), 28 deletions(-)

diff --git a/contrib/long_distance_matching/README.md b/contrib/long_distance_matching/README.md
index d9cb0895..e67bba71 100644
--- a/contrib/long_distance_matching/README.md
+++ b/contrib/long_distance_matching/README.md
@@ -28,12 +28,75 @@ The parameters are as follows and must all be defined:
 - `INSERT_BY_TAG` : insert entries into the hash table as a function of the hash. This increases speed by reducing the number of hash table lookups and match comparisons. Certain hashes will never be inserted.
 - `USE_CHECKSUM`  : store a checksum with the hash table entries for faster comparison. This halves the number of entries the hash table can contain.
 
+The optional parameter `HASH_ONLY_EVERY_LOG` is the log inverse frequency of insertion into the hash table. That is, an entry is inserted approximately every `1 << HASH_ONLY_EVERY_LOG` times. If this parameter is not defined, the value is computed as a function of the window size and memory usage to approximate a even coverage of the window.
+
+
+### Benchmark
+
+Below is a comparison of various compression methods on a tar of four versions of llvm (versions `3.9.0`, `3.9.1`, `4.0.0`, `4.0.1`) with a total size of `727900160` B.
+
+| Method | Size | Ratio |
+|:---|---:|---:|
+|lrzip -p 32 -n -w 1 | `369968714` | `1.97`|
+|ldm | `209391361` | `3.48`|
+|lz4 | `189954338` | `3.83`|
+|lrzip -p 32 -l -w 1 | `163940343` | `4.44`|
+|zstd -1 | `126080293` | `5.77`|
+|lrzip -p 32 -n | `124821009` | `5.83`|
+|lrzip -p 32 -n -w 1 & zstd -1 | `120317909` | `6.05`|
+|zstd -3 -o | `115290952` | `6.31`|
+|lrzip -p 32 -g -L 9 -w 1 | `107168979` | `6.79`|
+|zstd -6 -o | `102772098` | `7.08`|
+|zstd -T16 -9 | `98040470` | `7.42`|
+|lrzip -p 32 -n -w 1 & zstd -T32 -19 | `88050289` | `8.27`|
+|zstd -T32 -19 | `83626098` | `8.70`|
+|lrzip -p 32 -n & zstd -1 | `36335117` | `20.03`|
+|ldm & zstd -6 | `32856232` | `22.15`|
+|lrzip -p 32 -g -L 9 | `32243594` | `22.58`|
+|lrzip -p 32 -n & zstd -6 | `30954572` | `23.52`|
+|lrzip -p 32 -n & zstd -T32 -19 | `26472064` | `27.50`|
+
+The method marked `ldm` was run with the following parameters:
+
+| Parameter | Value |
+|:---|---:|
+| `LDM_MEMORY_USAGE`    |   `23`|
+|`HASH_BUCKET_SIZE_LOG` |    `3`|
+|`LDM_LAG`              |    `0`|
+|`LDM_WINDOW_SIZE_LOG`  |   `28`|
+|`LDM_MIN_MATCH_LENGTH`|   `64`|
+|`INSERT_BY_TAG`        |    `1`|
+|`USE_CHECKSUM`         |    `1`|
+
+The compression speed was `220.5 MB/s`.
+
+### Parameter selection
+
+Below is a brief discussion of the effects of the parameters on the speed and compression ratio.
+
+#### Speed
+
+A large bottleneck in terms of speed is finding the matches and comparing to see if they are greater than the minimum match length. Generally:
+- The fewer matches found (or the lower the percentage of the literals matched), the slower the algorithm will behave.
+- Increasing `HASH_ONLY_EVERY_LOG` results in fewer inserts and, if `INSERT_BY_TAG` is set, fewer lookups in the table. This has a large effect on speed, as well as compression ratio.
+- If `HASH_ONLY_EVERY_LOG` is not set, its value is calculated based on `LDM_WINDOW_SIZE_LOG` and `LDM_MEMORY_USAGE`. Increasing `LDM_WINDOW_SIZE_LOG` has the effect of increasing `HASH_ONLY_EVERY_LOG` and increasing `LDM_MEMORY_USAGE` decreases `HASH_ONLY_EVERY_LOG`.
+- `USE_CHECKSUM` generally improves speed with hash table lookups.
+
+#### Compression ratio
+
+The compression ratio is highly correlated with the coverage of matches. As a long distance matcher, the algorithm was designed to "optimize" for long distance matches outside the zstd compression window. The compression ratio after recompressing the output of the long-distance matcher with zstd was a more important signal in development than the raw compression ratio itself.
+
+Generally, increasing `LDM_MEMORY_USAGE` will improve the compression ratio. However when using the default computed value of `HASH_ONLY_EVERY_LOG`, this increases the frequency of insertion and lookup in the table and thus may result in a decrease in speed. 
+
+Below is a table showing the speed and compression ratio when compressing the llvm tar (as described above) using different settings for `LDM_MEMORY_USAGE`. The other parameters were the same as used in the benchmark above.
+
+| `LDM_MEMORY_USAGE` | Ratio | Speed (MB/s) | Ratio after zstd -6  |
+|---:| ---: | ---: | ---: |
+| `18` | `1.85` | `232.4` | `10.92` |
+| `21` | `2.79` | `233.9` | `15.92` |
+| `23` | `3.48` | `220.5` | `18.29` |
+| `25` | `4.56` | `140.8` | `19.21` |
+
 ### Compression statistics
 
 Compression statistics (and the configuration) can be enabled/disabled via `COMPUTE_STATS` and `OUTPUT_CONFIGURATION` in `ldm.h`.
-
-
-
-
-
-
diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index 25bf5c83..ff9d94d0 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -58,9 +58,6 @@ struct LDM_compressStats {
 
   U32 minOffset, maxOffset;
   U32 offsetHistogram[32];
-
-  U64 TMP_hashCount[1 << HASH_ONLY_EVERY_LOG];
-  U64 TMP_totalHashCount;
 };
 
 typedef struct LDM_hashTable LDM_hashTable;
@@ -398,17 +395,6 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
                    (double) stats->numMatches);
   }
   printf("\n");
-#if INSERT_BY_TAG
-/*
-  printf("Lower bit distribution\n");
-  for (i = 0; i < (1 << HASH_ONLY_EVERY_LOG); i++) {
-    printf("%5d %5llu %6.3f\n", i, stats->TMP_hashCount[i],
-           100.0 * (double) stats->TMP_hashCount[i] /
-                   (double) stats->TMP_totalHashCount);
-  }
-*/
-#endif
-
   printf("=====================\n");
 }
 
@@ -503,14 +489,6 @@ static void setNextHash(LDM_CCtx *cctx) {
       cctx->lastPosHashed[LDM_HASH_LENGTH]);
   cctx->nextPosHashed = cctx->nextIp;
 
-#if INSERT_BY_TAG
-  {
-    U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextHash);
-    cctx->stats.TMP_totalHashCount++;
-    cctx->stats.TMP_hashCount[hashEveryMask]++;
-  }
-#endif
-
 #if LDM_LAG
   if (cctx->ip - cctx->ibase > LDM_LAG) {
     cctx->lagHash = updateHash(
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index af35130e..456ec5aa 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -71,6 +71,9 @@
 #define LDM_OFFSET_SIZE 4
 
 #define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
+
+// TODO: Match lengths that are too small do not use the hash table efficiently.
+// There should be a minimum hash length given the hash table size.
 #define LDM_HASH_LENGTH LDM_MIN_MATCH_LENGTH
 
 typedef struct LDM_compressStats LDM_compressStats;

From 627621839cf39310793fc5c2358985929497d42b Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 27 Jul 2017 15:37:37 -0700
Subject: [PATCH 098/100] Add checks in initialization code

---
 contrib/long_distance_matching/ldm.c        | 33 ++++++++++++++++-----
 contrib/long_distance_matching/ldm.h        |  9 ++++--
 contrib/long_distance_matching/ldm_common.c | 20 +++++++++----
 contrib/long_distance_matching/main.c       | 18 ++++++++---
 4 files changed, 61 insertions(+), 19 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index ff9d94d0..c2cdb21e 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -111,13 +111,25 @@ struct LDM_hashTable {
 /**
  * Create a hash table that can contain size elements.
  * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
+ *
+ * Returns NULL if table creation failed.
  */
 static LDM_hashTable *HASH_createTable(U32 size) {
   LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
+  if (!table) return NULL;
+
   table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
   table->numEntries = size;
   table->entries = calloc(size, sizeof(LDM_hashEntry));
   table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
+
+  if (!table->entries || !table->bucketOffsets) {
+    free(table->bucketOffsets);
+    free(table->entries);
+    free(table);
+    return NULL;
+  }
+
   return table;
 }
 
@@ -566,9 +578,9 @@ static void LDM_putHashOfCurrentPosition(LDM_CCtx *cctx) {
   putHashOfCurrentPositionFromHash(cctx, hash);
 }
 
-void LDM_initializeCCtx(LDM_CCtx *cctx,
-                        const void *src, size_t srcSize,
-                        void *dst, size_t maxDstSize) {
+size_t LDM_initializeCCtx(LDM_CCtx *cctx,
+                          const void *src, size_t srcSize,
+                          void *dst, size_t maxDstSize) {
   cctx->isize = srcSize;
   cctx->maxOSize = maxDstSize;
 
@@ -590,16 +602,20 @@ void LDM_initializeCCtx(LDM_CCtx *cctx,
 #else
   cctx->hashTable = HASH_createTable(LDM_HASHTABLESIZE_U32);
 #endif
+
+  if (!cctx->hashTable) return 1;
+
   cctx->stats.minOffset = UINT_MAX;
   cctx->stats.windowSizeLog = LDM_WINDOW_SIZE_LOG;
   cctx->stats.hashTableSizeLog = LDM_MEMORY_USAGE;
 
-
   cctx->lastPosHashed = NULL;
 
   cctx->step = 1;   // Fixed to be 1 for now. Changing may break things.
   cctx->nextIp = cctx->ip + cctx->step;
   cctx->nextPosHashed = 0;
+
+  return 0;
 }
 
 void LDM_destroyCCtx(LDM_CCtx *cctx) {
@@ -726,7 +742,10 @@ size_t LDM_compress(const void *src, size_t srcSize,
   U64 forwardMatchLength = 0;
   U64 backwardsMatchLength = 0;
 
-  LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
+  if (LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize)) {
+    // Initialization failed.
+    return 0;
+  }
 
 #ifdef OUTPUT_CONFIGURATION
   LDM_outputConfiguration();
@@ -744,8 +763,8 @@ size_t LDM_compress(const void *src, size_t srcSize,
    * is less than the minimum match length), then stop searching for matches
    * and encode the final literals.
    */
-  while (LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
-         &backwardsMatchLength) == 0) {
+  while (!LDM_findBestMatch(&cctx, &match, &forwardMatchLength,
+                           &backwardsMatchLength)) {
 
 #ifdef COMPUTE_STATS
     cctx.stats.numMatches++;
diff --git a/contrib/long_distance_matching/ldm.h b/contrib/long_distance_matching/ldm.h
index 456ec5aa..4adadbd0 100644
--- a/contrib/long_distance_matching/ldm.h
+++ b/contrib/long_distance_matching/ldm.h
@@ -82,6 +82,7 @@ typedef struct LDM_DCtx LDM_DCtx;
 
 /**
  *  Compresses src into dst.
+ *  Returns the compressed size if successful, 0 otherwise.
  *
  *  NB: This currently ignores maxDstSize and assumes enough space is available.
  *
@@ -113,10 +114,12 @@ size_t LDM_compress(const void *src, size_t srcSize,
  * Initialize the compression context.
  *
  * Allocates memory for the hash table.
+ *
+ * Returns 0 if successful, 1 otherwise.
  */
-void LDM_initializeCCtx(LDM_CCtx *cctx,
-                        const void *src, size_t srcSize,
-                        void *dst, size_t maxDstSize);
+size_t LDM_initializeCCtx(LDM_CCtx *cctx,
+                          const void *src, size_t srcSize,
+                          void *dst, size_t maxDstSize);
 
 /**
  * Frees up memory allocated in LDM_initializeCCtx().
diff --git a/contrib/long_distance_matching/ldm_common.c b/contrib/long_distance_matching/ldm_common.c
index 26b716a1..8b34f8ad 100644
--- a/contrib/long_distance_matching/ldm_common.c
+++ b/contrib/long_distance_matching/ldm_common.c
@@ -2,19 +2,29 @@
 
 #include "ldm.h"
 
+/**
+ * This function reads the header at the beginning of src and writes
+ * the compressed and decompressed size to compressedSize and
+ * decompressedSize.
+ *
+ * The header consists of 16 bytes: 8 bytes each in little-endian format
+ * of the compressed size and the decompressed size.
+ */
 void LDM_readHeader(const void *src, U64 *compressedSize,
                     U64 *decompressedSize) {
   const BYTE *ip = (const BYTE *)src;
   *compressedSize = MEM_readLE64(ip);
-  ip += sizeof(U64);
-  *decompressedSize = MEM_readLE64(ip);
-  // ip += sizeof(U64);
+  *decompressedSize = MEM_readLE64(ip + 8);
 }
 
+/**
+ * Writes the 16-byte header (8-bytes each of the compressedSize and
+ * decompressedSize in little-endian format) to memPtr.
+ */
 void LDM_writeHeader(void *memPtr, U64 compressedSize,
                      U64 decompressedSize) {
-  MEM_write64(memPtr, compressedSize);
-  MEM_write64((BYTE *)memPtr + 8, decompressedSize);
+  MEM_writeLE64(memPtr, compressedSize);
+  MEM_writeLE64((BYTE *)memPtr + 8, decompressedSize);
 }
 
 struct LDM_DCtx {
diff --git a/contrib/long_distance_matching/main.c b/contrib/long_distance_matching/main.c
index d55e01d3..72af5404 100644
--- a/contrib/long_distance_matching/main.c
+++ b/contrib/long_distance_matching/main.c
@@ -12,7 +12,7 @@
 #include "ldm.h"
 #include "zstd.h"
 
-// #define DECOMPRESS_AND_VERIFY
+#define DECOMPRESS_AND_VERIFY
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
@@ -186,9 +186,18 @@ static int compare(FILE *fp0, FILE *fp1) {
 }
 
 /* Verify the input file is the same as the decompressed file. */
-static void verify(const char *inpFilename, const char *decFilename) {
-  FILE *inpFp = fopen(inpFilename, "rb");
-  FILE *decFp = fopen(decFilename, "rb");
+static int verify(const char *inpFilename, const char *decFilename) {
+  FILE *inpFp, *decFp;
+
+  if ((inpFp = fopen(inpFilename, "rb")) == NULL) {
+    perror("Could not open input file\n");
+    return 1;
+  }
+
+  if ((decFp = fopen(decFilename, "rb")) == NULL) {
+    perror("Could not open decompressed file\n");
+    return 1;
+  }
 
   printf("verify : %s <-> %s\n", inpFilename, decFilename);
   {
@@ -202,6 +211,7 @@ static void verify(const char *inpFilename, const char *decFilename) {
 
 	fclose(decFp);
 	fclose(inpFp);
+  return 0;
 }
 #endif
 

From 1294a4a897d696e9a1f999f465527de31479596d Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 27 Jul 2017 15:49:46 -0700
Subject: [PATCH 099/100] Fix typo

---
 contrib/long_distance_matching/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/long_distance_matching/README.md b/contrib/long_distance_matching/README.md
index e67bba71..771a6c3c 100644
--- a/contrib/long_distance_matching/README.md
+++ b/contrib/long_distance_matching/README.md
@@ -1,6 +1,6 @@
 This is a compression algorithm focused on finding long distance matches.
 
-It is based upon lz4 and uses nearly the same block format (github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md). The number of bytes to encode the offset is four instead of two in lz4 to reflect the longer distance matching. The block format is descriped in `ldm.h`.
+It is based upon lz4 and uses nearly the same block format (github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md). The number of bytes to encode the offset is four instead of two in lz4 to reflect the longer distance matching. The block format is described in `ldm.h`.
 
 ### Build
 
@@ -28,7 +28,7 @@ The parameters are as follows and must all be defined:
 - `INSERT_BY_TAG` : insert entries into the hash table as a function of the hash. This increases speed by reducing the number of hash table lookups and match comparisons. Certain hashes will never be inserted.
 - `USE_CHECKSUM`  : store a checksum with the hash table entries for faster comparison. This halves the number of entries the hash table can contain.
 
-The optional parameter `HASH_ONLY_EVERY_LOG` is the log inverse frequency of insertion into the hash table. That is, an entry is inserted approximately every `1 << HASH_ONLY_EVERY_LOG` times. If this parameter is not defined, the value is computed as a function of the window size and memory usage to approximate a even coverage of the window.
+The optional parameter `HASH_ONLY_EVERY_LOG` is the log inverse frequency of insertion into the hash table. That is, an entry is inserted approximately every `1 << HASH_ONLY_EVERY_LOG` times. If this parameter is not defined, the value is computed as a function of the window size and memory usage to approximate an even coverage of the window.
 
 
 ### Benchmark

From 8fae41c412d99c8069ea1bd98ffcf50e2c99cd1e Mon Sep 17 00:00:00 2001
From: Stella Lau <laus@fb.com>
Date: Thu, 27 Jul 2017 17:14:05 -0700
Subject: [PATCH 100/100] Return error code in verify() and minor code cleanup

---
 contrib/long_distance_matching/ldm.c  | 50 ++++++++++++++-------------
 contrib/long_distance_matching/main.c | 12 ++++---
 2 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/contrib/long_distance_matching/ldm.c b/contrib/long_distance_matching/ldm.c
index c2cdb21e..4dccd0bf 100644
--- a/contrib/long_distance_matching/ldm.c
+++ b/contrib/long_distance_matching/ldm.c
@@ -108,6 +108,12 @@ struct LDM_hashTable {
   BYTE *bucketOffsets;     // A pointer (per bucket) to the next insert position.
 };
 
+static void HASH_destroyTable(LDM_hashTable *table) {
+  free(table->entries);
+  free(table->bucketOffsets);
+  free(table);
+}
+
 /**
  * Create a hash table that can contain size elements.
  * The number of buckets is determined by size >> HASH_BUCKET_SIZE_LOG.
@@ -124,9 +130,7 @@ static LDM_hashTable *HASH_createTable(U32 size) {
   table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
 
   if (!table->entries || !table->bucketOffsets) {
-    free(table->bucketOffsets);
-    free(table->entries);
-    free(table);
+    HASH_destroyTable(table);
     return NULL;
   }
 
@@ -275,13 +279,13 @@ static LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
                                         U64 *pBackwardMatchLength) {
   LDM_hashTable *table = cctx->hashTable;
   LDM_hashEntry *bucket = getBucket(table, hash);
-  LDM_hashEntry *cur = bucket;
+  LDM_hashEntry *cur;
   LDM_hashEntry *bestEntry = NULL;
   U64 bestMatchLength = 0;
 #if !(USE_CHECKSUM)
   (void)checksum;
 #endif
-  for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
+  for (cur = bucket; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
     const BYTE *pMatch = cur->offset + cctx->ibase;
 
     // Check checksum for faster check.
@@ -336,12 +340,6 @@ static void HASH_insert(LDM_hashTable *table,
   table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
 }
 
-static void HASH_destroyTable(LDM_hashTable *table) {
-  free(table->entries);
-  free(table->bucketOffsets);
-  free(table);
-}
-
 static void HASH_outputTableOccupancy(const LDM_hashTable *table) {
   U32 ctr = 0;
   LDM_hashEntry *cur = table->entries;
@@ -360,8 +358,9 @@ static void HASH_outputTableOccupancy(const LDM_hashTable *table) {
          100.0 * (double)(ctr) / table->numEntries);
 }
 
-// TODO: This can be done more efficiently (but it is not that important as it
-// is only used for computing stats).
+// TODO: This can be done more efficiently, for example by using builtin
+// functions (but it is not that important as it is only used for computing
+// stats).
 static int intLog2(U64 x) {
   int ret = 0;
   while (x >>= 1) {
@@ -371,7 +370,6 @@ static int intLog2(U64 x) {
 }
 
 void LDM_printCompressStats(const LDM_compressStats *stats) {
-  int i = 0;
   printf("=====================\n");
   printf("Compression statistics\n");
   printf("Window size, hash table size (bytes): 2^%u, 2^%u\n",
@@ -395,16 +393,20 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
   printf("offset histogram | match length histogram\n");
   printf("offset/ML, num matches, %% of matches | num matches, %% of matches\n");
 
-  for (; i <= intLog2(stats->maxOffset); i++) {
-    printf("2^%*d: %10u    %6.3f%% |2^%*d:  %10u    %6.3f \n",
-           2, i,
-           stats->offsetHistogram[i],
-           100.0 * (double) stats->offsetHistogram[i] /
-                   (double) stats->numMatches,
-           2, i,
-           stats->matchLengthHistogram[i],
-           100.0 * (double) stats->matchLengthHistogram[i] /
-                   (double) stats->numMatches);
+  {
+    int i;
+    int logMaxOffset = intLog2(stats->maxOffset);
+    for (i = 0; i <= logMaxOffset; i++) {
+      printf("2^%*d: %10u    %6.3f%% |2^%*d:  %10u    %6.3f \n",
+             2, i,
+             stats->offsetHistogram[i],
+             100.0 * (double) stats->offsetHistogram[i] /
+                     (double) stats->numMatches,
+             2, i,
+             stats->matchLengthHistogram[i],
+             100.0 * (double) stats->matchLengthHistogram[i] /
+                     (double) stats->numMatches);
+    }
   }
   printf("\n");
   printf("=====================\n");
diff --git a/contrib/long_distance_matching/main.c b/contrib/long_distance_matching/main.c
index 72af5404..7c7086a5 100644
--- a/contrib/long_distance_matching/main.c
+++ b/contrib/long_distance_matching/main.c
@@ -12,7 +12,7 @@
 #include "ldm.h"
 #include "zstd.h"
 
-#define DECOMPRESS_AND_VERIFY
+// #define DECOMPRESS_AND_VERIFY
 
 /* Compress file given by fname and output to oname.
  * Returns 0 if successful, error code otherwise.
@@ -206,6 +206,7 @@ static int verify(const char *inpFilename, const char *decFilename) {
       printf("verify : OK\n");
     } else {
       printf("verify : NG\n");
+      return 1;
     }
   }
 
@@ -239,7 +240,7 @@ int main(int argc, const char *argv[]) {
   /* Compress */
   {
     if (compress(inpFilename, ldmFilename)) {
-        printf("Compress error");
+        printf("Compress error\n");
         return 1;
     }
   }
@@ -250,7 +251,7 @@ int main(int argc, const char *argv[]) {
     struct timeval tv1, tv2;
     gettimeofday(&tv1, NULL);
     if (decompress(ldmFilename, decFilename)) {
-        printf("Decompress error");
+        printf("Decompress error\n");
         return 1;
     }
     gettimeofday(&tv2, NULL);
@@ -259,7 +260,10 @@ int main(int argc, const char *argv[]) {
           (double) (tv2.tv_sec - tv1.tv_sec));
   }
   /* verify */
-  verify(inpFilename, decFilename);
+  if (verify(inpFilename, decFilename)) {
+    printf("Verification error\n");
+    return 1;
+  }
 #endif
   return 0;
 }