Merge pull request #434 from eustas/master

Update research

Makefile

@@ -22,10 +22,6 @@ all: test
 $(DIRS):
 	mkdir -p $@
 
-$(OBJECTS): $(DIRS)
-	$(CC) $(CFLAGS) $(CPPFLAGS) -Iinclude \
-        -c $(patsubst %.o,%.c,$(patsubst $(OBJDIR)/%,%,$@)) -o $@
-
 $(EXECUTABLE): $(OBJECTS)
 	$(CC) $(OBJECTS) -lm -o $(BINDIR)/$(EXECUTABLE)
 
@@ -39,3 +35,8 @@ test: $(EXECUTABLE)
 
 clean:
 	rm -rf $(BINDIR) $(LIB_A)
+
+.SECONDEXPANSION:
+$(OBJECTS): $$(patsubst %.o,%.c,$$(patsubst $$(OBJDIR)/%,%,$$@)) | $(DIRS)
+	$(CC) $(CFLAGS) $(CPPFLAGS) -Iinclude \
+        -c $(patsubst %.o,%.c,$(patsubst $(OBJDIR)/%,%,$@)) -o $@

research/Makefile

@@ -11,7 +11,7 @@ $(BINDIR):
 	mkdir -p $@
 
 $(EXECUTABLES): $(BINDIR)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(addsuffix .cc, $@) -o $(BINDIR)/$@
+	$(CC) $(CFLAGS) $(CPPFLAGS) $(addsuffix .cc, $@) -o $(BINDIR)/$@ -lgflags_nothreads
 
 clean:
 	rm -rf $(BINDIR)

research/draw_diff.cc

@@ -1,5 +1,5 @@
 /* Copyright 2016 Google Inc. All Rights Reserved.
-   Author: vanickulin@google.com (Ivan Nikulin)
+   Author: zip753@gmail.com (Ivan Nikulin)
 
    Distributed under MIT license.
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
@@ -13,16 +13,21 @@
 #include <cmath>
 #include <cstdint>
 #include <cstdio>
+#include <cstdlib>  /* exit, EXIT_FAILURE */
 #include <vector>
 
+#ifndef CHECK
+#define CHECK(X) if (!(X)) exit(EXIT_FAILURE);
+#endif
+
 void ReadPGM(FILE* f, uint8_t*** image, size_t* height, size_t* width) {
   int colors;
-  assert(fscanf(f, "P5\n%lu %lu\n%d\n", width, height, &colors) == 3);
+  CHECK(fscanf(f, "P5\n%lu %lu\n%d\n", width, height, &colors) == 3);
   assert(colors == 255);
   *image = new uint8_t*[*height];
   for (int i = *height - 1; i >= 0; --i) {
     (*image)[i] = new uint8_t[*width];
-    assert(fread((*image)[i], 1, *width, f) == *width);
+    CHECK(fread((*image)[i], 1, *width, f) == *width);
   }
 }
 

research/draw_histogram.cc

@@ -1,5 +1,5 @@
 /* Copyright 2016 Google Inc. All Rights Reserved.
-   Author: vanickulin@google.com (Ivan Nikulin)
+   Author: zip753@gmail.com (Ivan Nikulin)
 
    Distributed under MIT license.
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
@@ -15,18 +15,21 @@
 #include <cstdio> /* fscanf, fprintf */
 #include <cstdint>
 
+#include <gflags/gflags.h>
+using gflags::ParseCommandLineFlags;
+
 #include "./read_dist.h"
 
-const int FLAGS_height = 1000;  // Height of the resulting histogam.
-const int FLAGS_width = 1000;  // Width of the resulting histogam.
-int FLAGS_size;  // Size of the compressed file.
-const int FLAGS_brotli_window = 0;  // Size of brotli window in bits.
-const uint64_t FLAGS_min_distance = 0;  // Minimum distance.
-uint64_t FLAGS_max_distance = 0;  // Maximum distance.
-const bool FLAGS_with_copies = false;  // True if input contains copy length.
-const bool FLAGS_simple = false;  // True if using only black and white pixels.
-const bool FLAGS_linear = true;  // True if using linear distance mapping.
-const uint64_t FLAGS_skip = 0;  // Number of bytes to skip.
+DEFINE_int32(height, 1000, "Height of the resulting histogam.");
+DEFINE_int32(width, 8000, "Width of the resulting histogam.");
+DEFINE_int32(size, 1e8, "Size of the compressed file.");
+DEFINE_int32(brotli_window, -1, "Size of brotli window in bits.");
+DEFINE_uint64(min_distance, 0, "Minimum distance.");
+DEFINE_uint64(max_distance, 1 << 30, "Maximum distance.");
+DEFINE_bool(with_copies, false, "True if input contains copy length.");
+DEFINE_bool(simple, false, "True if using only black and white pixels.");
+DEFINE_bool(linear, false, "True if using linear distance mapping.");
+DEFINE_uint64(skip, 0, "Number of bytes to skip.");
 
 inline double DistanceTransform(double x) {
   static bool linear = FLAGS_linear;
@@ -83,7 +86,7 @@ void BuildHistogram(FILE* fin, int** histo) {
   while (ReadBackwardReference(fin, &copy, &pos, &distance)) {
     if (pos == -1) continue;  // In case when only insert is present.
     if (distance < min_distance || distance >= GetMaxDistance()) continue;
-    if (FLAGS_brotli_window != 0) {
+    if (FLAGS_brotli_window != -1) {
       AdjustPosition(&pos);
     }
     if (pos >= skip && distance <= pos) {
@@ -163,8 +166,9 @@ void DrawPixels(uint8_t** pixel, FILE* fout) {
 }
 
 int main(int argc, char* argv[]) {
-  if (argc != 4) {
-    printf("usage: draw_histogram.cc dist_file input_size output_file\n");
+  ParseCommandLineFlags(&argc, &argv, true);
+  if (argc != 3) {
+    printf("usage: draw_histogram.cc data output_file\n");
     return 1;
   }
 
@@ -172,11 +176,7 @@ int main(int argc, char* argv[]) {
   int width = FLAGS_width;
 
   FILE* fin = fopen(argv[1], "r");
-  FILE* fout = fopen(argv[3], "wb");
-
-  FLAGS_size = atoi(argv[2]);
-
-  if (FLAGS_max_distance == 0) FLAGS_max_distance = FLAGS_size;
+  FILE* fout = fopen(argv[2], "wb");
 
   uint8_t** pixel = new uint8_t*[height];
   int** histo = new int*[height];

research/find_opt_references.cc

@@ -1,5 +1,5 @@
 /* Copyright 2016 Google Inc. All Rights Reserved.
-   Author: vanickulin@google.com (Ivan Nikulin)
+   Author: zip753@gmail.com (Ivan Nikulin)
 
    Distributed under MIT license.
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
@@ -13,10 +13,13 @@
 #include <cstdio>
 #include <cstring>
 
+#include <gflags/gflags.h>
+using gflags::ParseCommandLineFlags;
+
 #include "./esaxx/sais.hxx"
 
-const int FLAGS_min_length = 1;  // Minimal length of found backward references.
-const int FLAGS_skip = 1;  // Number of bytes to skip.
+DEFINE_int32(min_length, 1, "Minimal length of found backward references.");
+DEFINE_int32(skip, 1, "Number of bytes to skip.");
 
 const size_t kFileBufferSize = (1 << 16);  // 64KB
 
@@ -112,6 +115,7 @@ void ProcessReferences(input_type* storage, sarray_type* sarray, lcp_type* lcp,
 }
 
 int main(int argc, char* argv[]) {
+  ParseCommandLineFlags(&argc, &argv, true);
   if (argc != 3) {
     printf("usage: %s input_file output_file\n", argv[0]);
     return 1;

research/read_dist.h

@@ -1,37 +1,47 @@
 /* Copyright 2016 Google Inc. All Rights Reserved.
-   Author: vanickulin@google.com (Ivan Nikulin)
+   Author: zip753@gmail.com (Ivan Nikulin)
 
    Distributed under MIT license.
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-/* API for reading distances from *.dist file. */
-
-#include <cassert>
-#include <cstdio>
+/* API for reading distances from *.dist file.
+   The format of *.dist file is as follows: for each backward reference there is
+   a position-distance pair, also a copy length may be specified. Copy length is
+   prefixed with flag byte 0, position-distance pair is prefixed with flag
+   byte 1. Each number is a 32-bit integer. Copy length always comes before
+   position-distance pair. Standalone copy length is allowed, in this case it is
+   ignored. */
 
 #ifndef BROTLI_RESEARCH_READ_DIST_H_
 #define BROTLI_RESEARCH_READ_DIST_H_
 
+#include <cstdio>
+#include <cstdlib>  /* exit, EXIT_FAILURE */
+
+#ifndef CHECK
+#define CHECK(X) if (!(X)) exit(EXIT_FAILURE);
+#endif
+
 /* Reads backwards reference from .dist file. Sets all missing fields to -1.
    Returns false when EOF is met or input is corrupt. */
 bool ReadBackwardReference(FILE* fin, int* copy, int* pos, int* dist) {
   int c = getc(fin);
   if (c == EOF) return false;
   if (c == 0) {
-    assert(fread(copy, sizeof(int), 1, fin) == 1);
+    CHECK(fread(copy, sizeof(int), 1, fin) == 1);
     if ((c = getc(fin)) != 1) {
       ungetc(c, fin);
       *pos = *dist = -1;
     } else {
-      assert(fread(pos, sizeof(int), 1, fin) == 1);
-      assert(fread(dist, sizeof(int), 1, fin) == 1);
+      CHECK(fread(pos, sizeof(int), 1, fin) == 1);
+      CHECK(fread(dist, sizeof(int), 1, fin) == 1);
     }
   } else if (c != 1) {
     return false;
   } else {
-    assert(fread(pos, sizeof(int), 1, fin) == 1);
-    assert(fread(dist, sizeof(int), 1, fin) == 1);
+    CHECK(fread(pos, sizeof(int), 1, fin) == 1);
+    CHECK(fread(dist, sizeof(int), 1, fin) == 1);
     *copy = -1;
   }
   return true;

tools/version.h

@@ -1,14 +0,0 @@
-/* Copyright 2015 Google Inc. All Rights Reserved.
-
-   Distributed under MIT license.
-   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
-*/
-
-/* Defines a common version string used by all of the brotli tools. */
-
-#ifndef BROTLI_TOOLS_VERSION_H_
-#define BROTLI_TOOLS_VERSION_H_
-
-#define BROTLI_VERSION "0.5.0"
-
-#endif  /* BROTLI_TOOLS_VERSION_H_ */