support for partial input (streaming)

dec/Makefile

@@ -4,7 +4,7 @@ include ../shared.mk
 
 CPPFLAGS += -Wall
 
-OBJS = bit_reader.o decode.o huffman.o safe_malloc.o streams.o
+OBJS = bit_reader.o decode.o huffman.o safe_malloc.o state.o streams.o
 
 all : $(OBJS)
 

dec/bit_reader.c

@@ -24,10 +24,13 @@
 extern "C" {
 #endif
 
-int BrotliInitBitReader(BrotliBitReader* const br, BrotliInput input) {
-  size_t i;
+void BrotliInitBitReader(BrotliBitReader* const br,
+                         BrotliInput input, int finish) {
   assert(br != NULL);
 
+  br->finish_ = finish;
+  br->tmp_bytes_read_ = 0;
+
   br->buf_ptr_ = br->buf_;
   br->input_ = input;
   br->val_ = 0;
@@ -35,6 +38,12 @@ int BrotliInitBitReader(BrotliBitReader* const br, BrotliInput input) {
   br->bit_pos_ = 0;
   br->bit_end_pos_ = 0;
   br->eos_ = 0;
+}
+
+
+int BrotliWarmupBitReader(BrotliBitReader* const br) {
+  size_t i;
+
   if (!BrotliReadMoreInput(br)) {
     return 0;
   }

dec/bit_reader.h

@@ -60,9 +60,23 @@ typedef struct {
   uint32_t    bit_pos_;      /* current bit-reading position in val_ */
   uint32_t    bit_end_pos_;  /* bit-reading end position from LSB of val_ */
   int         eos_;          /* input stream is finished */
+
+  /* Set to 0 to support partial data streaming. Set to 1 to expect full data or
+     for the last chunk of partial data. */
+  int         finish_;
+  /* indicates how much bytes already read when reading partial data */
+  int         tmp_bytes_read_;
 } BrotliBitReader;
 
-int BrotliInitBitReader(BrotliBitReader* const br, BrotliInput input);
+/* Initializes the bitreader fields. After this, BrotliWarmupBitReader must
+   be used. */
+void BrotliInitBitReader(BrotliBitReader* const br,
+                         BrotliInput input, int finish);
+
+/* Fetches data to fill up internal buffers. Returns 0 if there wasn't enough */
+/* data to read. It then buffers the read data and can be called again with */
+/* more data. If br->finish_ is 1, never fails. */
+int BrotliWarmupBitReader(BrotliBitReader* const br);
 
 /* Return the prefetched bits, so they can be looked up. */
 static BROTLI_INLINE uint32_t BrotliPrefetchBits(BrotliBitReader* const br) {
@@ -100,9 +114,12 @@ static BROTLI_INLINE void ShiftBytes32(BrotliBitReader* const br) {
 
    Does nothing if there are at least 32 bytes present after current position.
 
-   Returns 0 if either:
+   Returns 0 if one of:
     - the input callback returned an error, or
     - there is no more input and the position is past the end of the stream.
+    - finish is false and less than BROTLI_READ_SIZE are available - a next call
+      when more data is available makes it continue including the partially read
+      data
 
    After encountering the end of the input stream, 32 additional zero bytes are
    copied to the ringbuffer, therefore it is safe to call this function after
@@ -115,11 +132,18 @@ static BROTLI_INLINE int BrotliReadMoreInput(BrotliBitReader* const br) {
     return br->bit_pos_ <= br->bit_end_pos_;
   } else {
     uint8_t* dst = br->buf_ptr_;
-    int bytes_read = BrotliRead(br->input_, dst, BROTLI_READ_SIZE);
+    int bytes_read = BrotliRead(br->input_, dst + br->tmp_bytes_read_,
+        (size_t) (BROTLI_READ_SIZE - br->tmp_bytes_read_));
     if (bytes_read < 0) {
       return 0;
     }
+    bytes_read += br->tmp_bytes_read_;
+    br->tmp_bytes_read_ = 0;
     if (bytes_read < BROTLI_READ_SIZE) {
+      if (!br->finish_) {
+        br->tmp_bytes_read_ = bytes_read;
+        return 0;
+      }
       br->eos_ = 1;
       /* Store 32 bytes of zero after the stream end. */
 #if (BROTLI_USE_64_BITS)

dec/decode.h

@@ -18,6 +18,7 @@
 #ifndef BROTLI_DEC_DECODE_H_
 #define BROTLI_DEC_DECODE_H_
 
+#include "./state.h"
 #include "./streams.h"
 #include "./types.h"
 
@@ -30,8 +31,7 @@ extern "C" {
 /* or if it has two meta-blocks, where the first is uncompressed and the */
 /* second is empty. */
 /* Returns 1 on success, 0 on failure. */
-int BrotliDecompressedSize(size_t encoded_size,
-                           const uint8_t* encoded_buffer,
+int BrotliDecompressedSize(size_t encoded_size, const uint8_t* encoded_buffer,
                            size_t* decoded_size);
 
 /* Decompresses the data in encoded_buffer into decoded_buffer, and sets */
@@ -39,17 +39,75 @@ int BrotliDecompressedSize(size_t encoded_size,
 /* Returns 0 if there was either a bit stream error or memory allocation */
 /* error, and 1 otherwise. */
 /* If decoded size is zero, returns 1 and keeps decoded_buffer unchanged. */
-int BrotliDecompressBuffer(size_t encoded_size,
-                           const uint8_t* encoded_buffer,
-                           size_t* decoded_size,
-                           uint8_t* decoded_buffer);
+int BrotliDecompressBuffer(size_t encoded_size, const uint8_t* encoded_buffer,
+                           size_t* decoded_size, uint8_t* decoded_buffer);
 
 /* Same as above, but uses the specified input and output callbacks instead */
 /* of reading from and writing to pre-allocated memory buffers. */
 int BrotliDecompress(BrotliInput input, BrotliOutput output);
 
+/* Same as above, but supports the caller to call the decoder repeatedly with
+   partial data to support streaming. The state must be initialized with
+   BrotliStateInit and reused with every call for the same stream.
+   Return values:
+   0: failure.
+   1: success, and done.
+   2: success so far, end not reached so should call again with more input.
+   The finish parameter is used as follows, for a series of calls with the
+   same state:
+   0: Every call except the last one must be called with finish set to 0. The
+      last call may have finish set to either 0 or 1. Only if finish is 0, can
+      the function return 2. It may also return 0 or 1, in that case no more
+      calls (even with finish 1) may be made.
+   1: Only the last call may have finish set to 1. It's ok to give empty input
+      if all input was already given to previous calls. It is also ok to have
+      only one single call in total, with finish 1, and with all input
+      available immediately. That matches the non-streaming case. If finish is
+      1, the function can only return 0 or 1, never 2. After a finish, no more
+      calls may be done.
+   After everything is done, the state must be cleaned with BrotliStateCleanup
+   to free allocated resources.
+   The given BrotliOutput must always accept all output and make enough space,
+   it returning a smaller value than the amount of bytes to write always results
+   in an error.
+*/
+int BrotliDecompressStreaming(BrotliInput input, BrotliOutput output,
+                              int finish, BrotliState* s);
+
+/* Same as above, but with memory buffers.
+   Must be called with an allocated input buffer in *next_in and an allocated
+   output buffer in *next_out. The values *available_in and *available_out
+   must specify the allocated size in *next_in and *next_out respectively.
+   The value *total_out must be 0 initially, and will be summed with the
+   amount of output bytes written after each call, so that at the end it
+   gives the complete decoded size.
+   After each call, *available_in will be decremented by the amount of input
+   bytes consumed, and the *next_in pointer will be incremented by that amount.
+   Similarly, *available_out will be decremented by the amount of output
+   bytes written, and the *next_out pointer will be incremented by that
+   amount.
+
+   The input may be partial. With each next function call, *next_in and
+   *available_in must be updated to point to a next part of the compressed
+   input. The current implementation will always consume all input unless
+   an error occurs, so normally *available_in will always be 0 after
+   calling this function and the next adjacent part of input is desired.
+
+   In the current implementation, the function requires that there is enough
+   output buffer size to write all currently processed input, so
+   *available_out must be large enough. Since the function updates *next_out
+   each time, as long as the output buffer is large enough you can keep
+   reusing this variable. It is also possible to update *next_out and
+   *available_out yourself before a next call, e.g. to point to a new larger
+   buffer.
+*/
+int BrotliDecompressBufferStreaming(size_t* available_in,
+                                    const uint8_t** next_in, int finish,
+                                    size_t* available_out, uint8_t** next_out,
+                                    size_t* total_out, BrotliState* s);
+
 #if defined(__cplusplus) || defined(c_plusplus)
-}    /* extern "C" */
+} /* extern "C" */
 #endif
 
 #endif  /* BROTLI_DEC_DECODE_H_ */

dec/huffman.c

@@ -159,6 +159,24 @@ int BrotliBuildHuffmanTable(HuffmanCode* root_table,
   return total_size;
 }
 
+void BrotliHuffmanTreeGroupInit(HuffmanTreeGroup* group, int alphabet_size,
+                                int ntrees) {
+  group->alphabet_size = alphabet_size;
+  group->num_htrees = ntrees;
+  group->codes = (HuffmanCode*)malloc(
+      sizeof(HuffmanCode) * (size_t)(ntrees * BROTLI_HUFFMAN_MAX_TABLE_SIZE));
+  group->htrees = (HuffmanCode**)malloc(sizeof(HuffmanCode*) * (size_t)ntrees);
+}
+
+void BrotliHuffmanTreeGroupRelease(HuffmanTreeGroup* group) {
+  if (group->codes) {
+    free(group->codes);
+  }
+  if (group->htrees) {
+    free(group->htrees);
+  }
+}
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }    /* extern "C" */
 #endif

dec/huffman.h

@@ -25,6 +25,10 @@
 extern "C" {
 #endif
 
+/* Maximum possible Huffman table size for an alphabet size of 704, max code
+ * length 15 and root table bits 8. */
+#define BROTLI_HUFFMAN_MAX_TABLE_SIZE  1080
+
 typedef struct {
   uint8_t bits;     /* number of bits used for this symbol */
   uint16_t value;   /* symbol value or table offset */
@@ -37,6 +41,18 @@ int BrotliBuildHuffmanTable(HuffmanCode* root_table,
                             const uint8_t* const code_lengths,
                             int code_lengths_size);
 
+/* Contains a collection of huffman trees with the same alphabet size. */
+typedef struct {
+  int alphabet_size;
+  int num_htrees;
+  HuffmanCode* codes;
+  HuffmanCode** htrees;
+} HuffmanTreeGroup;
+
+void BrotliHuffmanTreeGroupInit(HuffmanTreeGroup* group,
+                                int alphabet_size, int ntrees);
+void BrotliHuffmanTreeGroupRelease(HuffmanTreeGroup* group);
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }    /* extern "C" */
 #endif

dec/state.c

@@ -0,0 +1,87 @@
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "./state.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+void BrotliStateInit(BrotliState* s) {
+  int i;
+
+  s->state = BROTLI_STATE_UNINITED;
+  s->sub_state[0] = BROTLI_STATE_SUB_NONE;
+  s->sub_state[1] = BROTLI_STATE_SUB_NONE;
+
+  s->block_type_trees = NULL;
+  s->block_len_trees = NULL;
+  s->ringbuffer = NULL;
+
+  s->context_map = NULL;
+  s->context_modes = NULL;
+  s->dist_context_map = NULL;
+  s->context_map_slice = NULL;
+  s->dist_context_map_slice = NULL;
+
+  for (i = 0; i < 3; ++i) {
+    s->hgroup[i].codes = NULL;
+    s->hgroup[i].htrees = NULL;
+  }
+
+  s->code_lengths = NULL;
+  s->context_map_table = NULL;
+}
+
+void BrotliStateCleanup(BrotliState* s) {
+  int i;
+
+  if (s->context_map_table != 0) {
+    free(s->context_map_table);
+  }
+  if (s->code_lengths != 0) {
+    free(s->code_lengths);
+  }
+
+  if (s->context_modes != 0) {
+    free(s->context_modes);
+  }
+  if (s->context_map != 0) {
+    free(s->context_map);
+  }
+  if (s->dist_context_map != 0) {
+    free(s->dist_context_map);
+  }
+  for (i = 0; i < 3; ++i) {
+    BrotliHuffmanTreeGroupRelease(&s->hgroup[i]);
+  }
+
+  if (s->ringbuffer != 0) {
+    free(s->ringbuffer);
+  }
+  if (s->block_type_trees != 0) {
+    free(s->block_type_trees);
+  }
+  if (s->block_len_trees != 0) {
+    free(s->block_len_trees);
+  }
+}
+
+#if defined(__cplusplus) || defined(c_plusplus)
+} /* extern "C" */
+#endif

dec/state.h

@@ -0,0 +1,163 @@
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/* Brotli state for partial streaming decoding. */
+
+#ifndef BROTLI_DEC_STATE_H_
+#define BROTLI_DEC_STATE_H_
+
+#include <stdio.h>
+#include "./bit_reader.h"
+#include "./huffman.h"
+#include "./streams.h"
+#include "./types.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+typedef enum {
+  BROTLI_STATE_UNINITED = 0,
+  BROTLI_STATE_BITREADER_WARMUP = 1,
+  BROTLI_STATE_METABLOCK_BEGIN = 10,
+  BROTLI_STATE_METABLOCK_HEADER_1 = 11,
+  BROTLI_STATE_METABLOCK_HEADER_2 = 12,
+  BROTLI_STATE_BLOCK_BEGIN = 13,
+  BROTLI_STATE_BLOCK_INNER = 14,
+  BROTLI_STATE_BLOCK_DISTANCE = 15,
+  BROTLI_STATE_BLOCK_POST = 16,
+  BROTLI_STATE_UNCOMPRESSED = 17,
+  BROTLI_STATE_METABLOCK_DONE = 20,
+  BROTLI_STATE_HUFFMAN_CODE_0 = 30,
+  BROTLI_STATE_HUFFMAN_CODE_1 = 31,
+  BROTLI_STATE_HUFFMAN_CODE_2 = 32,
+  BROTLI_STATE_CONTEXT_MAP_1 = 33,
+  BROTLI_STATE_CONTEXT_MAP_2 = 34,
+  BROTLI_STATE_TREE_GROUP = 35,
+  BROTLI_STATE_SUB_NONE = 50,
+  BROTLI_STATE_SUB_UNCOMPRESSED_SHORT = 51,
+  BROTLI_STATE_SUB_UNCOMPRESSED_FILL = 52,
+  BROTLI_STATE_SUB_UNCOMPRESSED_COPY = 53,
+  BROTLI_STATE_SUB_UNCOMPRESSED_WARMUP = 54,
+  BROTLI_STATE_SUB_HUFFMAN_LENGTH_BEGIN = 60,
+  BROTLI_STATE_SUB_HUFFMAN_LENGTH_SYMBOLS = 61,
+  BROTLI_STATE_SUB_HUFFMAN_DONE = 62,
+  BROTLI_STATE_SUB_TREE_GROUP = 70,
+  BROTLI_STATE_SUB_CONTEXT_MAP_HUFFMAN = 80,
+  BROTLI_STATE_SUB_CONTEXT_MAPS = 81,
+  BROTLI_STATE_DONE = 100
+} BrotliRunningState;
+
+typedef struct {
+  BrotliRunningState state;
+  BrotliRunningState sub_state[2];  /* State inside function call */
+
+  int pos;
+  int input_end;
+  int window_bits;
+  int max_backward_distance;
+  int max_distance;
+  int ringbuffer_size;
+  int ringbuffer_mask;
+  uint8_t* ringbuffer;
+  uint8_t* ringbuffer_end;
+  /* This ring buffer holds a few past copy distances that will be used by */
+  /* some special distance codes. */
+  int dist_rb[4];
+  int dist_rb_idx;
+  /* The previous 2 bytes used for context. */
+  uint8_t prev_byte1;
+  uint8_t prev_byte2;
+  HuffmanTreeGroup hgroup[3];
+  HuffmanCode* block_type_trees;
+  HuffmanCode* block_len_trees;
+  BrotliBitReader br;
+  /* This counter is reused for several disjoint loops. */
+  int loop_counter;
+
+  int meta_block_remaining_len;
+  int is_uncompressed;
+  int block_length[3];
+  int block_type[3];
+  int num_block_types[3];
+  int block_type_rb[6];
+  int block_type_rb_index[3];
+  int distance_postfix_bits;
+  int num_direct_distance_codes;
+  int distance_postfix_mask;
+  int num_distance_codes;
+  uint8_t* context_map;
+  uint8_t* context_modes;
+  int num_literal_htrees;
+  uint8_t* dist_context_map;
+  int num_dist_htrees;
+  int context_offset;
+  uint8_t* context_map_slice;
+  uint8_t literal_htree_index;
+  int dist_context_offset;
+  uint8_t* dist_context_map_slice;
+  uint8_t dist_htree_index;
+  int context_lookup_offset1;
+  int context_lookup_offset2;
+  uint8_t context_mode;
+  HuffmanCode* htree_command;
+
+  int cmd_code;
+  int range_idx;
+  int insert_code;
+  int copy_code;
+  int insert_length;
+  int copy_length;
+  int distance_code;
+  int distance;
+  const uint8_t* copy_src;
+  uint8_t* copy_dst;
+
+  /* For CopyUncompressedBlockToOutput */
+  int nbytes;
+
+  /* For HuffmanTreeGroupDecode */
+  int htrees_decoded;
+
+  /* For ReadHuffmanCodeLengths */
+  int symbol;
+  uint8_t prev_code_len;
+  int repeat;
+  uint8_t repeat_code_len;
+  int space;
+  HuffmanCode table[32];
+  uint8_t code_length_code_lengths[18];
+
+  /* For ReadHuffmanCode */
+  int simple_code_or_skip;
+  uint8_t* code_lengths;
+
+  /* For HuffmanTreeGroupDecode */
+  int htree_index;
+  HuffmanCode* next;
+
+  /* For DecodeContextMap */
+  int context_index;
+  HuffmanCode* context_map_table;
+} BrotliState;
+
+void BrotliStateInit(BrotliState* s);
+void BrotliStateCleanup(BrotliState* s);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+} /* extern "C" */
+#endif
+
+#endif  /* BROTLI_DEC_STATE_H_ */