mirror of
https://github.com/google/brotli.git
synced 2024-11-09 13:40:06 +00:00
Add go wrapper, streamline java decoder: (#524)
* add (c)brotli golang wrapper * remove (language-specific) enums in java decoder
This commit is contained in:
parent
8a06e02935
commit
a657d9969d
4
BUILD
4
BUILD
@ -100,3 +100,7 @@ cc_binary(
|
||||
":brotlienc",
|
||||
],
|
||||
)
|
||||
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_prefix")
|
||||
|
||||
go_prefix("github.com/google/brotli")
|
||||
|
@ -7,3 +7,12 @@ maven_jar(
|
||||
name = "junit_junit",
|
||||
artifact = "junit:junit:4.12",
|
||||
)
|
||||
|
||||
git_repository(
|
||||
name = "io_bazel_rules_go",
|
||||
remote = "https://github.com/bazelbuild/rules_go.git",
|
||||
tag = "0.4.1",
|
||||
)
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
|
||||
|
||||
go_repositories()
|
||||
|
23
go/cbrotli/BUILD
Executable file
23
go/cbrotli/BUILD
Executable file
@ -0,0 +1,23 @@
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
licenses(["notice"]) # MIT
|
||||
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_prefix", "go_library", "go_test")
|
||||
|
||||
go_prefix("github.com/google/brotli")
|
||||
|
||||
go_library(
|
||||
name = "cbrotli",
|
||||
srcs = ["cbrotli.go"],
|
||||
deps = [
|
||||
"//go/cbrotli/internal:decoder",
|
||||
"//go/cbrotli/internal:encoder",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "cbrotli_test",
|
||||
size = "small",
|
||||
srcs = ["cbrotli_test.go"],
|
||||
library = ":cbrotli",
|
||||
)
|
254
go/cbrotli/cbrotli.go
Executable file
254
go/cbrotli/cbrotli.go
Executable file
@ -0,0 +1,254 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Distributed under MIT license.
|
||||
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
|
||||
// Package cbrotli compresses and decompresses data with C-Brotli library.
|
||||
package cbrotli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
"github.com/google/brotli/go/cbrotli/internal/decoder"
|
||||
"github.com/google/brotli/go/cbrotli/internal/encoder"
|
||||
)
|
||||
|
||||
// An internalError reports an error in the (c-)brotli code itself.
|
||||
type internalError string
|
||||
|
||||
func (e internalError) Error() string {
|
||||
return "cbrotli: internal error: " + string(e)
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Encoder
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
// WriterOptions configures Writer.
|
||||
type WriterOptions struct {
|
||||
// Quality controls the compression-speed vs compression-density trade-offs.
|
||||
// The higher the quality, the slower the compression. Range is 0 to 11.
|
||||
Quality int
|
||||
// LGWin is the base 2 logarithm of the sliding window size.
|
||||
// Range is 10 to 24. 0 indicates automatic configuration based on Quality.
|
||||
LGWin int
|
||||
}
|
||||
|
||||
// Writer implements io.WriteCloser, an io.Writer decorator that produces
|
||||
// Brotli-encoded data.
|
||||
type Writer struct {
|
||||
dst io.Writer
|
||||
encoder encoder.Encoder
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewWriter initializes new Writer instance.
|
||||
// Close MUST be called to free resources.
|
||||
func NewWriter(dst io.Writer, options WriterOptions) *Writer {
|
||||
return &Writer{
|
||||
dst: dst,
|
||||
encoder: encoder.New(options.Quality, options.LGWin),
|
||||
}
|
||||
}
|
||||
|
||||
// Close implements io.Closer. Close MUST be invoked to free native resources.
|
||||
// Also Close implicitly flushes remaining data to the decorated writer.
|
||||
func (z *Writer) Close() error {
|
||||
if z.closed {
|
||||
return nil
|
||||
}
|
||||
defer z.encoder.Close()
|
||||
_, err := z.writeChunk(nil, encoder.Finish)
|
||||
z.closed = true
|
||||
return err
|
||||
}
|
||||
|
||||
func (z *Writer) writeChunk(p []byte, op encoder.Operation) (int, error) {
|
||||
if z.closed {
|
||||
return 0, internalError("write after close")
|
||||
}
|
||||
var totalBytesConsumed int
|
||||
var err error
|
||||
for {
|
||||
bytesConsumed, output, status := z.encoder.CompressStream(p, op)
|
||||
if status == encoder.Error {
|
||||
err = internalError("encoder failure")
|
||||
break
|
||||
}
|
||||
p = p[bytesConsumed:]
|
||||
totalBytesConsumed += bytesConsumed
|
||||
_, err = z.dst.Write(output)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if len(p) == 0 && status == encoder.Done {
|
||||
break
|
||||
}
|
||||
}
|
||||
return totalBytesConsumed, err
|
||||
}
|
||||
|
||||
// Write implements io.Writer.
|
||||
func (z *Writer) Write(p []byte) (int, error) {
|
||||
return z.writeChunk(p, encoder.Process)
|
||||
}
|
||||
|
||||
// Flush outputs encoded data for all input provided to Write. The resulting
|
||||
// output can be decoded to match all input before Flush, but the stream is
|
||||
// not yet complete until after Close.
|
||||
// Flush has a negative impact on compression.
|
||||
func (z *Writer) Flush() error {
|
||||
_, err := z.writeChunk(nil, encoder.Finish)
|
||||
return err
|
||||
}
|
||||
|
||||
// Encode returns content encoded with Brotli.
|
||||
func Encode(content []byte, options WriterOptions) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
writer := NewWriter(&buf, options)
|
||||
defer writer.Close()
|
||||
_, err := writer.Write(content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Decoder
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
// Reader implements io.ReadCloser, an io.Reader decorator that decodes
|
||||
// Brotli-encoded data.
|
||||
type Reader struct {
|
||||
src io.Reader
|
||||
decoder decoder.Decoder
|
||||
buf []byte // intermediate read buffer pointed to by next
|
||||
eof bool // true if all compressed stream is decoded
|
||||
next []byte // buffered data to be passed to decoder
|
||||
output []byte // data produced by decoder, but not yet consumed
|
||||
srcErr error // last source reader error
|
||||
err error // reader state; nil if it is OK to read further
|
||||
closed bool // true is stream is already closed
|
||||
}
|
||||
|
||||
// NewReader initializes new Reader instance.
|
||||
// Close MUST be called to free resources.
|
||||
func NewReader(src io.Reader) *Reader {
|
||||
return &Reader{
|
||||
src: src,
|
||||
decoder: decoder.New(),
|
||||
buf: make([]byte, 32*1024),
|
||||
eof: false,
|
||||
}
|
||||
}
|
||||
|
||||
// Close implements io.Closer. Close MUST be invoked to free native resources.
|
||||
func (z *Reader) Close() error {
|
||||
if z.closed {
|
||||
return nil
|
||||
}
|
||||
z.decoder.Close()
|
||||
z.err = internalError("read after close")
|
||||
z.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func isEOF(src io.Reader) bool {
|
||||
n, err := src.Read(make([]byte, 1))
|
||||
return n == 0 && err == io.EOF
|
||||
}
|
||||
|
||||
// Read implements io.Reader.
|
||||
func (z *Reader) Read(p []byte) (int, error) {
|
||||
// Any error state is unrecoverable.
|
||||
if z.err != nil {
|
||||
return 0, z.err
|
||||
}
|
||||
// See io.Reader documentation.
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var totalOutBytes int
|
||||
|
||||
// There is no practical limit for amount of bytes being consumed by decoder
|
||||
// before producing any output. Continue feeding decoder until some data is
|
||||
// produced
|
||||
for {
|
||||
// Push already produced output first.
|
||||
if outBytes := len(z.output); outBytes != 0 {
|
||||
outBytes = copy(p, z.output)
|
||||
p = p[outBytes:]
|
||||
z.output = z.output[outBytes:]
|
||||
totalOutBytes += outBytes
|
||||
// Output buffer is full.
|
||||
if len(p) == 0 {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
// No more produced output left.
|
||||
// If no more output is expected, then we are finished.
|
||||
if z.eof {
|
||||
z.err = io.EOF
|
||||
break
|
||||
}
|
||||
// Replenish buffer (might cause blocking read), only if necessary.
|
||||
if len(z.next) == 0 && totalOutBytes == 0 && z.srcErr != io.EOF {
|
||||
var n int
|
||||
n, z.srcErr = z.src.Read(z.buf)
|
||||
z.next = z.buf[:n]
|
||||
if z.srcErr != nil && z.srcErr != io.EOF {
|
||||
z.err = z.srcErr
|
||||
break
|
||||
}
|
||||
}
|
||||
// Do decoding.
|
||||
consumed, output, status := z.decoder.DecompressStream(z.next)
|
||||
z.output = output
|
||||
z.next = z.next[consumed:]
|
||||
if status == decoder.Error {
|
||||
// When error happens, the remaining output does not matter.
|
||||
z.err = internalError("decoder failure")
|
||||
break
|
||||
} else if status == decoder.Done {
|
||||
// Decoder stream is closed; no further input is expected.
|
||||
if len(z.next) != 0 || (z.srcErr != io.EOF && !isEOF(z.src)) {
|
||||
z.err = internalError("excessive input")
|
||||
break
|
||||
}
|
||||
// No more output is expected; keep pushing output.
|
||||
z.eof = true
|
||||
continue
|
||||
} else {
|
||||
// If can not move any further...
|
||||
if consumed == 0 && len(z.output) == 0 {
|
||||
// Unexpected end of input.
|
||||
if z.srcErr == io.EOF || totalOutBytes == 0 {
|
||||
z.err = io.ErrUnexpectedEOF
|
||||
}
|
||||
// Postpone blocking reads for the next invocation.
|
||||
break
|
||||
}
|
||||
// Continue pushing output.
|
||||
}
|
||||
}
|
||||
return totalOutBytes, z.err
|
||||
}
|
||||
|
||||
// Decode decodes Brotli encoded data.
|
||||
func Decode(encodedData []byte) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
reader := NewReader(bytes.NewReader(encodedData))
|
||||
defer reader.Close()
|
||||
_, err := io.Copy(&buf, reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
271
go/cbrotli/cbrotli_test.go
Executable file
271
go/cbrotli/cbrotli_test.go
Executable file
@ -0,0 +1,271 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Distributed under MIT license.
|
||||
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
|
||||
package cbrotli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func checkCompressedData(compressedData, wantOriginalData []byte) error {
|
||||
uncompressed, err := Decode(compressedData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("brotli decompress failed: %v", err)
|
||||
}
|
||||
if !bytes.Equal(uncompressed, wantOriginalData) {
|
||||
if len(wantOriginalData) != len(uncompressed) {
|
||||
return fmt.Errorf(""+
|
||||
"Data doesn't uncompress to the original value.\n"+
|
||||
"Length of original: %v\n"+
|
||||
"Length of uncompressed: %v",
|
||||
len(wantOriginalData), len(uncompressed))
|
||||
}
|
||||
for i := range wantOriginalData {
|
||||
if wantOriginalData[i] != uncompressed[i] {
|
||||
return fmt.Errorf(""+
|
||||
"Data doesn't uncompress to the original value.\n"+
|
||||
"Original at %v is %v\n"+
|
||||
"Uncompressed at %v is %v",
|
||||
i, wantOriginalData[i], i, uncompressed[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestEncoderNoWrite(t *testing.T) {
|
||||
out := bytes.Buffer{}
|
||||
e := NewWriter(&out, WriterOptions{Quality: 5})
|
||||
if err := e.Close(); err != nil {
|
||||
t.Errorf("Close()=%v, want nil", err)
|
||||
}
|
||||
// Check Write after close.
|
||||
if _, err := e.Write([]byte("hi")); err == nil {
|
||||
t.Errorf("No error after Close() + Write()")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncoderEmptyWrite(t *testing.T) {
|
||||
out := bytes.Buffer{}
|
||||
e := NewWriter(&out, WriterOptions{Quality: 5})
|
||||
n, err := e.Write([]byte(""))
|
||||
if n != 0 || err != nil {
|
||||
t.Errorf("Write()=%v,%v, want 0, nil", n, err)
|
||||
}
|
||||
if err := e.Close(); err != nil {
|
||||
t.Errorf("Close()=%v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriter(t *testing.T) {
|
||||
// Test basic encoder usage.
|
||||
input := []byte("<html><body><H1>Hello world</H1></body></html>")
|
||||
out := bytes.Buffer{}
|
||||
e := NewWriter(&out, WriterOptions{Quality: 1})
|
||||
in := bytes.NewReader([]byte(input))
|
||||
n, err := io.Copy(e, in)
|
||||
if err != nil {
|
||||
t.Errorf("Copy Error: %v", err)
|
||||
}
|
||||
if int(n) != len(input) {
|
||||
t.Errorf("Copy() n=%v, want %v", n, len(input))
|
||||
}
|
||||
if err := e.Close(); err != nil {
|
||||
t.Errorf("Close Error after copied %d bytes: %v", n, err)
|
||||
}
|
||||
if err := checkCompressedData(out.Bytes(), input); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncoderStreams(t *testing.T) {
|
||||
// Test that output is streamed.
|
||||
// Adjust window size to ensure the encoder outputs at least enough bytes
|
||||
// to fill the window.
|
||||
const lgWin = 16
|
||||
windowSize := int(math.Pow(2, lgWin))
|
||||
input := make([]byte, 2*windowSize)
|
||||
rand.Read(input)
|
||||
out := bytes.Buffer{}
|
||||
e := NewWriter(&out, WriterOptions{Quality: 11, LGWin: lgWin})
|
||||
halfInput := input[:len(input)/2]
|
||||
in := bytes.NewReader(halfInput)
|
||||
|
||||
n, err := io.Copy(e, in)
|
||||
if err != nil {
|
||||
t.Errorf("Copy Error: %v", err)
|
||||
}
|
||||
|
||||
// We've fed more data than the sliding window size. Check that some
|
||||
// compressed data has been output.
|
||||
if out.Len() == 0 {
|
||||
t.Errorf("Output length is after %d bytes written", n)
|
||||
}
|
||||
if err := e.Close(); err != nil {
|
||||
t.Errorf("Close Error after copied %d bytes: %v", n, err)
|
||||
}
|
||||
if err := checkCompressedData(out.Bytes(), halfInput); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncoderLargeInput(t *testing.T) {
|
||||
input := make([]byte, 1000000)
|
||||
rand.Read(input)
|
||||
out := bytes.Buffer{}
|
||||
e := NewWriter(&out, WriterOptions{Quality: 5})
|
||||
in := bytes.NewReader(input)
|
||||
|
||||
n, err := io.Copy(e, in)
|
||||
if err != nil {
|
||||
t.Errorf("Copy Error: %v", err)
|
||||
}
|
||||
if int(n) != len(input) {
|
||||
t.Errorf("Copy() n=%v, want %v", n, len(input))
|
||||
}
|
||||
if err := e.Close(); err != nil {
|
||||
t.Errorf("Close Error after copied %d bytes: %v", n, err)
|
||||
}
|
||||
if err := checkCompressedData(out.Bytes(), input); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncoderFlush(t *testing.T) {
|
||||
input := make([]byte, 1000)
|
||||
rand.Read(input)
|
||||
out := bytes.Buffer{}
|
||||
e := NewWriter(&out, WriterOptions{Quality: 5})
|
||||
in := bytes.NewReader(input)
|
||||
_, err := io.Copy(e, in)
|
||||
if err != nil {
|
||||
t.Fatalf("Copy Error: %v", err)
|
||||
}
|
||||
if err := e.Flush(); err != nil {
|
||||
t.Fatalf("Flush(): %v", err)
|
||||
}
|
||||
decompressed := make([]byte, 1000)
|
||||
reader := NewReader(bytes.NewReader(out.Bytes()))
|
||||
n, err := reader.Read(decompressed)
|
||||
if n != len(decompressed) || err != nil {
|
||||
t.Errorf("Expected <%v, nil>, but <%v, %v>", len(decompressed), n, err)
|
||||
}
|
||||
reader.Close()
|
||||
if !bytes.Equal(decompressed, input) {
|
||||
t.Errorf(""+
|
||||
"Decompress after flush: %v\n"+
|
||||
"%q\n"+
|
||||
"want:\n%q",
|
||||
err, decompressed, input)
|
||||
}
|
||||
if err := e.Close(); err != nil {
|
||||
t.Errorf("Close(): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("hello world!"), 10000)
|
||||
encoded, _ := Encode(content, WriterOptions{Quality: 5})
|
||||
r := NewReader(bytes.NewReader(encoded))
|
||||
var decodedOutput bytes.Buffer
|
||||
n, err := io.Copy(&decodedOutput, r)
|
||||
if err != nil {
|
||||
t.Fatalf("Copy(): n=%v, err=%v", n, err)
|
||||
}
|
||||
if err := r.Close(); err != nil {
|
||||
t.Errorf("Close(): %v", err)
|
||||
}
|
||||
if got := decodedOutput.Bytes(); !bytes.Equal(got, content) {
|
||||
t.Errorf(""+
|
||||
"Reader output:\n"+
|
||||
"%q\n"+
|
||||
"want:\n"+
|
||||
"%q",
|
||||
got, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecode(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("hello world!"), 10000)
|
||||
encoded, _ := Encode(content, WriterOptions{Quality: 5})
|
||||
decoded, err := Decode(encoded)
|
||||
if err != nil {
|
||||
t.Errorf("Decode: %v", err)
|
||||
}
|
||||
if !bytes.Equal(decoded, content) {
|
||||
t.Errorf(""+
|
||||
"Decode content:\n"+
|
||||
"%q\n"+
|
||||
"want:\n"+
|
||||
"%q",
|
||||
decoded, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeFuzz(t *testing.T) {
|
||||
// Test that the decoder terminates with corrupted input.
|
||||
content := bytes.Repeat([]byte("hello world!"), 100)
|
||||
src := rand.NewSource(0)
|
||||
encoded, _ := Encode(content, WriterOptions{Quality: 5})
|
||||
for i := 0; i < 100; i++ {
|
||||
enc := append([]byte{}, encoded...)
|
||||
for j := 0; j < 5; j++ {
|
||||
enc[int(src.Int63())%len(enc)] = byte(src.Int63() % 256)
|
||||
}
|
||||
Decode(enc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeTrailingData(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("hello world!"), 100)
|
||||
encoded, _ := Encode(content, WriterOptions{Quality: 5})
|
||||
_, err := Decode(append(encoded, 0))
|
||||
if err == nil {
|
||||
t.Errorf("Expected 'excessive input' error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecode(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
data []byte
|
||||
repeats int
|
||||
}{
|
||||
{nil, 0},
|
||||
{[]byte("A"), 1},
|
||||
{[]byte("<html><body><H1>Hello world</H1></body></html>"), 10},
|
||||
{[]byte("<html><body><H1>Hello world</H1></body></html>"), 1000},
|
||||
} {
|
||||
t.Logf("case %q x %d", test.data, test.repeats)
|
||||
input := bytes.Repeat(test.data, test.repeats)
|
||||
encoded, err := Encode(input, WriterOptions{Quality: 5})
|
||||
if err != nil {
|
||||
t.Errorf("Encode: %v", err)
|
||||
}
|
||||
// Inputs are compressible, but may be too small to compress.
|
||||
if maxSize := len(input)/2 + 20; len(encoded) >= maxSize {
|
||||
t.Errorf(""+
|
||||
"Encode returned %d bytes, want <%d\n"+
|
||||
"Encoded=%q",
|
||||
len(encoded), maxSize, encoded)
|
||||
}
|
||||
decoded, err := Decode(encoded)
|
||||
if err != nil {
|
||||
t.Errorf("Decode: %v", err)
|
||||
}
|
||||
if !bytes.Equal(decoded, input) {
|
||||
t.Errorf(""+
|
||||
"Decode content:\n"+
|
||||
"%q\n"+
|
||||
"want:\n"+
|
||||
"%q",
|
||||
decoded, input)
|
||||
}
|
||||
}
|
||||
}
|
19
go/cbrotli/internal/BUILD
Executable file
19
go/cbrotli/internal/BUILD
Executable file
@ -0,0 +1,19 @@
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
licenses(["notice"]) # MIT
|
||||
|
||||
load("@io_bazel_rules_go//go:def.bzl", "cgo_library")
|
||||
|
||||
cgo_library(
|
||||
name = "decoder",
|
||||
srcs = ["decoder.go"],
|
||||
visibility = ["//go/cbrotli:__subpackages__"],
|
||||
cdeps = ["//:brotlidec"],
|
||||
)
|
||||
|
||||
cgo_library(
|
||||
name = "encoder",
|
||||
srcs = ["encoder.go"],
|
||||
visibility = ["//go/cbrotli:__subpackages__"],
|
||||
cdeps = ["//:brotlienc"],
|
||||
)
|
110
go/cbrotli/internal/decoder.go
Executable file
110
go/cbrotli/internal/decoder.go
Executable file
@ -0,0 +1,110 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Distributed under MIT license.
|
||||
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
|
||||
// Package decoder wraps the brotli decoder C API used by package brotli.
|
||||
package decoder
|
||||
|
||||
/*
|
||||
#include <brotli/decode.h>
|
||||
|
||||
// Wrap BrotliDecoderDecompressStream so that it doesn't take variable (in-out)
|
||||
// pointers. Instead of updated pointer, deltas are saved in auxiliary struct.
|
||||
|
||||
struct DecompressStreamResult {
|
||||
size_t bytes_consumed;
|
||||
const uint8_t* output_data;
|
||||
size_t output_data_size;
|
||||
BrotliDecoderResult status;
|
||||
};
|
||||
|
||||
struct DecompressStreamResult DecompressStream(BrotliDecoderState* s,
|
||||
const uint8_t* encoded_data, size_t encoded_data_size) {
|
||||
struct DecompressStreamResult result;
|
||||
size_t available_in = encoded_data_size;
|
||||
const uint8_t* next_in = encoded_data;
|
||||
size_t available_out = 0;
|
||||
result.status = BrotliDecoderDecompressStream(s,
|
||||
&available_in, &next_in, &available_out, 0, 0);
|
||||
result.bytes_consumed = encoded_data_size - available_in;
|
||||
result.output_data = 0;
|
||||
result.output_data_size = 0;
|
||||
if (result.status != BROTLI_DECODER_RESULT_ERROR) {
|
||||
result.output_data = BrotliDecoderTakeOutput(s, &result.output_data_size);
|
||||
if (BrotliDecoderIsFinished(s)) {
|
||||
result.status = BROTLI_DECODER_RESULT_SUCCESS;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Status represents internal state after DecompressStream invokation
|
||||
type Status int
|
||||
|
||||
const (
|
||||
// Error happened
|
||||
Error Status = iota
|
||||
// Done means that no more output will be produced
|
||||
Done
|
||||
// Ok means that more output might be produced with no additional input
|
||||
Ok
|
||||
)
|
||||
|
||||
// Decoder is the Brotli c-decoder handle.
|
||||
type Decoder struct {
|
||||
state *C.BrotliDecoderState
|
||||
}
|
||||
|
||||
// New returns a new Brotli c-decoder handle.
|
||||
// Close MUST be called to free resources.
|
||||
func New() Decoder {
|
||||
return Decoder{state: C.BrotliDecoderCreateInstance(nil, nil, nil)}
|
||||
}
|
||||
|
||||
// Close frees resources used by decoder.
|
||||
func (z *Decoder) Close() {
|
||||
C.BrotliDecoderDestroyInstance(z.state)
|
||||
z.state = nil
|
||||
}
|
||||
|
||||
func goStatus(cStatus C.BrotliDecoderResult) (status Status) {
|
||||
switch cStatus {
|
||||
case C.BROTLI_DECODER_RESULT_SUCCESS:
|
||||
return Done
|
||||
case C.BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
|
||||
return Ok
|
||||
case C.BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
|
||||
return Ok
|
||||
}
|
||||
return Error
|
||||
}
|
||||
|
||||
// cBytes casts a Go []byte into a C uint8_t*. We pass &buf[0] directly to C,
|
||||
// which is legal because C doesn't save the pointer longer than the call and
|
||||
// the byte array itself doesn't contain any pointers.
|
||||
func cBytes(buf []byte) (*C.uint8_t, C.size_t) {
|
||||
if len(buf) == 0 {
|
||||
return (*C.uint8_t)(nil), 0
|
||||
}
|
||||
return (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf))
|
||||
}
|
||||
|
||||
// DecompressStream reads Brotli-encoded bytes from in, and returns produced
|
||||
// bytes. Output contents should not be modified. Liveness of output is
|
||||
// hard-limited by Decoder liveness; slice becomes invalid when any Decoder
|
||||
// method is invoked.
|
||||
func (z *Decoder) DecompressStream(in []byte) (
|
||||
bytesConsumed int, output []byte, status Status) {
|
||||
cin, cinSize := cBytes(in)
|
||||
result := C.DecompressStream(z.state, cin, cinSize)
|
||||
output = C.GoBytes(
|
||||
unsafe.Pointer(result.output_data), C.int(result.output_data_size))
|
||||
return int(result.bytes_consumed), output, goStatus(result.status)
|
||||
}
|
135
go/cbrotli/internal/encoder.go
Executable file
135
go/cbrotli/internal/encoder.go
Executable file
@ -0,0 +1,135 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Distributed under MIT license.
|
||||
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
|
||||
// Package encoder wraps the brotli encoder C API used by package brotli.
|
||||
package encoder
|
||||
|
||||
/*
|
||||
#include <brotli/encode.h>
|
||||
|
||||
// Wrap BrotliEncoderCompressStream so that it doesn't take variable (in-out)
|
||||
// pointers. Instead of updated pointer, deltas are saved in auxiliary struct.
|
||||
struct CompressStreamResult {
|
||||
size_t bytes_consumed;
|
||||
const uint8_t* output_data;
|
||||
size_t output_data_size;
|
||||
int success;
|
||||
int has_more;
|
||||
};
|
||||
|
||||
struct CompressStreamResult CompressStream(
|
||||
BrotliEncoderState* s, BrotliEncoderOperation op,
|
||||
const uint8_t* data, size_t data_size) {
|
||||
struct CompressStreamResult result;
|
||||
size_t available_in = data_size;
|
||||
const uint8_t* next_in = data;
|
||||
size_t available_out = 0;
|
||||
result.success = BrotliEncoderCompressStream(s, op,
|
||||
&available_in, &next_in, &available_out, 0, 0) ? 1 : 0;
|
||||
result.bytes_consumed = data_size - available_in;
|
||||
result.output_data = 0;
|
||||
result.output_data_size = 0;
|
||||
if (result.success) {
|
||||
result.output_data = BrotliEncoderTakeOutput(s, &result.output_data_size);
|
||||
}
|
||||
result.has_more = BrotliEncoderHasMoreOutput(s) ? 1 : 0;
|
||||
return result;
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Operation represents type of request to CompressStream
|
||||
type Operation int
|
||||
|
||||
const (
|
||||
// Process input
|
||||
Process Operation = iota
|
||||
// Flush input processed so far
|
||||
Flush
|
||||
// Finish stream
|
||||
Finish
|
||||
)
|
||||
|
||||
// Status represents internal state after CompressStream invocation
|
||||
type Status int
|
||||
|
||||
const (
|
||||
// Error happened
|
||||
Error Status = iota
|
||||
// Done means that no more output will be produced
|
||||
Done
|
||||
// Ok means that more output might be produced with no additional input
|
||||
Ok
|
||||
)
|
||||
|
||||
// Encoder is the Brotli c-encoder handle.
|
||||
type Encoder struct {
|
||||
state *C.BrotliEncoderState
|
||||
}
|
||||
|
||||
// New returns a new Brotli c-encoder handle.
|
||||
// quality and lgWin are described in third_party/Brotli/enc/encode.h.
|
||||
// Close MUST be called to free resources.
|
||||
func New(quality, lgWin int) Encoder {
|
||||
state := C.BrotliEncoderCreateInstance(nil, nil, nil)
|
||||
// TODO(b/18187008): Check if LGBLOCK or MODE are useful to Flywheel.
|
||||
C.BrotliEncoderSetParameter(
|
||||
state, C.BROTLI_PARAM_QUALITY, (C.uint32_t)(quality))
|
||||
C.BrotliEncoderSetParameter(
|
||||
state, C.BROTLI_PARAM_LGWIN, (C.uint32_t)(lgWin))
|
||||
return Encoder{state}
|
||||
}
|
||||
|
||||
// Close frees resources used by encoder.
|
||||
func (z *Encoder) Close() {
|
||||
C.BrotliEncoderDestroyInstance(z.state)
|
||||
z.state = nil
|
||||
}
|
||||
|
||||
// cBytes casts a Go []byte into a C uint8_t*. We pass &buf[0] directly to C,
|
||||
// which is legal because C doesn't save the pointer longer than the call and
|
||||
// the byte array itself doesn't contain any pointers.
|
||||
func cBytes(buf []byte) (*C.uint8_t, C.size_t) {
|
||||
if len(buf) == 0 {
|
||||
return (*C.uint8_t)(nil), 0
|
||||
}
|
||||
return (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf))
|
||||
}
|
||||
|
||||
func cOperation(op Operation) (cOp C.BrotliEncoderOperation) {
|
||||
switch op {
|
||||
case Flush:
|
||||
return C.BROTLI_OPERATION_FLUSH
|
||||
case Finish:
|
||||
return C.BROTLI_OPERATION_FINISH
|
||||
}
|
||||
return C.BROTLI_OPERATION_PROCESS
|
||||
}
|
||||
|
||||
// CompressStream processes data and produces Brotli-encoded bytes. Encoder may
|
||||
// consume considerable amount of input before the first output bytes come out.
|
||||
// Flush and Finish operations force Encoder to produce output that corresponds
|
||||
// to input consumed so far. Output contents should not be modified. Liveness of
|
||||
// output is hard-limited by Encoder liveness; slice becomes invalid when any
|
||||
// Encoder method is invoked.
|
||||
func (z *Encoder) CompressStream(in []byte, op Operation) (
|
||||
bytesConsumed int, output []byte, status Status) {
|
||||
cin, cinSize := cBytes(in)
|
||||
result := C.CompressStream(z.state, cOperation(op), cin, cinSize)
|
||||
output = C.GoBytes(
|
||||
unsafe.Pointer(result.output_data), C.int(result.output_data_size))
|
||||
var outcome Status
|
||||
if result.success == 0 {
|
||||
outcome = Error
|
||||
} else if result.has_more != 0 {
|
||||
outcome = Ok
|
||||
} else {
|
||||
outcome = Done
|
||||
}
|
||||
return int(result.bytes_consumed), output, outcome
|
||||
}
|
@ -38,6 +38,12 @@ java_test(
|
||||
runtime_deps = [":test_lib"],
|
||||
)
|
||||
|
||||
java_test(
|
||||
name = "EnumTest",
|
||||
test_class = "org.brotli.dec.EnumTest",
|
||||
runtime_deps = [":test_lib"],
|
||||
)
|
||||
|
||||
java_test(
|
||||
name = "SynthTest",
|
||||
test_class = "org.brotli.dec.SynthTest",
|
||||
|
53
java/org/brotli/dec/EnumTest.java
Executable file
53
java/org/brotli/dec/EnumTest.java
Executable file
@ -0,0 +1,53 @@
|
||||
/* Copyright 2017 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
package org.brotli.dec;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.TreeSet;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
/**
|
||||
* Tests for Enum-like classes.
|
||||
*/
|
||||
@RunWith(JUnit4.class)
|
||||
public class EnumTest {
|
||||
|
||||
private void checkEnumClass(Class<?> clazz) {
|
||||
TreeSet<Integer> values = new TreeSet<Integer>();
|
||||
for (Field f : clazz.getDeclaredFields()) {
|
||||
assertEquals("int", f.getType().getName());
|
||||
assertEquals(Modifier.FINAL | Modifier.STATIC, f.getModifiers());
|
||||
Integer value = null;
|
||||
try {
|
||||
value = f.getInt(null);
|
||||
} catch (IllegalAccessException ex) {
|
||||
fail("Inaccessible field");
|
||||
}
|
||||
assertFalse(values.contains(value));
|
||||
values.add(value);
|
||||
}
|
||||
assertEquals(0, values.first().intValue());
|
||||
assertEquals(values.size(), values.last() + 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRunningState() {
|
||||
checkEnumClass(RunningState.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWordTransformType() {
|
||||
checkEnumClass(WordTransformType.class);
|
||||
}
|
||||
}
|
@ -9,18 +9,18 @@ package org.brotli.dec;
|
||||
/**
|
||||
* Enumeration of decoding state-machine.
|
||||
*/
|
||||
enum RunningState {
|
||||
UNINITIALIZED,
|
||||
BLOCK_START,
|
||||
COMPRESSED_BLOCK_START,
|
||||
MAIN_LOOP,
|
||||
READ_METADATA,
|
||||
COPY_UNCOMPRESSED,
|
||||
INSERT_LOOP,
|
||||
COPY_LOOP,
|
||||
COPY_WRAP_BUFFER,
|
||||
TRANSFORM,
|
||||
FINISHED,
|
||||
CLOSED,
|
||||
WRITE
|
||||
final class RunningState {
|
||||
static final int UNINITIALIZED = 0;
|
||||
static final int BLOCK_START = 1;
|
||||
static final int COMPRESSED_BLOCK_START = 2;
|
||||
static final int MAIN_LOOP = 3;
|
||||
static final int READ_METADATA = 4;
|
||||
static final int COPY_UNCOMPRESSED = 5;
|
||||
static final int INSERT_LOOP = 6;
|
||||
static final int COPY_LOOP = 7;
|
||||
static final int COPY_WRAP_BUFFER = 8;
|
||||
static final int TRANSFORM = 9;
|
||||
static final int FINISHED = 10;
|
||||
static final int CLOSED = 11;
|
||||
static final int WRITE = 12;
|
||||
}
|
||||
|
@ -14,8 +14,8 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
final class State {
|
||||
RunningState runningState = UNINITIALIZED;
|
||||
RunningState nextRunningState;
|
||||
int runningState = UNINITIALIZED;
|
||||
int nextRunningState;
|
||||
final BitReader br = new BitReader();
|
||||
byte[] ringBuffer;
|
||||
final int[] blockTypeTrees = new int[3 * Huffman.HUFFMAN_MAX_TABLE_SIZE];
|
||||
|
@ -33,10 +33,10 @@ import static org.brotli.dec.WordTransformType.UPPERCASE_FIRST;
|
||||
final class Transform {
|
||||
|
||||
private final byte[] prefix;
|
||||
private final WordTransformType type;
|
||||
private final int type;
|
||||
private final byte[] suffix;
|
||||
|
||||
Transform(String prefix, WordTransformType type, String suffix) {
|
||||
Transform(String prefix, int type, String suffix) {
|
||||
this.prefix = readUniBytes(prefix);
|
||||
this.type = type;
|
||||
this.suffix = readUniBytes(suffix);
|
||||
@ -188,14 +188,14 @@ final class Transform {
|
||||
}
|
||||
|
||||
// Copy trimmed word.
|
||||
WordTransformType op = transform.type;
|
||||
tmp = op.omitFirst;
|
||||
int op = transform.type;
|
||||
tmp = WordTransformType.getOmitFirst(op);
|
||||
if (tmp > len) {
|
||||
tmp = len;
|
||||
}
|
||||
wordOffset += tmp;
|
||||
len -= tmp;
|
||||
len -= op.omitLast;
|
||||
len -= WordTransformType.getOmitLast(op);
|
||||
i = len;
|
||||
while (i > 0) {
|
||||
dst[offset++] = word[wordOffset++];
|
||||
|
@ -12,37 +12,34 @@ package org.brotli.dec;
|
||||
* <p>There are two simple types of transforms: omit X first/last symbols, two character-case
|
||||
* transforms and the identity transform.
|
||||
*/
|
||||
enum WordTransformType {
|
||||
IDENTITY(0, 0),
|
||||
OMIT_LAST_1(0, 1),
|
||||
OMIT_LAST_2(0, 2),
|
||||
OMIT_LAST_3(0, 3),
|
||||
OMIT_LAST_4(0, 4),
|
||||
OMIT_LAST_5(0, 5),
|
||||
OMIT_LAST_6(0, 6),
|
||||
OMIT_LAST_7(0, 7),
|
||||
OMIT_LAST_8(0, 8),
|
||||
OMIT_LAST_9(0, 9),
|
||||
UPPERCASE_FIRST(0, 0),
|
||||
UPPERCASE_ALL(0, 0),
|
||||
OMIT_FIRST_1(1, 0),
|
||||
OMIT_FIRST_2(2, 0),
|
||||
OMIT_FIRST_3(3, 0),
|
||||
OMIT_FIRST_4(4, 0),
|
||||
OMIT_FIRST_5(5, 0),
|
||||
OMIT_FIRST_6(6, 0),
|
||||
OMIT_FIRST_7(7, 0),
|
||||
/*
|
||||
* brotli specification doesn't use OMIT_FIRST_8(8, 0) transform.
|
||||
* Probably, it would be used in future format extensions.
|
||||
*/
|
||||
OMIT_FIRST_9(9, 0);
|
||||
final class WordTransformType {
|
||||
static final int IDENTITY = 0;
|
||||
static final int OMIT_LAST_1 = 1;
|
||||
static final int OMIT_LAST_2 = 2;
|
||||
static final int OMIT_LAST_3 = 3;
|
||||
static final int OMIT_LAST_4 = 4;
|
||||
static final int OMIT_LAST_5 = 5;
|
||||
static final int OMIT_LAST_6 = 6;
|
||||
static final int OMIT_LAST_7 = 7;
|
||||
static final int OMIT_LAST_8 = 8;
|
||||
static final int OMIT_LAST_9 = 9;
|
||||
static final int UPPERCASE_FIRST = 10;
|
||||
static final int UPPERCASE_ALL = 11;
|
||||
static final int OMIT_FIRST_1 = 12;
|
||||
static final int OMIT_FIRST_2 = 13;
|
||||
static final int OMIT_FIRST_3 = 14;
|
||||
static final int OMIT_FIRST_4 = 15;
|
||||
static final int OMIT_FIRST_5 = 16;
|
||||
static final int OMIT_FIRST_6 = 17;
|
||||
static final int OMIT_FIRST_7 = 18;
|
||||
static final int OMIT_FIRST_8 = 19;
|
||||
static final int OMIT_FIRST_9 = 20;
|
||||
|
||||
final int omitFirst;
|
||||
final int omitLast;
|
||||
static int getOmitFirst(int type) {
|
||||
return type >= OMIT_FIRST_1 ? (type - OMIT_FIRST_1 + 1) : 0;
|
||||
}
|
||||
|
||||
WordTransformType(int omitFirst, int omitLast) {
|
||||
this.omitFirst = omitFirst;
|
||||
this.omitLast = omitLast;
|
||||
static int getOmitLast(int type) {
|
||||
return type <= OMIT_LAST_9 ? (type - OMIT_LAST_1 + 1) : 0;
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user