AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2024-11-21 19:20:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify go brotli wrapper. (#540)

Browse Source 
Based on PR #533.

Kudos to Bryan (bcmillis@).
This commit is contained in:
Eugene Kliuchnikov 2017-04-13 20:05:36 +02:00 committed by GitHub
parent f2aa4d1e8c
commit 04de756ad2
8 changed files with 349 additions and 532 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
go/cbrotli/BUILD
View File

@ -2,16 +2,19 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # MIT
load("@io_bazel_rules_go//go:def.bzl", "go_prefix", "go_library", "go_test")
load("@io_bazel_rules_go//go:def.bzl", "go_prefix", "cgo_library", "go_test")
go_prefix("github.com/google/brotli")
go_library(
cgo_library(
name = "cbrotli",
srcs = ["cbrotli.go"],
deps = [
"//go/cbrotli/internal:decoder",
"//go/cbrotli/internal:encoder",
srcs = [
"reader.go",
"writer.go",
],
cdeps = [
"//:brotlidec",
"//:brotlienc",
],
)

254
go/cbrotli/cbrotli.go
View File

@ -1,254 +0,0 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Distributed under MIT license.
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
// Package cbrotli compresses and decompresses data with C-Brotli library.
package cbrotli
import (
"bytes"
"io"
"github.com/google/brotli/go/cbrotli/internal/decoder"
"github.com/google/brotli/go/cbrotli/internal/encoder"
)
// An internalError reports an error in the (c-)brotli code itself.
type internalError string
func (e internalError) Error() string {
return "cbrotli: internal error: " + string(e)
}
//------------------------------------------------------------------------------
// Encoder
//------------------------------------------------------------------------------
// WriterOptions configures Writer.
type WriterOptions struct {
// Quality controls the compression-speed vs compression-density trade-offs.
// The higher the quality, the slower the compression. Range is 0 to 11.
Quality int
// LGWin is the base 2 logarithm of the sliding window size.
// Range is 10 to 24. 0 indicates automatic configuration based on Quality.
LGWin int
}
// Writer implements io.WriteCloser, an io.Writer decorator that produces
// Brotli-encoded data.
type Writer struct {
dst io.Writer
encoder encoder.Encoder
closed bool
}
// NewWriter initializes new Writer instance.
// Close MUST be called to free resources.
func NewWriter(dst io.Writer, options WriterOptions) *Writer {
return &Writer{
dst: dst,
encoder: encoder.New(options.Quality, options.LGWin),
}
}
// Close implements io.Closer. Close MUST be invoked to free native resources.
// Also Close implicitly flushes remaining data to the decorated writer.
func (z *Writer) Close() error {
if z.closed {
return nil
}
defer z.encoder.Close()
_, err := z.writeChunk(nil, encoder.Finish)
z.closed = true
return err
}
func (z *Writer) writeChunk(p []byte, op encoder.Operation) (int, error) {
if z.closed {
return 0, internalError("write after close")
}
var totalBytesConsumed int
var err error
for {
bytesConsumed, output, status := z.encoder.CompressStream(p, op)
if status == encoder.Error {
err = internalError("encoder failure")
break
}
p = p[bytesConsumed:]
totalBytesConsumed += bytesConsumed
_, err = z.dst.Write(output)
if err != nil {
break
}
if len(p) == 0 && status == encoder.Done {
break
}
}
return totalBytesConsumed, err
}
// Write implements io.Writer.
func (z *Writer) Write(p []byte) (int, error) {
return z.writeChunk(p, encoder.Process)
}
// Flush outputs encoded data for all input provided to Write. The resulting
// output can be decoded to match all input before Flush, but the stream is
// not yet complete until after Close.
// Flush has a negative impact on compression.
func (z *Writer) Flush() error {
_, err := z.writeChunk(nil, encoder.Finish)
return err
}
// Encode returns content encoded with Brotli.
func Encode(content []byte, options WriterOptions) ([]byte, error) {
var buf bytes.Buffer
writer := NewWriter(&buf, options)
defer writer.Close()
_, err := writer.Write(content)
if err != nil {
return nil, err
}
if err := writer.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
//------------------------------------------------------------------------------
// Decoder
//------------------------------------------------------------------------------
// Reader implements io.ReadCloser, an io.Reader decorator that decodes
// Brotli-encoded data.
type Reader struct {
src io.Reader
decoder decoder.Decoder
buf []byte // intermediate read buffer pointed to by next
eof bool // true if all compressed stream is decoded
next []byte // buffered data to be passed to decoder
output []byte // data produced by decoder, but not yet consumed
srcErr error // last source reader error
err error // reader state; nil if it is OK to read further
closed bool // true is stream is already closed
}
// NewReader initializes new Reader instance.
// Close MUST be called to free resources.
func NewReader(src io.Reader) *Reader {
return &Reader{
src: src,
decoder: decoder.New(),
buf: make([]byte, 32*1024),
eof: false,
}
}
// Close implements io.Closer. Close MUST be invoked to free native resources.
func (z *Reader) Close() error {
if z.closed {
return nil
}
z.decoder.Close()
z.err = internalError("read after close")
z.closed = true
return nil
}
func isEOF(src io.Reader) bool {
n, err := src.Read(make([]byte, 1))
return n == 0 && err == io.EOF
}
// Read implements io.Reader.
func (z *Reader) Read(p []byte) (int, error) {
// Any error state is unrecoverable.
if z.err != nil {
return 0, z.err
}
// See io.Reader documentation.
if len(p) == 0 {
return 0, nil
}
var totalOutBytes int
// There is no practical limit for amount of bytes being consumed by decoder
// before producing any output. Continue feeding decoder until some data is
// produced
for {
// Push already produced output first.
if outBytes := len(z.output); outBytes != 0 {
outBytes = copy(p, z.output)
p = p[outBytes:]
z.output = z.output[outBytes:]
totalOutBytes += outBytes
// Output buffer is full.
if len(p) == 0 {
break
}
continue
}
// No more produced output left.
// If no more output is expected, then we are finished.
if z.eof {
z.err = io.EOF
break
}
// Replenish buffer (might cause blocking read), only if necessary.
if len(z.next) == 0 && totalOutBytes == 0 && z.srcErr != io.EOF {
var n int
n, z.srcErr = z.src.Read(z.buf)
z.next = z.buf[:n]
if z.srcErr != nil && z.srcErr != io.EOF {
z.err = z.srcErr
break
}
}
// Do decoding.
consumed, output, status := z.decoder.DecompressStream(z.next)
z.output = output
z.next = z.next[consumed:]
if status == decoder.Error {
// When error happens, the remaining output does not matter.
z.err = internalError("decoder failure")
break
} else if status == decoder.Done {
// Decoder stream is closed; no further input is expected.
if len(z.next) != 0 || (z.srcErr != io.EOF && !isEOF(z.src)) {
z.err = internalError("excessive input")
break
}
// No more output is expected; keep pushing output.
z.eof = true
continue
} else {
// If can not move any further...
if consumed == 0 && len(z.output) == 0 {
// Unexpected end of input.
if z.srcErr == io.EOF || totalOutBytes == 0 {
z.err = io.ErrUnexpectedEOF
}
// Postpone blocking reads for the next invocation.
break
}
// Continue pushing output.
}
}
return totalOutBytes, z.err
}
// Decode decodes Brotli encoded data.
func Decode(encodedData []byte) ([]byte, error) {
var buf bytes.Buffer
reader := NewReader(bytes.NewReader(encodedData))
defer reader.Close()
_, err := io.Copy(&buf, reader)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}

33
go/cbrotli/cbrotli_test.go
View File

@ -91,7 +91,7 @@ func TestEncoderStreams(t *testing.T) {
// to fill the window.
const lgWin = 16
windowSize := int(math.Pow(2, lgWin))
input := make([]byte, 2*windowSize)
input := make([]byte, 8*windowSize)
rand.Read(input)
out := bytes.Buffer{}
e := NewWriter(&out, WriterOptions{Quality: 11, LGWin: lgWin})
@ -106,7 +106,7 @@ func TestEncoderStreams(t *testing.T) {
// We've fed more data than the sliding window size. Check that some
// compressed data has been output.
if out.Len() == 0 {
t.Errorf("Output length is after %d bytes written", n)
t.Errorf("Output length is 0 after %d bytes written", n)
}
if err := e.Close(); err != nil {
t.Errorf("Close Error after copied %d bytes: %v", n, err)
@ -151,6 +151,9 @@ func TestEncoderFlush(t *testing.T) {
if err := e.Flush(); err != nil {
t.Fatalf("Flush(): %v", err)
}
if out.Len() == 0 {
t.Fatalf("0 bytes written after Flush()")
}
decompressed := make([]byte, 1000)
reader := NewReader(bytes.NewReader(out.Bytes()))
n, err := reader.Read(decompressed)
@ -187,8 +190,8 @@ func TestReader(t *testing.T) {
"Reader output:\n"+
"%q\n"+
"want:\n"+
"%q",
got, content)
"<%d bytes>",
got, len(content))
}
}
@ -204,8 +207,8 @@ func TestDecode(t *testing.T) {
"Decode content:\n"+
"%q\n"+
"want:\n"+
"%q",
decoded, content)
"<%d bytes>",
decoded, len(content))
}
}
@ -213,7 +216,13 @@ func TestDecodeFuzz(t *testing.T) {
// Test that the decoder terminates with corrupted input.
content := bytes.Repeat([]byte("hello world!"), 100)
src := rand.NewSource(0)
encoded, _ := Encode(content, WriterOptions{Quality: 5})
encoded, err := Encode(content, WriterOptions{Quality: 5})
if err != nil {
t.Fatalf("Encode(<%d bytes>, _) = _, %s", len(content), err)
}
if len(encoded) == 0 {
t.Fatalf("Encode(<%d bytes>, _) produced empty output", len(content))
}
for i := 0; i < 100; i++ {
enc := append([]byte{}, encoded...)
for j := 0; j < 5; j++ {
@ -260,12 +269,18 @@ func TestEncodeDecode(t *testing.T) {
t.Errorf("Decode: %v", err)
}
if !bytes.Equal(decoded, input) {
var want string
if len(input) > 320 {
want = fmt.Sprintf("<%d bytes>", len(input))
} else {
want = fmt.Sprintf("%q", input)
}
t.Errorf(""+
"Decode content:\n"+
"%q\n"+
"want:\n"+
"%q",
decoded, input)
"%s",
decoded, want)
}
}
}

19
go/cbrotli/internal/BUILD
View File

@ -1,19 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # MIT
load("@io_bazel_rules_go//go:def.bzl", "cgo_library")
cgo_library(
name = "decoder",
srcs = ["decoder.go"],
visibility = ["//go/cbrotli:__subpackages__"],
cdeps = ["//:brotlidec"],
)
cgo_library(
name = "encoder",
srcs = ["encoder.go"],
visibility = ["//go/cbrotli:__subpackages__"],
cdeps = ["//:brotlienc"],
)

110
go/cbrotli/internal/decoder.go
View File

@ -1,110 +0,0 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Distributed under MIT license.
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
// Package decoder wraps the brotli decoder C API used by package brotli.
package decoder
/*
#include <brotli/decode.h>
// Wrap BrotliDecoderDecompressStream so that it doesn't take variable (in-out)
// pointers. Instead of updated pointer, deltas are saved in auxiliary struct.
struct DecompressStreamResult {
size_t bytes_consumed;
const uint8_t* output_data;
size_t output_data_size;
BrotliDecoderResult status;
};
struct DecompressStreamResult DecompressStream(BrotliDecoderState* s,
const uint8_t* encoded_data, size_t encoded_data_size) {
struct DecompressStreamResult result;
size_t available_in = encoded_data_size;
const uint8_t* next_in = encoded_data;
size_t available_out = 0;
result.status = BrotliDecoderDecompressStream(s,
&available_in, &next_in, &available_out, 0, 0);
result.bytes_consumed = encoded_data_size - available_in;
result.output_data = 0;
result.output_data_size = 0;
if (result.status != BROTLI_DECODER_RESULT_ERROR) {
result.output_data = BrotliDecoderTakeOutput(s, &result.output_data_size);
if (BrotliDecoderIsFinished(s)) {
result.status = BROTLI_DECODER_RESULT_SUCCESS;
}
}
return result;
}
*/
import "C"
import (
"unsafe"
)
// Status represents internal state after DecompressStream invokation
type Status int
const (
// Error happened
Error Status = iota
// Done means that no more output will be produced
Done
// Ok means that more output might be produced with no additional input
Ok
)
// Decoder is the Brotli c-decoder handle.
type Decoder struct {
state *C.BrotliDecoderState
}
// New returns a new Brotli c-decoder handle.
// Close MUST be called to free resources.
func New() Decoder {
return Decoder{state: C.BrotliDecoderCreateInstance(nil, nil, nil)}
}
// Close frees resources used by decoder.
func (z *Decoder) Close() {
C.BrotliDecoderDestroyInstance(z.state)
z.state = nil
}
func goStatus(cStatus C.BrotliDecoderResult) (status Status) {
switch cStatus {
case C.BROTLI_DECODER_RESULT_SUCCESS:
return Done
case C.BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
return Ok
case C.BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
return Ok
}
return Error
}
// cBytes casts a Go []byte into a C uint8_t*. We pass &buf[0] directly to C,
// which is legal because C doesn't save the pointer longer than the call and
// the byte array itself doesn't contain any pointers.
func cBytes(buf []byte) (*C.uint8_t, C.size_t) {
if len(buf) == 0 {
return (*C.uint8_t)(nil), 0
}
return (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf))
}
// DecompressStream reads Brotli-encoded bytes from in, and returns produced
// bytes. Output contents should not be modified. Liveness of output is
// hard-limited by Decoder liveness; slice becomes invalid when any Decoder
// method is invoked.
func (z *Decoder) DecompressStream(in []byte) (
bytesConsumed int, output []byte, status Status) {
cin, cinSize := cBytes(in)
result := C.DecompressStream(z.state, cin, cinSize)
output = C.GoBytes(
unsafe.Pointer(result.output_data), C.int(result.output_data_size))
return int(result.bytes_consumed), output, goStatus(result.status)
}

134
go/cbrotli/internal/encoder.go
View File

@ -1,134 +0,0 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Distributed under MIT license.
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
// Package encoder wraps the brotli encoder C API used by package brotli.
package encoder
/*
#include <brotli/encode.h>
// Wrap BrotliEncoderCompressStream so that it doesn't take variable (in-out)
// pointers. Instead of updated pointer, deltas are saved in auxiliary struct.
struct CompressStreamResult {
size_t bytes_consumed;
const uint8_t* output_data;
size_t output_data_size;
int success;
int has_more;
};
struct CompressStreamResult CompressStream(
BrotliEncoderState* s, BrotliEncoderOperation op,
const uint8_t* data, size_t data_size) {
struct CompressStreamResult result;
size_t available_in = data_size;
const uint8_t* next_in = data;
size_t available_out = 0;
result.success = BrotliEncoderCompressStream(s, op,
&available_in, &next_in, &available_out, 0, 0) ? 1 : 0;
result.bytes_consumed = data_size - available_in;
result.output_data = 0;
result.output_data_size = 0;
if (result.success) {
result.output_data = BrotliEncoderTakeOutput(s, &result.output_data_size);
}
result.has_more = BrotliEncoderHasMoreOutput(s) ? 1 : 0;
return result;
}
*/
import "C"
import (
"unsafe"
)
// Operation represents type of request to CompressStream
type Operation int
const (
// Process input
Process Operation = iota
// Flush input processed so far
Flush
// Finish stream
Finish
)
// Status represents internal state after CompressStream invocation
type Status int
const (
// Error happened
Error Status = iota
// Done means that no more output will be produced
Done
// Ok means that more output might be produced with no additional input
Ok
)
// Encoder is the Brotli c-encoder handle.
type Encoder struct {
state *C.BrotliEncoderState
}
// New returns a new Brotli c-encoder handle.
// quality and lgWin are described in third_party/Brotli/enc/encode.h.
// Close MUST be called to free resources.
func New(quality, lgWin int) Encoder {
state := C.BrotliEncoderCreateInstance(nil, nil, nil)
C.BrotliEncoderSetParameter(
state, C.BROTLI_PARAM_QUALITY, (C.uint32_t)(quality))
C.BrotliEncoderSetParameter(
state, C.BROTLI_PARAM_LGWIN, (C.uint32_t)(lgWin))
return Encoder{state}
}
// Close frees resources used by encoder.
func (z *Encoder) Close() {
C.BrotliEncoderDestroyInstance(z.state)
z.state = nil
}
// cBytes casts a Go []byte into a C uint8_t*. We pass &buf[0] directly to C,
// which is legal because C doesn't save the pointer longer than the call and
// the byte array itself doesn't contain any pointers.
func cBytes(buf []byte) (*C.uint8_t, C.size_t) {
if len(buf) == 0 {
return (*C.uint8_t)(nil), 0
}
return (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf))
}
func cOperation(op Operation) (cOp C.BrotliEncoderOperation) {
switch op {
case Flush:
return C.BROTLI_OPERATION_FLUSH
case Finish:
return C.BROTLI_OPERATION_FINISH
}
return C.BROTLI_OPERATION_PROCESS
}
// CompressStream processes data and produces Brotli-encoded bytes. Encoder may
// consume considerable amount of input before the first output bytes come out.
// Flush and Finish operations force Encoder to produce output that corresponds
// to input consumed so far. Output contents should not be modified. Liveness of
// output is hard-limited by Encoder liveness; slice becomes invalid when any
// Encoder method is invoked.
func (z *Encoder) CompressStream(in []byte, op Operation) (
bytesConsumed int, output []byte, status Status) {
cin, cinSize := cBytes(in)
result := C.CompressStream(z.state, cOperation(op), cin, cinSize)
output = C.GoBytes(
unsafe.Pointer(result.output_data), C.int(result.output_data_size))
var outcome Status
if result.success == 0 {
outcome = Error
} else if result.has_more != 0 {
outcome = Ok
} else {
outcome = Done
}
return int(result.bytes_consumed), output, outcome
}

156
go/cbrotli/reader.go Executable file
View File

@ -0,0 +1,156 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Distributed under MIT license.
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
// Package cbrotli compresses and decompresses data with C-Brotli library.
package cbrotli
/*
#include <stddef.h>
#include <stdint.h>
#include <brotli/decode.h>
static BrotliDecoderResult DecompressStream(BrotliDecoderState* s,
uint8_t* out, size_t out_len,
const uint8_t* in, size_t in_len,
size_t* bytes_written,
size_t* bytes_consumed) {
size_t in_remaining = in_len;
size_t out_remaining = out_len;
BrotliDecoderResult result = BrotliDecoderDecompressStream(
s, &in_remaining, &in, &out_remaining, &out, NULL);
*bytes_written = out_len - out_remaining;
*bytes_consumed = in_len - in_remaining;
return result;
}
*/
import "C"
import (
"bytes"
"errors"
"io"
"io/ioutil"
)
type decodeError C.BrotliDecoderErrorCode
func (err decodeError) Error() string {
return "cbrotli: " +
C.GoString(C.BrotliDecoderErrorString(C.BrotliDecoderErrorCode(err)))
}
var errExcessiveInput = errors.New("cbrotli: excessive input")
var errInvalidState = errors.New("cbrotli: invalid state")
var errReaderClosed = errors.New("cbrotli: Reader is closed")
// Reader implements io.ReadCloser by reading Brotli-encoded data from an
// underlying Reader.
type Reader struct {
src io.Reader
state *C.BrotliDecoderState
buf []byte // scratch space for reading from src
in []byte // current chunk to decode; usually aliases buf
}
// readBufSize is a "good" buffer size that avoids excessive round-trips
// between C and Go but doesn't waste too much memory on buffering.
// It is arbitrarily chosen to be equal to the constant used in io.Copy.
const readBufSize = 32 * 1024
// NewReader initializes new Reader instance.
// Close MUST be called to free resources.
func NewReader(src io.Reader) *Reader {
return &Reader{
src: src,
state: C.BrotliDecoderCreateInstance(nil, nil, nil),
buf: make([]byte, readBufSize),
}
}
// Close implements io.Closer. Close MUST be invoked to free native resources.
func (r *Reader) Close() error {
if r.state == nil {
return errReaderClosed
}
// Close despite the state; i.e. there might be some unread decoded data.
C.BrotliDecoderDestroyInstance(r.state)
r.state = nil
return nil
}
func (r *Reader) Read(p []byte) (n int, err error) {
if int(C.BrotliDecoderHasMoreOutput(r.state)) == 0 && len(r.in) == 0 {
m, readErr := r.src.Read(r.buf)
if m == 0 {
// If readErr is `nil`, we just proxy underlying stream behavior.
return 0, readErr
}
r.in = r.buf[:m]
}
if len(p) == 0 {
return 0, nil
}
for n == 0 {
var written, consumed C.size_t
var data *C.uint8_t
if len(r.in) != 0 {
data = (*C.uint8_t)(&r.in[0])
}
result := C.DecompressStream(r.state,
(*C.uint8_t)(&p[0]), C.size_t(len(p)),
data, C.size_t(len(r.in)),
&written, &consumed)
r.in = r.in[int(consumed):]
n = int(written)
switch result {
case C.BROTLI_DECODER_RESULT_SUCCESS:
if len(r.in) > 0 {
return n, errExcessiveInput
}
return n, nil
case C.BROTLI_DECODER_RESULT_ERROR:
return n, decodeError(C.BrotliDecoderGetErrorCode(r.state))
case C.BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
if n == 0 {
return 0, io.ErrShortBuffer
}
return n, nil
case C.BROTLI_DECODER_NEEDS_MORE_INPUT:
}
if len(r.in) != 0 {
return 0, errInvalidState
}
// Top off the buffer.
encN, err := r.src.Read(r.buf)
if encN == 0 && n == 0 {
// Not enough data to complete decoding.
if err == io.EOF {
return 0, io.ErrUnexpectedEOF
}
return 0, err
}
r.in = r.buf[:encN]
}
return n, nil
}
// Decode decodes Brotli encoded data.
func Decode(encodedData []byte) ([]byte, error) {
r := &Reader{
src: bytes.NewReader(nil),
state: C.BrotliDecoderCreateInstance(nil, nil, nil),
buf: make([]byte, 4), // arbitrarily small but nonzero so that r.src.Read returns io.EOF
in: encodedData,
}
defer r.Close()
return ioutil.ReadAll(r)
}

160
go/cbrotli/writer.go Executable file
View File

@ -0,0 +1,160 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Distributed under MIT license.
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
package cbrotli
/*
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <brotli/encode.h>
struct CompressStreamResult {
size_t bytes_consumed;
const uint8_t* output_data;
size_t output_data_size;
int success;
int has_more;
};
static struct CompressStreamResult CompressStream(
BrotliEncoderState* s, BrotliEncoderOperation op,
const uint8_t* data, size_t data_size) {
struct CompressStreamResult result;
size_t available_in = data_size;
const uint8_t* next_in = data;
size_t available_out = 0;
result.success = BrotliEncoderCompressStream(s, op,
&available_in, &next_in, &available_out, 0, 0) ? 1 : 0;
result.bytes_consumed = data_size - available_in;
result.output_data = 0;
result.output_data_size = 0;
if (result.success) {
result.output_data = BrotliEncoderTakeOutput(s, &result.output_data_size);
}
result.has_more = BrotliEncoderHasMoreOutput(s) ? 1 : 0;
return result;
}
*/
import "C"
import (
"bytes"
"errors"
"io"
"unsafe"
)
// WriterOptions configures Writer.
type WriterOptions struct {
// Quality controls the compression-speed vs compression-density trade-offs.
// The higher the quality, the slower the compression. Range is 0 to 11.
Quality int
// LGWin is the base 2 logarithm of the sliding window size.
// Range is 10 to 24. 0 indicates automatic configuration based on Quality.
LGWin int
// BufferSize is the number of bytes to use to buffer encoded output.
// 0 indicates an implementation-defined default.
BufferSize int
}
// Writer implements io.WriteCloser by writing Brotli-encoded data to an
// underlying Writer.
type Writer struct {
dst io.Writer
state *C.BrotliEncoderState
buf, encoded []byte
}
var (
errEncode = errors.New("cbrotli: encode error")
errWriterClosed = errors.New("cbrotli: Writer is closed")
)
// NewWriter initializes new Writer instance.
// Close MUST be called to free resources.
func NewWriter(dst io.Writer, options WriterOptions) *Writer {
state := C.BrotliEncoderCreateInstance(nil, nil, nil)
C.BrotliEncoderSetParameter(
state, C.BROTLI_PARAM_QUALITY, (C.uint32_t)(options.Quality))
C.BrotliEncoderSetParameter(
state, C.BROTLI_PARAM_LGWIN, (C.uint32_t)(options.LGWin))
return &Writer{
dst: dst,
state: state,
}
}
func (w *Writer) writeChunk(p []byte, op C.BrotliEncoderOperation) (n int, err error) {
if w.state == nil {
return 0, errWriterClosed
}
for {
var data *C.uint8_t
if len(p) != 0 {
data = (*C.uint8_t)(&p[0])
}
result := C.CompressStream(w.state, op, data, C.size_t(len(p)))
if result.success == 0 {
return n, errEncode
}
p = p[int(result.bytes_consumed):]
n += int(result.bytes_consumed)
length := int(result.output_data_size)
if length != 0 {
// It is a workaround for non-copying-wrapping of native memory.
// C-encoder never pushes output block longer than ((2 << 25) + 502).
// TODO: use natural wrapper, when it becomes available, see
// https://golang.org/issue/13656.
output := (*[1 << 30]byte)(unsafe.Pointer(result.output_data))[:length:length]
_, err = w.dst.Write(output)
if err != nil {
return n, err
}
}
if len(p) == 0 && result.has_more == 0 {
return n, nil
}
}
}
// Flush outputs encoded data for all input provided to Write. The resulting
// output can be decoded to match all input before Flush, but the stream is
// not yet complete until after Close.
// Flush has a negative impact on compression.
func (w *Writer) Flush() error {
_, err := w.writeChunk(nil, C.BROTLI_OPERATION_FLUSH)
return err
}
// Close flushes remaining data to the decorated writer and frees C resources.
func (w *Writer) Close() error {
// If stream is already closed, it is reported by `writeChunk`.
_, err := w.writeChunk(nil, C.BROTLI_OPERATION_FINISH)
// C-Brotli tolerates `nil` pointer here.
C.BrotliEncoderDestroyInstance(w.state)
w.state = nil
return err
}
// Write implements io.Writer. Flush or Close must be called to ensure that the
// encoded bytes are actually flushed to the underlying Writer.
func (w *Writer) Write(p []byte) (n int, err error) {
return w.writeChunk(p, C.BROTLI_OPERATION_PROCESS)
}
// Encode returns content encoded with Brotli.
func Encode(content []byte, options WriterOptions) ([]byte, error) {
var buf bytes.Buffer
writer := NewWriter(&buf, options)
_, err := writer.Write(content)
if closeErr := writer.Close(); err == nil {
err = closeErr
}
return buf.Bytes(), err
}