Skip to content

Commit

Permalink
Merge pull request #37 from D3vl0per/36-compression-brotli-level-setu…
Browse files Browse the repository at this point in the history
…p-error

Compression brotli level setup error
  • Loading branch information
D3vl0per authored May 29, 2024
2 parents f4a4b1a + 9789e40 commit 71b44a8
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 58 deletions.
1 change: 0 additions & 1 deletion .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -283,4 +283,3 @@ linters:
- unused
- gomnd
- forcetypeassert
- golint
4 changes: 2 additions & 2 deletions aged/age_bind_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,15 @@ func TestRoundTrips(t *testing.T) {
name: "Compress with Zstd, no obfuscate",
parameter: aged.Parameters{
Data: config.plainData,
Compressor: &compression.Zstd{},
Compressor: &compression.Zstd{Level: compression.ZstdSpeedDefault},
},
},
{
name: "Compress with Zstd, obfuscate",
parameter: aged.Parameters{
Data: config.plainData,
Obfuscator: &aged.AgeV1Obf{},
Compressor: &compression.Zstd{},
Compressor: &compression.Zstd{Level: compression.ZstdSpeedDefault},
},
},
{
Expand Down
17 changes: 15 additions & 2 deletions compression/compression.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package compression

import (
"bytes"
"errors"
"io"

"github.com/andybalholm/brotli"
Expand Down Expand Up @@ -47,6 +48,8 @@ const (
BrotliBestSpeed int = 0
)

var ErrMissingCompressionLevel = errors.New("missing compression level parameter")

type Compressor interface {
Compress([]byte) ([]byte, error)
Decompress([]byte) ([]byte, error)
Expand Down Expand Up @@ -156,6 +159,10 @@ func (z *Zstd) Compress(in []byte) ([]byte, error) {
}

func (z *Zstd) CompressStream(in io.Reader, out io.Writer) error {
if z.Level == 0 {
return ErrMissingCompressionLevel
}

enc, err := zstd.NewWriter(out, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(z.Level)))
if err != nil {
return err
Expand Down Expand Up @@ -282,7 +289,6 @@ func (f *Flate) GetModes() []int {
DefaultCompression,
BestCompression,
HuffmanOnly,
StatelessCompression,
}
}

Expand Down Expand Up @@ -360,7 +366,6 @@ func (zl *Zlib) GetModes() []int {
DefaultCompression,
BestCompression,
HuffmanOnly,
StatelessCompression,
}
}

Expand All @@ -385,6 +390,10 @@ func (b *Brotli) Compress(in []byte) ([]byte, error) {
}

func (b *Brotli) CompressStream(in io.Reader, out io.Writer) error {
if b.bw == nil {
b.bw = brotli.NewWriterLevel(nil, b.Level)
}

b.bw.Reset(out)
_, err := io.Copy(b.bw, in)
if err != nil {
Expand All @@ -411,6 +420,10 @@ func (b *Brotli) Decompress(in []byte) ([]byte, error) {
}

func (b *Brotli) DecompressStream(in io.Reader, out io.Writer) error {
if b.br == nil {
b.br = brotli.NewReader(nil)
}

if err := b.br.Reset(in); err != nil {
return err
}
Expand Down
117 changes: 64 additions & 53 deletions compression/compression_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,45 +11,14 @@ import (
r "github.com/stretchr/testify/require"
)

type compressor struct {
name string
compressor compression.Compressor
modes []int
}

func compressionArlgorithms() []compressor {
genericModes := []int{compression.BestCompression, compression.BestSpeed, compression.NoCompression, compression.DefaultCompression, compression.HuffmanOnly}
zstdModes := []int{compression.ZstdSpeedBestCompression, compression.ZstdSpeedBetterCompression, compression.ZstdSpeedDefault, compression.ZstdSpeedFastest}
brotliModes := []int{compression.BrotliBestCompression, compression.BrotliDefaultCompression, compression.BrotliBestSpeed}

compressors := []compressor{
{
name: "zlib",
compressor: &compression.Zlib{},
modes: genericModes,
},
{
name: "gzip",
compressor: &compression.Gzip{},
modes: genericModes,
},
{
name: "zstd",
compressor: &compression.Zstd{},
modes: zstdModes,
},
{
name: "generic",
compressor: &compression.Flate{},
modes: genericModes,
},
{
name: "brotli",
compressor: &compression.Brotli{},
modes: brotliModes,
},
func compressors() []compression.Compressor {
return []compression.Compressor{
&compression.Zlib{},
&compression.Gzip{},
&compression.Zstd{},
&compression.Flate{},
&compression.Brotli{},
}
return compressors
}

type compressionSample struct {
Expand Down Expand Up @@ -93,16 +62,16 @@ func compressionSamples() []compressionSample {
}

func BenchmarkRoundTrip(b *testing.B) {
compressors := compressionArlgorithms()
compressors := compressors()
compressionSamples := compressionSamples()

for _, compressor := range compressors {
for _, mode := range compressor.modes {
compressor.compressor.SetLevel(mode)
for _, mode := range compressor.GetModes() {
compressor.SetLevel(mode)
for _, sample := range compressionSamples {
b.Run(compressor.name+"-"+strconv.Itoa(mode)+"-"+sample.name, func(b *testing.B) {
b.Run(compressor.GetName()+"/"+strconv.Itoa(mode)+"/"+sample.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
benchmarkRoundTrip(b, compressor.compressor, sample.data)
benchmarkRoundTrip(b, compressor, sample.data)
}
})
}
Expand Down Expand Up @@ -138,21 +107,38 @@ func benchmarkRoundTrip(b *testing.B, compressor compression.Compressor, data []
}

func TestRoundTrips(t *testing.T) {
compressors := compressionArlgorithms()
compressors := compressors()
compressionSamples := compressionSamples()

for _, compressor := range compressors {
for _, mode := range compressor.modes {
compressor.compressor.SetLevel(mode)
for _, mode := range compressor.GetModes() {
compressor.SetLevel(mode)
for _, sample := range compressionSamples {
t.Run(compressor.name+"-"+strconv.Itoa(mode)+"-"+sample.name, func(t *testing.T) {
testRoundTrip(t, compressor.compressor, sample.data)
t.Run(compressor.GetName()+"/"+strconv.Itoa(mode)+"/"+sample.name, func(t *testing.T) {
testRoundTrip(t, compressor, sample.data)
})
}
}
}
}

func TestInterfacelessRoundTrip(t *testing.T) {
compressors := []compression.Compressor{
&compression.Zlib{Level: compression.DefaultCompression},
&compression.Gzip{Level: compression.DefaultCompression},
&compression.Zstd{Level: compression.ZstdSpeedDefault},
&compression.Flate{Level: compression.DefaultCompression},
&compression.Brotli{Level: compression.BrotliDefaultCompression},
}
samples := compressionSamples()

for _, compressor := range compressors {
t.Run(compressor.GetName(), func(t *testing.T) {
testRoundTrip(t, compressor, samples[0].data)
})
}
}

func testRoundTrip(t *testing.T, compressor compression.Compressor, data []byte) {
compressed, err := compressor.Compress(data)
r.NoError(t, err)
Expand All @@ -166,11 +152,11 @@ func testRoundTrip(t *testing.T, compressor compression.Compressor, data []byte)

r.Equal(t, compressed, compressedBuff.Bytes())

t.Log("Compressor name: ", compressor.GetName())
t.Log("Data sample: ", data[:16])
t.Log("Orignal size: ", len(data))
t.Log("Compressed size: ", compressedBuff.Len())
t.Log("Compression mode: ", compressor.GetLevel())
t.Log("Compressor name:", compressor.GetName())
t.Log("Data sample:", data[:16])
t.Log("Orignal size:", len(data))
t.Log("Compressed size:", compressedBuff.Len())
t.Log("Compression mode:", compressor.GetLevel())
t.Log("---")
compressedReader := bytes.NewReader(compressedBuff.Bytes())

Expand Down Expand Up @@ -326,3 +312,28 @@ func TestZstdWrongDecompressData(t *testing.T) {
err = compressor.DecompressStream(reader, &compressedBuff)
r.Error(t, err)
}

func TestMissingCompressLevels(t *testing.T) {
compressors := []compression.Compressor{
&compression.Zstd{},
}
samples := compressionSamples()

for _, compressor := range compressors {
t.Run(compressor.GetName(), func(t *testing.T) {

out, err := compressor.Compress(samples[0].data)
r.ErrorIs(t, err, compression.ErrMissingCompressionLevel)
r.Nil(t, out)
})

t.Run("streaming/"+compressor.GetName(), func(t *testing.T) {
var out bytes.Buffer
reader := bytes.NewReader(samples[0].data)

err := compressor.CompressStream(reader, &out)
r.ErrorIs(t, err, compression.ErrMissingCompressionLevel)
r.Nil(t, out.Bytes())
})
}
}

0 comments on commit 71b44a8

Please sign in to comment.