Skip to content

Commit

Permalink
Merge pull request #2 from jancajthaml-openbank/feature/faster-read
Browse files Browse the repository at this point in the history 
updated tests, added makefile, spedup read, created storage fascade
  • Loading branch information
@jancajthaml
jancajthaml authored Jan 21, 2019
2 parents 1c048fb + 753c6f7 commit dfba8d3
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 39 deletions.
9 changes: 9 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
.ONESHELL:

.PHONY: all
all: test

.PHONY: test
test:
GOMAXPROCS=1 \
go test -v ./... -benchmem -bench=. -timeout=20s
51 changes: 32 additions & 19 deletions storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package utils
package storage

import (
"bytes"
Expand Down Expand Up @@ -49,9 +49,22 @@ func nameFromDirent(dirent *syscall.Dirent) []byte {
return name
}

type Storage struct {
Root string
}

func NewStorage(root string) Storage {
if root == "" || os.MkdirAll(filepath.Clean(root), os.ModePerm) != nil {
panic("unable to assert root storage directory")
}
return Storage{
Root: root,
}
}

// ListDirectory returns sorted slice of item names in given absolute path
// default sorting is ascending
func ListDirectory(absPath string, ascending bool) (result []string, err error) {
func (storage Storage) ListDirectory(absPath string, ascending bool) (result []string, err error) {
defer func() {
if r := recover(); r != nil {
if err == nil {
Expand All @@ -69,7 +82,7 @@ func ListDirectory(absPath string, ascending bool) (result []string, err error)
de *syscall.Dirent
)

dh, err = os.Open(filepath.Clean(absPath))
dh, err = os.Open(filepath.Clean(storage.Root + "/" + absPath))
if err != nil {
return
}
Expand Down Expand Up @@ -135,7 +148,7 @@ func ListDirectory(absPath string, ascending bool) (result []string, err error)
}

// CountFiles returns number of items in directory
func CountFiles(absPath string) (result int, err error) {
func (storage Storage) CountFiles(absPath string) (result int, err error) {
defer func() {
if r := recover(); r != nil {
if err == nil {
Expand All @@ -153,7 +166,7 @@ func CountFiles(absPath string) (result int, err error) {
de *syscall.Dirent
)

dh, err = os.Open(filepath.Clean(absPath))
dh, err = os.Open(filepath.Clean(storage.Root + "/" + absPath))
if err != nil {
return
}
Expand Down Expand Up @@ -194,10 +207,10 @@ func CountFiles(absPath string) (result int, err error) {
}

// Exists returns true if absolute path exists
func Exists(absPath string) (bool, error) {
func (storage Storage) Exists(absPath string) (bool, error) {
var (
trusted = new(syscall.Stat_t)
cleaned = filepath.Clean(absPath)
cleaned = filepath.Clean(storage.Root + "/" + absPath)
err error
)
err = syscall.Stat(cleaned, trusted)
Expand All @@ -211,8 +224,8 @@ func Exists(absPath string) (bool, error) {
}

// TouchFile creates files given absolute path if file does not already exist
func TouchFile(absPath string) error {
cleanedPath := filepath.Clean(absPath)
func (storage Storage) TouchFile(absPath string) error {
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
if err := os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm); err != nil {
return err
}
Expand All @@ -225,8 +238,8 @@ func TouchFile(absPath string) error {
}

// ReadFileFully reads whole file given absolute path
func ReadFileFully(absPath string) ([]byte, error) {
f, err := os.OpenFile(filepath.Clean(absPath), os.O_RDONLY, os.ModePerm)
func (storage Storage) ReadFileFully(absPath string) ([]byte, error) {
f, err := os.OpenFile(filepath.Clean(storage.Root+"/"+absPath), os.O_RDONLY, os.ModePerm)
if err != nil {
return nil, err
}
Expand All @@ -245,8 +258,8 @@ func ReadFileFully(absPath string) ([]byte, error) {

// WriteFile writes data given absolute path to a file if that file does not
// already exists
func WriteFile(absPath string, data []byte) error {
cleanedPath := filepath.Clean(absPath)
func (storage Storage) WriteFile(absPath string, data []byte) error {
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
if err := os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm); err != nil {
return err
}
Expand All @@ -262,14 +275,14 @@ func WriteFile(absPath string, data []byte) error {
}

// DeleteFile removes file given absolute path if that file does exists
func DeleteFile(absPath string) error {
return os.Remove(filepath.Clean(absPath))
func (storage Storage) DeleteFile(absPath string) error {
return os.Remove(filepath.Clean(storage.Root + "/" + absPath))
}

// UpdateFile rewrite file with data given absolute path to a file if that file
// exist
func UpdateFile(absPath string, data []byte) (err error) {
cleanedPath := filepath.Clean(absPath)
func (storage Storage) UpdateFile(absPath string, data []byte) (err error) {
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
var f *os.File
f, err = os.OpenFile(cleanedPath, os.O_WRONLY|os.O_TRUNC, os.ModePerm)
if err != nil {
Expand All @@ -282,8 +295,8 @@ func UpdateFile(absPath string, data []byte) (err error) {

// AppendFile appens data given absolute path to a file, creates it if it does
// not exist
func AppendFile(absPath string, data []byte) (err error) {
cleanedPath := filepath.Clean(absPath)
func (storage Storage) AppendFile(absPath string, data []byte) (err error) {
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
err = os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm)
if err != nil {
return err
Expand Down
57 changes: 37 additions & 20 deletions storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
Expand All @@ -22,16 +23,16 @@ func TestExists(t *testing.T) {
filename := file.Name()
defer os.Remove(filename)

var (
ok bool
fail error
)
storage := NewStorage(tmpDir)

ok, fail = Exists(filename)
var ok bool
var fail error

ok, fail = storage.Exists(filepath.Base(filename))
assert.Nil(t, fail)
assert.True(t, ok)

ok, fail = Exists(filename + "xxx")
ok, fail = storage.Exists(filepath.Base(filename + "xxx"))
assert.Nil(t, fail)
assert.False(t, ok)
}
Expand All @@ -41,6 +42,8 @@ func TestListDirectory(t *testing.T) {
require.Nil(t, err)
defer os.RemoveAll(tmpdir)

storage := NewStorage(tmpDir)

NewSlice := func(start, end, step int) []int {
if step <= 0 || end < start {
return []int{}
Expand All @@ -63,7 +66,7 @@ func TestListDirectory(t *testing.T) {
file.Close()
}

list, err := ListDirectory(tmpdir, true)
list, err := storage.ListDirectory(filepath.Base(tmpdir), true)
require.Nil(t, err)

assert.NotNil(t, list)
Expand All @@ -77,8 +80,7 @@ func TestCountFiles(t *testing.T) {
require.Nil(t, err)
defer os.RemoveAll(tmpdir)

require.Nil(t, os.MkdirAll(tmpdir, os.ModePerm))
defer os.RemoveAll(tmpdir)
storage := NewStorage(tmpDir)

for i := 0; i < 60; i++ {
file, err := os.Create(tmpdir + "/" + testPad(i) + "F")
Expand All @@ -91,7 +93,7 @@ func TestCountFiles(t *testing.T) {
require.Nil(t, err)
}

numberOfFiles, err := CountFiles(tmpdir)
numberOfFiles, err := storage.CountFiles(filepath.Base(tmpdir))
require.Nil(t, err)
assert.Equal(t, 60, numberOfFiles)
}
Expand All @@ -101,19 +103,20 @@ func BenchmarkCountFiles(b *testing.B) {
require.Nil(b, err)
defer os.RemoveAll(tmpdir)

os.MkdirAll(tmpdir, os.ModePerm)
defer os.RemoveAll(tmpdir)
storage := NewStorage(tmpDir)

for i := 0; i < 1000; i++ {
file, err := os.Create(fmt.Sprintf("%s%010d", tmpdir, i))
require.Nil(b, err)
file.Close()
}

basePath := filepath.Base(tmpdir)

b.ResetTimer()
b.ReportAllocs()
for n := 0; n < b.N; n++ {
CountFiles(tmpdir)
storage.CountFiles(basePath)
}
}

Expand All @@ -122,16 +125,20 @@ func BenchmarkListDirectory(b *testing.B) {
require.Nil(b, err)
defer os.RemoveAll(tmpdir)

storage := NewStorage(tmpDir)

for i := 0; i < 1000; i++ {
file, err := os.Create(fmt.Sprintf("%s%010d", tmpdir, i))
require.Nil(b, err)
file.Close()
}

basePath := filepath.Base(tmpdir)

b.ResetTimer()
b.ReportAllocs()
for n := 0; n < b.N; n++ {
ListDirectory(tmpdir, true)
storage.ListDirectory(basePath, true)
}
}

Expand All @@ -141,10 +148,13 @@ func BenchmarkExists(b *testing.B) {
filename := file.Name()
defer os.Remove(filename)

storage := NewStorage(tmpDir)
basePath := filepath.Base(filename)

b.ResetTimer()
b.ReportAllocs()
for n := 0; n < b.N; n++ {
Exists(filename)
storage.Exists(basePath)
}
}

Expand All @@ -154,12 +164,14 @@ func BenchmarkUpdateFile(b *testing.B) {
filename := file.Name()
defer os.Remove(filename)

storage := NewStorage(tmpDir)
basePath := filepath.Base(filename)
data := []byte("abcd")

b.ResetTimer()
b.ReportAllocs()
for n := 0; n < b.N; n++ {
UpdateFile(filename, data)
storage.UpdateFile(basePath, data)
}
}

Expand All @@ -169,26 +181,31 @@ func BenchmarkAppendFile(b *testing.B) {
filename := file.Name()
defer os.Remove(filename)

storage := NewStorage(tmpDir)
basePath := filepath.Base(filename)
data := []byte("abcd")

b.ResetTimer()
b.ReportAllocs()
for n := 0; n < b.N; n++ {
AppendFile(filename, data)
storage.AppendFile(basePath, data)
}
}

func BenchmarkReadFileFully(b *testing.B) {
file, err := ioutil.TempFile(tmpDir, "appended.*")
file, err := ioutil.TempFile(tmpDir, "updated.*")
require.Nil(b, err)
filename := file.Name()
defer os.Remove(filename)

require.Nil(b, UpdateFile(filename, []byte("abcd")))
storage := NewStorage(tmpDir)
basePath := filepath.Base(filename)

require.Nil(b, storage.UpdateFile(basePath, []byte("abcd")))

b.ResetTimer()
b.ReportAllocs()
for n := 0; n < b.N; n++ {
ReadFileFully(filename)
storage.ReadFileFully(basePath)
}
}

0 comments on commit dfba8d3

Please sign in to comment.