From b168686d3be29e6f70d24e32c7269f94054b88b7 Mon Sep 17 00:00:00 2001 From: Jan Cajthaml Date: Mon, 1 Jul 2019 09:24:20 +0200 Subject: [PATCH] collect test coverage --- .depignore | 9 ++++++ .gitignore | 3 ++ Makefile | 17 +++++++++-- dev/lifecycle/lint | 53 ++++++++++++++++++++++++++++++++++ dev/lifecycle/sec | 33 +++++++++++++++++++++ dev/lifecycle/sync | 48 +++++++++++++++++++++++++++++++ dev/lifecycle/test | 65 ++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 53 ++++++++++++++++++++++++++++++++++ storage.go | 71 +++++++++++++++++----------------------------- storage_test.go | 4 +-- 10 files changed, 306 insertions(+), 50 deletions(-) create mode 100644 .depignore create mode 100644 .gitignore create mode 100755 dev/lifecycle/lint create mode 100755 dev/lifecycle/sec create mode 100755 dev/lifecycle/sync create mode 100755 dev/lifecycle/test create mode 100644 docker-compose.yml diff --git a/.depignore b/.depignore new file mode 100644 index 0000000..1c67adc --- /dev/null +++ b/.depignore @@ -0,0 +1,9 @@ +.git +vendor +dev +reports +.gitignore +.editorconfig +.env +.depignore +docker-compose.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..467362a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +vendor +reports +*.test diff --git a/Makefile b/Makefile index efefd5d..2eaa5c9 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,20 @@ .ONESHELL: .PHONY: all -all: test +all: sync test lint sec + +.PHONY: lint +lint: + @docker-compose run --rm lint --pkg local-fs || : + +.PHONY: sec +sec: + @docker-compose run --rm sec --pkg local-fs || : + +.PHONY: sync +sync: + @docker-compose run --rm sync --pkg local-fs .PHONY: test test: - GOMAXPROCS=1 \ - go test -v ./... -benchmem -bench=. -timeout=20s + @docker-compose run --rm test --pkg local-fs diff --git a/dev/lifecycle/lint b/dev/lifecycle/lint new file mode 100755 index 0000000..52435fb --- /dev/null +++ b/dev/lifecycle/lint @@ -0,0 +1,53 @@ +#!/bin/bash + +set -eu +trap exit INT TERM + +TARGET_PACKAGE="" + +while [ $# -gt 0 ] ; do +key="$1" + +case $key in + + --pkg) + TARGET_PACKAGE="$2" + shift + shift + ;; + + *) + shift + ;; + +esac +done + +if [ ! "${TARGET_PACKAGE}" ] ; then + (>&2 echo "[error] target package not provided") + exit 1 +fi + +scan() { + if [ ! -f ${1} ] ; then + return + fi + (gofmt -s -w ${1} || :) + (/go/bin/golint -min_confidence 0.5 ${1} || :) + (/go/bin/misspell ${1} || :) + (/go/bin/prealloc ${1} || :) + (/go/bin/gocyclo -over 15 ${1} || :) + (/go/bin/prealloc ${1} || :) + (/go/bin/goconst ${1} || :) + (/go/bin/ineffassign ${1} || :) +} + +find /go/src/github.com/jancajthaml-openbank/${TARGET_PACKAGE} \ + -name "*.go" \ + -not \ + -path "*vendor/*" \ +\ +| sort -u \ +| while read f ; do + scan ${f} +done diff --git a/dev/lifecycle/sec b/dev/lifecycle/sec new file mode 100755 index 0000000..b948a48 --- /dev/null +++ b/dev/lifecycle/sec @@ -0,0 +1,33 @@ +#!/bin/bash + +set -eu +trap exit INT TERM + +TARGET_PACKAGE="" + +while [ $# -gt 0 ] ; do +key="$1" + +case $key in + + --pkg) + TARGET_PACKAGE="$2" + shift + shift + ;; + + *) + shift + ;; + +esac +done + +if [ ! "${TARGET_PACKAGE}" ] ; then + (>&2 echo "[error] target package not provided") + exit 1 +fi + +cd /go/src/github.com/jancajthaml-openbank/${TARGET_PACKAGE} + +gosec ./... diff --git a/dev/lifecycle/sync b/dev/lifecycle/sync new file mode 100755 index 0000000..533d411 --- /dev/null +++ b/dev/lifecycle/sync @@ -0,0 +1,48 @@ +#!/bin/bash + +set -eu +trap exit INT TERM + +TARGET_PACKAGE="" + +while [ $# -gt 0 ] ; do +key="$1" + +case $key in + + --pkg) + TARGET_PACKAGE="$2" + shift + shift + ;; + + *) + shift + ;; + +esac +done + +if [ ! "${TARGET_PACKAGE}" ] ; then + (>&2 echo "[error] target package not provided") + exit 1 +fi + +cd /go/src/github.com/jancajthaml-openbank/${TARGET_PACKAGE} + +if [ -d vendor ] ; then + rm -rf vendor +fi + +if [ -f go.sum ] ; then + rm -f go.sum +fi + +GO111MODULE=on \ + go mod verify + +GO111MODULE=on \ + go mod tidy + +GO111MODULE=on \ + go mod vendor diff --git a/dev/lifecycle/test b/dev/lifecycle/test new file mode 100755 index 0000000..b743fd9 --- /dev/null +++ b/dev/lifecycle/test @@ -0,0 +1,65 @@ +#!/bin/bash + +set -e +trap exit INT TERM + +TARGET_PACKAGE="" + +while [ $# -gt 0 ] ; do +key="$1" + +case $key in + + --pkg) + TARGET_PACKAGE="$2" + shift + shift + ;; + + *) + shift + ;; + +esac +done + +if [ ! "${TARGET_PACKAGE}" ] ; then + (>&2 echo "[error] target package not provided") + exit 1 +fi + +coverage_out=$(tempfile) +test_out=$(tempfile) +reports_out=$(pwd)/reports + +mkdir -p ${reports_out} ${reports_out}/unit-tests-${TARGET_PACKAGE} + +cd /go/src/github.com/jancajthaml-openbank/${TARGET_PACKAGE} + +# run unit tests and collect coverage + +2>&1 \ + go test \ + -v ./... \ + -coverprofile=${coverage_out} \ + -coverpkg=./... \ + -timeout=20s | tee ${test_out} + +go2xunit \ + -fail \ + -input ${test_out} \ + -output ${reports_out}/unit-tests-${TARGET_PACKAGE}/results.xml + +go tool cover \ + --html=${coverage_out} \ + -o ${reports_out}/unit-tests-${TARGET_PACKAGE}/coverage.html + +# run only benchmarks + +GOMAXPROCS=1 \ + go test \ + -v ./... \ + -run=^$ \ + -bench=. \ + -benchmem \ + -timeout=1m diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8f5a95f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,53 @@ +version: '2' + +# ---------------------------------------------------------------------------- # + +services: + + # -------------------------------------------------------------------------- # + + go: + image: jancajthaml/go + volumes: + - .:/project + - .:/go/src/github.com/jancajthaml-openbank/local-fs + working_dir: /go/src/github.com/jancajthaml-openbank/local-fs + environment: + - GOOS + - GOARCH + - GOPATH=/go + - COMPOSE_PROJECT_NAME + entrypoint: ["go"] + privileged: true + + # -------------------------------------------------------------------------- # + + sync: + extends: go + entrypoint: ["/project/dev/lifecycle/sync"] + + # -------------------------------------------------------------------------- # + + update: + extends: go + entrypoint: ["/project/dev/lifecycle/update"] + + # -------------------------------------------------------------------------- # + + lint: + extends: go + entrypoint: ["/project/dev/lifecycle/lint"] + + # -------------------------------------------------------------------------- # + + sec: + extends: go + entrypoint: ["/project/dev/lifecycle/sec"] + + # -------------------------------------------------------------------------- # + + test: + extends: go + entrypoint: ["/project/dev/lifecycle/test"] + +# ---------------------------------------------------------------------------- # diff --git a/storage.go b/storage.go index 7ea111e..20a6f04 100644 --- a/storage.go +++ b/storage.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "sort" "syscall" "unsafe" @@ -46,19 +47,21 @@ func nameFromDirent(dirent *syscall.Dirent) []byte { return name } +// Storage is a fascade to access storage type Storage struct { Root string encryptionKey []byte bufferSize int } +// NewStorage returns new storage over given root func NewStorage(root string) Storage { if root == "" || os.MkdirAll(filepath.Clean(root), os.ModePerm) != nil { panic("unable to assert root storage directory") } return Storage{ Root: root, - bufferSize: 2 * os.Getpagesize(), + bufferSize: 8192, } } @@ -72,25 +75,14 @@ func (storage *Storage) SetEncryptionKey(key []byte) { // ListDirectory returns sorted slice of item names in given absolute path // default sorting is ascending -func (storage Storage) ListDirectory(absPath string, ascending bool) (result []string, err error) { - defer func() { - if r := recover(); r != nil { - if err == nil { - err = fmt.Errorf("ListDirectory paniced: %+v", r) - } - } - if err != nil { - result = nil - } - }() - +func (storage Storage) ListDirectory(path string, ascending bool) (result []string, err error) { var ( n int dh *os.File de *syscall.Dirent ) - dh, err = os.Open(filepath.Clean(storage.Root + "/" + absPath)) + dh, err = os.Open(filepath.Clean(storage.Root + "/" + path)) if err != nil { return } @@ -102,6 +94,7 @@ func (storage Storage) ListDirectory(absPath string, ascending bool) (result []s for { n, err = syscall.ReadDirent(fd, scratchBuffer) + runtime.KeepAlive(dh) if err != nil { if r := dh.Close(); r != nil { err = r @@ -156,25 +149,14 @@ func (storage Storage) ListDirectory(absPath string, ascending bool) (result []s } // CountFiles returns number of items in directory -func (storage Storage) CountFiles(absPath string) (result int, err error) { - defer func() { - if r := recover(); r != nil { - if err == nil { - err = fmt.Errorf("CountFiles paniced: %+v", r) - } - } - if err != nil { - result = -1 - } - }() - +func (storage Storage) CountFiles(path string) (result int, err error) { var ( n int dh *os.File de *syscall.Dirent ) - dh, err = os.Open(filepath.Clean(storage.Root + "/" + absPath)) + dh, err = os.Open(filepath.Clean(storage.Root + "/" + path)) if err != nil { return } @@ -185,6 +167,7 @@ func (storage Storage) CountFiles(absPath string) (result int, err error) { for { n, err = syscall.ReadDirent(fd, scratchBuffer) + runtime.KeepAlive(dh) if err != nil { if r := dh.Close(); r != nil { err = r @@ -198,11 +181,9 @@ func (storage Storage) CountFiles(absPath string) (result int, err error) { for len(buf) > 0 { de = (*syscall.Dirent)(unsafe.Pointer(&buf[0])) buf = buf[de.Reclen:] - if de.Ino == 0 || de.Type != syscall.DT_REG { continue } - result++ } } @@ -215,10 +196,10 @@ func (storage Storage) CountFiles(absPath string) (result int, err error) { } // Exists returns true if absolute path exists -func (storage Storage) Exists(absPath string) (bool, error) { +func (storage Storage) Exists(path string) (bool, error) { var ( trusted = new(syscall.Stat_t) - cleaned = filepath.Clean(storage.Root + "/" + absPath) + cleaned = filepath.Clean(storage.Root + "/" + path) err error ) err = syscall.Stat(cleaned, trusted) @@ -232,8 +213,8 @@ func (storage Storage) Exists(absPath string) (bool, error) { } // TouchFile creates files given absolute path if file does not already exist -func (storage Storage) TouchFile(absPath string) error { - cleanedPath := filepath.Clean(storage.Root + "/" + absPath) +func (storage Storage) TouchFile(path string) error { + cleanedPath := filepath.Clean(storage.Root + "/" + path) if err := os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm); err != nil { return err } @@ -246,8 +227,8 @@ func (storage Storage) TouchFile(absPath string) error { } // GetFileReader creates file io.Reader -func (storage Storage) GetFileReader(absPath string) (*fileReader, error) { - f, err := os.OpenFile(filepath.Clean(storage.Root+"/"+absPath), os.O_RDONLY, os.ModePerm) +func (storage Storage) GetFileReader(path string) (*fileReader, error) { + f, err := os.OpenFile(filepath.Clean(storage.Root+"/"+path), os.O_RDONLY, os.ModePerm) if err != nil { return nil, err } @@ -259,8 +240,8 @@ func (storage Storage) GetFileReader(absPath string) (*fileReader, error) { } // ReadFileFully reads whole file given absolute path -func (storage Storage) ReadFileFully(absPath string) ([]byte, error) { - f, err := os.OpenFile(filepath.Clean(storage.Root+"/"+absPath), os.O_RDONLY, os.ModePerm) +func (storage Storage) ReadFileFully(path string) ([]byte, error) { + f, err := os.OpenFile(filepath.Clean(storage.Root+"/"+path), os.O_RDONLY, os.ModePerm) if err != nil { return nil, err } @@ -279,8 +260,8 @@ func (storage Storage) ReadFileFully(absPath string) ([]byte, error) { // WriteFile writes data given absolute path to a file if that file does not // already exists -func (storage Storage) WriteFile(absPath string, data []byte) error { - cleanedPath := filepath.Clean(storage.Root + "/" + absPath) +func (storage Storage) WriteFile(path string, data []byte) error { + cleanedPath := filepath.Clean(storage.Root + "/" + path) if err := os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm); err != nil { return err } @@ -296,14 +277,14 @@ func (storage Storage) WriteFile(absPath string, data []byte) error { } // DeleteFile removes file given absolute path if that file does exists -func (storage Storage) DeleteFile(absPath string) error { - return os.Remove(filepath.Clean(storage.Root + "/" + absPath)) +func (storage Storage) DeleteFile(path string) error { + return os.Remove(filepath.Clean(storage.Root + "/" + path)) } // UpdateFile rewrite file with data given absolute path to a file if that file // exist -func (storage Storage) UpdateFile(absPath string, data []byte) (err error) { - cleanedPath := filepath.Clean(storage.Root + "/" + absPath) +func (storage Storage) UpdateFile(path string, data []byte) (err error) { + cleanedPath := filepath.Clean(storage.Root + "/" + path) var f *os.File f, err = os.OpenFile(cleanedPath, os.O_WRONLY|os.O_TRUNC, os.ModePerm) if err != nil { @@ -316,8 +297,8 @@ func (storage Storage) UpdateFile(absPath string, data []byte) (err error) { // AppendFile appens data given absolute path to a file, creates it if it does // not exist -func (storage Storage) AppendFile(absPath string, data []byte) (err error) { - cleanedPath := filepath.Clean(storage.Root + "/" + absPath) +func (storage Storage) AppendFile(path string, data []byte) (err error) { + cleanedPath := filepath.Clean(storage.Root + "/" + path) err = os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm) if err != nil { return err diff --git a/storage_test.go b/storage_test.go index 7854918..14a4917 100644 --- a/storage_test.go +++ b/storage_test.go @@ -19,7 +19,7 @@ func testPad(version int) string { func TestExists(t *testing.T) { tmpDir := os.TempDir() - file, err := ioutil.TempFile(tmpDir, "existant.*.tmp") + file, err := ioutil.TempFile(tmpDir, "existent.*.tmp") require.Nil(t, err) filename := file.Name() defer os.Remove(filename) @@ -138,7 +138,7 @@ func BenchmarkCountFiles(b *testing.B) { storage := NewStorage(tmpDir) - for i := 0; i < 1000; i++ { + for i := 0; i < 10000; i++ { file, err := os.Create(fmt.Sprintf("%s%010d", tmpdir, i)) require.Nil(b, err) file.Close()