From 62c4633e93123a1d70260c3cf6baa7e22743afd5 Mon Sep 17 00:00:00 2001 From: tunabay <55529452+tunabay@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:31:29 +0900 Subject: [PATCH] Initial version. --- .ci/golangci-lint.yml | 99 +++++ .github/workflows/go-test.yml | 34 ++ .github/workflows/golangci-lint.yml | 31 ++ LICENSE | 18 + README.md | 25 +- cache.go | 639 ++++++++++++++++++++++++++++ config.go | 66 +++ doc.go | 9 + error.go | 14 + example/README.md | 18 + example/imgsv/image.go | 109 +++++ example/imgsv/index.html | 46 ++ example/imgsv/main.go | 73 ++++ example/imgsv/param.go | 62 +++ example/imgsv/server.go | 264 ++++++++++++ file.go | 88 ++++ go.mod | 8 + go.sum | 5 + hex.go | 15 + key.go | 71 ++++ 20 files changed, 1693 insertions(+), 1 deletion(-) create mode 100644 .ci/golangci-lint.yml create mode 100644 .github/workflows/go-test.yml create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 LICENSE create mode 100644 cache.go create mode 100644 config.go create mode 100644 doc.go create mode 100644 error.go create mode 100644 example/README.md create mode 100644 example/imgsv/image.go create mode 100644 example/imgsv/index.html create mode 100644 example/imgsv/main.go create mode 100644 example/imgsv/param.go create mode 100644 example/imgsv/server.go create mode 100644 file.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hex.go create mode 100644 key.go diff --git a/.ci/golangci-lint.yml b/.ci/golangci-lint.yml new file mode 100644 index 0000000..866a525 --- /dev/null +++ b/.ci/golangci-lint.yml @@ -0,0 +1,99 @@ +run: + timeout: 5m + tests: true + fast: false + skip-dirs-use-default: true + print-issued-lines: true + print-linter-name: true + +linters: + disable-all: true + fast: false + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + + - asasalint + - asciicheck + - bidichk + - bodyclose + - containedctx + - contextcheck + - dogsled + - dupword + - errchkjson + - errname + - errorlint + - exportloopref + - forbidigo + - forcetypeassert + - goconst + - gocritic + - goerr113 + - gofmt + - gofumpt + - goimports + - gomodguard + - goprintffuncname + - gosec + - importas + - misspell + - noctx + - nolintlint + - nosprintfhostport + - prealloc + - predeclared + - reassign + - revive + - stylecheck + - testpackage + - testableexamples + - thelper + - unconvert + - unparam + - usestdlibvars + - whitespace + - wrapcheck + +linters-settings: + gofumpt: + lang-version: "1.19" + gosimple: + go: "1.19" + staticcheck: + go: "1.19" + stylecheck: + go: "1.19" + unused: + go: "1.19" + + misspell: + locale: US + + errcheck: + exclude-functions: + - io/ioutil.ReadFile + - io.Copy(*bytes.Buffer) + - io.Copy(os.Stdout) + - (*github.com/tunabay/go-bitarray.Builder).WriteBit + - (*github.com/tunabay/go-bitarray.Builder).WriteByte + - (*github.com/tunabay/go-bitarray.Builder).WriteBitArray + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 + fix: false + + exclude-use-default: true + exclude-rules: + + # ignore in unit tests + - linters: [ gosec, goerr113, ifshort ] + path: "_test\\.go$" + - linters: [ staticcheck ] + text: "^SA9003: empty branch" diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml new file mode 100644 index 0000000..7b5144b --- /dev/null +++ b/.github/workflows/go-test.yml @@ -0,0 +1,34 @@ +name: go-test +on: + push: + tags: + - v* + branches-ignore: + - 'doc-*' + - 'doc/*' + pull_request: + branches: + - main + - master + - release +jobs: + go-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: ^1.19 + - uses: actions/checkout@v3 + - name: go-test + run: | + go test -v -count=1 \ + -covermode=count \ + -coverpkg=github.com/tunabay/go-filecache \ + -coverprofile=cover.out \ + ./... + go tool cover -func=cover.out + go tool cover -html=cover.out -o go-test-coverage.html + - uses: actions/upload-artifact@v3 + with: + path: go-test-coverage.html + retention-days: 3 diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..a6286b9 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,31 @@ +name: golangci-lint +on: + push: + tags: + - v* + branches-ignore: + - 'doc-*' + - 'doc/*' + pull_request: + branches: + - main + - master + - release +jobs: + golangci: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: ^1.19 + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.50 + only-new-issues: true + skip-go-installation: true + args: >- + --verbose + --issues-exit-code=1 + --config=.ci/golangci-lint.yml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f74946a --- /dev/null +++ b/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2022 Hirotsuna Mizuno - https://github.com/tunabay + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index fc81063..f9e1b91 100644 --- a/README.md +++ b/README.md @@ -1 +1,24 @@ -# go-filecache \ No newline at end of file +# go-filecache + +[![Go Reference](https://pkg.go.dev/badge/github.com/tunabay/go-filecache.svg)](https://pkg.go.dev/github.com/tunabay/go-filecache) +[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE) + + +## Overview + +Package filecache provides a LRU file caching mechanism to cache resources to +the local disk that take a long time to generate or download from the network. + +The creation process runs only once even if multiple go-routines concurrently +request for a key that does not exist in the cache. + + +## Documentation + +- Read the [documentation](https://pkg.go.dev/github.com/tunabay/go-filecache). + + +## License + +go-filecache is available under the MIT license. See the [LICENSE](LICENSE) file +for more information. diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..364058f --- /dev/null +++ b/cache.go @@ -0,0 +1,639 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package filecache + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "github.com/petar/GoLLRB/llrb" + "github.com/tunabay/go-infounit" +) + +// Cache represents the file cache directory. +type Cache[K Key] struct { + dir string + create CreateFunc[K] + maxFiles uint64 + maxSize infounit.ByteCount + maxAge time.Duration + gcInterval time.Duration + + numFiles uint64 + totalSize infounit.ByteCount + numRequested uint64 + numHit uint64 + numCreated uint64 + numFailed uint64 + numRemoved uint64 + + opMap map[Hash]*opEntry + refMap map[Hash]int + cond *sync.Cond + mu sync.Mutex + + log Logger + debugLog bool +} + +func (c *Cache[_]) ref(hash Hash) { + c.mu.Lock() + defer c.mu.Unlock() + v := c.refMap[hash] + c.refMap[hash] = v + 1 +} + +func (c *Cache[K]) unref(hash Hash) { + c.mu.Lock() + defer c.mu.Unlock() + v := c.refMap[hash] + if v == 1 { + delete(c.refMap, hash) + } else { + c.refMap[hash] = v - 1 + } +} + +// CreateFunc represents a function for file creation. It will be called when +// Get is called for a new key that does not exist in the cache. It should +// create the file using the pre-opened file argument passed. It does not have +// to close the file, while it will be closed automatically after return. +type CreateFunc[K Key] func(K, *os.File) error + +// New create a cache with the default configuration. +func New[K Key](dir string, create CreateFunc[K]) (*Cache[K], error) { + return NewWithConfig[K]( + &Config[K]{ + Dir: dir, + Create: create, + MaxSize: infounit.Gigabyte, + MaxFiles: 512, + MaxAge: time.Hour * 24, + }, + ) +} + +// NewWithConfig create a cache using the given configuration parameters. +func NewWithConfig[K Key](conf *Config[K]) (*Cache[K], error) { + switch { + case conf.Dir == "": + return nil, fmt.Errorf("%w: empty Dir", ErrInvalidConfig) + case conf.Create == nil: + return nil, fmt.Errorf("%w: nil Create", ErrInvalidConfig) + case conf.MaxAge < 0: + return nil, fmt.Errorf("%w: negative MaxAge", ErrInvalidConfig) + case conf.GCInterval < 0: + return nil, fmt.Errorf("%w: negative GCInterval", ErrInvalidConfig) + } + + c := &Cache[K]{ + dir: conf.Dir, + create: conf.Create, + maxFiles: conf.MaxFiles, + maxSize: conf.MaxSize, + maxAge: conf.MaxAge, + gcInterval: conf.GCInterval, + + opMap: make(map[Hash]*opEntry), + refMap: make(map[Hash]int), + + log: conf.Logger, + debugLog: conf.DebugLog, + } + c.cond = sync.NewCond(&c.mu) + + if c.gcInterval == 0 { + c.gcInterval = defaultGCInterval + } + + if c.dir == "" { + c.dir = filepath.Base(os.Args[0]) + } + if !filepath.IsAbs(c.dir) { + ucd, err := os.UserCacheDir() + if err != nil { + return nil, fmt.Errorf("%s: can not resolve relative cache dir: %w", c.dir, err) + } + c.dir = filepath.Join(ucd, c.dir) + } + + if err := os.MkdirAll(c.dir, 0o0700); err != nil { + return nil, fmt.Errorf("%s: %w", c.dir, err) + } + c.logPrintf("Cache directory: %s", c.dir) + + // read dir, count total size, total files. + var ( + numRemoved uint64 + sizeRemoved infounit.ByteCount + ) + walker := func(path string, d fs.DirEntry, err error) error { + switch { + case err != nil: + c.logPrintf("%s: Skip unreadable file.", path) + return fs.SkipDir + case d.IsDir(): + return nil + } + fname := d.Name() + if len(fname) != HashSize*2 { + c.logPrintf("%s: Skip unexpected file in cache dir.", fname) + return nil + } + if _, err := hex.DecodeString(fname); err != nil { + c.logPrintf("%s: Skip unexpected file in cache dir.", fname) + return nil + } + finfo, err := d.Info() + sz := infounit.ByteCount(finfo.Size()) + if err != nil { + c.logPrintf("%s: Failed to stat: %v", path, err) + return nil + } + age := time.Since(finfo.ModTime()) + if c.maxAge < age { + if err := os.Remove(path); err != nil { + c.logPrintf("%s: Failed to remove expired cache: %v", path, err) + return nil + } + c.logPrintf("%s: Removed expired cache. size=%.1S, age=%v", fname, sz, age) + numRemoved++ + sizeRemoved += sz + return nil + } + c.numFiles++ + c.totalSize += sz + c.logDebugf("%s: Cache found. size=%.1S, age=%v", fname, sz, age) + + return nil + } + if err := filepath.WalkDir(c.dir, walker); err != nil { + c.logPrintf("%s: Failed to read cache dir: %v", c.dir, err) + return nil, fmt.Errorf("%s: failed to read cache dir: %w", c.dir, err) + } + if numRemoved != 0 { + c.logPrintf("Removed %d expired cache files. total=%.1S", numRemoved, sizeRemoved) + } + if c.numFiles != 0 { + c.logPrintf("Found %d cache files. total=%.1S", c.numFiles, c.totalSize) + } + + return c, nil +} + +// Serve serves the Cache instance. It performs find and delete old cache files. +func (c *Cache[K]) Serve(ctx context.Context) error { + rmCache := func(hash Hash, path string, lastMod time.Time) error { + c.mu.Lock() + if _, refed := c.refMap[hash]; refed { + c.mu.Unlock() + return nil // concurrently read + } + finfo, err := os.Stat(path) + if err != nil { + return nil // file disappeared? + } + if !lastMod.Equal(finfo.ModTime()) { + return nil // concurrently accessed + } + op := &opEntry{opType: 1, done: make(chan struct{})} + c.opMap[hash] = op + c.mu.Unlock() + + if err := os.Remove(path); err != nil { + c.mu.Lock() + delete(c.opMap, hash) + c.mu.Unlock() + close(op.done) + + return fmt.Errorf("%x: %w", hash[:], err) + } + c.logPrintf("%x: Removed.", hash[:]) // successfully removed + + c.mu.Lock() + delete(c.opMap, hash) + c.numRemoved++ + c.numFiles-- + c.totalSize -= infounit.ByteCount(finfo.Size()) + c.mu.Unlock() + close(op.done) + + return nil + } + + zeroCand := &candidate{} + for { + c.mu.Lock() + for c.numFiles <= c.maxFiles && c.totalSize <= c.maxSize && ctx.Err() == nil { + c.cond.Wait() + } + c.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil + } + + c.logDebugf("Started GC...") + + // build candidates + var maxCands uint64 = 64 + for c.numFiles+maxCands < c.maxFiles { + maxCands <<= 1 + } + tree := llrb.New() + + walker := func(path string, d fs.DirEntry, err error) error { + switch { + case err != nil: + return fs.SkipDir + case d.IsDir(): + return nil + } + fname := d.Name() + if len(fname) != HashSize*2 { + return nil + } + fhashb, err := hex.DecodeString(fname) + if err != nil { + return nil + } + var fhash Hash + copy(fhash[:], fhashb) + + finfo, err := d.Info() + if err != nil { + return nil + } + if age := time.Since(finfo.ModTime()); c.maxAge < age { + if err := rmCache(fhash, path, finfo.ModTime()); err != nil { + c.logPrintf("%s: Failed to remove expired cache: %v", path, err) + } + return nil + } + cand := &candidate{ + hash: fhash, + path: path, + lastMod: finfo.ModTime(), + } + tree.InsertNoReplace(cand) + if maxCands < uint64(tree.Len()) { + tree.DeleteMax() + } + return nil + } + if err := filepath.WalkDir(c.dir, walker); err != nil { + continue // failed to read + } + + candList := make([]*candidate, tree.Len()) + n := 0 + iterator := func(iif llrb.Item) bool { + candList[n] = iif.(*candidate) //nolint:forcetypeassert + n++ + return true + } + tree.AscendGreaterOrEqual(zeroCand, iterator) + + for _, cand := range candList { + c.mu.Lock() + overflow := c.maxFiles < c.numFiles || c.maxSize < c.totalSize + if !overflow { + c.mu.Unlock() + break + } + c.mu.Unlock() + + if err := rmCache(cand.hash, cand.path, cand.lastMod); err != nil { + c.logPrintf("%s: Failed to remove expired cache: %v", cand.path, err) + continue + } + } + c.logDebugf("GC finished.") + + // wait for the next + timer := time.NewTimer(c.gcInterval) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return nil + case <-timer.C: + } + } +} + +// candidate represents a candidate file for deletion in the cache directory. +// Among these candidates, those with the oldest lastMod will be deleted in +// order. +type candidate struct { + hash Hash + path string + lastMod time.Time +} + +// Less compares the lastMod values of the two candidates and reports the +// result. +func (c *candidate) Less(xif llrb.Item) bool { + x := xif.(*candidate) //nolint:forcetypeassert + return c.lastMod.Before(x.lastMod) +} + +// Get gets the file for the key from the cache. If the file for the specified +// key does not exist in the cache, it will call the CreateFunc to create the +// new file. It returns the file opened for read, cached or not. +// +// The returned file is guaranteed to remain referenced until it is closed and +// not removed from the cache during that time. After using the file, it is the +// caller's responsibility to call the File.Close() method of the returned file. +// Otherwise the file will remain in the cache, and the reference will remain in +// the memory. +func (c *Cache[K]) Get(key K) (*File[K], bool, error) { + hash := key.Hash() + + c.logDebugf("Get: key=%q", key.String()) + + dir, path := c.filePath(hash) + + var ( + created bool + lastMod time.Time + ) + for isRetry := false; ; isRetry = true { + c.mu.Lock() + if !isRetry { + c.numRequested++ + } + op, ok := c.opMap[hash] + switch { + case ok && op.opType == 0: + // concurrently being created + c.numHit++ + c.mu.Unlock() + c.logDebugf("Get: File is being created concurrently, waiting for completion...") + <-op.done + if op.err != nil { + return nil, false, op.err + } + // file exists, which is just created + + case ok: + // concurrently being removed + c.mu.Unlock() + if isRetry { + return nil, false, fmt.Errorf("%w: removal twice", ErrInternal) + } + c.logDebugf("Get: File is being deleted concurrently, waiting for completion...") + <-op.done + continue + + default: + // no concurrent operation + if cinfo, err := os.Stat(path); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + c.numFailed++ + c.mu.Unlock() + return nil, false, fmt.Errorf("internal error, stat failed: %w", err) + } + + // file does not exist + c.logDebugf("Get: File does not exist, creating...") + op = &opEntry{done: make(chan struct{})} + c.opMap[hash] = op + c.mu.Unlock() + + if err := os.MkdirAll(dir, 0o0700); err != nil { + op.err = fmt.Errorf("%s: failed to create: %w", dir, err) + _ = os.Remove(path) + c.mu.Lock() + delete(c.opMap, hash) + c.numFailed++ + c.mu.Unlock() + close(op.done) + + return nil, false, op.err + } + + tmpPath := path + ".tmp" + + f, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o0644) + if err != nil { + op.err = fmt.Errorf("failed to open file: %w", err) + _ = os.Remove(tmpPath) + _ = os.Remove(path) + c.mu.Lock() + delete(c.opMap, hash) + c.numFailed++ + c.mu.Unlock() + close(op.done) + + return nil, false, op.err + } + if err := c.create(key, f); err != nil { + op.err = fmt.Errorf("failed to create file: %w", err) + _ = f.Close() + _ = os.Remove(tmpPath) + _ = os.Remove(path) + c.mu.Lock() + delete(c.opMap, hash) + c.numFailed++ + c.mu.Unlock() + close(op.done) + + return nil, false, op.err + } + _ = f.Close() + + var sz infounit.ByteCount + finfo, err := os.Stat(tmpPath) + if err != nil { + op.err = fmt.Errorf("failed to stat file: %w", err) + _ = os.Remove(tmpPath) + _ = os.Remove(path) + c.mu.Lock() + delete(c.opMap, hash) + c.numFailed++ + c.mu.Unlock() + close(op.done) + + return nil, false, op.err + } + sz = infounit.ByteCount(finfo.Size()) + + if err := os.Rename(tmpPath, path); err != nil { + op.err = fmt.Errorf("failed to write file: %w", err) + _ = os.Remove(tmpPath) + _ = os.Remove(path) + c.mu.Lock() + delete(c.opMap, hash) + c.numFailed++ + c.mu.Unlock() + close(op.done) + + return nil, false, op.err + } + + // file created + c.logPrintf("Get: File successfully created and cached. size=%d", sz) + c.mu.Lock() + c.numFiles++ + c.totalSize += sz + delete(c.opMap, hash) + c.cond.Broadcast() + c.numCreated++ + c.mu.Unlock() + close(op.done) + created = true + } else { + // file exists + c.logDebugf("Get: Cache exists.") + lastMod = cinfo.ModTime() + tnow := time.Now() + _ = os.Chtimes(path, tnow, tnow) + c.numHit++ + c.mu.Unlock() + } + } + break + } + + osFile, err := os.Open(path) // O_RDONLY + if err != nil { + _ = osFile.Close() + return nil, !created, fmt.Errorf("failed to open: %w", err) + } + finfo, err := osFile.Stat() + if err != nil { + _ = osFile.Close() + return nil, !created, fmt.Errorf("failed to stat: %w", err) + } + if lastMod.IsZero() { + lastMod = finfo.ModTime() + } + + file := &File[K]{ + parent: c, + key: key, + hash: hash, + file: osFile, + size: finfo.Size(), + lastMod: lastMod, + } + c.ref(hash) + + return file, !created, nil +} + +// opEntry represents the currently processing operation on a cache entry. When +// accessing an entry, if an another goroutine is processing it, it uses the +// done channel to wait for that processing to complete. +type opEntry struct { + opType uint8 // 0: creating, 1: removing + done chan struct{} // closed when operation done + err error +} + +// filePath returns the full path of the cache file corresponding to the given +// hash value. +func (c *Cache[_]) filePath(hash Hash) (dir, path string) { + dir = filepath.Join(c.dir, b2hex(hash[HashSize-1]), b2hex(hash[HashSize-2])) + path = filepath.Join(dir, hashHex(hash)) + return +} + +// Status represents the cache status and statistics. +type Status struct { + NumFiles uint64 // number of files currently in cache. + TotalSize infounit.ByteCount // total size of files currently in cache. + NumRequested uint64 // total number of files requested. + NumHit uint64 // total number of cache hits. + NumCreated uint64 // total number of newly created cache files. + NumFailed uint64 // total number of operation failures. + NumRemoved uint64 // total number of removed cache files. + NumOps int // number of operations currently being processed. + NumRefs int // number of currently referenced cache files. +} + +// String returns the string representation of Status. +func (s Status) String() string { + return fmt.Sprintf( + "files=%d, size=%.1S, req=%d, hit=%d, new=%d, fail=%d, del=%d, op=%d, ref=%d", + s.NumFiles, + s.TotalSize, + s.NumRequested, + s.NumHit, + s.NumCreated, + s.NumFailed, + s.NumRemoved, + s.NumOps, + s.NumRefs, + ) +} + +// Status returns the current cache status and statistics. +func (c *Cache[_]) Status() *Status { + c.mu.Lock() + defer c.mu.Unlock() + return &Status{ + NumFiles: c.numFiles, + TotalSize: c.totalSize, + NumRequested: c.numRequested, + NumHit: c.numHit, + NumCreated: c.numCreated, + NumFailed: c.numFailed, + NumRemoved: c.numRemoved, + NumOps: len(c.opMap), + NumRefs: len(c.refMap), + } +} + +// logPrefix returns the prefix string for log messages, according to the +// current configuration. +func (c *Cache[_]) logPrefix() string { + if !c.debugLog { + return "" + } + if _, file, line, ok := runtime.Caller(2); ok { + return fmt.Sprintf("%s:%d:", filepath.Base(file), line) + } + return "(unknown):" +} + +// logPrintf outputs a log message according to the current configuration. +func (c *Cache[_]) logPrintf(format string, v ...any) { + if c.log == nil { + return + } + s := make([]string, 0, 2) + if prefix := c.logPrefix(); prefix != "" { + s = append(s, prefix) + } + s = append(s, fmt.Sprintf(format, v...)) + + c.log.FileCacheLog(strings.Join(s, " ")) +} + +// logDebugf outputs a debug log message according to the current configuration. +func (c *Cache[_]) logDebugf(format string, v ...any) { + if c.log == nil || !c.debugLog { + return + } + + s := make([]string, 0, 2) + if prefix := c.logPrefix(); prefix != "" { + s = append(s, prefix) + } + s = append(s, fmt.Sprintf(format, v...)) + + c.log.FileCacheLog(strings.Join(s, " ")) +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..fc2fbe2 --- /dev/null +++ b/config.go @@ -0,0 +1,66 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package filecache + +import ( + "time" + + "github.com/tunabay/go-infounit" +) + +// Config represents the parameters to configure Cache creation. +type Config[K Key] struct { + // The path to the directory for cache files. It should be a dedicated + // directory used exclusively for this cache. The directory will be + // automatically created if it does not exist. Both absolute and + // relative paths are allowed. A relative path is treated as relative + // from the user-specific cache directory returned by os.UserCacheDir(). + // If it is empty, use the program name directory. + Dir string + + // The callback function that is called when a not-cached resource is + // requested. + Create CreateFunc[K] + + // The upper limit on the number of files that can be cached. Zero + // value means unlimited. When more than this number of files are + // cached, the oldest files will be removed. Note that more than this + // number of files may be cached temporarily. + MaxFiles uint64 + + // The limit on the total size of files that can be cached. Zero + // value means unlimited. When more than this total size of files are + // cached, the oldest files will be removed. Note that more than this + // size of files may be cached temporarily. There is no guarantee that + // more disk space than this will not be used. + MaxSize infounit.ByteCount + + // The maximum age of cache files. Note that it is the time since + // last access, not the time since creation. Also the cache is not + // removed immediately after this age. It is still possible that an + // aged cache file will continue to be hit and reused. Zero value + // means unlimited. + MaxAge time.Duration + + // The interval between GC processing to find and remove old cache + // files that exceed the configured limits. + GCInterval time.Duration + + // If not nil, Cache outputs log messages to this Logger object. + Logger Logger + + // If true, Cache outputs debug log messages. Only effective if + // Logger is not nil. + DebugLog bool +} + +// defaultGCInterval defines the default value for Config.GCInterval. +const defaultGCInterval = time.Minute + +// Logger is the interface implemented to receive log messages from the running +// Cache instance. +type Logger interface { + FileCacheLog(string) +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..79ac815 --- /dev/null +++ b/doc.go @@ -0,0 +1,9 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +/* +Package filecache provides a LRU file caching mechanism to cache resources to +the local disk that take a long time to generate or download from the network. +*/ +package filecache diff --git a/error.go b/error.go new file mode 100644 index 0000000..4b4e930 --- /dev/null +++ b/error.go @@ -0,0 +1,14 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package filecache + +import "errors" + +// ErrInvalidConfig is the error thrown when the passed configuration parameter +// is not valid. +var ErrInvalidConfig = errors.New("invalid config") + +// ErrInternal is the error thrown when an internal error occurred. +var ErrInternal = errors.New("internal error") diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..70e75e7 --- /dev/null +++ b/example/README.md @@ -0,0 +1,18 @@ +# Example image server + +## Run the image server + +``` +go run ./imgsv :8080 +``` + +or + +``` +go build -o ./imgsv.bin ./imgsv +./imgsv.bin :8080 +``` + +## Access with your browser + +http://127.0.0.1:8080/ diff --git a/example/imgsv/image.go b/example/imgsv/image.go new file mode 100644 index 0000000..48f48ad --- /dev/null +++ b/example/imgsv/image.go @@ -0,0 +1,109 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package main + +import ( + "fmt" + "image" + "image/color" + "image/png" + "math" + "os" + "sync" +) + +// createImage is the callback function that will be called when an image that +// is not in the cache is requested. It receives an image parameter and an +// opened os.File object as arguments. It generates an image and writes it to +// the file. The passed file is automatically closed after return, so there is +// no need to close it here. +// +// The calculation code in this function simply generates a deterministic +// geometric pattern based on the given parameters and output it as a PNG, and +// has nothing to do with caching. The important point is that it takes a long +// time to process and outputs the result to a file. Data written to the file +// will be automatically cached and reused for requests with the same +// parameters. +func createImage(p *imgParam, file *os.File) error { + // Calculate coefficients from the parameters. + var ( + w, h = int(p.width), int(p.height) + tc = float64(p.twist) * math.Pi / 180 + sc = float64(p.stripe) / math.Pi + cc = float64(p.circleWidth) * .005 + ic = make([]color.NRGBA, 257) + bc float64 + ) + xc, yc, zc := float64(-1), -float64(h)/float64(w), .125/float64(w) + if w < h { + xc, yc, zc = -float64(w)/float64(h), -1, .125/float64(h) + } + + // Prepare colors for the image pixels. + eo := func(i int) float64 { + ev := float64(p.color[i]) / 255 + if ev <= .04045 { + return ev / 12.92 + } + return math.Pow(math.FMA(ev, 1/1.055, .055/1.055), 2.4) + } + const ax, ay, az = .2126755, .71513641, .072188085 + if eo(0)*ax+eo(1)*ay+eo(2)*az+.05 < math.Sqrt(.0525) { + bc = 255 + } + c := func(i, t int) byte { + v := math.FMA(float64(p.color[i])-bc, float64(t)/256, bc) + return byte(math.Round(v)) + } + for i := range ic { + ic[i] = color.NRGBA{R: c(0, i), G: c(1, i), B: c(2, i), A: 0xff} + } + + // Create image and paint pixels. + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + pc := func(x, y float64) int { + r := math.Hypot(x, y) + a := math.Mod(math.Atan2(y, x)+r*tc, math.Pi*2) + math.Pi*2 + s := int(math.Floor(a * sc)) + if r < .05 { + return s & 1 + } + for cr := float64(0); cr < 1.5; cr += .25 { + if math.Abs(r-cr) < cc { + return s & 1 + } + } + return ^s & 1 + } + paintRow := func(py int) { + py4 := py << 4 + for px := 0; px < w; px++ { + px4, n := px<<4, 0 + for v := 0; v < 16; v++ { + y := math.FMA(float64(py4+v), zc, yc) + for u := 0; u < 16; u++ { + n += pc(math.FMA(float64(px4+u), zc, xc), y) + } + } + img.SetNRGBA(px, py, ic[n]) + } + } + var wg sync.WaitGroup + for y := 0; y < h; y++ { + wg.Add(1) + go func(py int) { + defer wg.Done() + paintRow(py) + }(y) + } + wg.Wait() + + // Encode image as PNG and write to the file. + if err := png.Encode(file, img); err != nil { + return fmt.Errorf("failed to encode PNG: %w", err) + } + + return nil +} diff --git a/example/imgsv/index.html b/example/imgsv/index.html new file mode 100644 index 0000000..04ee1de --- /dev/null +++ b/example/imgsv/index.html @@ -0,0 +1,46 @@ + + + +go-filecache example + + + +

go-filecache example

+

This is an example image server. /image.png generates and caches images.

+ +

Examples:

+ +

Parameters:

+
width
Image width in pixel. [16..]
+
height
Image height in pixel. [16..]
+
color
Key color in RRGGBB hex format.
+
twist
Amount to twist in degrees, can be negative.
+
circle-width
Line thickness of the concentric circles. [1..16]
+
stripe
Number of radiation stripes. [1..90]
+
+ +

Inline images:

+ + + +
+ + + +
+ + + + + + diff --git a/example/imgsv/main.go b/example/imgsv/main.go new file mode 100644 index 0000000..4dc5071 --- /dev/null +++ b/example/imgsv/main.go @@ -0,0 +1,73 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "strings" + "time" +) + +// main is the main function of this example program. A simple image web server +// that serves dynamically generated images over HTTP. +// +// Generating the image takes some time, so the response to the first request is +// a bit delayed. However, subsequent requests with the same parameters will +// return the cached image, resulting in a faster response. +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + // Parse command parameters. + listenAddr := ":8080" + switch { + case len(os.Args) == 1: + // use default addr + + case 2 < len(os.Args), strings.HasPrefix(strings.TrimLeft(os.Args[1], "-"), "h"): + fmt.Fprintf(os.Stderr, "USAGE: %s [ [host]:port ]\n", os.Args[0]) + return + + default: + listenAddr = os.Args[1] + } + + // Create and run the image server. + sv, err := newServer() + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: server: %v\n", err) + return + } + go func() { + if err := sv.serve(ctx); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: server: %v\n", err) + os.Exit(1) + } + }() + + // Create and run the HTTP server. + httpd := &http.Server{ + Addr: listenAddr, + Handler: sv, + ReadTimeout: time.Second * 10, + WriteTimeout: time.Minute, + MaxHeaderBytes: 512, + } + go func() { + <-ctx.Done() + sdctx, sdcancel := context.WithTimeout(context.Background(), time.Second*5) + defer sdcancel() + if err := httpd.Shutdown(sdctx); err != nil { //nolint:contextcheck + fmt.Fprintf(os.Stderr, "ERROR: httpd: %v\n", err) + } + }() + if err := httpd.ListenAndServe(); err != nil { + fmt.Fprintf(os.Stderr, "httpd: %v\n", err) + } +} diff --git a/example/imgsv/param.go b/example/imgsv/param.go new file mode 100644 index 0000000..9e87829 --- /dev/null +++ b/example/imgsv/param.go @@ -0,0 +1,62 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package main + +import ( + "crypto/sha512" + "encoding/binary" + "fmt" + + "github.com/tunabay/go-filecache" +) + +// imgParam represents the set of parameters to generate image. Since the same +// image is always generated from the same parameter set, the image is cached +// using this imgParam as a key. +type imgParam struct { + color [3]byte // RRGGBB value of the key color. + width, height uint16 // image dimension in pixel, at least 16x16. + twist int16 // amount to twist in degrees, can be negative. + circleWidth uint16 // line thickness of the concentric circles, 1..16. + stripe uint16 // number of radiation stripes, 1..90. +} + +// String returns the string representation of the parameter set. It implements +// filecache.Key interface so that it can be used as a cache key. +// +// It is only used for logging and file information, and is not that important. +// Still, it is required to implement the filecache.Key interface. Also not all +// fields need to be included in the return value, as it is not used for hash +// calculation. +func (p *imgParam) String() string { + return fmt.Sprintf( + "size=%dx%d, color=#%x, twist=%d, circle-width=%d, stripe=%d", + p.width, + p.height, + p.color, + p.twist, + p.circleWidth, + p.stripe, + ) +} + +// Hash computes and returns the hash value of the parameter set using +// SHA-512/256. The same set of parameters will always return the same hash +// value. It implements the filecache.Key interface. +func (p *imgParam) Hash() filecache.Hash { + b := make([]byte, 13) + copy(b, p.color[:]) + binary.BigEndian.PutUint16(b[3:], uint16(p.twist)) + binary.BigEndian.PutUint16(b[5:], p.circleWidth) + binary.BigEndian.PutUint16(b[7:], p.stripe) + binary.BigEndian.PutUint16(b[9:], p.width) + binary.BigEndian.PutUint16(b[11:], p.height) + + // Since this is an example, this calculates and returns a SHA-512/256 + // hash value. However, since this data length fits in 32 bytes, it may + // be better to return the byte string itself that represents the + // parameter values directly. + return sha512.Sum512_256(b) +} diff --git a/example/imgsv/server.go b/example/imgsv/server.go new file mode 100644 index 0000000..738977d --- /dev/null +++ b/example/imgsv/server.go @@ -0,0 +1,264 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package main + +import ( + "context" + "embed" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" + + "github.com/tunabay/go-filecache" + "github.com/tunabay/go-infounit" +) + +// cacheDir is the path to the cache directory. +const cacheDir = "/tmp/go-filecache-example" + +// server represents the example image server. It holds one filecache.Cache +// instance. +type server struct { + cache *filecache.Cache[*imgParam] +} + +// newServer creates an image server instance. +func newServer() (*server, error) { + sv := &server{} + cacheConf := &filecache.Config[*imgParam]{ + Dir: cacheDir, + Create: createImage, + MaxFiles: 16, + MaxSize: infounit.Megabyte * 2, + MaxAge: time.Minute * 10, + GCInterval: time.Minute, + Logger: sv, + DebugLog: true, + } + cache, err := filecache.NewWithConfig[*imgParam](cacheConf) + if err != nil { + return nil, fmt.Errorf("failed to create cache: %w", err) + } + sv.cache = cache + + return sv, nil +} + +// FileCacheLog implements filecache.Logger to receive log messages from the +// filecache package. +func (sv *server) FileCacheLog(line string) { + fmt.Fprintf(os.Stderr, "filecache: %s\n", line) +} + +// serve serves the image server. It calls filecache.Cache.Serve to perform its +// expiration process. +func (sv *server) serve(ctx context.Context) error { + // Log the cache status every 30 seconds. + go func() { + ticker := time.NewTicker(time.Second * 30) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + cstat := sv.cache.Status() + fmt.Fprintf(os.Stderr, "cache status: %v\n", cstat) + } + }() + + if err := sv.cache.Serve(ctx); err != nil { + return fmt.Errorf("cache: %w", err) + } + + return nil +} + +// ServeHTTP responds to incoming HTTP requests. It extracts the image +// parameter set from the URL requested, and use it as a key to lookup the cache +// for the image. +// +// It immediately sends the image to the client if the cached image exists. +// If the cache does not exist, the filecache.Cache calls the callback function +// createImage() to generate the image, and then returns the image as if the +// cache already exists. +func (sv *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + errf := func(code int, format string, v ...any) { + b := []byte(fmt.Sprintf(format, v...) + "\n") + w.Header().Add("Content-Type", "text/plain") + w.Header().Add("Content-Length", strconv.FormatInt(int64(len(b)), 10)) + w.WriteHeader(code) + if _, err := w.Write(b); err != nil { + fmt.Fprintf(os.Stderr, "WARN: ResponseWriter.Write: %v\n", err) + } + } + + // Check the request. + switch { + case r.Method != http.MethodGet: + errf(http.StatusMethodNotAllowed, "Method %s not allowed.", r.Method) + return + + case r.URL.Path == "/": + if err := sv.serveIndex(w, r); err != nil { + fmt.Fprintf(os.Stderr, "WARN: index.html: %v\n", err) + } + return + + case r.URL.Path == "/image.png": + case r.URL.Path == "/favicon.ico": + + default: + errf(http.StatusNotFound, "Resource %s not found.", r.URL.Path) + return + } + fmt.Fprintf(os.Stderr, "REQUEST: %v\n", r.URL) + + // Parse parameters in the query string. + param := &imgParam{ + width: 1280, + height: 720, + color: [3]byte{0x00, 0x99, 0x00}, + twist: 120, + circleWidth: 12, + stripe: 8, + } + qvals := r.URL.Query() + if s := qvals.Get("width"); s != "" { + v, err := strconv.ParseUint(s, 10, 16) + switch { + case err != nil: + errf(http.StatusNotFound, "Invalid width %q: %v", s, err) + return + case v < 16: + errf(http.StatusNotFound, "Too small width %d, at least 16", v) + return + } + param.width = uint16(v) + } + if s := qvals.Get("height"); s != "" { + v, err := strconv.ParseUint(s, 10, 16) + switch { + case err != nil: + errf(http.StatusNotFound, "Invalid height %q: %v", s, err) + return + case v < 16: + errf(http.StatusNotFound, "Too small height %d, at least 16", v) + return + } + param.height = uint16(v) + } + if s := qvals.Get("color"); s != "" { + v, err := hex.DecodeString(s) + switch { + case err != nil: + errf(http.StatusNotFound, "Invalid color %q: %v", s, err) + return + case len(v) != 3: + errf(http.StatusNotFound, "Invalid color %q", s) + return + } + copy(param.color[:], v) + } + if s := qvals.Get("twist"); s != "" { + v, err := strconv.ParseInt(s, 10, 16) + if err != nil { + errf(http.StatusNotFound, "Invalid twist %q: %v", s, err) + return + } + param.twist = int16(v) + } + if s := qvals.Get("circle-width"); s != "" { + v, err := strconv.ParseUint(s, 10, 16) + switch { + case err != nil: + errf(http.StatusNotFound, "Invalid circle-width %q: %v", s, err) + return + case v == 0, 16 < v: + errf(http.StatusNotFound, "Invalid circle-width: %v, must be 1..16", v) + return + } + param.circleWidth = uint16(v) + } + if s := qvals.Get("stripe"); s != "" { + v, err := strconv.ParseUint(s, 10, 16) + switch { + case err != nil: + errf(http.StatusNotFound, "Invalid stripe %q: %v", s, err) + return + case v == 0, 90 < v: + errf(http.StatusNotFound, "Invalid stripe %v, must be 1..90", v) + return + } + param.stripe = uint16(v) + } + if r.URL.Path == "/favicon.ico" { + param.width = 64 + param.height = 64 + param.color = [3]byte{0xcc, 0x00, 0x00} + param.twist = 15 + param.circleWidth = 12 + param.stripe = 4 + } + + startedAt := time.Now() + + file, cached, err := sv.cache.Get(param) + if err != nil { + fmt.Fprintf(os.Stderr, "WARN: Cache.Get: %v\n", err) + return + } + // IMPORTANT: It's the caller's responsibility to call the Close() + // method of the file returned by Get(). + defer func() { + if err := file.Close(); err != nil { + fmt.Fprintf(os.Stderr, "WARN: File.Close: %v\n", err) + } + }() + + elapsed := time.Since(startedAt) + + finfo, _ := file.Stat() + w.Header().Add("Content-Length", strconv.FormatInt(finfo.Size(), 10)) + w.Header().Add("Content-Type", "image/png") + if _, err := io.Copy(w, file); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: io.Copy: %v\n", err) + return + } + + tag := "newly created" + if cached { + tag = "cached" + } + fmt.Fprintf(os.Stderr, "Served [%s] %s (elapsed %v)\n", finfo.Name(), tag, elapsed) +} + +//go:embed index.html +var efs embed.FS + +func (sv *server) serveIndex(w http.ResponseWriter, _ *http.Request) error { + file, err := efs.Open("index.html") + if err != nil { + return fmt.Errorf("index.html: %w", err) + } + defer file.Close() + + fstat, err := file.Stat() + if err != nil { + return fmt.Errorf("index.html: %w", err) + } + + w.Header().Add("Content-Length", strconv.FormatInt(fstat.Size(), 10)) + w.Header().Add("Content-Type", "text/html") + if _, err := io.Copy(w, file); err != nil { + return fmt.Errorf("index.html: %w", err) + } + + return nil +} diff --git a/file.go b/file.go new file mode 100644 index 0000000..b429729 --- /dev/null +++ b/file.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package filecache + +import ( + "io/fs" + "os" + "time" +) + +// File represents the cached file returned by Get() method. +type File[K Key] struct { + parent *Cache[K] + key K + hash Hash + file *os.File + size int64 + lastMod time.Time +} + +// Name returns the string representation of the associated key. +func (f *File[_]) Name() string { return f.key.String() } + +// Read implements io.Reader interface. +func (f *File[_]) Read(b []byte) (int, error) { + return f.file.Read(b) //nolint:wrapcheck +} + +// ReadAt implements io.ReaderAt interface. +func (f *File[_]) ReadAt(b []byte, off int64) (int, error) { + return f.file.ReadAt(b, off) //nolint:wrapcheck +} + +// Seek implements io.Seeker interface. +func (f *File[_]) Seek(offset int64, whence int) (int64, error) { + return f.file.Seek(offset, whence) //nolint:wrapcheck +} + +// Close implements io.Closer interface. +func (f *File[_]) Close() error { + f.parent.unref(f.hash) + + return f.file.Close() //nolint:wrapcheck +} + +// OSFile returns the underlying os.File object. Use with caution, as all writes +// will fail and operations such as deleting, renaming or closing the file will +// cause unexpected results. Only use if it is required to pass the file to a +// package that accepts only os.File or the file descriptor. Also use File.Close +// instead of os.File.Close even if this method is called. +func (f *File[_]) OSFile() *os.File { return f.file } + +// Stat returns the file information. +func (f *File[K]) Stat() (os.FileInfo, error) { + return &FileInfo[K]{ + key: f.key, + size: f.size, + lastMod: f.lastMod, + }, nil +} + +// FileInfo implements fs.FileInfo interface. +type FileInfo[K Key] struct { + key K + size int64 + lastMod time.Time +} + +// Name returns the string representation of the key. Note that it is not the +// underlying file path. +func (i *FileInfo[K]) Name() string { return i.key.String() } + +// Size returns the file size in byte. +func (i *FileInfo[_]) Size() int64 { return i.size } + +// Mode always returns 0400. +func (i *FileInfo[_]) Mode() fs.FileMode { return 0o0400 } + +// ModTime returns the last access time or created time of the cache entry. +func (i *FileInfo[_]) ModTime() time.Time { return i.lastMod } + +// IsDir always returns false, since a directory can not be cached. +func (*FileInfo[_]) IsDir() bool { return false } + +// Sys returns the associated key. +func (i *FileInfo[_]) Sys() any { return i.key } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..12b1000 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/tunabay/go-filecache + +go 1.19 + +require ( + github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 + github.com/tunabay/go-infounit v1.1.3 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..16e4ef0 --- /dev/null +++ b/go.sum @@ -0,0 +1,5 @@ +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= +github.com/tunabay/go-infounit v1.1.3 h1:3Tjl60DnWLLyYJc1mlEp+JGORA++tbRVfURNHvpO6+s= +github.com/tunabay/go-infounit v1.1.3/go.mod h1:XLnA60NwPAzZAgPFLngLiQ6oQ9ibk4iRum0hwv2ykrM= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/hex.go b/hex.go new file mode 100644 index 0000000..5a9dcae --- /dev/null +++ b/hex.go @@ -0,0 +1,15 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package filecache + +import ( + "encoding/hex" +) + +// b2hex converts a byte into a hex two letters string. +func b2hex(b byte) string { return hex.EncodeToString([]byte{b}) } + +// hashHex returns the hex representation of the hash. +func hashHex(hash Hash) string { return hex.EncodeToString(hash[:]) } diff --git a/key.go b/key.go new file mode 100644 index 0000000..81b8acf --- /dev/null +++ b/key.go @@ -0,0 +1,71 @@ +// Copyright (c) 2022 Hirotsuna Mizuno. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +package filecache + +import ( + "crypto/sha512" + "encoding/binary" + "encoding/hex" + "fmt" + "strconv" +) + +// HashSize is the size, in bytes, of the key hash. +const HashSize = 32 + +// Hash represents a 32-bytes hash value. +type Hash [HashSize]byte + +// Key is the interface implemented by the key type to identify a file cache +// entry. +type Key interface { + fmt.Stringer + + Hash() Hash +} + +// Uint64Key is a wrapper type to attach the Hash method to uint64. +type Uint64Key uint64 + +// Hash returns a byte sequence in which the LSBs represent the uint64 value +// itself as a hash value. +func (k Uint64Key) Hash() (b Hash) { + binary.BigEndian.PutUint64(b[HashSize-8:], uint64(k)) + return +} + +// String returns the string representation of the uint64 value. +func (k Uint64Key) String() string { return strconv.FormatUint(uint64(k), 10) } + +// Uint32Key is a wrapper type to attach the Hash method to uint32. +type Uint32Key uint32 + +// Hash returns a byte sequence in which the LSBs represent the uint32 value +// itself as a hash value. +func (k Uint32Key) Hash() (b Hash) { + binary.BigEndian.PutUint32(b[HashSize-4:], uint32(k)) + return +} + +// String returns the string representation of the uint32 value. +func (k Uint32Key) String() string { return strconv.FormatUint(uint64(k), 10) } + +// StringKey is a wrapper type to attach the Hash method to string. +type StringKey string + +// Hash calculates and returns a hash value of the string using SHA-512/256. +func (k StringKey) Hash() Hash { return sha512.Sum512_256([]byte(k)) } + +// String returns the string value itself of StringKey. +func (k StringKey) String() string { return string(k) } + +// ByteSliceKey is a wrapper type to attach the Hash method to []byte. +type ByteSliceKey []byte + +// Hash calculates and returns a hash value of the []byte using SHA-512/256. +func (k ByteSliceKey) Hash() Hash { return sha512.Sum512_256([]byte(k)) } + +// String returns the string representation of the []byte key. +func (k ByteSliceKey) String() string { return hex.EncodeToString([]byte(k)) }