From 2cf848e752c4dfb92c1f3ff32d7518920f7b9c26 Mon Sep 17 00:00:00 2001 From: darkweak Date: Tue, 2 Jul 2024 22:17:45 +0200 Subject: [PATCH] fix(tests): unit tests on surrogate, add default provider name constant --- .github/workflows/non-regression.yml | 21 +- .github/workflows/plugins-master.yml | 2 +- .github/workflows/plugins.yml | 4 +- .../workflows/workflow_plugins_generator.sh | 4 +- Makefile | 10 +- pkg/storage/defaultProvider.go | 2 +- pkg/storage/types/types.go | 2 + pkg/surrogate/providers/common.go | 2 +- pkg/surrogate/providers/common_test.go | 40 +- pkg/surrogate/providers/factory_test.go | 15 +- plugins/caddy/Caddyfile | 342 +++++++++++++++++- plugins/caddy/httpcache_test.go | 3 +- plugins/echo/souin_test.go | 4 + plugins/goa/souin_test.go | 2 + plugins/souin/go.sum | 2 + plugins/traefik/Makefile | 3 +- .../traefik/override/middleware/middleware.go | 2 +- plugins/traefik/override/rfc/revalidation.go | 108 ++++++ .../traefik/override/storage/cacheProvider.go | 8 +- .../traefik/override/storage/types/types.go | 3 +- .../souin/pkg/middleware/middleware.go | 2 +- .../darkweak/souin/pkg/rfc/revalidation.go | 55 ++- .../souin/pkg/storage/cacheProvider.go | 8 +- .../darkweak/souin/pkg/storage/types/types.go | 3 +- 24 files changed, 569 insertions(+), 78 deletions(-) create mode 100644 plugins/traefik/override/rfc/revalidation.go diff --git a/.github/workflows/non-regression.yml b/.github/workflows/non-regression.yml index 2152713e8..aac5740b7 100644 --- a/.github/workflows/non-regression.yml +++ b/.github/workflows/non-regression.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: env: - GO_VERSION: '1.21' + GO_VERSION: '1.22' jobs: lint-validation: @@ -34,24 +34,7 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - name: Run unit static tests - run: go test -v -race $(go list ./... | grep -v pkg/storage) - unit-test-golang-with-services: - needs: lint-validation - name: Unit tests with external services - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: ${{ env.GO_VERSION }} - - name: Build and run the docker stack - run: | - docker network create your_network || true - docker compose -f docker-compose.yml.test up -d --build --force-recreate --remove-orphans - - name: Run pkg storage tests - run: docker compose -f docker-compose.yml.test exec -T souin go test -v -race ./pkg/storage + run: go test -v validate-prod-container-building: needs: unit-test-golang-with-services name: Validate that the container build for prod diff --git a/.github/workflows/plugins-master.yml b/.github/workflows/plugins-master.yml index 2469f4b67..51e5cbbc4 100644 --- a/.github/workflows/plugins-master.yml +++ b/.github/workflows/plugins-master.yml @@ -30,4 +30,4 @@ jobs: run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - name: Build current Souin as caddy module with referenced Souin core version when merge on master - run: cd plugins/caddy && xcaddy build --with github.com/${{ github.repository }}/plugins/caddy@$(git rev-parse --short "$GITHUB_SHA") + run: cd plugins/caddy && xcaddy build --with github.com/${{ github.repository }}/plugins/caddy@$(git rev-parse --short "$GITHUB_SHA") --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 07bbabe8c..701d5e0f0 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -45,7 +45,7 @@ jobs: run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - name: Build Souin as caddy module - run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. + run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy - name: Run Caddy tests run: cd plugins/caddy && go test -v ./... @@ -113,7 +113,7 @@ jobs: run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - name: Build Souin as caddy module for current commit - run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. + run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy - name: Run detached caddy run: cd souin/plugins/caddy && ./caddy run --config ../../docs/cache-tests/cache-tests-caddyfile --adapter caddyfile & diff --git a/.github/workflows/workflow_plugins_generator.sh b/.github/workflows/workflow_plugins_generator.sh index 9aec3e8f7..69e61d0bf 100644 --- a/.github/workflows/workflow_plugins_generator.sh +++ b/.github/workflows/workflow_plugins_generator.sh @@ -53,7 +53,7 @@ jobs: run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - name: Build Souin as caddy module - run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. + run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy - name: Run Caddy tests run: cd plugins/caddy && go test -v ./... @@ -121,7 +121,7 @@ jobs: run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - name: Build Souin as caddy module for current commit - run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. + run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy - name: Run detached caddy run: cd souin/plugins/caddy && ./caddy run --config ../../docs/cache-tests/cache-tests-caddyfile --adapter caddyfile & diff --git a/Makefile b/Makefile index 2fc251357..8a8a0931b 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,15 @@ build-caddy: ## Build caddy binary cd plugins/caddy && \ go mod tidy && \ go mod download && \ - XCADDY_RACE_DETECTOR=1 XCADDY_DEBUG=1 xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. + XCADDY_RACE_DETECTOR=1 XCADDY_DEBUG=1 xcaddy build \ + --with github.com/darkweak/souin/plugins/caddy=./ \ + --with github.com/darkweak/souin=../.. \ + --with github.com/darkweak/storages/badger/caddy \ + --with github.com/darkweak/storages/etcd/caddy \ + --with github.com/darkweak/storages/nuts/caddy \ + --with github.com/darkweak/storages/olric/caddy \ + --with github.com/darkweak/storages/otter/caddy \ + --with github.com/darkweak/storages/redis/caddy build-dev: env-dev ## Build containers with dev env vars $(DC_BUILD) souin diff --git a/pkg/storage/defaultProvider.go b/pkg/storage/defaultProvider.go index cb6c1385e..120fa2d4f 100644 --- a/pkg/storage/defaultProvider.go +++ b/pkg/storage/defaultProvider.go @@ -29,7 +29,7 @@ func Factory(c configurationtypes.AbstractConfigurationInterface) (types.Storer, // Name returns the storer name func (provider *Default) Name() string { - return "DEFAULT" + return types.DefaultStorageName } // MapKeys method returns a map with the key and value diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 05fc16d4c..2ebba32a8 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -7,6 +7,8 @@ import ( "github.com/darkweak/storages/core" ) +const DefaultStorageName = "DEFAULT" + type Storer interface { MapKeys(prefix string) map[string]string ListKeys() []string diff --git a/pkg/surrogate/providers/common.go b/pkg/surrogate/providers/common.go index b3d3f9f1e..640d15651 100644 --- a/pkg/surrogate/providers/common.go +++ b/pkg/surrogate/providers/common.go @@ -103,7 +103,7 @@ func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterf if configuration, ok := config.GetSurrogateKeys()["_configuration"]; ok { storer := core.GetRegisteredStorer(configuration.SurrogateConfiguration.Storer) if storer == nil { - storer = core.GetRegisteredStorer("DEFAULT") + storer = core.GetRegisteredStorer(types.DefaultStorageName) if storer == nil { config.GetLogger().Sugar().Errorf("Impossible to retrieve the storers %s, nuts neither for the surrogate-keys", configuration.SurrogateConfiguration.Storer) } diff --git a/pkg/surrogate/providers/common_test.go b/pkg/surrogate/providers/common_test.go index 789523672..8eb7fa50f 100644 --- a/pkg/surrogate/providers/common_test.go +++ b/pkg/surrogate/providers/common_test.go @@ -1,19 +1,32 @@ package providers -// const ( -// baseHeaderValue = "test0, test1, test2, test3, test4" -// emptyHeaderValue = "" -// ) - -/* -func mockCommonProvider() *baseStorage /*, func() error { - instanciator, _ := storage.NewStorageFromName("nuts") +import ( + "fmt" + "net/http" + "strings" + "sync" + "testing" + + "github.com/darkweak/souin/configurationtypes" + "github.com/darkweak/souin/pkg/storage" + "github.com/darkweak/souin/tests" + "github.com/darkweak/storages/core" + "go.uber.org/zap" +) + +const ( + baseHeaderValue = "test0, test1, test2, test3, test4" + emptyHeaderValue = "" +) + +func mockCommonProvider() *baseStorage { + memoryStorer, _ := storage.Factory(mockConfiguration(tests.BaseConfiguration)) + core.RegisterStorage(memoryStorer) config := tests.MockConfiguration(tests.NutsConfiguration) config.DefaultCache.Badger.Configuration = nil - storer, _ := instanciator(config) sss := &SouinSurrogateStorage{ baseStorage: &baseStorage{ - Storage: storer, + Storage: memoryStorer, Keys: make(map[string]configurationtypes.SurrogateKeys), keysRegexp: make(map[string]keysRegexpInner), dynamic: true, @@ -24,7 +37,7 @@ func mockCommonProvider() *baseStorage /*, func() error { sss.baseStorage.parent = sss - return sss.baseStorage /*, storer.Close + return sss.baseStorage } func TestBaseStorage_ParseHeaders(t *testing.T) { @@ -126,8 +139,8 @@ func TestBaseStorage_Store(t *testing.T) { _ = bs.Store(&res, "/some") storageSize := len(bs.Storage.MapKeys(surrogatePrefix)) - if storageSize != 9 { - t.Errorf("The surrogate storage should contain 9 stored elements, %v given: %#v.\n", storageSize, bs.Storage.ListKeys()) + if storageSize != 6 { + t.Errorf("The surrogate storage should contain 6 stored elements, %v given: %#v.\n", storageSize, bs.Storage.MapKeys("")) } value = bs.Storage.Get(surrogatePrefix + "something") @@ -159,4 +172,3 @@ func TestBaseStorage_Store_Load(t *testing.T) { t.Errorf("The surrogate storage should contain %d stored elements, %d given.", length+1, len(strings.Split(string(v), ","))) } } -*/ diff --git a/pkg/surrogate/providers/factory_test.go b/pkg/surrogate/providers/factory_test.go index b88a3f580..6620eb599 100644 --- a/pkg/surrogate/providers/factory_test.go +++ b/pkg/surrogate/providers/factory_test.go @@ -4,6 +4,9 @@ import ( "testing" "github.com/darkweak/souin/configurationtypes" + "github.com/darkweak/souin/pkg/storage" + "github.com/darkweak/souin/pkg/storage/types" + "github.com/darkweak/storages/core" "go.uber.org/zap" "go.uber.org/zap/zapcore" yaml "gopkg.in/yaml.v3" @@ -40,7 +43,7 @@ default_cache: } type testConfiguration struct { - defaultCache *configurationtypes.DefaultCache `yaml:"default_cache"` + DefaultCache *configurationtypes.DefaultCache `yaml:"default_cache"` } func (*testConfiguration) GetUrls() map[string]configurationtypes.URL { @@ -50,7 +53,7 @@ func (*testConfiguration) GetPluginName() string { return "" } func (t *testConfiguration) GetDefaultCache() configurationtypes.DefaultCacheInterface { - return t.defaultCache + return t.DefaultCache } func (*testConfiguration) GetAPI() configurationtypes.API { return configurationtypes.API{} @@ -106,10 +109,12 @@ func TestSurrogateFactory(t *testing.T) { akamaiConfiguration := mockConfiguration(cdnConfigurationAkamai) fastlyConfiguration := mockConfiguration(cdnConfigurationFastly) souinConfiguration := mockConfiguration(cdnConfigurationSouin) + memoryStorer, _ := storage.Factory(souinConfiguration) + core.RegisterStorage(memoryStorer) - akamaiProvider := SurrogateFactory(akamaiConfiguration, "nuts") - fastlyProvider := SurrogateFactory(fastlyConfiguration, "nuts") - souinProvider := SurrogateFactory(souinConfiguration, "nuts") + akamaiProvider := SurrogateFactory(akamaiConfiguration, types.DefaultStorageName) + fastlyProvider := SurrogateFactory(fastlyConfiguration, types.DefaultStorageName) + souinProvider := SurrogateFactory(souinConfiguration, types.DefaultStorageName) if akamaiProvider == nil { t.Error("Impossible to create the Akamai surrogate provider instance") diff --git a/plugins/caddy/Caddyfile b/plugins/caddy/Caddyfile index f9cfcf9bb..108384901 100644 --- a/plugins/caddy/Caddyfile +++ b/plugins/caddy/Caddyfile @@ -1,20 +1,346 @@ { debug + log { + level debug + } + cache { + allowed_http_verbs GET POST + api { + prometheus + souin + } + cdn { + dynamic + strategy hard + } + regex { + exclude /test2.* + } + ttl 1000s + timeout { + backend 10s + cache 100ms + } + default_cache_control public + } +} + +:4443 +respond "Hello World!" + +@match path /test1* +@match2 path /test2* +@matchdefault path /default +@souin-api path /souin-api* + +cache @match { + ttl 5s +} + +cache @match2 { + ttl 50s +} + +cache @matchdefault { + ttl 5s +} + +route /badger-configuration { cache { - badger + ttl 15s + badger { + configuration { + Dir /tmp/badger-configuration + ValueDir match2 + ValueLogFileSize 16777216 + MemTableSize 4194304 + ValueThreshold 524288 + } + } } + respond "Hello badger" } -http://localhost { - route /hello { - cache +route /etcd { + cache { + ttl 5s + etcd { + configuration { + Endpoints etcd1:2379 etcd2:2379 etcd3:2379 + AutoSyncInterval 1s + DialTimeout 1s + DialKeepAliveTime 1s + DialKeepAliveTimeout 1s + MaxCallSendMsgSize 10000000 + MaxCallRecvMsgSize 10000000 + Username john + Password doe + RejectOldCluster false + PermitWithoutStream false + } + } } + respond "Hello etcd" +} - route /any { - cache { - badger { - path /tmp/something +route /etcd-configuration { + cache { + ttl 5s + etcd { + configuration { + Endpoints etcd:2379 etcd:2379 + AutoSyncInterval 1s + DialTimeout 1s + DialKeepAliveTime 1s + DialKeepAliveTimeout 1s + MaxCallSendMsgSize 10000000 + MaxCallRecvMsgSize 10000000 + RejectOldCluster false + PermitWithoutStream false } } } + respond "Hello etcd" +} + +route /nuts-configuration { + cache { + ttl 15s + nuts { + configuration { + Dir /tmp/nuts-configuration + EntryIdxMode 1 + RWMode 0 + SegmentSize 1024 + NodeNum 42 + SyncEnable true + StartFileLoadingMode 1 + } + } + } + respond "Hello nuts" +} + +route /redis { + cache { + ttl 5s + redis { + configuration { + ClientName souin-redis + InitAddress 127.0.0.1:6379 + SelectDB 0 + } + } + } + respond "Hello redis" +} + +route /redis-configuration { + cache { + ttl 5s + redis { + configuration { + ClientName souin-redis + InitAddress 127.0.0.1:6379 + SelectDB 0 + } + } + } + respond "Hello redis" +} + +route /redis-url { + cache { + ttl 5s + redis { + url 127.0.0.1:6379 + } + } + respond "Hello redis url" +} + +route /vary { + cache { + ttl 15s + } + header Vary X-Something + respond "Hello {http.request.header.X-Something}" +} + +route /cache-s-maxage { + cache + header Cache-Control "s-maxage=10" + respond "Hello, s-maxage!" +} + +route /cache-maxage { + cache + header Cache-Control "max-age=5" + respond "Hello, max-age!" +} + +route /cache-maxstale { + cache { + ttl 3s + stale 5s + } + header Cache-Control "max-age=5" + respond "Hello, max-age!" +} + +route /not-modified { + cache { + ttl 5s + } + reverse_proxy 127.0.0.1:9000 +} + +route /no-reverse-proxy { + cache + reverse_proxy 127.0.0.1:9000 +} + +route /surrogate-keys { + cache + header Surrogate-Key "KEY-{http.request.header.X-Surrogate-Key-Suffix}" + header Vary X-Surrogate-Key-Suffix,Accept-Encoding + respond "Hello {http.request.header.X-Surrogate-Key-Suffix}" +} + +route /another-cache-status-name { + cache { + cache_name Another + } +} + +route /backend-timeout { + cache { + timeout { + backend 1s + cache 1ms + } + } + reverse_proxy 127.0.0.1:8081 +} + +route /stream { + cache + reverse_proxy 127.0.0.1:81 +} + +route /gzip { + cache + encode { + gzip + minimum_length 5 + } + header Content-Type text/plain + respond "Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip. Hello, gzip." +} + +route /custom-key/without-* { + cache { + cache_keys { + body { + disable_body + } + host { + disable_host + } + method { + disable_method + } + everything-with-content-type { + disable_method + headers Content-Type + } + } + } + respond "Hello to the authenticated user." +} + +route /must-revalidate { + cache { + ttl 5s + stale 5s + } + header Cache-Control "must-revalidate" + reverse_proxy 127.0.0.1:81 +} + +route /cache-authorization { + cache { + cache_keys { + /.+ { + headers Authorization + } + } + } + header Souin-Cache-Control public + respond "Hello to the authenticated user." +} + +route /bypass { + cache { + mode bypass + } + + header Cache-Control "no-store" + respond "Hello bypass" +} + +route /bypass_request { + cache { + mode bypass_request + } + + respond "Hello bypass_request" +} + +route /bypass_response { + cache { + mode bypass_response + } + + header Cache-Control "no-cache, no-store" + respond "Hello bypass_response" +} + +route /strict_request { + cache { + mode strict + } + + respond "Hello strict" +} + +route /strict_response { + cache { + mode strict + } + + header Cache-Control "no-cache, no-store" + respond "Hello strict" +} + +cache @souin-api { +} + +# ESI part +route /esi-include { + cache + header Content-Type text/html + respond "

ESI INCLUDE

" +} + +route /alt-esi-include { + cache + header Content-Type text/html + respond "

ALTERNATE ESI INCLUDE

" +} + +route /esi { + cache + header Content-Type text/html + respond `` } \ No newline at end of file diff --git a/plugins/caddy/httpcache_test.go b/plugins/caddy/httpcache_test.go index 85205b38c..2e1a972a0 100644 --- a/plugins/caddy/httpcache_test.go +++ b/plugins/caddy/httpcache_test.go @@ -889,7 +889,8 @@ func TestVaryHandler(t *testing.T) { t.Error("The object is not type of *http.Response") } - if rs.Header.Get("Cache-Status") != fmt.Sprintf("Souin; hit; ttl=%d; key=GET-http-localhost:9080-/vary-multiple", ttl) || rs.Header.Get("Age") != fmt.Sprint(120-ttl) { + if (rs.Header.Get("Cache-Status") != fmt.Sprintf("Souin; hit; ttl=%d; key=GET-http-localhost:9080-/vary-multiple", ttl) || rs.Header.Get("Age") != fmt.Sprint(120-ttl)) && + (rs.Header.Get("Cache-Status") != fmt.Sprintf("Souin; hit; ttl=%d; key=GET-http-localhost:9080-/vary-multiple", ttl-1) || rs.Header.Get("Age") != fmt.Sprint(120-ttl-1)) { t.Errorf("The response doesn't match the expected header or age: %s => %s", rs.Header.Get("Cache-Status"), rs.Header.Get("Age")) } } diff --git a/plugins/echo/souin_test.go b/plugins/echo/souin_test.go index b28e2fdec..bad21d885 100644 --- a/plugins/echo/souin_test.go +++ b/plugins/echo/souin_test.go @@ -2,6 +2,7 @@ package souin import ( "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -10,6 +11,7 @@ import ( "github.com/darkweak/souin/configurationtypes" "github.com/darkweak/souin/pkg/middleware" + "github.com/darkweak/storages/core" "github.com/labstack/echo/v4" ) @@ -91,6 +93,7 @@ func Test_SouinEchoPlugin_Process_CannotHandle(t *testing.T) { } func Test_SouinEchoPlugin_Process_APIHandle(t *testing.T) { + core.ResetRegisteredStorages() req := httptest.NewRequest(http.MethodGet, "/souin-api/souin", nil) req.Header = http.Header{} res := httptest.NewRecorder() @@ -131,6 +134,7 @@ func Test_SouinEchoPlugin_Process_APIHandle(t *testing.T) { var payload []string _ = json.Unmarshal(b, &payload) if len(payload) != 1 { + fmt.Printf("%#v\n\n", payload) t.Error("The system must store 1 item, excluding the mapping") } if payload[0] != "GET-http-example.com-/handled" { diff --git a/plugins/goa/souin_test.go b/plugins/goa/souin_test.go index d55776bfe..81e42e4a3 100644 --- a/plugins/goa/souin_test.go +++ b/plugins/goa/souin_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/darkweak/souin/pkg/middleware" + "github.com/darkweak/storages/core" ) func Test_NewHTTPCache(t *testing.T) { @@ -86,6 +87,7 @@ func Test_SouinGoaPlugin_Middleware_CannotHandle(t *testing.T) { } func Test_SouinGoaPlugin_Middleware_APIHandle(t *testing.T) { + core.ResetRegisteredStorages() time.Sleep(DevDefaultConfiguration.DefaultCache.GetTTL() + DevDefaultConfiguration.GetDefaultCache().GetStale()) res, res2, handler := prepare() req := httptest.NewRequest(http.MethodGet, "/souin-api/souin", nil) diff --git a/plugins/souin/go.sum b/plugins/souin/go.sum index 0fb8834b7..747a49ab7 100644 --- a/plugins/souin/go.sum +++ b/plugins/souin/go.sum @@ -98,6 +98,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/darkweak/go-esi v0.0.5 h1:b9LHI8Tz46R+i6p8avKPHAIBRQUCZDebNmKm5w/Zrns= github.com/darkweak/go-esi v0.0.5/go.mod h1:koCJqwum1u6mslyZuq/Phm6hfG1K3ZK5Y7jrUBTH654= +github.com/darkweak/storages/core v0.0.2 h1:+265yc/GdvisZuqehl7E/7awWU6f17wL9jnF2B6kKAc= +github.com/darkweak/storages/core v0.0.2/go.mod h1:wLp1cOAB4WUd46BBOtV4Lwot4GD+8fZbtIw6QM7fYuc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/plugins/traefik/Makefile b/plugins/traefik/Makefile index fb74e054f..cece085bf 100644 --- a/plugins/traefik/Makefile +++ b/plugins/traefik/Makefile @@ -20,7 +20,7 @@ load-checker: ## Ensure Souin is running prepare: vendor ## Prepare traefik plugin $(DC) down --remove-orphans - $(DC) up -d + $(DC) up --build # TODO find another way to do that replace: ## Replace sources in the vendor folder deeper than the go mod replace @@ -32,6 +32,7 @@ replace: ## Replace sources in the vendor folder deeper than the go mod replace $(MAKE) copy-to base=$(PKG) target=api $(MAKE) copy-to base=$(PKG) target=storage $(MAKE) copy-file-to base=$(PKG) target=rfc/vary.go + $(MAKE) copy-file-to base=$(PKG) target=rfc/revalidation.go $(MAKE) copy-file-to base=$(PKG) target=middleware/configuration.go $(MAKE) copy-file-to base=$(PKG) target=middleware/middleware.go $(MAKE) copy-file-to base=$(PKG) target=middleware/writer.go diff --git a/plugins/traefik/override/middleware/middleware.go b/plugins/traefik/override/middleware/middleware.go index b9cd155da..2fe9f0551 100644 --- a/plugins/traefik/override/middleware/middleware.go +++ b/plugins/traefik/override/middleware/middleware.go @@ -220,7 +220,7 @@ func (s *SouinBaseHandler) Store( wg.Add(1) go func(currentStorer types.Storer) { defer wg.Done() - if currentStorer.Set(cachedKey, response, currentMatchedURL, ma) != nil { + if currentStorer.Set(cachedKey, response, ma) != nil { mu.Lock() fails = append(fails, fmt.Sprintf("; detail=%s-INSERTION-ERROR", currentStorer.Name())) mu.Unlock() diff --git a/plugins/traefik/override/rfc/revalidation.go b/plugins/traefik/override/rfc/revalidation.go new file mode 100644 index 000000000..c0eb363b3 --- /dev/null +++ b/plugins/traefik/override/rfc/revalidation.go @@ -0,0 +1,108 @@ +package rfc + +import ( + "net/http" + "strings" + "time" +) + +type Revalidator struct { + Matched bool + IfNoneMatchPresent bool + IfMatchPresent bool + IfModifiedSincePresent bool + IfUnmodifiedSincePresent bool + IfUnmotModifiedSincePresent bool + NeedRevalidation bool + NotModified bool + IfModifiedSince time.Time + IfUnmodifiedSince time.Time + IfNoneMatch []string + IfMatch []string + RequestETags []string + ResponseETag string +} + +func ValidateETagFromHeader(etag string, validator *Revalidator) { + validator.ResponseETag = etag + validator.NeedRevalidation = validator.NeedRevalidation || validator.ResponseETag != "" + validator.Matched = validator.ResponseETag == "" || (validator.ResponseETag != "" && len(validator.RequestETags) == 0) + + if len(validator.RequestETags) == 0 { + validator.NotModified = false + return + } + + // If-None-Match + if validator.IfNoneMatchPresent { + for _, ifNoneMatch := range validator.IfNoneMatch { + // Asrterisk special char to match any of ETag + if ifNoneMatch == "*" { + validator.Matched = true + return + } + if ifNoneMatch == validator.ResponseETag { + validator.Matched = true + return + } + } + + validator.Matched = false + return + } + + // If-Match + if validator.IfMatchPresent { + validator.Matched = false + if validator.ResponseETag == "" { + return + } + + for _, ifMatch := range validator.IfMatch { + // Asrterisk special char to match any of ETag + if ifMatch == "*" { + validator.Matched = true + return + } + if ifMatch == validator.ResponseETag { + validator.Matched = true + return + } + } + } +} + +func ParseRequest(req *http.Request) *Revalidator { + var rqEtags []string + if len(req.Header.Get("If-None-Match")) > 0 { + rqEtags = strings.Split(req.Header.Get("If-None-Match"), ",") + } + for i, tag := range rqEtags { + rqEtags[i] = strings.Trim(tag, " ") + } + validator := Revalidator{ + NotModified: len(rqEtags) > 0, + RequestETags: rqEtags, + } + // If-Modified-Since + if ifModifiedSince := req.Header.Get("If-Modified-Since"); ifModifiedSince != "" { + validator.IfModifiedSincePresent = true + validator.IfModifiedSince, _ = time.Parse(time.RFC1123, ifModifiedSince) + validator.NeedRevalidation = true + } + + // If-Unmodified-Since + if ifUnmodifiedSince := req.Header.Get("If-Unmodified-Since"); ifUnmodifiedSince != "" { + validator.IfUnmodifiedSincePresent = true + validator.IfUnmodifiedSince, _ = time.Parse(time.RFC1123, ifUnmodifiedSince) + validator.NeedRevalidation = true + } + + // If-None-Match + if ifNoneMatches := req.Header.Values("If-None-Match"); len(ifNoneMatches) > 0 { + validator.IfNoneMatchPresent = true + validator.IfNoneMatch = ifNoneMatches + } + + return &validator +} diff --git a/plugins/traefik/override/storage/cacheProvider.go b/plugins/traefik/override/storage/cacheProvider.go index ab5d9fb8a..8331486c6 100644 --- a/plugins/traefik/override/storage/cacheProvider.go +++ b/plugins/traefik/override/storage/cacheProvider.go @@ -85,7 +85,7 @@ func (provider *Cache) Prefix(key string, req *http.Request, validator *rfc.Reva if k == key || varyVoter(key, req, k.(string)) { if res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(v.([]byte))), req); err == nil { - rfc.ValidateETag(res, validator) + rfc.ValidateETagFromHeader(res.Header.Get("etag"), validator) if validator.Matched { result = res return false @@ -101,11 +101,7 @@ func (provider *Cache) Prefix(key string, req *http.Request, validator *rfc.Reva } // Set method will store the response in Cache provider -func (provider *Cache) Set(key string, value []byte, url t.URL, duration time.Duration) error { - if duration == 0 { - duration = url.TTL.Duration - } - +func (provider *Cache) Set(key string, value []byte, duration time.Duration) error { provider.Cache.Set(key, value, duration) provider.Cache.Set(StalePrefix+key, value, provider.stale+duration) diff --git a/plugins/traefik/override/storage/types/types.go b/plugins/traefik/override/storage/types/types.go index 699352a84..67031c15e 100644 --- a/plugins/traefik/override/storage/types/types.go +++ b/plugins/traefik/override/storage/types/types.go @@ -4,7 +4,6 @@ import ( "net/http" "time" - "github.com/darkweak/souin/configurationtypes" "github.com/darkweak/souin/pkg/rfc" ) @@ -13,7 +12,7 @@ type Storer interface { ListKeys() []string Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response Get(key string) []byte - Set(key string, value []byte, url configurationtypes.URL, duration time.Duration) error + Set(key string, value []byte, duration time.Duration) error Delete(key string) DeleteMany(key string) Init() error diff --git a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/middleware/middleware.go b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/middleware/middleware.go index b9cd155da..2fe9f0551 100644 --- a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/middleware/middleware.go +++ b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/middleware/middleware.go @@ -220,7 +220,7 @@ func (s *SouinBaseHandler) Store( wg.Add(1) go func(currentStorer types.Storer) { defer wg.Done() - if currentStorer.Set(cachedKey, response, currentMatchedURL, ma) != nil { + if currentStorer.Set(cachedKey, response, ma) != nil { mu.Lock() fails = append(fails, fmt.Sprintf("; detail=%s-INSERTION-ERROR", currentStorer.Name())) mu.Unlock() diff --git a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/rfc/revalidation.go b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/rfc/revalidation.go index c2d63e4aa..c0eb363b3 100644 --- a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/rfc/revalidation.go +++ b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/rfc/revalidation.go @@ -4,8 +4,6 @@ import ( "net/http" "strings" "time" - - "github.com/darkweak/storages/core" ) type Revalidator struct { @@ -25,7 +23,56 @@ type Revalidator struct { ResponseETag string } -func ParseRequest(req *http.Request) *core.Revalidator { +func ValidateETagFromHeader(etag string, validator *Revalidator) { + validator.ResponseETag = etag + validator.NeedRevalidation = validator.NeedRevalidation || validator.ResponseETag != "" + validator.Matched = validator.ResponseETag == "" || (validator.ResponseETag != "" && len(validator.RequestETags) == 0) + + if len(validator.RequestETags) == 0 { + validator.NotModified = false + return + } + + // If-None-Match + if validator.IfNoneMatchPresent { + for _, ifNoneMatch := range validator.IfNoneMatch { + // Asrterisk special char to match any of ETag + if ifNoneMatch == "*" { + validator.Matched = true + return + } + if ifNoneMatch == validator.ResponseETag { + validator.Matched = true + return + } + } + + validator.Matched = false + return + } + + // If-Match + if validator.IfMatchPresent { + validator.Matched = false + if validator.ResponseETag == "" { + return + } + + for _, ifMatch := range validator.IfMatch { + // Asrterisk special char to match any of ETag + if ifMatch == "*" { + validator.Matched = true + return + } + if ifMatch == validator.ResponseETag { + validator.Matched = true + return + } + } + } +} + +func ParseRequest(req *http.Request) *Revalidator { var rqEtags []string if len(req.Header.Get("If-None-Match")) > 0 { rqEtags = strings.Split(req.Header.Get("If-None-Match"), ",") @@ -33,7 +80,7 @@ func ParseRequest(req *http.Request) *core.Revalidator { for i, tag := range rqEtags { rqEtags[i] = strings.Trim(tag, " ") } - validator := core.Revalidator{ + validator := Revalidator{ NotModified: len(rqEtags) > 0, RequestETags: rqEtags, } diff --git a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/cacheProvider.go b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/cacheProvider.go index ab5d9fb8a..8331486c6 100644 --- a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/cacheProvider.go +++ b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/cacheProvider.go @@ -85,7 +85,7 @@ func (provider *Cache) Prefix(key string, req *http.Request, validator *rfc.Reva if k == key || varyVoter(key, req, k.(string)) { if res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(v.([]byte))), req); err == nil { - rfc.ValidateETag(res, validator) + rfc.ValidateETagFromHeader(res.Header.Get("etag"), validator) if validator.Matched { result = res return false @@ -101,11 +101,7 @@ func (provider *Cache) Prefix(key string, req *http.Request, validator *rfc.Reva } // Set method will store the response in Cache provider -func (provider *Cache) Set(key string, value []byte, url t.URL, duration time.Duration) error { - if duration == 0 { - duration = url.TTL.Duration - } - +func (provider *Cache) Set(key string, value []byte, duration time.Duration) error { provider.Cache.Set(key, value, duration) provider.Cache.Set(StalePrefix+key, value, provider.stale+duration) diff --git a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/types/types.go b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/types/types.go index 699352a84..67031c15e 100644 --- a/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/types/types.go +++ b/plugins/traefik/vendor/github.com/darkweak/souin/pkg/storage/types/types.go @@ -4,7 +4,6 @@ import ( "net/http" "time" - "github.com/darkweak/souin/configurationtypes" "github.com/darkweak/souin/pkg/rfc" ) @@ -13,7 +12,7 @@ type Storer interface { ListKeys() []string Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response Get(key string) []byte - Set(key string, value []byte, url configurationtypes.URL, duration time.Duration) error + Set(key string, value []byte, duration time.Duration) error Delete(key string) DeleteMany(key string) Init() error