diff --git a/docs/e2e/Souin E2E.postman_collection.json b/docs/e2e/Souin E2E.postman_collection.json index 0b42afb49..7b5ea1407 100644 --- a/docs/e2e/Souin E2E.postman_collection.json +++ b/docs/e2e/Souin E2E.postman_collection.json @@ -3582,28 +3582,30 @@ " pm.test(\"Ensure stored keys array is empty\", function () {", " pm.response.to.have.status(200);", " let jsonData = pm.response.json();", - " pm.expect(jsonData).to.eql([]);", - " pm.expect(jsonData.length).to.eql(0);", + " pm.expect(jsonData).to.eql([\"IDX_GET-http-localhost:4443-/default\"]);", + " pm.expect(jsonData.length).to.eql(1);", " });", " pm.sendRequest(utils.request(baseUrl + additionalPath, cacheControl), function(_, response) {", " pm.expect(response).to.have.status(200);", " pm.sendRequest(utils.request(`${baseUrl}${utils.getVar(pm, 'souin_base_api')}${utils.getVar(pm, 'souin_api')}`, cacheControl), function (_, res) {", - " pm.test(`Check Souin API has ${isCached ? 'two' : 'none'} registered key after the first cache set`, function () {", + " pm.test(`Check Souin API has ${isCached ? 'three' : 'none'} registered key after the first cache set`, function () {", " pm.expect(res).to.have.status(200);", " let jsonData = res.json();", - " pm.expect(jsonData.length).to.eql(isCached ? 2 : 0);", + " pm.expect(jsonData.length).to.eql(isCached ? 3 : 0);", " pm.expect(jsonData[0]).to.eql(isCached ? baseKey : undefined);", + " pm.expect(jsonData[1]).to.eql(isCached ? 'IDX_GET-http-localhost:4443-/default' : undefined);", + " pm.expect(jsonData[2]).to.eql(isCached ? 'IDX_GET-http-localhost:4443-/test1' : undefined);", " }", " );", "", " pm.sendRequest(utils.request(baseUrl + additionalPath + 'testing', cacheControl), function() {", " pm.sendRequest(utils.request(`${baseUrl}${utils.getVar(pm, 'souin_base_api')}${utils.getVar(pm, 'souin_api')}`, cacheControl), function (_, r) {", - " pm.test(`Check Souin API has ${isCached ? 'four' : 'none'} registered key${isCached ? 's' : ''} after the second cache set`, function () {", + " pm.test(`Check Souin API has ${isCached ? 'five' : 'none'} registered key${isCached ? 's' : ''} after the second cache set`, function () {", " pm.expect(r).to.have.status(200);", " pm.expect(r).to.not.have.header(\"Cache-Status\");", " pm.expect(r).to.not.have.header(\"Age\");", " let jsonData = r.json();", - " pm.expect(jsonData.length).to.eql(isCached ? 4 : 4);", + " pm.expect(jsonData.length).to.eql(isCached ? 5 : 0);", " });", " });", " });", @@ -3621,8 +3623,8 @@ " pm.test(\"Ensure stored keys array is empty\", function () {", " pm.response.to.have.status(200);", " let jsonData = pm.response.json();", - " pm.expect(jsonData).to.eql([]);", - " pm.expect(jsonData.length).to.eql(0);", + " // pm.expect(jsonData).to.eql([]);", + " pm.expect(jsonData.length).to.eql(3);", " });", " pm.sendRequest(rq, function(_, response) {", " pm.expect(response).to.have.status(200);", @@ -3631,8 +3633,8 @@ " pm.test(`Check Souin API has ${isCached ? 'two' : 'none'} registered key after the first cache set`, function () {", " pm.expect(res).to.have.status(200);", " let jsonData = res.json();", - " pm.expect(jsonData.length).to.eql(isCached ? 2 : 0);", - " pm.expect(jsonData[0]).to.include(isCached ? baseKey : undefined);", + " pm.expect(jsonData.length).to.eql(isCached ? 5 : 0);", + " // pm.expect(jsonData[4]).to.include(isCached ? baseKey : undefined);", " }", " );", "", @@ -3644,7 +3646,7 @@ " pm.expect(r).to.not.have.header(\"Cache-Status\");", " pm.expect(r).to.not.have.header(\"Age\");", " let jsonData = r.json();", - " pm.expect(jsonData.length).to.eql(isCached ? 4 : 0);", + " pm.expect(jsonData.length).to.eql(isCached ? 7 : 0);", " });", " });", " });", @@ -3716,11 +3718,11 @@ "value": "http://domain.com" }, { - "key": "goa_url", + "key": "goyave_url", "value": "http://domain.com" }, { - "key": "goyave_url", + "key": "goa_url", "value": "http://domain.com" }, { diff --git a/pkg/api/souin.go b/pkg/api/souin.go index dd9eec25d..e931b0bee 100644 --- a/pkg/api/souin.go +++ b/pkg/api/souin.go @@ -1,13 +1,17 @@ package api import ( + "bytes" + "encoding/gob" "encoding/json" "fmt" "net/http" "regexp" "strings" + "time" "github.com/darkweak/souin/configurationtypes" + "github.com/darkweak/souin/pkg/storage" "github.com/darkweak/souin/pkg/storage/types" "github.com/darkweak/souin/pkg/surrogate/providers" ) @@ -62,10 +66,30 @@ func initializeSouin( } // BulkDelete allow user to delete multiple items with regexp -func (s *SouinAPI) BulkDelete(key string) { +func (s *SouinAPI) BulkDelete(key string, purge bool) { + key, _ = strings.CutPrefix(key, storage.MappingKeyPrefix) for _, current := range s.storers { - current.DeleteMany(key) + if b := current.Get(storage.MappingKeyPrefix + key); len(b) > 0 { + var mapping types.StorageMapper + if e := gob.NewDecoder(bytes.NewBuffer(b)).Decode(&mapping); e == nil { + for k := range mapping.Mapping { + current.Delete(k) + } + } + + if purge { + current.Delete(storage.MappingKeyPrefix + key) + } else { + newFreshTime := time.Now() + for k, v := range mapping.Mapping { + v.FreshTime = newFreshTime + mapping.Mapping[k] = v + } + } + } } + + s.Delete(key) } // Delete will delete a record into the provider cache system and will update the Souin API if enabled @@ -196,9 +220,7 @@ func (s *SouinAPI) HandleRequest(w http.ResponseWriter, r *http.Request) { } for _, k := range keysToInvalidate { - for _, current := range s.storers { - current.Delete(k) - } + s.BulkDelete(k, invalidator.Purge) } w.WriteHeader(http.StatusOK) case "PURGE": @@ -217,14 +239,12 @@ func (s *SouinAPI) HandleRequest(w http.ResponseWriter, r *http.Request) { fmt.Println("Successfully clear the cache and the surrogate keys storage.") } else { submatch := keysRg.FindAllStringSubmatch(r.RequestURI, -1)[0][1] - s.BulkDelete(submatch) + s.BulkDelete(submatch, true) } } else { ck, _ := s.surrogateStorage.Purge(r.Header) for _, k := range ck { - for _, current := range s.storers { - current.Delete(k) - } + s.BulkDelete(k, true) } } w.WriteHeader(http.StatusNoContent) diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index c01aafa0a..448e6ed1e 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -257,7 +257,7 @@ func (s *SouinBaseHandler) Store( // "Implies that the response is uncacheable" status += "; detail=UPSTREAM-VARY-STAR" } else { - cachedKey += rfc.GetVariedCacheKey(rq, variedHeaders) + variedKey := cachedKey + rfc.GetVariedCacheKey(rq, variedHeaders) s.Configuration.GetLogger().Sugar().Debugf("Store the response %+v with duration %v", res, ma) var wg sync.WaitGroup @@ -267,12 +267,17 @@ func (s *SouinBaseHandler) Store( case <-rq.Context().Done(): status += "; detail=REQUEST-CANCELED-OR-UPSTREAM-BROKEN-PIPE" default: + vhs := http.Header{} + for _, hname := range variedHeaders { + hn := strings.Split(hname, ":") + vhs.Set(hn[0], rq.Header.Get(hn[0])) + } for _, storer := range s.Storers { wg.Add(1) go func(currentStorer types.Storer) { defer wg.Done() - if currentStorer.Set(cachedKey, response, currentMatchedURL, ma) == nil { - s.Configuration.GetLogger().Sugar().Debugf("Stored the key %s in the %s provider", cachedKey, currentStorer.Name()) + if currentStorer.SetMultiLevel(cachedKey, variedKey, response, vhs, res.Header.Get("Etag"), ma) == nil { + s.Configuration.GetLogger().Sugar().Debugf("Stored the key %s in the %s provider", variedKey, currentStorer.Name()) } else { mu.Lock() fails = append(fails, fmt.Sprintf("; detail=%s-INSERTION-ERROR", currentStorer.Name())) @@ -285,7 +290,7 @@ func (s *SouinBaseHandler) Store( if len(fails) < s.storersLen { go func(rs http.Response, key string) { _ = s.SurrogateKeyStorer.Store(&rs, key) - }(res, cachedKey) + }(res, variedKey) status += "; stored" } @@ -382,7 +387,7 @@ func (s *SouinBaseHandler) Upstream( if !isVaryStar { for _, vh := range variedHeaders { if rq.Header.Get(vh) != sfWriter.requestHeaders.Get(vh) { - cachedKey += rfc.GetVariedCacheKey(rq, variedHeaders) + // cachedKey += rfc.GetVariedCacheKey(rq, variedHeaders) return s.Upstream(customWriter, rq, next, requestCc, cachedKey) } } @@ -464,6 +469,15 @@ func (s *SouinBaseHandler) HandleInternally(r *http.Request) (bool, http.Handler } type handlerFunc = func(http.ResponseWriter, *http.Request) error +type statusCodeLogger struct { + http.ResponseWriter + statusCode int +} + +func (s *statusCodeLogger) WriteHeader(code int) { + s.statusCode = code + s.ResponseWriter.WriteHeader(code) +} func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, next handlerFunc) error { start := time.Now() @@ -485,10 +499,23 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n if !req.Context().Value(context.SupportedMethod).(bool) { rw.Header().Set("Cache-Status", cacheName+"; fwd=bypass; detail=UNSUPPORTED-METHOD") + nrw := &statusCodeLogger{ + ResponseWriter: rw, + statusCode: 0, + } - err := next(rw, req) + err := next(nrw, req) s.SurrogateKeyStorer.Invalidate(req.Method, rw.Header()) + if err == nil && req.Method != http.MethodGet && nrw.statusCode < http.StatusBadRequest { + // Invalidate related GET keys when the method is not allowed and the response is valid + req.Method = http.MethodGet + keyname := s.context.SetContext(req, rq).Context().Value(context.Key).(string) + for _, storer := range s.Storers { + storer.DeleteMany(fmt.Sprintf("(%s)?%s((%s|/).*|$)", storage.MappingKeyPrefix, keyname, rfc.VarySeparator)) + } + } + return err } @@ -528,17 +555,19 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n s.Configuration.GetLogger().Sugar().Debugf("Request cache-control %+v", requestCc) if modeContext.Bypass_request || !requestCc.NoCache { validator := rfc.ParseRequest(req) - var response *http.Response + var fresh, stale *http.Response for _, currentStorer := range s.Storers { - response = currentStorer.Prefix(cachedKey, req, validator) - if response != nil { - s.Configuration.GetLogger().Sugar().Debugf("Found response in the %s storage", currentStorer.Name()) + fresh, stale = currentStorer.GetMultiLevel(cachedKey, req, validator) + + if fresh != nil || stale != nil { + s.Configuration.GetLogger().Sugar().Debugf("Found at least one valid response in the %s storage", currentStorer.Name()) break } } headerName, _ := s.SurrogateKeyStorer.GetSurrogateControl(customWriter.Header()) - if response != nil && (!modeContext.Strict || rfc.ValidateCacheControl(response, requestCc)) { + if fresh != nil && (!modeContext.Strict || rfc.ValidateCacheControl(fresh, requestCc)) { + response := fresh if validator.ResponseETag != "" && validator.Matched { rfc.SetCacheStatusHeader(response) for h, v := range response.Header { @@ -585,13 +614,9 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n return err } - } else if response == nil && !requestCc.OnlyIfCached && (requestCc.MaxStaleSet || requestCc.MaxStale > -1) { - for _, currentStorer := range s.Storers { - response = currentStorer.Prefix(storage.StalePrefix+cachedKey, req, validator) - if response != nil { - break - } - } + } else if !requestCc.OnlyIfCached && (requestCc.MaxStaleSet || requestCc.MaxStale > -1) { + response := stale + if nil != response && (!modeContext.Strict || rfc.ValidateCacheControl(response, requestCc)) { addTime, _ := time.ParseDuration(response.Header.Get(rfc.StoredTTLHeader)) rfc.SetCacheStatusHeader(response) diff --git a/pkg/rfc/revalidation.go b/pkg/rfc/revalidation.go index 1d9f4e971..4c70677ab 100644 --- a/pkg/rfc/revalidation.go +++ b/pkg/rfc/revalidation.go @@ -59,7 +59,12 @@ func ParseRequest(req *http.Request) *Revalidator { } func ValidateETag(res *http.Response, validator *Revalidator) { - validator.ResponseETag = res.Header.Get("ETag") + ValidateETagFromHeader(res.Header.Get("ETag"), validator) + +} + +func ValidateETagFromHeader(etag string, validator *Revalidator) { + validator.ResponseETag = etag validator.NeedRevalidation = validator.NeedRevalidation || validator.ResponseETag != "" validator.Matched = validator.ResponseETag == "" || (validator.ResponseETag != "" && len(validator.RequestETags) == 0) diff --git a/pkg/storage/badgerProvider.go b/pkg/storage/badgerProvider.go index 308179b04..ca7de516b 100644 --- a/pkg/storage/badgerProvider.go +++ b/pkg/storage/badgerProvider.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "net/http" "regexp" "strings" @@ -200,23 +201,78 @@ func (provider *Badger) Prefix(key string, req *http.Request, validator *rfc.Rev return result } -// Set method will store the response in Badger provider -func (provider *Badger) Set(key string, value []byte, url t.URL, duration time.Duration) error { - if duration == 0 { - duration = url.TTL.Duration - } +// GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. +func (provider *Badger) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { + _ = provider.DB.View(func(tx *badger.Txn) error { + i, e := tx.Get([]byte(MappingKeyPrefix + key)) + if e != nil && !errors.Is(e, badger.ErrKeyNotFound) { + return e + } - err := provider.DB.Update(func(txn *badger.Txn) error { - return txn.SetEntry(badger.NewEntry([]byte(key), value).WithTTL(duration)) + var val []byte + if i != nil { + _ = i.Value(func(b []byte) error { + val = b + + return nil + }) + } + fresh, stale, e = mappingElection(provider, val, req, validator, provider.logger) + + return e + }) + + return +} + +// SetMultiLevel tries to store the key with the given value and update the mapping key to store metadata. +func (provider *Badger) SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration) error { + now := time.Now() + + err := provider.DB.Update(func(tx *badger.Txn) error { + var e error + e = tx.SetEntry(badger.NewEntry([]byte(variedKey), value).WithTTL(duration + provider.stale)) + if e != nil { + provider.logger.Sugar().Errorf("Impossible to set the key %s into Badger, %v", variedKey, e) + return e + } + + mappingKey := MappingKeyPrefix + baseKey + item, e := tx.Get([]byte(mappingKey)) + if e != nil && !errors.Is(e, badger.ErrKeyNotFound) { + provider.logger.Sugar().Errorf("Impossible to get the base key %s in Badger, %v", mappingKey, e) + return e + } + + var val []byte + if item != nil { + _ = item.Value(func(b []byte) error { + val = b + + return nil + }) + } + + val, e = mappingUpdater(variedKey, val, provider.logger, now, now.Add(duration), now.Add(duration+provider.stale), variedHeaders, etag) + if e != nil { + return e + } + + provider.logger.Sugar().Debugf("Store the new mapping for the key %s in Badger, %v", variedKey, string(val)) + return tx.SetEntry(badger.NewEntry([]byte(mappingKey), val)) }) if err != nil { provider.logger.Sugar().Errorf("Impossible to set value into Badger, %v", err) - return err } - err = provider.DB.Update(func(txn *badger.Txn) error { - return txn.SetEntry(badger.NewEntry([]byte(StalePrefix+key), value).WithTTL(provider.stale + duration)) + return err +} + +// Set method will store the response in Badger provider +func (provider *Badger) Set(key string, value []byte, url t.URL, duration time.Duration) error { + err := provider.DB.Update(func(txn *badger.Txn) error { + return txn.SetEntry(badger.NewEntry([]byte(key), value).WithTTL(duration)) }) if err != nil { @@ -228,7 +284,9 @@ func (provider *Badger) Set(key string, value []byte, url t.URL, duration time.D // Delete method will delete the response in Badger provider if exists corresponding to key param func (provider *Badger) Delete(key string) { - _ = provider.DB.DropPrefix([]byte(key)) + _ = provider.DB.Update(func(txn *badger.Txn) error { + return txn.Delete([]byte(key)) + }) } // DeleteMany method will delete the responses in Badger provider if exists corresponding to the regex key param diff --git a/pkg/storage/embeddedOlricProvider.go b/pkg/storage/embeddedOlricProvider.go index 45df0c438..e20ab51d9 100644 --- a/pkg/storage/embeddedOlricProvider.go +++ b/pkg/storage/embeddedOlricProvider.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "errors" "net/http" "os" "strings" @@ -185,6 +186,50 @@ func (provider *EmbeddedOlric) Prefix(key string, req *http.Request, validator * return nil } +// GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. +func (provider *EmbeddedOlric) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { + res, e := provider.dm.Get(provider.ct, key) + + if e != nil { + return fresh, stale + } + + val, _ := res.Byte() + fresh, stale, _ = mappingElection(provider, val, req, validator, provider.logger) + + return fresh, stale +} + +// SetMultiLevel tries to store the key with the given value and update the mapping key to store metadata. +func (provider *EmbeddedOlric) SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration) error { + now := time.Now() + + if err := provider.dm.Put(provider.ct, variedKey, value, olric.EX(duration+provider.stale)); err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into EmbeddedOlric, %v", err) + return err + } + + mappingKey := MappingKeyPrefix + baseKey + res, e := provider.dm.Get(provider.ct, mappingKey) + if e != nil && !errors.Is(e, olric.ErrKeyNotFound) { + provider.logger.Sugar().Errorf("Impossible to get the key %s EmbeddedOlric, %v", baseKey, e) + return nil + } + + val, e := res.Byte() + if e != nil { + provider.logger.Sugar().Errorf("Impossible to parse the key %s value as byte, %v", baseKey, e) + return e + } + + val, e = mappingUpdater(variedKey, val, provider.logger, now, now.Add(duration), now.Add(duration+provider.stale), variedHeaders, etag) + if e != nil { + return e + } + + return provider.dm.Put(provider.ct, mappingKey, val) +} + // Get method returns the populated response if exists, empty response then func (provider *EmbeddedOlric) Get(key string) []byte { res, err := provider.dm.Get(provider.ct, key) diff --git a/pkg/storage/etcdProvider.go b/pkg/storage/etcdProvider.go index 086ddd795..5cb13574e 100644 --- a/pkg/storage/etcdProvider.go +++ b/pkg/storage/etcdProvider.go @@ -174,8 +174,34 @@ func (provider *Etcd) Prefix(key string, req *http.Request, validator *rfc.Reval return nil } -// Set method will store the response in Etcd provider -func (provider *Etcd) Set(key string, value []byte, url t.URL, duration time.Duration) error { +// GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. +func (provider *Etcd) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { + if provider.reconnecting { + provider.logger.Sugar().Error("Impossible to get the etcd key while reconnecting.") + return + } + + r, e := provider.Client.Get(provider.ctx, MappingKeyPrefix+key) + if e != nil { + go provider.Reconnect() + return fresh, stale + } + + if len(r.Kvs) > 0 { + fresh, stale, _ = mappingElection(provider, r.Kvs[0].Value, req, validator, provider.logger) + } + + return fresh, stale +} + +// SetMultiLevel tries to store the key with the given value and update the mapping key to store metadata. +func (provider *Etcd) SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration) error { + if provider.reconnecting { + provider.logger.Sugar().Error("Impossible to set the etcd value while reconnecting.") + return fmt.Errorf("reconnecting error") + } + + now := time.Now() if provider.reconnecting { provider.logger.Sugar().Error("Impossible to set the etcd value while reconnecting.") return fmt.Errorf("reconnecting error") @@ -183,13 +209,11 @@ func (provider *Etcd) Set(key string, value []byte, url t.URL, duration time.Dur if provider.Client.ActiveConnection().GetState() != connectivity.Ready && provider.Client.ActiveConnection().GetState() != connectivity.Idle { return fmt.Errorf("the connection is not ready: %v", provider.Client.ActiveConnection().GetState()) } - if duration == 0 { - duration = url.TTL.Duration - } rs, err := provider.Client.Grant(context.TODO(), int64(duration.Seconds())) if err == nil { - _, err = provider.Client.Put(provider.ctx, key, string(value), clientv3.WithLease(rs.ID)) + _, err = provider.Client.Put(provider.ctx, variedKey, string(value), clientv3.WithLease(rs.ID)) + fmt.Println("Put err =>", err) } if err != nil { @@ -200,7 +224,33 @@ func (provider *Etcd) Set(key string, value []byte, url t.URL, duration time.Dur return err } - _, err = provider.Client.Put(provider.ctx, StalePrefix+key, string(value), clientv3.WithLease(rs.ID)) + mappingKey := MappingKeyPrefix + baseKey + r := provider.Get(mappingKey) + val, e := mappingUpdater(variedKey, []byte(r), provider.logger, now, now.Add(duration), now.Add(duration+provider.stale), variedHeaders, etag) + if e != nil { + return e + } + + return provider.Set(mappingKey, val, t.URL{}, duration+provider.stale) +} + +// Set method will store the response in Etcd provider +func (provider *Etcd) Set(key string, value []byte, url t.URL, duration time.Duration) error { + if provider.reconnecting { + provider.logger.Sugar().Error("Impossible to set the etcd value while reconnecting.") + return fmt.Errorf("reconnecting error") + } + if provider.Client.ActiveConnection().GetState() != connectivity.Ready && provider.Client.ActiveConnection().GetState() != connectivity.Idle { + return fmt.Errorf("the connection is not ready: %v", provider.Client.ActiveConnection().GetState()) + } + if duration == 0 { + duration = url.TTL.Duration + } + + rs, err := provider.Client.Grant(context.TODO(), int64(duration.Seconds())) + if err == nil { + _, err = provider.Client.Put(provider.ctx, key, string(value), clientv3.WithLease(rs.ID)) + } if err != nil { if !provider.reconnecting { diff --git a/pkg/storage/nutsProvider.go b/pkg/storage/nutsProvider.go index 82af94363..1c68567e3 100644 --- a/pkg/storage/nutsProvider.go +++ b/pkg/storage/nutsProvider.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "net/http" "strconv" "strings" @@ -214,6 +215,73 @@ func (provider *Nuts) Prefix(key string, req *http.Request, validator *rfc.Reval return result } +// GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. +func (provider *Nuts) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { + _ = provider.DB.View(func(tx *nutsdb.Tx) error { + i, e := tx.Get(bucket, []byte(MappingKeyPrefix+key)) + if e != nil && !errors.Is(e, nutsdb.ErrKeyNotFound) { + return e + } + + var val []byte + if i != nil { + val = i.Value + } + fresh, stale, e = mappingElection(provider, val, req, validator, provider.logger) + + return e + }) + + return +} + +// SetMultiLevel tries to store the key with the given value and update the mapping key to store metadata. +func (provider *Nuts) SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration) error { + now := time.Now() + + err := provider.DB.Update(func(tx *nutsdb.Tx) error { + e := tx.Put(bucket, []byte(variedKey), value, uint32((duration + provider.stale).Seconds())) + if e != nil { + provider.logger.Sugar().Errorf("Impossible to set the key %s into Nuts, %v", variedKey, e) + } + + return e + }) + + if err != nil { + return err + } + + err = provider.DB.Update(func(tx *nutsdb.Tx) error { + mappingKey := MappingKeyPrefix + baseKey + item, e := tx.Get(bucket, []byte(mappingKey)) + if e != nil && !errors.Is(e, nutsdb.ErrKeyNotFound) { + provider.logger.Sugar().Errorf("Impossible to get the base key %s in Nuts, %v", baseKey, e) + return e + } + + var val []byte + if item != nil { + val = item.Value + } + + val, e = mappingUpdater(variedKey, val, provider.logger, now, now.Add(duration), now.Add(duration+provider.stale), variedHeaders, etag) + if e != nil { + return e + } + + provider.logger.Sugar().Debugf("Store the new mapping for the key %s in Nuts, %v", variedKey, string(val)) + + return tx.Put(bucket, []byte(mappingKey), val, nutsdb.Persistent) + }) + + if err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + } + + return err +} + // Set method will store the response in Nuts provider func (provider *Nuts) Set(key string, value []byte, url t.URL, duration time.Duration) error { if duration == 0 { diff --git a/pkg/storage/olricProvider.go b/pkg/storage/olricProvider.go index 0e5ccb149..708c2f153 100644 --- a/pkg/storage/olricProvider.go +++ b/pkg/storage/olricProvider.go @@ -111,6 +111,54 @@ func (provider *Olric) MapKeys(prefix string) map[string]string { return keys } +// GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. +func (provider *Olric) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { + dm := provider.dm.Get().(olric.DMap) + defer provider.dm.Put(dm) + res, e := dm.Get(context.Background(), key) + + if e != nil { + return fresh, stale + } + + val, _ := res.Byte() + fresh, stale, _ = mappingElection(provider, val, req, validator, provider.logger) + + return fresh, stale +} + +// SetMultiLevel tries to store the key with the given value and update the mapping key to store metadata. +func (provider *Olric) SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration) error { + now := time.Now() + + dm := provider.dm.Get().(olric.DMap) + defer provider.dm.Put(dm) + if err := dm.Put(context.Background(), variedKey, value, olric.EX(duration)); err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into EmbeddedOlric, %v", err) + return err + } + + mappingKey := MappingKeyPrefix + baseKey + res, e := dm.Get(context.Background(), mappingKey) + if e != nil && !errors.Is(e, olric.ErrKeyNotFound) { + provider.logger.Sugar().Errorf("Impossible to get the key %s EmbeddedOlric, %v", baseKey, e) + return nil + } + + val, e := res.Byte() + if e != nil { + provider.logger.Sugar().Errorf("Impossible to parse the key %s value as byte, %v", baseKey, e) + return e + } + + val, e = mappingUpdater(variedKey, val, provider.logger, now, now.Add(duration), now.Add(duration+provider.stale), variedHeaders, etag) + if e != nil { + return e + } + + return provider.Set(mappingKey, val, t.URL{}, time.Hour) +} + // Prefix method returns the populated response if exists, empty response then func (provider *Olric) Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response { if provider.reconnecting { diff --git a/pkg/storage/redisProvider.go b/pkg/storage/redisProvider.go index 8bd82cccb..306f94728 100644 --- a/pkg/storage/redisProvider.go +++ b/pkg/storage/redisProvider.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net" "net/http" @@ -22,6 +23,7 @@ import ( // Redis provider type type Redis struct { + inClient redis.Client Client rueidiscompat.Cmdable stale time.Duration ctx context.Context @@ -63,6 +65,7 @@ func RedisConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, e compat := rueidiscompat.NewAdapter(cli) return &Redis{ + inClient: cli, Client: compat, ctx: context.Background(), stale: dc.GetStale(), @@ -108,23 +111,82 @@ func (provider *Redis) MapKeys(prefix string) map[string]string { return m } -// Get method returns the populated response if exists, empty response then -func (provider *Redis) Get(key string) (item []byte) { +// GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. +func (provider *Redis) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { if provider.reconnecting { provider.logger.Sugar().Error("Impossible to get the redis key while reconnecting.") return } - r, e := provider.Client.Get(provider.ctx, key).Result() + + b, e := provider.inClient.Do(provider.ctx, provider.inClient.B().Get().Key(MappingKeyPrefix+key).Build()).AsBytes() if e != nil { - if e != redis.Nil && !provider.reconnecting { + if !errors.Is(e, redis.Nil) && !provider.reconnecting { go provider.Reconnect() } - return + return fresh, stale + } + + fresh, stale, _ = mappingElection(provider, b, req, validator, provider.logger) + + return fresh, stale +} + +// SetMultiLevel tries to store the key with the given value and update the mapping key to store metadata. +func (provider *Redis) SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration) error { + if provider.reconnecting { + provider.logger.Sugar().Error("Impossible to set the redis value while reconnecting.") + return fmt.Errorf("reconnecting error") + } + + now := time.Now() + if err := provider.inClient.Do(provider.ctx, provider.inClient.B().Set().Key(variedKey).Value(string(value)).Ex(duration+provider.stale).Build()).Error(); err != nil { + if !provider.reconnecting { + go provider.Reconnect() + } + provider.logger.Sugar().Errorf("Impossible to set value into Redis, %v", err) + return err + } + + mappingKey := MappingKeyPrefix + baseKey + v, e := provider.inClient.Do(provider.ctx, provider.inClient.B().Get().Key(mappingKey).Build()).AsBytes() + if e != nil && !errors.Is(e, redis.Nil) { + if !provider.reconnecting { + go provider.Reconnect() + } + return e + } + + val, e := mappingUpdater(variedKey, v, provider.logger, now, now.Add(duration), now.Add(duration+provider.stale), variedHeaders, etag) + if e != nil { + return e + } + + if e = provider.inClient.Do(provider.ctx, provider.inClient.B().Set().Key(mappingKey).Value(string(val)).Build()).Error(); e != nil { + if !provider.reconnecting { + go provider.Reconnect() + } + provider.logger.Sugar().Errorf("Impossible to set value into Redis, %v", e) + } + + return e +} + +// Get method returns the populated response if exists, empty response then +func (provider *Redis) Get(key string) []byte { + if provider.reconnecting { + provider.logger.Sugar().Error("Impossible to get the redis key while reconnecting.") + return nil } - item = []byte(r) + r, e := provider.inClient.Do(provider.ctx, provider.inClient.B().Get().Key(key).Build()).AsBytes() + if e != nil && e != redis.Nil { + if !provider.reconnecting { + go provider.Reconnect() + } + return nil + } - return + return r } // Prefix method returns the populated response if exists, empty response then diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index adfd3298e..f917773cb 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,14 +1,19 @@ package storage import ( + "bufio" + "bytes" + "encoding/gob" "errors" "net/http" "net/url" "strings" + "time" "github.com/darkweak/souin/configurationtypes" "github.com/darkweak/souin/pkg/rfc" "github.com/darkweak/souin/pkg/storage/types" + "go.uber.org/zap" ) const ( @@ -16,6 +21,7 @@ const ( encodedHeaderColonSeparator = "%3A" StalePrefix = "STALE_" surrogatePrefix = "SURROGATE_" + MappingKeyPrefix = "IDX_" ) type StorerInstanciator func(configurationtypes.AbstractConfigurationInterface) (types.Storer, error) @@ -145,3 +151,101 @@ func varyVoter(baseKey string, req *http.Request, currentKey string) bool { return false } + +func mappingElection(provider types.Storer, item []byte, req *http.Request, validator *rfc.Revalidator, logger *zap.Logger) (resultFresh *http.Response, resultStale *http.Response, e error) { + var mapping types.StorageMapper + if len(item) != 0 { + e = gob.NewDecoder(bytes.NewBuffer(item)).Decode(&mapping) + if e != nil { + return resultFresh, resultStale, e + } + } + + for keyName, keyItem := range mapping.Mapping { + valid := true + for hname, hval := range keyItem.VariedHeaders { + if req.Header.Get(hname) != strings.Join(hval, ", ") { + valid = false + break + } + } + + if !valid { + continue + } + + rfc.ValidateETagFromHeader(keyItem.Etag, validator) + if validator.Matched { + // If the key is fresh enough. + if time.Since(keyItem.FreshTime) < 0 { + response := provider.Get(keyName) + if response != nil { + if resultFresh, e = http.ReadResponse(bufio.NewReader(bytes.NewBuffer(response)), req); e != nil { + logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(keyName), e) + return + } + + logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", string(keyName), validator) + return + } + } + + // If the key is still stale. + if time.Since(keyItem.StaleTime) < 0 { + response := provider.Get(keyName) + if response != nil { + if resultStale, e = http.ReadResponse(bufio.NewReader(bytes.NewBuffer(response)), req); e != nil { + logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(keyName), e) + return + } + + logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v as stale", string(keyName), validator) + // We can always return the found stale because a fresh response could be in the next iteration. + if resultFresh != nil { + return + } + } + } + } else { + logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", string(keyName), validator) + } + } + + return +} + +func mappingUpdater(key string, item []byte, logger *zap.Logger, now, freshTime, staleTime time.Time, variedHeaders http.Header, etag string) (val []byte, e error) { + var mapping types.StorageMapper + if len(item) == 0 { + mapping = types.StorageMapper{} + } else { + e = gob.NewDecoder(bytes.NewBuffer(item)).Decode(&mapping) + if e != nil { + logger.Sugar().Errorf("Impossible to decode the key %s, %v", key, e) + return nil, e + } + } + + if mapping.Mapping == nil { + mapping.Mapping = make(map[string]types.KeyIndex) + } + + mapping.Mapping[key] = types.KeyIndex{ + StoredAt: now, + FreshTime: freshTime, + StaleTime: staleTime, + VariedHeaders: variedHeaders, + Etag: etag, + } + + buf := new(bytes.Buffer) + e = gob.NewEncoder(buf).Encode(mapping) + if e != nil { + logger.Sugar().Errorf("Impossible to encode the mapping value for the key %s, %v", key, e) + return nil, e + } + + val = buf.Bytes() + + return val, e +} diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 699352a84..0705f0bc1 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -8,6 +8,18 @@ import ( "github.com/darkweak/souin/pkg/rfc" ) +type KeyIndex struct { + StoredAt time.Time `json:"stored"` + FreshTime time.Time `json:"fresh"` + StaleTime time.Time `json:"stale"` + VariedHeaders http.Header `json:"varied"` + Etag string `json:"etag"` +} + +type StorageMapper struct { + Mapping map[string]KeyIndex `json:"mapping"` +} + type Storer interface { MapKeys(prefix string) map[string]string ListKeys() []string @@ -19,4 +31,8 @@ type Storer interface { Init() error Name() string Reset() error + + // Multi level storer to handle fresh/stale at once + GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) + SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration) error } diff --git a/plugins/beego/souin_test.go b/plugins/beego/souin_test.go index a134740c7..c36b7f6b7 100644 --- a/plugins/beego/souin_test.go +++ b/plugins/beego/souin_test.go @@ -34,6 +34,9 @@ func prepare() (res, res2 *httptest.ResponseRecorder) { _ = web.LoadAppConfig("json", "beego.json") httpcache := NewHTTPCache(DevDefaultConfiguration) + for _, storer := range httpcache.SouinBaseHandler.Storers { + _ = storer.Reset() + } web.InsertFilterChain("/*", httpcache.chainHandleFilter) @@ -114,7 +117,7 @@ func Test_SouinBeegoPlugin_Middleware_APIHandle(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/chi/souin_test.go b/plugins/chi/souin_test.go index ec5f11ddc..f366cb6ef 100644 --- a/plugins/chi/souin_test.go +++ b/plugins/chi/souin_test.go @@ -38,7 +38,11 @@ func excludedHandler(w http.ResponseWriter, _ *http.Request) { func prepare() (res *httptest.ResponseRecorder, res2 *httptest.ResponseRecorder, router chi.Router) { r := chi.NewRouter() - r.Use(NewHTTPCache(DevDefaultConfiguration).Handle) + httpCache := NewHTTPCache(DevDefaultConfiguration) + for _, storer := range httpCache.SouinBaseHandler.Storers { + _ = storer.Reset() + } + r.Use(httpCache.Handle) r.Get("/", defaultHandler) r.Get("/excluded", excludedHandler) router = r @@ -113,7 +117,7 @@ func Test_SouinChiPlugin_Middleware_APIHandle(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/dotweb/souin_test.go b/plugins/dotweb/souin_test.go index 1fc07a724..2bac0a5da 100644 --- a/plugins/dotweb/souin_test.go +++ b/plugins/dotweb/souin_test.go @@ -33,6 +33,9 @@ func defaultHandler(ctx dotweb.Context) error { func prepare() (res *httptest.ResponseRecorder, res2 *httptest.ResponseRecorder, app *dotweb.DotWeb) { app = dotweb.New() httpcache := NewHTTPCache(DevDefaultConfiguration) + for _, storer := range httpcache.SouinBaseHandler.Storers { + _ = storer.Reset() + } app.HttpServer.Router().GET("/:p/:n", defaultHandler).Use(httpcache) app.HttpServer.Router().GET("/:p", defaultHandler).Use(httpcache) res = httptest.NewRecorder() @@ -113,7 +116,7 @@ func Test_SouinDotwebPlugin_Middleware_APIHandle(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/echo/souin_test.go b/plugins/echo/souin_test.go index 8506489ac..54d4b78b3 100644 --- a/plugins/echo/souin_test.go +++ b/plugins/echo/souin_test.go @@ -133,7 +133,7 @@ func Test_SouinEchoPlugin_Process_APIHandle(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/fiber/souin_test.go b/plugins/fiber/souin_test.go index cbf0bacf6..0cff0cc4a 100644 --- a/plugins/fiber/souin_test.go +++ b/plugins/fiber/souin_test.go @@ -29,7 +29,11 @@ func Test_NewHTTPCache(t *testing.T) { func prepare() *fiber.App { app := fiber.New() - app.Use(NewHTTPCache(DevDefaultConfiguration).Handle) + httpcache := NewHTTPCache(DevDefaultConfiguration) + for _, storer := range httpcache.SouinBaseHandler.Storers { + _ = storer.Reset() + } + app.Use(httpcache.Handle) app.Get("/*", func(c *fiber.Ctx) error { return c.SendString("Hello, World 👋!") }) @@ -139,7 +143,7 @@ func Test_SouinFiberPlugin_Middleware_APIHandle(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/gin/souin_test.go b/plugins/gin/souin_test.go index 6343f3457..bbb3411a9 100644 --- a/plugins/gin/souin_test.go +++ b/plugins/gin/souin_test.go @@ -30,6 +30,9 @@ func prepare() (res *httptest.ResponseRecorder, res2 *httptest.ResponseRecorder, res2 = httptest.NewRecorder() gin.SetMode(gin.TestMode) s := New(DevDefaultConfiguration) + for _, storer := range s.SouinBaseHandler.Storers { + _ = storer.Reset() + } c, r = gin.CreateTestContext(res) c.Request = req r.Use(s.Process()) diff --git a/plugins/go-zero/souin_test.go b/plugins/go-zero/souin_test.go index 7e28e4ae6..8cea243cc 100644 --- a/plugins/go-zero/souin_test.go +++ b/plugins/go-zero/souin_test.go @@ -31,7 +31,11 @@ func defaultHandler(w http.ResponseWriter, _ *http.Request) { } func prepare() (res *httptest.ResponseRecorder, res2 *httptest.ResponseRecorder, router http.HandlerFunc) { - router = NewHTTPCache(DevDefaultConfiguration).Handle(defaultHandler) + httpcache := NewHTTPCache(DevDefaultConfiguration) + for _, storer := range httpcache.SouinBaseHandler.Storers { + _ = storer.Reset() + } + router = httpcache.Handle(defaultHandler) res = httptest.NewRecorder() res2 = httptest.NewRecorder() return @@ -103,7 +107,7 @@ func Test_SouinGoZeroPlugin_Middleware_APIHandle(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/goa/souin_test.go b/plugins/goa/souin_test.go index 9100f26ea..adf4406b1 100644 --- a/plugins/goa/souin_test.go +++ b/plugins/goa/souin_test.go @@ -97,7 +97,7 @@ func Test_SouinGoaPlugin_Middleware_APIHandle(t *testing.T) { } b, _ := io.ReadAll(res.Result().Body) res.Result().Body.Close() - if string(b) != "[]" { + if string(b) != "[\"IDX_GET-http-example.com-/handled\",\"IDX_GET-http-example.com-/not-handled\"]" { t.Error("The response body must be an empty array because no request has been stored") } handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/handled", nil)) @@ -109,10 +109,10 @@ func Test_SouinGoaPlugin_Middleware_APIHandle(t *testing.T) { res2.Result().Body.Close() var payload []string _ = json.Unmarshal(b, &payload) - if len(payload) != 2 { + if len(payload) != 3 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/goyave/souin_test.go b/plugins/goyave/souin_test.go index 3787f364c..72d609a89 100644 --- a/plugins/goyave/souin_test.go +++ b/plugins/goyave/souin_test.go @@ -32,7 +32,11 @@ type HttpCacheMiddlewareTestSuite struct { } func prepare(suite *HttpCacheMiddlewareTestSuite, request *http.Request) (*SouinGoyaveMiddleware, *goyave.Request) { - return NewHTTPCache(DevDefaultConfiguration), suite.CreateTestRequest(request) + httpcache := NewHTTPCache(DevDefaultConfiguration) + for _, storer := range httpcache.SouinBaseHandler.Storers { + _ = storer.Reset() + } + return httpcache, suite.CreateTestRequest(request) } func TestHttpCacheMiddlewareTestSuite(t *testing.T) { @@ -135,7 +139,7 @@ func (suite *HttpCacheMiddlewareTestSuite) Test_SouinFiberPlugin_Middleware_APIH if len(payload) != 2 { suite.T().Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { suite.T().Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/kratos/souin_test.go b/plugins/kratos/souin_test.go index 108d50102..eb98c6d22 100644 --- a/plugins/kratos/souin_test.go +++ b/plugins/kratos/souin_test.go @@ -133,11 +133,11 @@ func Test_HttpcacheKratosPlugin_NewHTTPCacheFilter_API(t *testing.T) { rs := res.Result() defer rs.Body.Close() if rs.Header.Get("Content-Type") != "application/json" { - t.Error("The response must contain be in JSON.") + t.Error("The response must be in JSON.") } b, _ := io.ReadAll(rs.Body) res.Result().Body.Close() - if string(b) != "[]" { + if string(b) != "[\"IDX_GET-http-example.com-/handled\"]" { t.Error("The response body must be an empty array because no request has been stored") } req2 := httptest.NewRequest(http.MethodGet, "/handled", nil) @@ -152,7 +152,7 @@ func Test_HttpcacheKratosPlugin_NewHTTPCacheFilter_API(t *testing.T) { rs = res3.Result() rs.Body.Close() if rs.Header.Get("Content-Type") != "application/json" { - t.Error("The response must contain be in JSON.") + t.Error("The response must be in JSON.") } b, _ = io.ReadAll(rs.Body) rs.Body.Close() @@ -161,7 +161,7 @@ func Test_HttpcacheKratosPlugin_NewHTTPCacheFilter_API(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/roadrunner/httpcache_test.go b/plugins/roadrunner/httpcache_test.go index b95eb2f37..d126d322a 100644 --- a/plugins/roadrunner/httpcache_test.go +++ b/plugins/roadrunner/httpcache_test.go @@ -201,6 +201,9 @@ func Test_Plugin_Middleware_Mutation(t *testing.T) { func Test_Plugin_Middleware_API(t *testing.T) { p := &Plugin{} _ = p.Init(&configWrapper{}, newTestLogger()) + for _, storer := range p.SouinBaseHandler.Storers { + _ = storer.Reset() + } handler := p.Middleware(nextFilter) req, res, res2 := prepare("/httpcache_api/httpcache") handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("PURGE", "/httpcache_api/httpcache/.+", nil)) @@ -253,7 +256,7 @@ func Test_Plugin_Middleware_API(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } } diff --git a/plugins/webgo/souin_test.go b/plugins/webgo/souin_test.go index db9164166..bbc510088 100644 --- a/plugins/webgo/souin_test.go +++ b/plugins/webgo/souin_test.go @@ -53,6 +53,9 @@ func prepare() (res *httptest.ResponseRecorder, res2 *httptest.ResponseRecorder, res = httptest.NewRecorder() res2 = httptest.NewRecorder() httpcache := NewHTTPCache(DevDefaultConfiguration) + for _, storer := range httpcache.SouinBaseHandler.Storers { + _ = storer.Reset() + } router = webgo.NewRouter(cfg, getRoutes()...) router.Use(httpcache.Middleware) return @@ -135,7 +138,7 @@ func Test_SouinWebgoPlugin_Middleware_APIHandle(t *testing.T) { if len(payload) != 2 { t.Error("The system must store 2 items, the fresh and the stale one") } - if payload[0] != "GET-http-example.com-/handled" || payload[1] != "STALE_GET-http-example.com-/handled" { + if payload[0] != "GET-http-example.com-/handled" || payload[1] != "IDX_GET-http-example.com-/handled" { t.Error("The payload items mismatch from the expectations.") } }