feat(core): default cache control behaviour (#184)

* Add default_cache_control for core and traefik * Move adding default Cache-Control header to bridge.go * Add to core docs * Add to traefik plugin docs * Add for Caddy plugin * Add to Caddy plugin docs * Add code and docs for Tyk plugin * Add Caddy plugin docs to top-level README * Set unit test traefik & caddy configuration to match old behavior * Fix interface{} is nil error of traefik plugin * Update without unnecessary assign * Update README + bump caddy to the next release * Bump caddy version * Stable olric container version * Bump olric version Co-authored-by: Menci <huanghaorui301@gmail.com>
darkweak · Jan 31, 2022 · 8962eb6 · 8962eb6
1 parent 8f2ac3a
commit 8962eb6
Show file tree

Hide file tree

Showing 89 changed files with 1,210 additions and 1,623 deletions.
diff --git a/README.md b/README.md
@@ -62,6 +62,8 @@ reverse_proxy_url: 'http://traefik' # If it's in the same network you can use ht
 | `default_cache.ttl` | Duration to cache request (in seconds)                      | 10                                                                                                                        |
 | `reverse_proxy_url` | The reverse-proxy's instance URL (Apache, Nginx, Træfik...) | - `http://yourservice` (Container way)<br/>`http://localhost:81` (Local way)<br/>`http://yourdomain.com:81` (Network way) |
 
+Besides, it's highly recommended to set `default_cache.default_cache_control` (see it below) to avoid undesired caching for responses without `Cache-Control` header.
+
 ### Optional configuration
 ```yaml
 # /anywhere/configuration.yml
@@ -94,6 +96,7 @@ default_cache:
     exclude: 'ARegexHere' # Regex to exclude from cache
   stale: 1000s # Stale duration
   ttl: 1000s # Default TTL
+  default_cache_control: no-store # Set default value for Cache-Control response header if not set by upstream
 log_level: INFO # Logs verbosity [ DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL ], case do not matter
 ssl_providers: # The {providers}.json to use
   - traefik
@@ -109,6 +112,7 @@ urls:
     headers: # Override default headers
     - Authorization
     - 'Content-Type'
+    default_cache_control: public, max-age=86400 # Override default default Cache-Control
 ykeys:
   The_First_Test:
     headers:
@@ -157,10 +161,12 @@ surrogate_keys:
 | `default_cache.regex.exclude`                     | The regex used to prevent paths being cached                                                   | `^[A-z]+.*$`                                                                                                            |
 | `default_cache.stale`                             | The stale duration                                                                             | `25m`                                                                                                                   |
 | `default_cache.ttl`                               | The TTL duration                                                                               | `120s`                                                                                                                  |
+| `default_cache.default_cache_control`             | Set the default value of `Cache-Control` response header if not set by upstream (Souin treats empty `Cache-Control` as `public` if omitted) | `no-store`                                                                                                                  |
 | `log_level`                                       | The log level                                                                                  | `One of DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL it's case insensitive`                                           |
 | `ssl_providers`                                   | List of your providers handling certificates                                                   | `- traefik`<br/><br/>`- nginx`<br/><br/>`- apache`                                                                      |
 | `urls.{your url or regex}`                        | List of your custom configuration depending each URL or regex                                  | 'https:\/\/yourdomain.com'                                                                                              |
 | `urls.{your url or regex}.ttl`                    | Override the default TTL if defined                                                            | `90s`<br/><br/>`10m`                                                                                                    |
+| `urls.{your url or regex}.default_cache_control`  | Override the default default `Cache-Control` if defined                                                            | `public, max-age=86400`                                                                                                    |
 | `urls.{your url or regex}.headers`                | Override the default headers if defined                                                        | `- Authorization`<br/><br/>`- 'Content-Type'`                                                                           |
 | `surrogate_keys.{key name}.headers`               | Headers that should match to be part of the surrogate key group                                | `Authorization: ey.+`<br/><br/>`Content-Type: json`                                                                     |
 | `surrogate_keys.{key name}.headers.{header name}` | Header name that should be present a match the regex to be part of the surrogate key group     | `Content-Type: json`                                                                                                    |
@@ -341,6 +347,7 @@ There is the fully configuration below
         }
         stale 200s
         ttl 1000s
+        default_cache_control no-store
     }
 }
 
@@ -359,6 +366,7 @@ cache @match {
 cache @match2 {
     ttl 50s
     headers Authorization
+    default_cache_control "public, max-age=86400"
 }
 
 cache @matchdefault {
@@ -476,6 +484,7 @@ http:
             regex:
               exclude: '/test_exclude.*'
             ttl: 5s
+            default_cache_control: no-store
           log_level: debug
           urls:
             'domain.com/testing':
@@ -487,6 +496,7 @@ http:
               headers:
                 - Authorization
                 - 'Content-Type'
+              default_cache_control: public, max-age=86400
           ykeys:
             The_First_Test:
               headers:
@@ -578,7 +588,8 @@ You have to define the use of Souin as `post` and `response` custom middleware.
       }
     },
     "default_cache": {
-      "ttl": "5s"
+      "ttl": "5s",
+      "default_cache_control": "no-store"
     }
   }
 }
@@ -607,3 +618,4 @@ Thanks to these users for contributing or helping this project in any way
 * [Massimiliano Cannarozzo](https://github.com/maxcanna/)
 * [Kevin Pollet](https://github.com/kevinpollet)
 * [Choelzl](https://github.com/choelzl)
+* [Menci](https://github.com/menci)
diff --git a/cache/types/souin.go b/cache/types/souin.go
@@ -84,8 +84,9 @@ func (r *RetrieverResponseProperties) SetMatchedURLFromRequest(req *http.Request
 	path := req.Host + req.URL.Path
 	regexpURL := r.GetRegexpUrls().FindString(path)
 	url := configurationtypes.URL{
-		TTL:     configurationtypes.Duration{Duration: r.GetConfiguration().GetDefaultCache().GetTTL()},
-		Headers: r.GetConfiguration().GetDefaultCache().GetHeaders(),
+		TTL:                 configurationtypes.Duration{Duration: r.GetConfiguration().GetDefaultCache().GetTTL()},
+		Headers:             r.GetConfiguration().GetDefaultCache().GetHeaders(),
+		DefaultCacheControl: r.GetConfiguration().GetDefaultCache().GetDefaultCacheControl(),
 	}
 	if regexpURL != "" {
 		u := r.GetConfiguration().GetUrls()[regexpURL]

diff --git a/configurationtypes/types.go b/configurationtypes/types.go
@@ -57,8 +57,9 @@ type Regex struct {
 
 // URL configuration
 type URL struct {
-	TTL     Duration `json:"ttl" yaml:"ttl"`
-	Headers []string `json:"headers" yaml:"headers"`
+	TTL                 Duration `json:"ttl" yaml:"ttl"`
+	Headers             []string `json:"headers" yaml:"headers"`
+	DefaultCacheControl string   `json:"default_cache_control" yaml:"default_cache_control"`
 }
 
 // CacheProvider config
@@ -83,15 +84,16 @@ type CDN struct {
 
 // DefaultCache configuration
 type DefaultCache struct {
-	Badger      CacheProvider `json:"badger" yaml:"badger"`
-	CDN         CDN           `json:"cdn" yaml:"cdn"`
-	Distributed bool          `json:"distributed" yaml:"distributed"`
-	Headers     []string      `json:"headers" yaml:"headers"`
-	Olric       CacheProvider `json:"olric" yaml:"olric"`
-	Port        Port          `json:"port" yaml:"port"`
-	Regex       Regex         `json:"regex" yaml:"regex"`
-	TTL         Duration      `json:"ttl" yaml:"ttl"`
-	Stale       Duration      `json:"stale" yaml:"stale"`
+	Badger              CacheProvider `json:"badger" yaml:"badger"`
+	CDN                 CDN           `json:"cdn" yaml:"cdn"`
+	Distributed         bool          `json:"distributed" yaml:"distributed"`
+	Headers             []string      `json:"headers" yaml:"headers"`
+	Olric               CacheProvider `json:"olric" yaml:"olric"`
+	Port                Port          `json:"port" yaml:"port"`
+	Regex               Regex         `json:"regex" yaml:"regex"`
+	TTL                 Duration      `json:"ttl" yaml:"ttl"`
+	Stale               Duration      `json:"stale" yaml:"stale"`
+	DefaultCacheControl string        `json:"default_cache_control" yaml:"default_cache_control"`
 }
 
 // GetBadger returns the Badger configuration
@@ -134,6 +136,11 @@ func (d *DefaultCache) GetStale() time.Duration {
 	return d.Stale.Duration
 }
 
+// GetDefaultCacheControl returns the default Cache-Control response header value when empty
+func (d *DefaultCache) GetDefaultCacheControl() string {
+	return d.DefaultCacheControl
+}
+
 // DefaultCacheInterface interface
 type DefaultCacheInterface interface {
 	GetBadger() CacheProvider
@@ -144,6 +151,7 @@ type DefaultCacheInterface interface {
 	GetRegex() Regex
 	GetTTL() time.Duration
 	GetStale() time.Duration
+	GetDefaultCacheControl() string
 }
 
 // APIEndpoint is the minimal structure to define an endpoint

diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/darkweak/souin
 go 1.16
 
 require (
-	github.com/buraksezer/olric v0.4.1-0.20211130141003-a889ee19dcd9
+	github.com/buraksezer/olric v0.4.2
 	github.com/dgraph-io/badger/v3 v3.2103.2
 	github.com/dgraph-io/ristretto v0.1.0
 	github.com/fsnotify/fsnotify v1.5.1

diff --git a/go.sum b/go.sum
@@ -55,8 +55,8 @@ github.com/buraksezer/connpool v0.4.0 h1:fNLvWu0FOtJxL7Sqetm6+040mhZWVs8c6vrke14
 github.com/buraksezer/connpool v0.4.0/go.mod h1:qPiG7gKXo+EjrwG/yqn2StZM4ek6gcYnnGgFIVKN6b0=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
-github.com/buraksezer/olric v0.4.1-0.20211130141003-a889ee19dcd9 h1:oyFU+vboRDFCfHjecXS6GleoBwNoxG6Bk2tAAHl+kBA=
-github.com/buraksezer/olric v0.4.1-0.20211130141003-a889ee19dcd9/go.mod h1:jSgaKtv7MphQyVPqS2qGoecZKGqiL2YYmyQx9LUJk6Y=
+github.com/buraksezer/olric v0.4.2 h1:1W5UCYFYrkeD6OHGyTi6R9YTLJ3vXcVkgas4ZbFQMfw=
+github.com/buraksezer/olric v0.4.2/go.mod h1:xNt+/QiiVuqioJzgTMuiV2LynTNu87L1Tp5289XuU78=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

diff --git a/olric/Dockerfile-olric b/olric/Dockerfile-olric
@@ -1,15 +1,15 @@
-FROM docker.io/olricio/olric-dev:latest as olric
+FROM docker.io/olricio/olricd:latest as olric
 
 ENV OLRICD_CONFIG /etc/olricd.yaml
 ENV CGO_ENABLED 1
 
-WORKDIR /go
+# WORKDIR /go
 
-RUN go get -u github.com/buraksezer/olric
-
-WORKDIR /go/src/github.com/buraksezer/olric
-
-RUN go build -v -o /go/bin/olricd /go/src/github.com/buraksezer/olric/cmd/olricd/main.go
+# RUN go get -u github.com/buraksezer/olric
+# 
+# WORKDIR /go/src/github.com/buraksezer/olric
+# 
+# RUN go build -v -o /go/bin/olricd /go/src/github.com/buraksezer/olric/cmd/olricd/main.go
 COPY ./olric.yml /etc/olricd.yaml
 
-CMD ["/go/bin/olricd", "-c", "/go/src/github.com/buraksezer/olric/cmd/olricd/olricd.yaml"]
+CMD ["/go/bin/olricd", "-c", "/etc/olricd.yaml"]
diff --git a/plugins/base.go b/plugins/base.go
@@ -208,8 +208,9 @@ func DefaultSouinPluginInitializerFromConfiguration(c configurationtypes.Abstrac
 
 	retriever := &types.RetrieverResponseProperties{
 		MatchedURL: configurationtypes.URL{
-			TTL:     configurationtypes.Duration{Duration: c.GetDefaultCache().GetTTL()},
-			Headers: c.GetDefaultCache().GetHeaders(),
+			TTL:                 configurationtypes.Duration{Duration: c.GetDefaultCache().GetTTL()},
+			Headers:             c.GetDefaultCache().GetHeaders(),
+			DefaultCacheControl: c.GetDefaultCache().GetDefaultCacheControl(),
 		},
 		Provider:      provider,
 		Configuration: c,

diff --git a/plugins/caddy/Caddyfile b/plugins/caddy/Caddyfile
@@ -18,6 +18,7 @@
         headers Content-Type Authorization
         log_level debug
         ttl 1000s
+        default_cache_control public
     }
 }
 

diff --git a/plugins/caddy/README.md b/plugins/caddy/README.md
@@ -58,6 +58,7 @@ There is the fully configuration below
         }
         stale 200s
         ttl 1000s
+        default_cache_control no-store
     }
 }
 
@@ -76,6 +77,7 @@ cache @match {
 cache @match2 {
     ttl 50s
     headers Authorization
+    default_cache_control "public, max-age=86400"
 }
 
 cache @matchdefault {

diff --git a/plugins/caddy/configuration.go b/plugins/caddy/configuration.go
@@ -9,14 +9,15 @@ import (
 
 // DefaultCache the struct
 type DefaultCache struct {
-	Badger      configurationtypes.CacheProvider
-	CDN         configurationtypes.CDN
-	Distributed bool
-	Headers     []string
-	Olric       configurationtypes.CacheProvider
-	Regex       configurationtypes.Regex
-	TTL         configurationtypes.Duration
-	Stale       configurationtypes.Duration
+	Badger              configurationtypes.CacheProvider
+	CDN                 configurationtypes.CDN
+	Distributed         bool
+	Headers             []string
+	Olric               configurationtypes.CacheProvider
+	Regex               configurationtypes.Regex
+	TTL                 configurationtypes.Duration
+	Stale               configurationtypes.Duration
+	DefaultCacheControl string
 }
 
 // GetBadger returns the Badger configuration
@@ -59,6 +60,11 @@ func (d *DefaultCache) GetStale() time.Duration {
 	return d.Stale.Duration
 }
 
+// GetStale returns the stale duration
+func (d *DefaultCache) GetDefaultCacheControl() string {
+	return d.DefaultCacheControl
+}
+
 //Configuration holder
 type Configuration struct {
 	DefaultCache *DefaultCache

diff --git a/plugins/caddy/examples/Caddyfile-embedded-olric b/plugins/caddy/examples/Caddyfile-embedded-olric
@@ -7,6 +7,7 @@
             path path/to/olric.yml
         }
         ttl 1000s
+        default_cache_control no-store
     }
 }
 
@@ -24,6 +25,7 @@ cache @match {
 cache @match2 {
     ttl 50s
     headers Authorization
+    default_cache_control "public, max-age=86400"
 }
 
 cache @matchdefault {

diff --git a/plugins/caddy/examples/Caddyfile-file-configuration-olric b/plugins/caddy/examples/Caddyfile-file-configuration-olric
@@ -7,6 +7,7 @@
             path path/to/olric.yml
         }
         ttl 1000s
+        default_cache_control no-store
     }
 }
 
@@ -24,6 +25,7 @@ cache @match {
 cache @match2 {
     ttl 50s
     headers Authorization
+    default_cache_control "public, max-age=86400"
 }
 
 cache @matchdefault {

diff --git a/plugins/caddy/examples/Caddyfile-not-distributed b/plugins/caddy/examples/Caddyfile-not-distributed
@@ -4,6 +4,7 @@
         headers Content-Type Authorization
         log_level info
         ttl 1000s
+        default_cache_control no-store
     }
 }
 
@@ -21,6 +22,7 @@ cache @match {
 cache @match2 {
     ttl 50s
     headers Authorization
+    default_cache_control "public, max-age=86400"
 }
 
 cache @matchdefault {

diff --git a/plugins/caddy/examples/Caddyfile-remote-olric-cluster b/plugins/caddy/examples/Caddyfile-remote-olric-cluster
@@ -7,6 +7,7 @@
             url olric:3320
         }
         ttl 1000s
+        default_cache_control no-store
     }
 }
 
@@ -24,6 +25,7 @@ cache @match {
 cache @match2 {
     ttl 50s
     headers Authorization
+    default_cache_control "public, max-age=86400"
 }
 
 cache @matchdefault {

diff --git a/plugins/caddy/examples/configuration-embedded-olric.json b/plugins/caddy/examples/configuration-embedded-olric.json
@@ -9,7 +9,8 @@
       "olric": {
         "path": "path/to/olric.yml"
       },
-      "ttl": "1000s"
+      "ttl": "1000s",
+      "default_cache_control": "no-store"
     },
     "http": {
       "servers": {
@@ -30,7 +31,8 @@
               "handle": [
                 {
                   "handler": "cache",
-                  "ttl": "30s"
+                  "ttl": "30s",
+                  "default_cache_control": "public, max-age=86400"
                 }
               ]
             },

diff --git a/plugins/caddy/examples/configuration-file-configuration-olric.json b/plugins/caddy/examples/configuration-file-configuration-olric.json
@@ -9,7 +9,8 @@
       "olric": {
         "path": "path/to/olric.yml"
       },
-      "ttl": "1000s"
+      "ttl": "1000s",
+      "default_cache_control": "no-store"
     },
     "http": {
       "servers": {
@@ -30,7 +31,8 @@
               "handle": [
                 {
                   "handler": "cache",
-                  "ttl": "30s"
+                  "ttl": "30s",
+                  "default_cache_control": "public, max-age=86400"
                 }
               ]
             },

diff --git a/plugins/caddy/examples/configuration-not-distributed.json b/plugins/caddy/examples/configuration-not-distributed.json
@@ -6,7 +6,8 @@
         "Authorization"
       ],
       "log_level": "info",
-      "ttl": "1000s"
+      "ttl": "1000s",
+      "default_cache_control": "no-store"
     },
     "http": {
       "servers": {
@@ -27,7 +28,8 @@
               "handle": [
                 {
                   "handler": "cache",
-                  "ttl": "30s"
+                  "ttl": "30s",
+                  "default_cache_control": "public, max-age=86400"
                 }
               ]
             },