From 3550b8b51f305f56f24a8e82e7b53401adfcdedb Mon Sep 17 00:00:00 2001 From: Soner Sayakci Date: Tue, 10 Sep 2024 10:53:48 +0200 Subject: [PATCH 1/4] feat: cleanup varnish config --- compose.yml | 12 ++++ rootfs/etc/varnish/default.vcl | 118 +++++++++++++-------------------- 2 files changed, 57 insertions(+), 73 deletions(-) create mode 100644 compose.yml diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..ca13ee8 --- /dev/null +++ b/compose.yml @@ -0,0 +1,12 @@ +services: + varnish: + image: local/varnish + build: . + environment: + SHOPWARE_BACKEND_HOST: host.docker.internal + ports: + - "80:80" + develop: + watch: + - path: rootfs + action: rebuild diff --git a/rootfs/etc/varnish/default.vcl b/rootfs/etc/varnish/default.vcl index e60907c..1239bd6 100644 --- a/rootfs/etc/varnish/default.vcl +++ b/rootfs/etc/varnish/default.vcl @@ -17,26 +17,6 @@ acl purgers { } sub vcl_recv { - # Ignore query strings that are only necessary for the js on the client. Customize as needed. - if (req.url ~ "(\?|&)(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=") { - # see rfc3986#section-2.3 "Unreserved Characters" for regex - set req.url = regsuball(req.url, "(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=[A-Za-z0-9\-\_\.\~]+&?", ""); - } - set req.url = regsub(req.url, "(\?|\?&|&)$", ""); - - # Normalize query arguments - set req.url = std.querysort(req.url); - - # Set a header announcing Surrogate Capability to the origin - set req.http.Surrogate-Capability = "shopware=ESI/1.0"; - - # Make sure that the client ip is forward to the client. - if (req.http.x-forwarded-for) { - set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; - } else { - set req.http.X-Forwarded-For = client.ip; - } - # Handle PURGE if (req.method == "PURGE") { if (client.ip !~ purgers) { @@ -60,32 +40,16 @@ sub vcl_recv { return (synth(200, "BAN URLs containing (" + req.url + ") done.")); } - # Normalize Accept-Encoding header - # straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html - if (req.http.Accept-Encoding) { - if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") { - # No point in compressing these - unset req.http.Accept-Encoding; - } elsif (req.http.Accept-Encoding ~ "gzip") { - set req.http.Accept-Encoding = "gzip"; - } elsif (req.http.Accept-Encoding ~ "deflate") { - set req.http.Accept-Encoding = "deflate"; - } else { - # unknown algorithm - unset req.http.Accept-Encoding; - } - } - + # Only handle relevant HTTP request methods if (req.method != "GET" && req.method != "HEAD" && req.method != "PUT" && req.method != "POST" && + req.method != "PATCH" && req.method != "TRACE" && req.method != "OPTIONS" && - req.method != "PATCH" && req.method != "DELETE") { - /* Non-RFC2616 or CONNECT which is weird. */ - return (pipe); + return (pipe); } # We only deal with GET and HEAD by default @@ -104,6 +68,30 @@ sub vcl_recv { return (pass); } + # Collapse multiple cookie headers into one + std.collect(req.http.Cookie); + + # Ignore query strings that are only necessary for the js on the client. Customize as needed. + if (req.url ~ "(\?|&)(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=") { + # see rfc3986#section-2.3 "Unreserved Characters" for regex + set req.url = regsuball(req.url, "(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=[A-Za-z0-9\-\_\.\~]+&?", ""); + } + + set req.url = regsub(req.url, "(\?|\?&|&)$", ""); + + # Normalize query arguments + set req.url = std.querysort(req.url); + + # Set a header announcing Surrogate Capability to the origin + set req.http.Surrogate-Capability = "shopware=ESI/1.0"; + + # Make sure that the client ip is forward to the client. + if (req.http.x-forwarded-for) { + set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; + } else { + set req.http.X-Forwarded-For = client.ip; + } + return (hash); } @@ -132,15 +120,12 @@ sub vcl_hit { } sub vcl_backend_response { - # Fix Vary Header in some cases - # https://www.varnish-cache.org/trac/wiki/VCLExampleFixupVary - if (beresp.http.Vary ~ "User-Agent") { - set beresp.http.Vary = regsub(beresp.http.Vary, ",? *User-Agent *", ""); - set beresp.http.Vary = regsub(beresp.http.Vary, "^, *", ""); - if (beresp.http.Vary == "") { - unset beresp.http.Vary; - } - } + # Serve stale content for three days after object expiration + # Perform asynchronous revalidation while stale content is served + set beresp.grace = 3d; + + unset beresp.http.X-Powered-By; + unset beresp.http.Server; if (beresp.http.Surrogate-Control ~ "ESI/1.0") { unset beresp.http.Surrogate-Control; @@ -148,40 +133,27 @@ sub vcl_backend_response { return (deliver); } - # Respect the Cache-Control=private header from the backend - if ( - beresp.http.Pragma ~ "no-cache" || - beresp.http.Cache-Control ~ "no-cache" || - beresp.http.Cache-Control ~ "private" - ) { - set beresp.ttl = 0s; - set beresp.http.X-Cacheable = "NO:Cache-Control=private"; - set beresp.uncacheable = true; - return (deliver); + if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") { + set beresp.do_gzip = true; } - # strip the cookie before the image is inserted into cache. - if (bereq.url ~ "\.(png|gif|jpg|swf|css|js|webp)$") { - unset beresp.http.set-cookie; + if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) { + unset beresp.http.Set-Cookie; } - - # Allow items to be stale if needed. - set beresp.grace = 24h; - - # Save the bereq.url so bans work efficiently - set beresp.http.x-url = bereq.url; - set beresp.http.X-Cacheable = "YES"; - - return (deliver); } sub vcl_deliver { ## we don't want the client to cache - set resp.http.Cache-Control = "max-age=0, private"; + if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(theme|media|thumbnail|bundles)/") { + set resp.http.Pragma = "no-cache"; + set resp.http.Expires = "-1"; + set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0"; + } # invalidation headers are only for internal use unset resp.http.sw-invalidation-states; unset resp.http.xkey; - - set resp.http.X-Cache-Hits = obj.hits; + unset resp.http.X-Varnish; + unset resp.http.Via; + unset resp.http.Link; } From f6426c27f6f34a05102f927965a23ccf2151a4e2 Mon Sep 17 00:00:00 2001 From: Soner Sayakci Date: Tue, 10 Sep 2024 11:29:37 +0200 Subject: [PATCH 2/4] feat: use cookie vmod to parse cookies --- rootfs/etc/varnish/default.vcl | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/rootfs/etc/varnish/default.vcl b/rootfs/etc/varnish/default.vcl index 1239bd6..ab3baff 100644 --- a/rootfs/etc/varnish/default.vcl +++ b/rootfs/etc/varnish/default.vcl @@ -2,6 +2,7 @@ vcl 4.1; import std; import xkey; +import cookie; # Specify your app nodes here. Use round-robin balancing to add more than one. backend default { @@ -14,6 +15,7 @@ acl purgers { "127.0.0.1"; "localhost"; "::1"; + "172.17.0.1"; } sub vcl_recv { @@ -33,7 +35,7 @@ sub vcl_recv { if (req.method == "BAN") { if (!client.ip ~ purgers) { - return (synth(405, "Method not allowed")); + return (synth(403, "Forbidden")); } ban("req.url ~ "+req.url); @@ -52,13 +54,12 @@ sub vcl_recv { return (pipe); } - # We only deal with GET and HEAD by default - if (req.method != "GET" && req.method != "HEAD") { + if (req.http.Authorization) { return (pass); } - # Don't cache Authenticate & Authorization - if (req.http.Authenticate || req.http.Authorization) { + # We only deal with GET and HEAD by default + if (req.method != "GET" && req.method != "HEAD") { return (pass); } @@ -68,8 +69,11 @@ sub vcl_recv { return (pass); } - # Collapse multiple cookie headers into one - std.collect(req.http.Cookie); + cookie.parse(req.http.cookie); + + set req.http.cache-hash = cookie.get("sw-cache-hash"); + set req.http.currency = cookie.get("sw-currency"); + set req.http.states = cookie.get("sw-states"); # Ignore query strings that are only necessary for the js on the client. Customize as needed. if (req.url ~ "(\?|&)(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=") { @@ -97,18 +101,16 @@ sub vcl_recv { sub vcl_hash { # Consider Shopware HTTP cache cookies - if (req.http.cookie ~ "sw-cache-hash=") { - hash_data("+context=" + regsub(req.http.cookie, "^.*?sw-cache-hash=([^;]*);*.*$", "\1")); - } elseif (req.http.cookie ~ "sw-currency=") { - hash_data("+currency=" + regsub(req.http.cookie, "^.*?sw-currency=([^;]*);*.*$", "\1")); + if (req.http.cache-hash != "") { + hash_data("+context=" + req.http.cache-hash); + } elseif (req.http.currency != "") { + hash_data("+currency=" + req.http.currency); } } sub vcl_hit { # Consider client states for response headers - if (req.http.cookie ~ "sw-states=") { - set req.http.states = regsub(req.http.cookie, "^.*?sw-states=([^;]*);*.*$", "\1"); - + if (req.http.states) { if (req.http.states ~ "logged-in" && obj.http.sw-invalidation-states ~ "logged-in" ) { return (pass); } @@ -119,6 +121,12 @@ sub vcl_hit { } } +sub vcl_backend_fetch { + unset bereq.http.cache-hash; + unset bereq.http.currency; + unset bereq.http.states; +} + sub vcl_backend_response { # Serve stale content for three days after object expiration # Perform asynchronous revalidation while stale content is served From a50360120d1a87859904535be661fd037c631291 Mon Sep 17 00:00:00 2001 From: Soner Sayakci Date: Tue, 10 Sep 2024 11:30:04 +0200 Subject: [PATCH 3/4] feat: handle checkout cart widget without talking to backend when not filled --- rootfs/etc/varnish/default.vcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rootfs/etc/varnish/default.vcl b/rootfs/etc/varnish/default.vcl index ab3baff..71b9590 100644 --- a/rootfs/etc/varnish/default.vcl +++ b/rootfs/etc/varnish/default.vcl @@ -75,6 +75,10 @@ sub vcl_recv { set req.http.currency = cookie.get("sw-currency"); set req.http.states = cookie.get("sw-states"); + if (req.url == "/widgets/checkout/info" && !req.http.states ~ "cart-filled") { + return (synth(204, "")); + } + # Ignore query strings that are only necessary for the js on the client. Customize as needed. if (req.url ~ "(\?|&)(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=") { # see rfc3986#section-2.3 "Unreserved Characters" for regex From 259ec5ebeaebea7fbfb5c1e06a4e7f893ae40361 Mon Sep 17 00:00:00 2001 From: Soner Sayakci Date: Tue, 10 Sep 2024 14:13:14 +0200 Subject: [PATCH 4/4] feat: introduce SHOPWARE_ALLOWED_PURGER_IP --- Dockerfile | 7 ++++--- README.md | 3 +++ compose.yml | 1 + rootfs/etc/varnish/default.vcl | 3 +-- rootfs/usr/local/bin/docker-varnish-entrypoint | 1 + 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 80dc06a..3cdef9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,13 +6,14 @@ RUN set -e; \ apk upgrade --no-cache; \ apk add --no-cache $VMOD_DEPS; \ \ -# install one, possibly multiple vmods + # install one, possibly multiple vmods install-vmod https://github.com/varnish/varnish-modules/releases/download/0.24.0/varnish-modules-0.24.0.tar.gz; \ \ -# clean up + # clean up apk del --no-network $VMOD_DEPS USER varnish ENV SHOPWARE_BACKEND_HOST=localhost \ - SHOPWARE_BACKEND_PORT=8000 + SHOPWARE_BACKEND_PORT=8000 \ + SHOPWARE_ALLOWED_PURGER_IP='"127.0.0.1"' COPY --chown=1000 rootfs / diff --git a/README.md b/README.md index d29f44d..b71d142 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ docker run \ - `SHOPWARE_BACKEND_HOST` - The host of the Shopware backend. Default: `localhost` - `SHOPWARE_BACKEND_PORT` - The port of the Shopware backend. Default: `8000` - `SHOPWARE_SOFT_PURGE` - If set to `1`, the soft purge feature is enabled. Default: `0` +- `SHOPWARE_ALLOWED_PURGER_IP` - The IP address of the allowed purger. Default: `"127.0.0.1"` + +The `SHOPWARE_ALLOWED_PURGER_IP` can be a single IP like `"172.17.0.1"` or a subnet like `"172.17.0.0"/24`. Take care that the ip address inside the environment variable needs to be double quoted. ## Further information diff --git a/compose.yml b/compose.yml index ca13ee8..db6df4f 100644 --- a/compose.yml +++ b/compose.yml @@ -4,6 +4,7 @@ services: build: . environment: SHOPWARE_BACKEND_HOST: host.docker.internal + SHOPWARE_ALLOWED_PURGER_IP: '"172.17.0.0"/24' ports: - "80:80" develop: diff --git a/rootfs/etc/varnish/default.vcl b/rootfs/etc/varnish/default.vcl index 71b9590..a9ea8f1 100644 --- a/rootfs/etc/varnish/default.vcl +++ b/rootfs/etc/varnish/default.vcl @@ -15,7 +15,7 @@ acl purgers { "127.0.0.1"; "localhost"; "::1"; - "172.17.0.1"; + __SHOPWARE_ALLOWED_PURGER_IP__; } sub vcl_recv { @@ -133,7 +133,6 @@ sub vcl_backend_fetch { sub vcl_backend_response { # Serve stale content for three days after object expiration - # Perform asynchronous revalidation while stale content is served set beresp.grace = 3d; unset beresp.http.X-Powered-By; diff --git a/rootfs/usr/local/bin/docker-varnish-entrypoint b/rootfs/usr/local/bin/docker-varnish-entrypoint index 88f6ae2..cfc6262 100755 --- a/rootfs/usr/local/bin/docker-varnish-entrypoint +++ b/rootfs/usr/local/bin/docker-varnish-entrypoint @@ -4,6 +4,7 @@ set -eo pipefail sed -i "s|__SHOPWARE_BACKEND_HOST__|${SHOPWARE_BACKEND_HOST}|g" /etc/varnish/default.vcl sed -i "s|__SHOPWARE_BACKEND_PORT__|${SHOPWARE_BACKEND_PORT}|g" /etc/varnish/default.vcl +sed -i 's|__SHOPWARE_ALLOWED_PURGER_IP__|'"${SHOPWARE_ALLOWED_PURGER_IP}"'|g' /etc/varnish/default.vcl if [[ "${SHOPWARE_SOFT_PURGE}" ]]; then sed -i "s|xkey.purge|xkey.softpurge|g" /etc/varnish/default.vcl