Skip to content

Commit

Permalink
feat(metrics): Prometheus support (#174)
Browse files Browse the repository at this point in the history
* feat(metrics): Prometheus support

* Bump Træfik

* Bump caddy

* Run prometheus registry during the configuration

* Fix prometheus already started

* Update newman tests

* Update documentation according to the new souin_ prefix on the prometheus metrics

* Add Skipper and Gin E2E
  • Loading branch information
darkweak authored Jan 26, 2022
1 parent 8ea71de commit d5f46f9
Show file tree
Hide file tree
Showing 348 changed files with 51,592 additions and 4,679 deletions.
84 changes: 84 additions & 0 deletions .github/workflows/plugins.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,90 @@ jobs:
folder: Echo
reporters: cli
delayRequest: 5000
build-gin-validator:
name: Check that Souin build as Gin middleware
runs-on: ubuntu-latest
steps:
-
name: Add domain.com host to /etc/hosts
run: |
sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts
-
name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.16
-
name: Checkout code
uses: actions/checkout@v2
-
name: Build Souin as Gin plugin
run: make build-and-run-gin
-
name: Wait for Souin is really loaded inside Gin as middleware
uses: jakejarvis/wait-action@master
with:
time: 30s
-
name: Set Gin logs configuration result as environment variable
run: cd plugins/gin/examples && echo "GIN_MIDDLEWARE_RESULT=$(docker-compose logs gin | grep Souin)" >> $GITHUB_ENV
-
name: Check if the configuration is loaded to define if Souin is loaded too
uses: nick-invision/assert-action@v1
with:
expected: '"message":"Souin configuration is now loaded."'
actual: ${{ env.GIN_MIDDLEWARE_RESULT }}
comparison: contains
-
name: Run Gin E2E tests
uses: anthonyvscode/newman-action@v1
with:
collection: "docs/e2e/Souin E2E.postman_collection.json"
folder: Gin
reporters: cli
delayRequest: 5000
build-skipper-validator:
name: Check that Souin build as Skipper middleware
runs-on: ubuntu-latest
steps:
-
name: Add domain.com host to /etc/hosts
run: |
sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts
-
name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.16
-
name: Checkout code
uses: actions/checkout@v2
-
name: Build Souin as Skipper plugin
run: make build-and-run-skipper
-
name: Wait for Souin is really loaded inside Skipper as middleware
uses: jakejarvis/wait-action@master
with:
time: 40s
-
name: Set Skipper logs configuration result as environment variable
run: cd plugins/skipper/examples && echo "SKIPPER_MIDDLEWARE_RESULT=$(docker-compose logs skipper | grep Souin)" >> $GITHUB_ENV
-
name: Check if the configuration is loaded to define if Souin is loaded too
uses: nick-invision/assert-action@v1
with:
expected: '"message":"Souin configuration is now loaded."'
actual: ${{ env.SKIPPER_MIDDLEWARE_RESULT }}
comparison: contains
-
name: Run Skipper E2E tests
uses: anthonyvscode/newman-action@v1
with:
collection: "docs/e2e/Souin E2E.postman_collection.json"
folder: Skipper
reporters: cli
delayRequest: 5000
build-tyk-validator:
name: Check that Souin build as Tyk middleware
runs-on: ubuntu-latest
Expand Down
41 changes: 30 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
2.1.2. [Souin out-of-the-box](#souin-out-of-the-box)
2.2. [Optional configuration](#optional-configuration)
3. [APIs](#apis)
3.1. [Souin API](#souin-api)
3.2. [Security API](#security-api)
3.1. [Prometheus API](#prometheus-api)
3.2. [Souin API](#souin-api)
3.3. [Security API](#security-api)
4. [Diagrams](#diagrams)
4.1. [Sequence diagram](#sequence-diagram)
5. [Cache systems](#cache-systems)
Expand Down Expand Up @@ -56,16 +57,19 @@ default_cache: # Required
reverse_proxy_url: 'http://traefik' # If it's in the same network you can use http://your-service, otherwise just use https://yourdomain.com
```
| Key | Description | Value example |
|:------------------------------:|:------------------------------------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------:|
| `default_cache.ttl` | Duration to cache request (in seconds) | 10 |
| `reverse_proxy_url` | The reverse-proxy's instance URL (Apache, Nginx, Træfik...) | - `http://yourservice` (Container way)<br/>`http://localhost:81` (Local way)<br/>`http://yourdomain.com:81` (Network way) |
| Key | Description | Value example |
|:--------------------|:------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------|
| `default_cache.ttl` | Duration to cache request (in seconds) | 10 |
| `reverse_proxy_url` | The reverse-proxy's instance URL (Apache, Nginx, Træfik...) | - `http://yourservice` (Container way)<br/>`http://localhost:81` (Local way)<br/>`http://yourdomain.com:81` (Network way) |

### Optional configuration
```yaml
# /anywhere/configuration.yml
api:
basepath: /souin-api # Default route basepath for every additional APIs to avoid conflicts with existing routes
prometheus: # Prometheus exposed metrics
security: true # Enable JWT Authentication token
enable: true # Enable the endpoints
security: # Secure your APIs
secret: your_secret_key # JWT secret key
enable: true # Required to enable the endpoints
Expand Down Expand Up @@ -124,10 +128,10 @@ surrogate_keys:
```

| Key | Description | Value example |
|:-------------------------------------------------:|:----------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------:|
|:--------------------------------------------------|:-----------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
| `api` | The cache-handler API cache management | |
| `api.basepath` | BasePath for all APIs to avoid conflicts | `/your-non-conflicting-route`<br/><br/>`(default: /souin-api)` |
| `api.{api}.enable` | Enable the new API with related routes | `true`<br/><br/>`(default: false)` |
| `api.{api}.enable` | Enable the API with related routes | `true`<br/><br/>`(default: false)` |
| `api.{api}.security` | Enable the JWT Authentication token verification | `true`<br/><br/>`(default: false)` |
| `api.security.secret` | JWT secret key | `Any_charCanW0rk123` |
| `api.security.users` | Array of authorized users with username x password combo | `- username: admin`<br/><br/>` password: admin` |
Expand Down Expand Up @@ -164,19 +168,34 @@ surrogate_keys:
| `ykeys.{key name}.headers` | (DEPRECATED) Headers that should match to be part of the ykey group | `Authorization: ey.+`<br/><br/>`Content-Type: json` |
| `ykeys.{key name}.headers.{header name}` | (DEPRECATED) Header name that should be present a match the regex to be part of the ykey group | `Content-Type: json` |
| `ykeys.{key name}.url` | (DEPRECATED) Url that should match to be part of the ykey group | `.+` |
| Key | Description | Value example |

## APIs
All endpoints are accessible through the `api.basepath` configuration line or by default through `/souin-api` to avoid named route conflicts. Be sure to define an unused route to not break your existing application.

### Prometheus API
Prometheus API expose some metrics about the cache.
The base path for the prometheus API is `/metrics`.
**Not supported inside Træfik because the deny the unsafe library usage inside plugins**

| Method | Endpoint | Description |
|:--------|:---------|:----------------------------------------|
| `GET` | `/` | Expose the different keys listed below. |

| Key | Definition |
|:-----------------------------------|:--------------------------------|
| `souin_request_counter` | Count the incoming requests |
| `souin_no_cached_response_counter` | Count the uncacheable responses |
| `souin_cached_response_counter` | Count the cacheable responses |
| `souin_avg_response_time` | Average response time |

### Souin API
Souin API allow users to manage the cache.
The base path for the souin API is `/souin`.
The Souin API supports the invalidation by surrogate keys such as Fastly which will replace the Varnish system. You can read the doc [about this system](https://github.com/darkweak/souin/blob/master/cache/surrogate/README.md).
This system is able to invalidate by tags your cloud provider cache. Actually it supports Akamai and Fastly but in a near future some other providers would be implemented like Cloudflare or Varnish.

| Method | Endpoint | Description |
|:-------:|:-----------------:|:-----------------------------------------------------------------------------------------|
|:--------|:------------------|:-----------------------------------------------------------------------------------------|
| `GET` | `/` | List stored keys cache |
| `PURGE` | `/{id or regexp}` | Purge selected item(s) depending. The parameter can be either a specific key or a regexp |
| `PURGE` | `/?ykey={key}` | Purge selected item(s) corresponding to the target ykey such as Varnish (deprecated) |
Expand All @@ -186,7 +205,7 @@ Security API allows users to protect other APIs with JWT authentication.
The base path for the security API is `/authentication`.

| Method | Endpoint | Body | Headers | Description |
|:------:|:----------:|:------------------------------------------:|:-------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------:|
|:-------|:-----------|:-------------------------------------------|:--------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------|
| `POST` | `/login` | `{"username":"admin", "password":"admin"}` | `['Content-Type' => 'json']` | Try to login, it returns a response which contains the cookie name `souin-authorization-token` with the JWT if succeed |
| `POST` | `/refresh` | `-` | `['Content-Type' => 'json', 'Cookie' => 'souin-authorization-token=the-token']` | Refreshes the token, replaces the old with a new one |

Expand Down
5 changes: 3 additions & 2 deletions api/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"net/http"

"github.com/darkweak/souin/api/auth"
"github.com/darkweak/souin/api/prometheus"
"github.com/darkweak/souin/cache/types"
"github.com/darkweak/souin/configurationtypes"
)
Expand All @@ -30,7 +31,7 @@ func GenerateHandlerMap(
for _, endpoint := range Initialize(transport, configuration) {
if endpoint.IsEnabled() {
shouldEnable = true
hm[basePathAPIS+endpoint.GetBasePath()] = endpoint.(*SouinAPI).HandleRequest
hm[basePathAPIS+endpoint.GetBasePath()] = endpoint.HandleRequest
}
}

Expand All @@ -44,5 +45,5 @@ func GenerateHandlerMap(
// Initialize contains all apis that should be enabled
func Initialize(transport types.TransportInterface, c configurationtypes.AbstractConfigurationInterface) []EndpointInterface {
security := auth.InitializeSecurity(c)
return []EndpointInterface{security, initializeSouin(c, security, transport)}
return []EndpointInterface{security, initializeSouin(c, security, transport), prometheus.InitializePrometheus(c, security)}
}
4 changes: 2 additions & 2 deletions api/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ func TestInitialize(t *testing.T) {

endpoints := Initialize(transport, config)

if len(endpoints) != 2 {
errors.GenerateError(t, fmt.Sprintf("Endpoints length should be 1, %d received", len(endpoints)))
if len(endpoints) != 3 {
errors.GenerateError(t, fmt.Sprintf("Endpoints length should be 3, %d received", len(endpoints)))
}
if !endpoints[0].IsEnabled() {
errors.GenerateError(t, "Endpoint should be enabled")
Expand Down
110 changes: 110 additions & 0 deletions api/prometheus/prometheus.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package prometheus

import (
"net/http"

"github.com/darkweak/souin/api/auth"
"github.com/darkweak/souin/configurationtypes"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

const (
counter = "counter"
average = "average"

RequestCounter = "souin_request_counter"
NoCachedResponseCounter = "souin_no_cached_response_counter"
CachedResponseCounter = "souin_cached_response_counter"
AvgResponseTime = "souin_avg_response_time"
)

// PrometheusAPI object contains informations related to the endpoints
type PrometheusAPI struct {
basePath string
enabled bool
security *auth.SecurityAPI
}

// InitializePrometheus initialize the prometheus endpoints
func InitializePrometheus(configuration configurationtypes.AbstractConfigurationInterface, api *auth.SecurityAPI) *PrometheusAPI {
basePath := configuration.GetAPI().Prometheus.BasePath
enabled := configuration.GetAPI().Prometheus.Enable
var security *auth.SecurityAPI
if configuration.GetAPI().Souin.Security {
security = api
}
if basePath == "" {
basePath = "/metrics"
}

if registered == nil {
run()
}
return &PrometheusAPI{
basePath,
enabled,
security,
}
}

// GetBasePath will return the basepath for this resource
func (p *PrometheusAPI) GetBasePath() string {
return p.basePath
}

// IsEnabled will return enabled status
func (p *PrometheusAPI) IsEnabled() bool {
return p.enabled
}

// HandleRequest will handle the request
func (p *PrometheusAPI) HandleRequest(w http.ResponseWriter, r *http.Request) {
promhttp.Handler().ServeHTTP(w, r)
}

var registered map[string]interface{}

// Increment will increment the counter.
func Increment(name string) {
registered[name].(prometheus.Counter).Inc()
}

// Increment will add the referred value the counter.
func Add(name string, value float64) {
if c, ok := registered[name].(prometheus.Counter); ok {
c.Add(value)
}
if g, ok := registered[name].(prometheus.Histogram); ok {
g.Observe(value)
}
}

func push(promType, name, help string) {
switch promType {
case counter:
registered[name] = promauto.NewCounter(prometheus.CounterOpts{
Name: name,
Help: help,
})

return
case average:
avg := prometheus.NewHistogram(prometheus.HistogramOpts{
Name: name,
Help: help,
})
prometheus.MustRegister(avg)
registered[name] = avg
}
}

// Run populate and prepare the map with the default values.
func run() {
registered = make(map[string]interface{})
push(counter, RequestCounter, "Total request counter")
push(counter, NoCachedResponseCounter, "No cached response counter")
push(counter, CachedResponseCounter, "Cached response counter")
push(average, AvgResponseTime, "Average Bidswitch response time")
}
Loading

0 comments on commit d5f46f9

Please sign in to comment.