Issue/#10 (#11)

* Performance update, remove useless dependencies, improve cache speed

* Remove useless env var and optimise imports

* Update makefile for travis

* Fix configuration pass and linter

* Add tests

* Enhance tests

* Tests and typo enhancement

* Lint and add headers feature in key

* Update documentation

* Fix documentation

* Remove duplicated line

* Improve doc, thanks to @oxodao

Dockerfile-dev

@@ -10,11 +10,16 @@ ADD ./cache /app/src/github.com/darkweak/souin/cache
 ADD ./errors /app/src/github.com/darkweak/souin/errors
 ADD ./providers /app/src/github.com/darkweak/souin/providers
 ADD ./default/server.* /app/src/github.com/darkweak/souin/
+ADD ./configuration/* /app/src/github.com/darkweak/souin/configuration/
 
 WORKDIR /app/src/github.com/darkweak/souin
 ENV GOPATH /app
-RUN go get ./...
-RUN go get -u golang.org/x/lint/golint
+RUN go get -u \
+    golang.org/x/lint/golint \
+    github.com/allegro/bigcache \
+    github.com/fsnotify/fsnotify \
+    github.com/go-redis/redis \
+    gopkg.in/yaml.v2
 
 EXPOSE 80
 

Dockerfile-prod

@@ -11,15 +11,19 @@ ADD ./cache /app/src/github.com/darkweak/souin/cache
 ADD ./errors /app/src/github.com/darkweak/souin/errors
 ADD ./providers /app/src/github.com/darkweak/souin/providers
 ADD ./default/server.* /app/src/github.com/darkweak/souin/
+ADD ./configuration/* /app/src/github.com/darkweak/souin/configuration/
 ADD ./entrypoint.sh /app/src/github.com/darkweak/souin/entrypoint.sh
 
 WORKDIR /app/src/github.com/darkweak/souin
 ENV GOPATH /app
 ENV GOOS linux
 ENV GOARCH arm
-RUN go get ./...
-RUN go get -u golang.org/x/lint/golint
-RUN go install
+RUN go get -u \
+    golang.org/x/lint/golint \
+    github.com/allegro/bigcache \
+    github.com/fsnotify/fsnotify \
+    github.com/go-redis/redis \
+    gopkg.in/yaml.v2
 RUN chmod 755 ./entrypoint.sh
 
 EXPOSE 80

Makefile

@@ -21,12 +21,10 @@ down: ## Down containers
 env-dev: ## Up container with dev env vars
 	cp Dockerfile-dev Dockerfile
 	cp docker-compose.yml.dev docker-compose.yml
-	cp .env.dev .env
 
 env-prod: ## Up container with prod env vars
 	cp Dockerfile-prod Dockerfile
 	cp docker-compose.yml.prod docker-compose.yml
-	cp .env.prod .env
 
 generate-plantUML: ## Generate plantUML diagrams
 	cd ./docs/plantUML && sh generate.sh && cd ../..

README.md

@@ -2,9 +2,9 @@
 
 # Souin Table of Contents
 1. [Souin reverse-proxy cache](#project-description)
-2. [Environment variables](#environment-variables)  
-  2.1. [Required variables](#required-variables)  
-  2.2. [Optional variables](#optional-variables)
+2. [Configuration](#configuration)  
+  2.1. [Required configuration](#required-configuration)  
+  2.2. [Optional configuration](#optional-configuration)
 3. [Diagrams](#diagrams)  
   3.1. [Sequence diagram](#sequence-diagram)
 4. [Cache systems](#cache-systems)
@@ -19,38 +19,67 @@
 
 ## Project description
 Souin is a new cache system suitable for every reverse-proxy. It will be placed on top of your current reverse-proxy whether it's Apache, Nginx or Traefik.  
-As it's written in go, it can be deployed on any server and thanks docker integration, it will be easy to install it on top of a Swarm or a kubernetes instance.
+As it's written in go, it can be deployed on any server and thanks to the docker integration, it will be easy to install on top of a Swarm or a kubernetes instance.
 
-## Environment variables
+## Configuration
+The configuration file is stored at `configuration/configuration.yml`. You can edit it provided you fill at least the required parameters as shown below.
 
-### Required variables
-|  Variable  |  Description  |  Value example  |
+### Required configuration
+```yaml
+ttl: 100 #TTL in second
+reverse_proxy_url: 'http://traefik' # The reverse-proxy http address
+cache:
+  port:
+    web: 80
+    tls: 443
+```
+This is a fully working minimal configuration for a Souin instance
+
+|  Key  |  Description  |  Value example  |
 |:---:|:---:|:---:|
-|`CACHE_PORT`|The HTTP port Souin will be listening on |`80`|
-|`CACHE_TLS_PORT`|The TLS port Souin will be listening on|`443`|
-|`REDIS_URL`|The redis instance URL|- `http://redis` (Container way)<br/>`http://localhost:6379` (Local way)|
-|`TTL`|Duration to cache request (in seconds)|10|
-|`REVERSE_PROXY`|The reverse-proxy's instance URL (Apache, Nginx, Træfik...)|- `http://yourservice` (Container way)<br/>`http://localhost:81` (Local way)|
-
-### Optional variables
-|  Variable  |  Description  |  Value example  |
+|`ttl`|Duration to cache request (in seconds)|10|
+|`reverse_proxy_url`|The reverse-proxy's instance URL (Apache, Nginx, Træfik...)|- `http://yourservice` (Container way)<br/>`http://localhost:81` (Local way)|
+|`cache.port.{web,tls}`|The device's local HTTP/TLS port that Souin should be listening on |Respectively `80` and `443`|
+
+### Optional configuration
+```yaml
+redis:
+  url: 'redis:6379' # Redis http address, only used for redis provider
+regex:
+  exclude: 'ARegexHere'
+ssl_providers: # Must match your volumes to /ssl/{provider}.json
+  - traefik
+cache:
+  headers:
+    - Authorization # Can be any other headers
+  providers: # By default it will use in-memory and redis cache. It can be either `all`, `redis` or `memory`. 
+    - all # Can be set to all if you want to enable all providers instead of specifying each one
+#    - memory
+#    - redis
+```
+
+|  Key  |  Description  |  Value example  |
 |:---:|:---:|:---:|
-|`REGEX`|The regex that matches URLs not to store in cache|`http://domain.com/mypath`|
+|`redis.url`|The redis url, used if you enabled it in the provider section|`redis:6379` (container way) and `http://yourdomain.com:6379` (network way)|
+|`regex.exclude`|The regex used to prevent paths being cached|`^[A-z]+.*$`|
+|`ssl_providers`|List of your providers handling certificates|`- traefik`<br/><br/>`- nginx`<br/><br/>`- apache`|
+|`cache.headers`|List of headers to include to the cache|`- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header`|
+|`cache.providers`|Your providers list to cache your data, by default it will use all systems|`- all`<br/><br/>`- memory`<br/><br/>`- redis`|
 
 ## Diagrams
 
 ### Sequence diagram
 <img src="docs/plantUML/sequenceDiagram.svg?sanitize=true" alt="Sequence diagram">
 
 ## Cache systems
-The cache system sits on two providers at the moment. It provides an in-memory and redis cache systems because setting, getting, updating and deleting keys in Redis is as easy as it gets.  
-In order to do that, Redis could be on the same network than the Souin instance when using docker-compose or over the internet then it will use by default in-memory to avoid network latency as much as possible. 
+The cache system sits on top of two providers at the moment. It provides an in-memory and redis cache systems because setting, getting, updating and deleting keys in Redis is as easy as it gets.  
+In order to do that, Redis needs to be either on the same network than the Souin instance when using docker-compose or over the internet, then it will use by default in-memory to avoid network latency as much as possible. 
 Souin will return at first the in-memory response when it gives a non-empty response, then the redis one will be used with same condition, or fallback to the reverse proxy otherwise.
 
 ### Cache invalidation
 The cache invalidation is made for CRUD requests, if you're doing a GET HTTP request, it will serve the cached response when it exists, otherwise the reverse-proxy response will be served.  
-If you're doing a POST, PUT, PATCH or DELETE HTTP request, the related cache GET request will be dropped and the list endpoint will be dropped too  
-It works very well with plain [API Platform](https://api-platform.com) integration (not for custom actions for now) and CRUD routes.
+If you're doing a POST, PUT, PATCH or DELETE HTTP request, the related cache GET request and the list endpoint will be dropped.  
+It works very well with plain [API Platform](https://api-platform.com) integration (not for custom actions at the moment) and CRUD routes.
 
 ## Examples
 
@@ -97,17 +126,11 @@ services:
     build:
       context: .
     ports:
-      - ${CACHE_PORT}:80
-      - ${CACHE_TLS_PORT}:443
+      - 80:80
+      - 443:443
     depends_on:
       - redis
     environment:
-      REDIS_URL: ${REDIS_URL}
-      TTL: ${TTL}
-      CACHE_PORT: ${CACHE_PORT}
-      CACHE_TLS_PORT: ${CACHE_TLS_PORT}
-      REVERSE_PROXY: ${REVERSE_PROXY}
-      REGEX: ${REGEX}
       GOPATH: /app
     volumes:
       - ./cmd:/app/cmd
@@ -127,25 +150,25 @@ networks:
 
 ### Træfik
 As Souin is compatible with Træfik, it can use (and it should use) `traefik.json` provided on træfik. Souin will get new/updated certs from Træfik, then your SSL certs will be up to date as far as Træfik will be too
-To provide, acme, use just have to map volume as above
+To provide, acme, you just have to map volume as above
 ```yaml
     volumes:
       - /anywhere/traefik.json:/ssl/traefik.json
 ```
 ### Apache
-Souin will listen `apache.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too
+Souin will listen to the `apache.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too
 ```yaml
     volumes:
       - /anywhere/apache.json:/ssl/apache.json
 ```
 ### Nginx
-Souin will listen `nginx.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too
+Souin will listen to the `nginx.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too
 ```yaml
     volumes:
       - /anywhere/nginx.json:/ssl/nginx.json
 ```
-At the moment you can't choose the path for the `*.json` in the container, they have to be placed in the `/ssl` folder. In the future you'll be able to do that just setting one env var
-If none `*.json` is provided to container, a default cert will be served.
+At the moment you can't choose the path for the `*.json` file in the container, they have to be placed in the `/ssl` folder. In the future you'll be able to do that by setting one env var
+If none `*.json` file is provided to container, a default cert will be served.
 
 
 ## Credits

cache/providers/abstract.go

@@ -1,10 +1,10 @@
 package providers
 
 import (
-	"os"
 	"regexp"
 
 	"github.com/darkweak/souin/cache/types"
+	"github.com/darkweak/souin/configuration"
 )
 
 // AbstractProviderInterface should be implemented in any providers
@@ -16,7 +16,34 @@ type AbstractProviderInterface interface {
 }
 
 // PathnameNotInRegex check if pathname is in parameter regex var
-func PathnameNotInRegex(pathname string) bool {
-	b, _ := regexp.Match(os.Getenv("REGEX"), []byte(pathname))
+func PathnameNotInRegex(pathname string, configuration configuration.Configuration) bool {
+	b, _ := regexp.Match(configuration.Regex.Exclude, []byte(pathname))
 	return !b
 }
+
+func contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
+
+// InitializeProviders allow to generate the providers array according to the configuration
+func InitializeProviders(configuration configuration.Configuration) *[]AbstractProviderInterface {
+	var providers []AbstractProviderInterface
+
+	if len(configuration.Cache.Providers) == 0 || contains(configuration.Cache.Providers, "all") {
+		providers = append(providers, MemoryConnectionFactory(configuration), RedisConnectionFactory(configuration))
+	} else {
+		if contains(configuration.Cache.Providers, "redis") {
+			providers = append(providers, RedisConnectionFactory(configuration))
+		}
+		if contains(configuration.Cache.Providers, "memory") {
+			providers = append(providers, MemoryConnectionFactory(configuration))
+		}
+	}
+
+	return &providers
+}

cache/providers/memory.go

@@ -1,11 +1,12 @@
 package providers
 
 import (
-	"os"
 	"time"
 
 	"github.com/allegro/bigcache"
 	"github.com/darkweak/souin/cache/types"
+	"github.com/darkweak/souin/configuration"
+	"strconv"
 )
 
 // Memory provider type
@@ -14,9 +15,9 @@ type Memory struct {
 }
 
 // MemoryConnectionFactory function create new Memory instance
-func MemoryConnectionFactory() *Memory {
-	t, _ := time.ParseDuration(os.Getenv("TTL"))
-	bc, _ := bigcache.NewBigCache(bigcache.DefaultConfig(t * time.Second))
+func MemoryConnectionFactory(configuration configuration.Configuration) *Memory {
+	t, _ := strconv.Atoi(configuration.TTL)
+	bc, _ := bigcache.NewBigCache(bigcache.DefaultConfig(time.Second * time.Duration(t)))
 	return &Memory{
 		bc,
 	}

cache/providers/memory_test.go

@@ -0,0 +1,26 @@
+package providers
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/darkweak/souin/errors"
+	"github.com/darkweak/souin/configuration"
+)
+
+const MEMORYVALUE = "My first data"
+
+func TestIShouldBeAbleToReadAndWriteDataInMemory(t *testing.T) {
+	client := MemoryConnectionFactory(configuration.GetConfig())
+	err := client.Set("Test", []byte(MEMORYVALUE))
+	if err != nil {
+		errors.GenerateError(t, "Impossible to set memory variable")
+	}
+	res, err := client.Get("Test")
+	if err != nil {
+		errors.GenerateError(t, "Retrieving data from memory")
+	}
+	if MEMORYVALUE != string(res) {
+		errors.GenerateError(t, fmt.Sprintf("%s not corresponding to %s", res, MEMORYVALUE))
+	}
+}

cache/providers/redis.go

@@ -1,33 +1,35 @@
 package providers
 
 import (
-	"os"
 	"strconv"
 	"time"
 
 	"github.com/darkweak/souin/cache/types"
+	"github.com/darkweak/souin/configuration"
 	"github.com/go-redis/redis"
 )
 
 // Redis provider type
 type Redis struct {
 	*redis.Client
+	configuration.Configuration
 }
 
 // RedisConnectionFactory function create new Redis instance
-func RedisConnectionFactory() *Redis {
+func RedisConnectionFactory(configuration configuration.Configuration) *Redis {
 	return &Redis{
 		redis.NewClient(&redis.Options{
-			Addr:     os.Getenv("REDIS_URL"),
+			Addr:     configuration.Redis.URL,
 			DB:       0,
 			Password: "",
 		}),
+		configuration,
 	}
 }
 
 // GetRequestInCache method returns the populated response if exists, empty response then
 func (provider *Redis) GetRequestInCache(key string) types.ReverseResponse {
-	val2, err := provider.Get(key).Result()
+	val2, err := provider.Get(provider.Context(), key).Result()
 
 	if err != nil {
 		return types.ReverseResponse{Response: "", Proxy: nil, Request: nil}
@@ -38,23 +40,23 @@ func (provider *Redis) GetRequestInCache(key string) types.ReverseResponse {
 
 // SetRequestInCache method will store the response in Redis provider
 func (provider *Redis) SetRequestInCache(key string, value []byte) {
-	ttl, _ := strconv.Atoi(os.Getenv("TTL"))
+	ttl, _ := strconv.Atoi(provider.Configuration.TTL)
 
-	err := provider.Set(key, string(value), time.Duration(ttl)*time.Second).Err()
+	err := provider.Set(provider.Context(), key, string(value), time.Duration(ttl)*time.Second).Err()
 	if err != nil {
 		panic(err)
 	}
 }
 
 // DeleteRequestInCache method will delete the response in Redis provider if exists corresponding to key param
 func (provider *Redis) DeleteRequestInCache(key string) {
-	provider.Do("del", key)
+	provider.Do(provider.Context(), "del", key)
 }
 
 // DeleteManyRequestInCache method will delete the response in Redis provider if exists corresponding to regex param
 func (provider *Redis) DeleteManyRequestInCache(regex string) {
-	for _, i := range provider.Keys(regex).Val() {
-		provider.Do("del", i)
+	for _, i := range provider.Keys(provider.Context(), regex).Val() {
+		provider.Do(provider.Context(), provider, "del", i)
 	}
 }