code(pkg/references) - add reference connector and placeholders

README.md

@@ -77,7 +77,7 @@ func main() {
 				},
 			},
 		},
-	}, nil, nil)
+	}, nil, nil, nil)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -160,6 +160,7 @@ type ConnectorConfig struct {
     ServerConfig          *ServerConnectorConfig  `json:"server_config" yaml:"server_config"`
     BrowserConfig         *BrowserConnectorConfig `yaml:"browser_config" json:"browser_config"`
     PluginConnectorConfig *PluginConnectorConfig  `json:"plugin_connector_config" yaml:"plugin_connector_config"`
+    ReferenceConfig       *ReferenceConnectorConfig `yaml:"reference_config" json:"reference_config"`
 }
 ```
 
@@ -172,6 +173,7 @@ Config can be one of:
 - [BrowserConfig](#browserconnectorconfig)
 - [StaticConfig](#staticconnectorconfig)
 - [PluginConnectorConfig](#pluginconnectorconfig)
+- [ReferenceConfig](#referenceconnectorconfig)
 
 Example:
 ```json
@@ -274,6 +276,21 @@ func (pl *plugin) SetConfig(cfg *config.PluginConnectorConfig, logger logger.Log
 }
 ```
 
+### ReferenceConnectorConfig
+Connector which allow get prefetched data from [references](#references)
+
+```go
+type ReferenceConnectorConfig struct {
+	Name string `yaml:"name" json:"name"`
+}
+```
+
+Example
+
+https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L66
+
+- Name - reference name from [references](#references) map
+
 ### StaticConnectorConfig
 Connector type which fetch data from provided string
 ```go
@@ -1005,6 +1022,86 @@ Examples:
 2. {INDEX} - for inject index in parent array
 3. {HUMAN_INDEX} - for inject index in parent array in human way
 4. {{{json_path}}} - will get information from propagated "object"/"array" field
+5. {{{RefName=SomeName}}} - get [reference](#references) value by name. [Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L54)
+6. {{{RefName=SomeName json.path}}} - get [reference](#references) value by name and extract value by json path. [Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L54)
+
+## References
+Special map which **prefetched**(before any processing) and can be user for [connector](#referenceconnectorconfig) or for [placeholder](#placeholder-list)
+
+Can be used for:
+1. Cache jwt token and use them in headers
+2. Cache values
+3. Etc
+
+For [Fitter](#how-to-use-fitter)
+```go
+type Config struct {
+    // Other Config Fields
+
+    Limits     *Limits                `yaml:"limits" json:"limits"`
+    References map[string]*ModelField `json:"references" yaml:"references"`
+}
+```
+
+For [Fitter Cli](#how-to-use-fittercli)
+
+```go
+type CliItem struct {
+    // Other Config Fields
+
+	Limits     *Limits                `yaml:"limits" json:"limits"`
+	References map[string]*ModelField `json:"references" yaml:"references"`
+}
+```
+
+Example
+
+https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L2
+```json
+{
+  "references": {
+    "TokenRef": {
+      "connector_config": {
+        "response_type": "json",
+        "static_config": {
+          "value": "\"plain token\""
+        }
+      },
+      "model": {
+        "base_field": {
+          "type": "string"
+        }
+      }
+    },
+    "TokenObjectRef": {
+      "connector_config": {
+        "response_type": "json",
+        "static_config": {
+          "value": "{\"token\":\"token from object\"}"
+        }
+      },
+      "model": {
+        "object_config": {
+          "fields": {
+            "token": {
+              "base_field": {
+                "type": "string",
+                "path": "token"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+- References - map[string]*[ModelField](#model-field) - object where is key if ReferenceName (can be user for [connector](#referenceconnectorconfig) or [placeholder](#placeholder-list)) and value is [ModelField](#model-field)
+- [Limits](#limits)
+
+
+[Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json)
 
 ## Limits
 Provide limitation for prevent DDOS, big usage of memory

cmd/cli/main.go

@@ -66,7 +66,7 @@ func main() {
 	}
 
 	cfg := getConfig(*filePath)
-	res, err := lib.Parse(cfg.Item, cfg.Limits, log)
+	res, err := lib.Parse(cfg.Item, cfg.Limits, cfg.References, log)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err.Error())
 		return

examples/cli/config_ref.json

@@ -0,0 +1,83 @@
+{
+  "references": {
+    "TokenRef": {
+      "connector_config": {
+        "response_type": "json",
+        "static_config": {
+          "value": "\"plain token\""
+        }
+      },
+      "model": {
+        "base_field": {
+          "type": "string"
+        }
+      }
+    },
+    "TokenObjectRef": {
+      "connector_config": {
+        "response_type": "json",
+        "static_config": {
+          "value": "{\"token\":\"token from object\"}"
+        }
+      },
+      "model": {
+        "object_config": {
+          "fields": {
+            "token": {
+              "base_field": {
+                "type": "string",
+                "path": "token"
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "item": {
+    "connector_config": {
+      "response_type": "HTML",
+      "url": "https://time.is",
+      "server_config": {
+        "method": "GET"
+      }
+    },
+    "model": {
+      "object_config": {
+        "fields": {
+          "first_field": {
+            "base_field": {
+              "type": "string",
+              "path": "time#clock",
+              "generated": {
+                "formatted": {
+                  "template": "Current time is: {PL} with token from TokenRef={{{RefName=TokenRef}}} and TokenObjectRef={{{RefName=TokenObjectRef token}}}"
+                }
+              }
+            }
+          },
+          "second_field": {
+            "base_field": {
+              "generated": {
+                "model": {
+                  "connector_config": {
+                    "response_type": "json",
+                    "reference_config": {
+                      "name": "TokenObjectRef"
+                    }
+                  },
+                  "model": {
+                    "base_field": {
+                      "type": "string",
+                      "path": "token"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

examples/go/main.go

@@ -46,7 +46,7 @@ func main() {
 				},
 			},
 		},
-	}, nil, nil)
+	}, nil, nil, nil)
 	if err != nil {
 		log.Fatal(err)
 	}

lib/lib.go

@@ -8,10 +8,11 @@ import (
 	"github.com/google/uuid"
 )
 
-func Parse(item *config.Item, limits *config.Limits, log logger.Logger) (*parser.ParseResult, error) {
+func Parse(item *config.Item, limits *config.Limits, refMap map[string]*config.ModelField, log logger.Logger) (*parser.ParseResult, error) {
 	cfg := &config.CliItem{
-		Item:   item,
-		Limits: limits,
+		Item:       item,
+		Limits:     limits,
+		References: refMap,
 	}
 	name := uuid.New().String()
 	cfg.Item.Name = name

pkg/builder/pure_string.go

@@ -1,6 +1,8 @@
 package builder
 
-import "strings"
+import (
+	"strings"
+)
 
 const (
 	EmptyString = `""`
@@ -29,6 +31,9 @@ func PureString(value string) *pureStringField {
 }
 
 func (s *pureStringField) ToJson() string {
+	if s.value == EmptyString {
+		return ""
+	}
 	return s.value
 }
 

pkg/builder/string.go

@@ -14,7 +14,14 @@ var (
 	_ Jsonable = &stringField{}
 )
 
-func String(value string) *stringField {
+func String(value string, trim ...bool) *stringField {
+	if len(trim) > 0 {
+		if !trim[0] {
+			return &stringField{
+				value: value,
+			}
+		}
+	}
 	return &stringField{
 		value: strings.TrimSpace(value),
 	}

pkg/config/config.go

@@ -19,6 +19,7 @@ const (
 )
 
 type HostRequestLimiter map[string]int64
+type RefMap map[string]*ModelField
 
 type Limits struct {
 	HostRequestLimiter HostRequestLimiter `yaml:"host_request_limiter" json:"host_request_limiter"`
@@ -28,16 +29,16 @@ type Limits struct {
 }
 
 type Config struct {
-	Items  []*Item `yaml:"items" json:"items"`
-	Limits *Limits `yaml:"limits" json:"limits"`
+	Items []*Item `yaml:"items" json:"items"`
 
-	References map[string]*ModelField `json:"references" yaml:"references"`
+	Limits     *Limits `yaml:"limits" json:"limits"`
+	References RefMap  `json:"references" yaml:"references"`
 }
 
 type CliItem struct {
-	Item   *Item   `yaml:"item" json:"item"`
-	Limits *Limits `yaml:"limits" json:"limits"`
+	Item *Item `yaml:"item" json:"item"`
 
+	Limits     *Limits                `yaml:"limits" json:"limits"`
 	References map[string]*ModelField `json:"references" yaml:"references"`
 }
 

pkg/parser/engine.go

@@ -7,10 +7,12 @@ import (
 	"github.com/PxyUp/fitter/pkg/connectors"
 	"github.com/PxyUp/fitter/pkg/logger"
 	"github.com/PxyUp/fitter/pkg/plugins/store"
+	"github.com/PxyUp/fitter/pkg/references"
 )
 
 var (
-	errInvalid = errors.New("invalid engine")
+	errMissingModelConfig = errors.New("missing model config")
+	errInvalid            = errors.New("invalid engine")
 
 	_          Engine = &engine{}
 	nullEngine Engine = &null{}
@@ -34,6 +36,9 @@ func (n *null) Get(model *config.Model, parsedValue builder.Jsonable, index *uin
 }
 
 func (e *engine) Get(model *config.Model, parsedValue builder.Jsonable, index *uint32) (*ParseResult, error) {
+	if model == nil {
+		return nil, errMissingModelConfig
+	}
 	body, err := e.connector.Get(parsedValue, index)
 	if err != nil {
 		e.logger.Errorw("connector return error during fetch data", "error", err.Error())
@@ -62,7 +67,9 @@ func NewEngine(cfg *config.ConnectorConfig, logger logger.Logger) Engine {
 		connector = store.Store.GetConnectorPlugin(cfg.PluginConnectorConfig.Name, cfg.PluginConnectorConfig, logger.With("connector", cfg.PluginConnectorConfig.Name))
 	}
 	if cfg.ReferenceConfig != nil {
-		connector = refStoreImpl.Get(cfg.ReferenceConfig.Name)
+		connector = connectors.NewStatic(&config.StaticConnectorConfig{
+			Value: references.Get(cfg.ReferenceConfig.Name).ToJson(),
+		})
 	}
 
 	var parserFactory Factory