code(pkg/parser/engine) - add reference connector html type

PxyUp · Jan 14, 2024 · 01fca8e · 01fca8e
1 parent 9dbcf0c
commit 01fca8e
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -636,7 +636,7 @@ type BaseField struct {
 }
 ```
 
-- FieldType - enum["null", "boolean", "string", "int", "int64", "float", "float64", "array", "object"] - static field for parse
+- FieldType - enum["null", "boolean", "string", "int", "int64", "float", "float64", "array", "object", "html"] - static field for parse. **Important**: type html will only works from connector which return HTML (HTMLAttribute - have no effect in this case). [Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L25) 
 - Path - selector(relative in case it is array child) for parsing
 - HTMLAttribute - extra value which have effect only in HTML parsing via **goquery**. Here you can specify which attribute need to be parsed.
 
@@ -1022,8 +1022,8 @@ Examples:
 2. {INDEX} - for inject index in parent array
 3. {HUMAN_INDEX} - for inject index in parent array in human way
 4. {{{json_path}}} - will get information from propagated "object"/"array" field
-5. {{{RefName=SomeName}}} - get [reference](#references) value by name. [Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L54)
-6. {{{RefName=SomeName json.path}}} - get [reference](#references) value by name and extract value by json path. [Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L54)
+5. {{{RefName=SomeName}}} - get [reference](#references) value by name. [Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L67)
+6. {{{RefName=SomeName json.path}}} - get [reference](#references) value by name and extract value by json path. [Example](https://github.com/PxyUp/fitter/blob/master/examples/cli/config_ref.json#L67)
 
 ## References
 Special map which **prefetched**(before any processing) and can be user for [connector](#referenceconnectorconfig) or for [placeholder](#placeholder-list)

diff --git a/examples/cli/config_ref.json b/examples/cli/config_ref.json
@@ -13,6 +13,19 @@
         }
       }
     },
+    "TokenHtmlRef": {
+      "connector_config": {
+        "response_type": "HTML",
+        "static_config": {
+          "value": "<html><title>Hello</title></html>"
+        }
+      },
+      "model": {
+        "base_field": {
+          "type": "html"
+        }
+      }
+    },
     "TokenObjectRef": {
       "connector_config": {
         "response_type": "json",
@@ -75,6 +88,26 @@
                 }
               }
             }
+          },
+          "third_field": {
+            "base_field": {
+              "generated": {
+                "model": {
+                  "connector_config": {
+                    "response_type": "HTML",
+                    "reference_config": {
+                      "name": "TokenHtmlRef"
+                    }
+                  },
+                  "model": {
+                    "base_field": {
+                      "type": "string",
+                      "path": "title"
+                    }
+                  }
+                }
+              }
+            }
           }
         }
       }

diff --git a/pkg/config/field.go b/pkg/config/field.go
@@ -5,13 +5,14 @@ import "encoding/json"
 type FieldType string
 
 const (
-	Null    FieldType = "null"
-	Bool    FieldType = "boolean"
-	String  FieldType = "string"
-	Int     FieldType = "int"
-	Int64   FieldType = "int64"
-	Float   FieldType = "float"
-	Float64 FieldType = "float64"
+	Null       FieldType = "null"
+	Bool       FieldType = "boolean"
+	String     FieldType = "string"
+	Int        FieldType = "int"
+	Int64      FieldType = "int64"
+	Float      FieldType = "float"
+	Float64    FieldType = "float64"
+	HtmlString FieldType = "html"
 
 	Array  FieldType = "array"
 	Object FieldType = "object"

diff --git a/pkg/parser/engine.go b/pkg/parser/engine.go
@@ -67,9 +67,25 @@ func NewEngine(cfg *config.ConnectorConfig, logger logger.Logger) Engine {
 		connector = store.Store.GetConnectorPlugin(cfg.PluginConnectorConfig.Name, cfg.PluginConnectorConfig, logger.With("connector", cfg.PluginConnectorConfig.Name))
 	}
 	if cfg.ReferenceConfig != nil {
-		connector = connectors.NewStatic(&config.StaticConnectorConfig{
-			Value: references.Get(cfg.ReferenceConfig.Name).ToJson(),
-		})
+		logger.Debugw("get value from reference store", "type", string(cfg.ResponseType), "name", cfg.ReferenceConfig.Name)
+		if cfg.ResponseType == config.Json {
+			connector = connectors.NewStatic(&config.StaticConnectorConfig{
+				Value: references.Get(cfg.ReferenceConfig.Name).ToJson(),
+			})
+		}
+		if cfg.ResponseType == config.XPath || cfg.ResponseType == config.HTML {
+			var htmlValue string
+			rawValue, ok := references.Get(cfg.ReferenceConfig.Name).Raw().(string)
+			if ok {
+				htmlValue = rawValue
+			} else {
+				htmlValue = "<html></html>"
+			}
+			connector = connectors.NewStatic(&config.StaticConnectorConfig{
+				Value: htmlValue,
+			})
+		}
+
 	}
 
 	var parserFactory Factory

diff --git a/pkg/parser/html.go b/pkg/parser/html.go
@@ -228,6 +228,14 @@ func (h *htmlParser) fillUpBaseField(source *goquery.Selection, field *config.Ba
 		return builder.Null()
 	}
 
+	if field.Type == config.HtmlString {
+		htmlString, err := source.Html()
+		if err != nil {
+			return builder.Null()
+		}
+		return builder.String(htmlString)
+	}
+
 	var text string
 
 	if field.HTMLAttribute != "" {