[core] support query parameters in apricot URIs in JIT

Makefile

@@ -71,7 +71,7 @@ INSTALL_WHAT:=$(patsubst %, install_%, $(WHAT))
 
 GENERATE_DIRS := ./apricot ./coconut/cmd ./common/runtype ./common/system ./core ./core/integration/ccdb ./core/integration/dcs ./core/integration/ddsched ./core/integration/kafka ./core/integration/odc ./executor ./walnut ./core/integration/trg ./core/integration/bookkeeping
 SRC_DIRS := ./apricot ./cmd/* ./core ./coconut ./executor ./common ./configuration ./occ/peanut ./walnut
-TEST_DIRS := ./configuration/cfgbackend
+TEST_DIRS := ./configuration/cfgbackend ./configuration/componentcfg
 GO_TEST_DIRS := ./core/repos ./core/integration/dcs
 
 # Use linker flags to provide version/build settings to the target

configuration/componentcfg/query.go

@@ -26,7 +26,9 @@ package componentcfg
 
 import (
 	"errors"
+	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
 
 	apricotpb "github.com/AliceO2Group/Control/apricot/protos"
@@ -42,8 +44,9 @@ var (
 	//                                          component        /RUNTYPE          /rolename             /entry                @timestamp
 	inputFullRegex = regexp.MustCompile(`^([a-zA-Z0-9-_]+)(\/[A-Z0-9-_]+){1}(\/[a-z-A-Z0-9-_]+){1}(\/[a-z-A-Z0-9-_]+){1}(\@[0-9]+)?$`)
 	//                                          component        /RUNTYPE          /rolename
-	inputEntriesRegex = regexp.MustCompile(`^([a-zA-Z0-9-_]+)(\/[A-Z0-9-_]+){1}(\/[a-z-A-Z0-9-_]+){1}$`)
-	E_BAD_KEY         = errors.New("bad component configuration key format")
+	inputEntriesRegex    = regexp.MustCompile(`^([a-zA-Z0-9-_]+)(\/[A-Z0-9-_]+){1}(\/[a-z-A-Z0-9-_]+){1}$`)
+	inputParametersRegex = regexp.MustCompile(`^([a-zA-Z0-9-_]+=[a-zA-Z0-9-_,]+)(&[a-zA-Z0-9-_]+=[a-zA-Z0-9-_,]+)*$`)
+	E_BAD_KEY            = errors.New("bad component configuration key format")
 )
 
 func IsStringValidQueryPathWithOptionalTimestamp(input string) bool {
@@ -52,6 +55,9 @@ func IsStringValidQueryPathWithOptionalTimestamp(input string) bool {
 func IsStringValidEntriesQueryPath(input string) bool {
 	return inputEntriesRegex.MatchString(input)
 }
+func IsStringValidQueryParameters(input string) bool {
+	return inputParametersRegex.MatchString(input)
+}
 
 type EntriesQuery struct {
 	Component string
@@ -192,3 +198,42 @@ func (p *Query) AbsoluteRaw() string {
 func (p *Query) AbsoluteWithoutTimestamp() string {
 	return ConfigComponentsPath + p.WithoutTimestamp()
 }
+
+type QueryParameters struct {
+	ProcessTemplates bool
+	VarStack         map[string]string
+}
+
+func NewQueryParameters(parameters string) (p *QueryParameters, err error) {
+	p = &QueryParameters{
+		ProcessTemplates: false,
+		VarStack:         make(map[string]string),
+	}
+	parameters = strings.TrimSpace(parameters)
+
+	if !IsStringValidQueryParameters(parameters) {
+		err = E_BAD_KEY
+		return
+	}
+	keyValues, err := url.ParseQuery(parameters)
+	if err != nil {
+		return
+	}
+	// in our case, we support just one value per key, thus we map the returned keyValues accordingly
+	for key, values := range keyValues {
+		if len(values) != 1 {
+			err = E_BAD_KEY
+			return
+		}
+		if key == "process" {
+			p.ProcessTemplates, err = strconv.ParseBool(values[0])
+			if err != nil {
+				return
+			}
+		} else {
+			p.VarStack[key] = values[0]
+		}
+	}
+
+	return p, nil
+}

configuration/componentcfg/query_test.go

@@ -0,0 +1,253 @@
+package componentcfg_test
+
+import (
+	apricotpb "github.com/AliceO2Group/Control/apricot/protos"
+	"github.com/AliceO2Group/Control/configuration/componentcfg"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"testing"
+)
+
+var _ = Describe("query", func() {
+	Describe("Component configuration query", func() {
+		var (
+			q   *componentcfg.Query
+			err error
+		)
+
+		When("creating a new query with the full path and timestamp", func() {
+			BeforeEach(func() {
+				q, err = componentcfg.NewQuery("qc/PHYSICS/pp/ctp-raw-qc@1234")
+			})
+			It("should be parsed without reporting errors", func() {
+				Expect(err).To(BeNil())
+			})
+			It("should return the path correctly", func() {
+				Expect(q.Path()).To(Equal("qc/PHYSICS/pp/ctp-raw-qc@1234"))
+			})
+			It("should return the raw query correctly", func() {
+				Expect(q.Raw()).To(Equal("qc/PHYSICS/pp/ctp-raw-qc/1234"))
+			})
+			It("should return the query without the timestamp correctly", func() {
+				Expect(q.WithoutTimestamp()).To(Equal("qc/PHYSICS/pp/ctp-raw-qc"))
+			})
+			It("should return the absolute raw query correctly", func() {
+				Expect(q.AbsoluteRaw()).To(Equal(componentcfg.ConfigComponentsPath + "qc/PHYSICS/pp/ctp-raw-qc/1234"))
+			})
+			It("should return the query without the timestamp correctly", func() {
+				Expect(q.AbsoluteWithoutTimestamp()).To(Equal(componentcfg.ConfigComponentsPath + "qc/PHYSICS/pp/ctp-raw-qc"))
+			})
+			It("should be able to generalize to a query for any run type", func() {
+				Expect(q.WithFallbackRunType().Path()).To(Equal("qc/ANY/pp/ctp-raw-qc@1234"))
+			})
+			It("should be able to generalize to a query for any role name", func() {
+				Expect(q.WithFallbackRunType().Path()).To(Equal("qc/ANY/pp/ctp-raw-qc@1234"))
+			})
+		})
+
+		When("creating a new query with the full path but no timestamp", func() {
+			BeforeEach(func() {
+				q, err = componentcfg.NewQuery("qc/PHYSICS/pp/ctp-raw-qc")
+			})
+			It("should be parsed without reporting errors", func() {
+				Expect(err).To(BeNil())
+			})
+			It("should return the path correctly", func() {
+				Expect(q.Path()).To(Equal("qc/PHYSICS/pp/ctp-raw-qc"))
+			})
+			It("should return the raw query correctly", func() {
+				Expect(q.Raw()).To(Equal("qc/PHYSICS/pp/ctp-raw-qc"))
+			})
+			It("should return the query without the timestamp correctly", func() {
+				Expect(q.WithoutTimestamp()).To(Equal("qc/PHYSICS/pp/ctp-raw-qc"))
+			})
+		})
+
+		When("creating a new query with all the supported types of characters", func() {
+			BeforeEach(func() {
+				q, err = componentcfg.NewQuery("aAzZ09-_/ANY/aAzZ09-_/aAzZ09-_")
+			})
+			It("should be parsed without reporting errors", func() {
+				Expect(err).To(BeNil())
+			})
+		})
+
+		Describe("dealing with incorrectly formatted queries", func() {
+			When("query is empty", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQuery("")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("query has not enough / separators", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQuery("qc/ANY/any")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("query has an empty token", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQuery("qc/ANY//ctp-raw-qc")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("query has a timestamp separator, but the timestamp itself is missing", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQuery("qc/ANY/any/ctp-raw-qc@")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			// I don't know whether we are expected to support it or not, but the current behaviour is that we don't,
+			// even though one can write a key with a space to consul
+			When("query has a space in the entry key", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQuery("qc/ANY/any/ctp qc")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("run type uses lower case letters", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQuery("qc/physics/any/ctp-raw-qc")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("run type is unknown", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQuery("qc/FOO/any/ctp-raw-qc")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+		})
+	})
+
+	Describe("Component configuration entries query", func() {
+		var (
+			q   *componentcfg.EntriesQuery
+			err error
+		)
+
+		When("creating a new valid query", func() {
+			BeforeEach(func() {
+				q, err = componentcfg.NewEntriesQuery("qc/PHYSICS/pp")
+			})
+			It("should be parsed without reporting errors", func() {
+				Expect(err).To(BeNil())
+			})
+			It("should have parsed the component correctly", func() {
+				Expect(q.Component).To(Equal("qc"))
+			})
+			It("should have parsed the run type correctly", func() {
+				Expect(q.RunType).To(Equal(apricotpb.RunType_PHYSICS))
+			})
+			It("should have parsed the role name correctly", func() {
+				Expect(q.RoleName).To(Equal("pp"))
+			})
+		})
+
+		When("creating a new query with all the supported types of characters", func() {
+			BeforeEach(func() {
+				q, err = componentcfg.NewEntriesQuery("aAzZ09-_/ANY/aAzZ09-_")
+			})
+			It("should be parsed without reporting errors", func() {
+				Expect(err).To(BeNil())
+			})
+		})
+
+		Describe("dealing with incorrectly formatted queries", func() {
+			When("query is empty", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewEntriesQuery("")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("query has not enough / separators", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewEntriesQuery("qc/ANY")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("query has an empty token", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewEntriesQuery("qc/ANY/")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+		})
+	})
+
+	Describe("Component configuration query parameters", func() {
+		var (
+			q   *componentcfg.QueryParameters
+			err error
+		)
+
+		When("creating new valid query parameters", func() {
+			BeforeEach(func() {
+				q, err = componentcfg.NewQueryParameters("process=true&a=aaa&b=123&C_D3=C,C,C")
+			})
+			It("should be parsed without reporting errors", func() {
+				Expect(err).To(BeNil())
+			})
+			It("should have parsed the process variable correctly", func() {
+				Expect(q.ProcessTemplates).To(BeTrue())
+			})
+			It("should have parsed the var stack correctly", func() {
+				Expect(q.VarStack["a"]).To(Equal("aaa"))
+				Expect(q.VarStack["b"]).To(Equal("123"))
+				Expect(q.VarStack["C_D3"]).To(Equal("C,C,C"))
+			})
+		})
+
+		Describe("dealing with incorrectly formatted query parameters", func() {
+			When("query parameters are empty", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQueryParameters("")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("a key has empty value", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQueryParameters("process=")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+			When("there are two identical keys", func() {
+				BeforeEach(func() {
+					q, err = componentcfg.NewQueryParameters("a=33&a=34")
+				})
+				It("should return an error", func() {
+					Expect(err).To(MatchError(componentcfg.E_BAD_KEY))
+				})
+			})
+		})
+	})
+})
+
+func TestQuery(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Query Suite")
+}

configuration/template/dplutil.go

@@ -47,7 +47,7 @@ func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, wo
 	var payloads []string
 
 	// Match any consul URL
-	re := regexp.MustCompile(`(consul-json|apricot)://[^ |\n]*`)
+	re := regexp.MustCompile(`(consul-json|apricot)://[^ |"\n]*`)
 	matches := re.FindAllStringSubmatch(dplCommand, nMaxExpectedQcPayloads)
 	matches = append(matches)
 
@@ -57,13 +57,28 @@ func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, wo
 		keyRe := regexp.MustCompile(`components/[^']*`)
 		consulKeyMatch := keyRe.FindAllStringSubmatch(match[0], 1)
 		consulKey := strings.SplitAfter(consulKeyMatch[0][0], "components/")
+		// split between the query and its parameters if there are any
+		consulKeyTokens := strings.Split(consulKey[1], "?")
 
-		// And query Apricot for the configuration payload
-		newQ, err := componentcfg.NewQuery(consulKey[1])
+		// Query Apricot for the configuration payload
+		query, err := componentcfg.NewQuery(consulKeyTokens[0])
 		if err != nil {
 			return "", fmt.Errorf("JIT could not create a query out of path '%s'. error: %w", consulKey[1], err)
 		}
-		payload, err := confSvc.GetComponentConfiguration(newQ)
+		// parse parameters if they are present
+		queryParams := &componentcfg.QueryParameters{ProcessTemplates: false, VarStack: nil}
+		if len(consulKeyTokens) == 2 {
+			queryParams, err = componentcfg.NewQueryParameters(consulKeyTokens[1])
+			if err != nil {
+				return "", fmt.Errorf("JIT could not parse query parameters of path '%s', error: %w", consulKey[1], err)
+			}
+		}
+		var payload string
+		if queryParams.ProcessTemplates {
+			payload, err = confSvc.GetAndProcessComponentConfiguration(query, queryParams.VarStack)
+		} else {
+			payload, err = confSvc.GetComponentConfiguration(query)
+		}
 
 		if err != nil {
 			return "", fmt.Errorf("JIT failed trying to query QC payload '%s', error: %w", match, err)