support JSON Merge Patch and Strategic Merge Patch in Resource Modifiers

changelogs/unreleased/6917-27149chen

@@ -0,0 +1 @@
+Support JSON Merge Patch and Strategic Merge Patch in Resource Modifiers

go.mod

@@ -58,6 +58,7 @@ require (
 	k8s.io/metrics v0.25.6
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
 	sigs.k8s.io/controller-runtime v0.12.2
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -163,7 +164,6 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/component-base v0.24.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
 

internal/resourcemodifiers/json_merge_patch.go

@@ -0,0 +1,45 @@
+package resourcemodifiers
+
+import (
+	"fmt"
+
+	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/yaml"
+)
+
+type JSONMergePatch struct {
+	PatchBytes []byte `json:"patchBytes,omitempty"`
+}
+
+type JSONMergePatcher struct {
+	patches []JSONMergePatch
+}
+
+func (p *JSONMergePatcher) Patch(u *unstructured.Unstructured, _ logrus.FieldLogger) (*unstructured.Unstructured, error) {
+	objBytes, err := u.MarshalJSON()
+	if err != nil {
+		return nil, fmt.Errorf("error in marshaling object %s", err)
+	}
+
+	for _, patch := range p.patches {
+		patchBytes, err := yaml.YAMLToJSON(patch.PatchBytes)
+		if err != nil {
+			return nil, fmt.Errorf("error in converting YAML to JSON %s", err)
+		}
+
+		objBytes, err = jsonpatch.MergePatch(objBytes, patchBytes)
+		if err != nil {
+			return nil, fmt.Errorf("error in applying JSON Patch %s", err.Error())
+		}
+	}
+
+	updated := &unstructured.Unstructured{}
+	err = updated.UnmarshalJSON(objBytes)
+	if err != nil {
+		return nil, fmt.Errorf("error in unmarshalling modified object %s", err.Error())
+	}
+
+	return updated, nil
+}

internal/resourcemodifiers/json_patch.go

@@ -0,0 +1,96 @@
+package resourcemodifiers
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+type JSONPatch struct {
+	Operation string `json:"operation"`
+	From      string `json:"from,omitempty"`
+	Path      string `json:"path"`
+	Value     string `json:"value,omitempty"`
+}
+
+func (p *JSONPatch) ToString() string {
+	if addQuotes(p.Value) {
+		return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": "%s"}`, p.Operation, p.From, p.Path, p.Value)
+	}
+	return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": %s}`, p.Operation, p.From, p.Path, p.Value)
+}
+
+func addQuotes(value string) bool {
+	if value == "" {
+		return true
+	}
+	// if value is null, then don't add quotes
+	if value == "null" {
+		return false
+	}
+	// if value is a boolean, then don't add quotes
+	if _, err := strconv.ParseBool(value); err == nil {
+		return false
+	}
+	// if value is a json object or array, then don't add quotes.
+	if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
+		return false
+	}
+	// if value is a number, then don't add quotes
+	if _, err := strconv.ParseFloat(value, 64); err == nil {
+		return false
+	}
+	return true
+}
+
+type JSONPatcher struct {
+	patches []JSONPatch `yaml:"patches"`
+}
+
+func (p *JSONPatcher) Patch(u *unstructured.Unstructured, logger logrus.FieldLogger) (*unstructured.Unstructured, error) {
+	modifiedObjBytes, err := p.applyPatch(u)
+	if err != nil {
+		if errors.Is(err, jsonpatch.ErrTestFailed) {
+			logger.Infof("Test operation failed for JSON Patch %s", err.Error())
+			return u.DeepCopy(), nil
+		}
+		return nil, fmt.Errorf("error in applying JSON Patch %s", err.Error())
+	}
+
+	updated := &unstructured.Unstructured{}
+	err = updated.UnmarshalJSON(modifiedObjBytes)
+	if err != nil {
+		return nil, fmt.Errorf("error in unmarshalling modified object %s", err.Error())
+	}
+
+	return updated, nil
+}
+
+func (p *JSONPatcher) applyPatch(u *unstructured.Unstructured) ([]byte, error) {
+	patchBytes := p.patchArrayToByteArray()
+	jsonPatch, err := jsonpatch.DecodePatch(patchBytes)
+	if err != nil {
+		return nil, fmt.Errorf("error in decoding json patch %s", err.Error())
+	}
+
+	objBytes, err := u.MarshalJSON()
+	if err != nil {
+		return nil, fmt.Errorf("error in marshaling object %s", err.Error())
+	}
+
+	return jsonPatch.Apply(objBytes)
+}
+
+func (p *JSONPatcher) patchArrayToByteArray() []byte {
+	var patches []string
+	for _, patch := range p.patches {
+		patches = append(patches, patch.ToString())
+	}
+	patchesStr := strings.Join(patches, ",\n\t")
+	return []byte(fmt.Sprintf(`[%s]`, patchesStr))
+}

internal/resourcemodifiers/resource_modifiers.go

@@ -3,16 +3,16 @@
 import (
 	"fmt"
 	"regexp"
-	"strconv"
-	"strings"
 
 	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/gobwas/glob"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/yaml"
 
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
@@ -23,23 +23,20 @@
 	ResourceModifierSupportedVersionV1 = "v1"
 )
 
-type JSONPatch struct {
-	Operation string `json:"operation"`
-	From      string `json:"from,omitempty"`
-	Path      string `json:"path"`
-	Value     string `json:"value,omitempty"`
-}
-
 type Conditions struct {
 	Namespaces        []string              `json:"namespaces,omitempty"`
 	GroupResource     string                `json:"groupResource"`
 	ResourceNameRegex string                `json:"resourceNameRegex,omitempty"`
 	LabelSelector     *metav1.LabelSelector `json:"labelSelector,omitempty"`
+	Matches           []JSONPatch           `json:"matches,omitempty"`
 }
 
 type ResourceModifierRule struct {
-	Conditions Conditions  `json:"conditions"`
-	Patches    []JSONPatch `json:"patches"`
+	Conditions       Conditions            `json:"conditions"`
+	PatchData        string                `json:"patchData,omitempty"`
+	Patches          []JSONPatch           `json:"patches,omitempty"`
+	MergePatches     []JSONMergePatch      `json:"mergePatches,omitempty"`
+	StrategicPatches []StrategicMergePatch `json:"strategicPatches,omitempty"`
 }
 
 type ResourceModifiers struct {
@@ -68,10 +65,10 @@
 	return resModifiers, nil
 }
 
-func (p *ResourceModifiers) ApplyResourceModifierRules(obj *unstructured.Unstructured, groupResource string, log logrus.FieldLogger) []error {
+func (p *ResourceModifiers) ApplyResourceModifierRules(obj *unstructured.Unstructured, groupResource string, scheme *runtime.Scheme, log logrus.FieldLogger) []error {
 	var errs []error
 	for _, rule := range p.ResourceModifierRules {
-		err := rule.Apply(obj, groupResource, log)
+		err := rule.apply(obj, groupResource, scheme, log)
 		if err != nil {
 			errs = append(errs, err)
 		}
@@ -80,13 +77,19 @@
 	return errs
 }
 
-func (r *ResourceModifierRule) Apply(obj *unstructured.Unstructured, groupResource string, log logrus.FieldLogger) error {
+func (r *ResourceModifierRule) apply(obj *unstructured.Unstructured, groupResource string, scheme *runtime.Scheme, log logrus.FieldLogger) error {
 	namespaceInclusion := collections.NewIncludesExcludes().Includes(r.Conditions.Namespaces...)
 	if !namespaceInclusion.ShouldInclude(obj.GetNamespace()) {
 		return nil
 	}
 
-	if r.Conditions.GroupResource != groupResource {
+	g, err := glob.Compile(r.Conditions.GroupResource)
+	if err != nil {
+		log.Errorf("bad glob pattern %s, err: %s", r.Conditions.GroupResource, err)
+		return err
+	}
+
+	if !g.Match(groupResource) {
 		return nil
 	}
 
@@ -110,57 +113,44 @@
 		}
 	}
 
-	patches, err := r.PatchArrayToByteArray()
+	match, err := matchConditions(obj, r.Conditions.Matches, log)
 	if err != nil {
 		return err
+	} else if !match {
+		log.Info("Conditions do not match, skip it")
+		return nil
 	}
+
 	log.Infof("Applying resource modifier patch on %s/%s", obj.GetNamespace(), obj.GetName())
-	err = ApplyPatch(patches, obj, log)
+	err = r.applyPatch(obj, scheme, log)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-// PatchArrayToByteArray converts all JsonPatch to string array with the format of jsonpatch.Patch and then convert it to byte array
-func (r *ResourceModifierRule) PatchArrayToByteArray() ([]byte, error) {
-	var patches []string
-	for _, patch := range r.Patches {
-		patches = append(patches, patch.ToString())
+func matchConditions(u *unstructured.Unstructured, patches []JSONPatch, _ logrus.FieldLogger) (bool, error) {
+	if len(patches) == 0 {
+		return true, nil
 	}
-	patchesStr := strings.Join(patches, ",\n\t")
-	return []byte(fmt.Sprintf(`[%s]`, patchesStr)), nil
-}
 
-func (p *JSONPatch) ToString() string {
-	if addQuotes(p.Value) {
-		return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": "%s"}`, p.Operation, p.From, p.Path, p.Value)
+	var fixed []JSONPatch
+	for _, patch := range patches {
+		patch.From = ""
+		patch.Operation = "test"
+		fixed = append(fixed, patch)
 	}
-	return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": %s}`, p.Operation, p.From, p.Path, p.Value)
-}
 
-func ApplyPatch(patch []byte, obj *unstructured.Unstructured, log logrus.FieldLogger) error {
-	jsonPatch, err := jsonpatch.DecodePatch(patch)
-	if err != nil {
-		return fmt.Errorf("error in decoding json patch %s", err.Error())
-	}
-	objBytes, err := obj.MarshalJSON()
-	if err != nil {
-		return fmt.Errorf("error in marshaling object %s", err.Error())
-	}
-	modifiedObjBytes, err := jsonPatch.Apply(objBytes)
+	p := &JSONPatcher{patches: fixed}
+	_, err := p.applyPatch(u)
 	if err != nil {
 		if errors.Is(err, jsonpatch.ErrTestFailed) {
-			log.Infof("Test operation failed for JSON Patch %s", err.Error())
-			return nil
+			return false, nil
 		}
-		return fmt.Errorf("error in applying JSON Patch %s", err.Error())
-	}
-	err = obj.UnmarshalJSON(modifiedObjBytes)
-	if err != nil {
-		return fmt.Errorf("error in unmarshalling modified object %s", err.Error())
+		return false, err
 	}
-	return nil
+
+	return true, nil
 }
 
 func unmarshalResourceModifiers(yamlData []byte) (*ResourceModifiers, error) {
@@ -172,25 +162,27 @@
 	return resModifiers, nil
 }
 
-func addQuotes(value string) bool {
-	if value == "" {
-		return true
-	}
-	// if value is null, then don't add quotes
-	if value == "null" {
-		return false
-	}
-	// if value is a boolean, then don't add quotes
-	if _, err := strconv.ParseBool(value); err == nil {
-		return false
-	}
-	// if value is a json object or array, then don't add quotes.
-	if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
-		return false
+type patcher interface {
+	Patch(u *unstructured.Unstructured, logger logrus.FieldLogger) (*unstructured.Unstructured, error)
+}
+
+func (r *ResourceModifierRule) applyPatch(u *unstructured.Unstructured, scheme *runtime.Scheme, logger logrus.FieldLogger) error {
+	var p patcher
+	if len(r.Patches) > 0 {
+		p = &JSONPatcher{patches: r.Patches}
+	} else if len(r.MergePatches) > 0 {
+		p = &JSONMergePatcher{patches: r.MergePatches}
+	} else if len(r.StrategicPatches) > 0 {
+		p = &StrategicMergePatcher{patches: r.StrategicPatches, scheme: scheme}
+	} else {
+		return fmt.Errorf("no patch data found")
 	}
-	// if value is a number, then don't add quotes
-	if _, err := strconv.ParseFloat(value, 64); err == nil {
-		return false
+
+	updated, err := p.Patch(u, logger)
+	if err != nil {
+		return fmt.Errorf("error in applying patch %s", err)
 	}
-	return true
+
+	u.SetUnstructuredContent(updated.Object)
+	return nil
 }