Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't include excluded items in ItemBlocks #8572

Merged
merged 1 commit into from
Jan 7, 2025
Merged

Don't include excluded items in ItemBlocks #8572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelogs/unreleased/8572-sseago
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Don't include excluded items in ItemBlocks
39 changes: 23 additions & 16 deletions pkg/backup/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,8 +455,8 @@
"name": items[i].name,
}).Infof("Processing item")

// Skip if this item has already been added to an ItemBlock
if items[i].inItemBlock {
// Skip if this item has already been processed (in a block or previously excluded)
if items[i].inItemBlockOrExcluded {
log.Debugf("Not creating new ItemBlock for %s %s/%s because it's already in an ItemBlock", items[i].groupResource.String(), items[i].namespace, items[i].name)
} else {
if itemBlock == nil {
Expand Down Expand Up @@ -639,12 +639,23 @@
continue
}
itemsMap[relatedItem] = append(itemsMap[relatedItem], &kubernetesResource{
groupResource: relatedItem.GroupResource,
preferredGVR: gvr,
namespace: relatedItem.Namespace,
name: relatedItem.Name,
inItemBlock: true,
groupResource: relatedItem.GroupResource,
preferredGVR: gvr,
namespace: relatedItem.Namespace,
name: relatedItem.Name,
inItemBlockOrExcluded: true,
})

relatedItemMetadata, err := meta.Accessor(item)
if err != nil {
log.WithError(errors.WithStack(err)).Warn("Failed to get object metadata.")
continue

Check warning on line 652 in pkg/backup/backup.go

View check run for this annotation

Codecov / codecov/patch

pkg/backup/backup.go#L651-L652 

Added lines #L651 - L652 were not covered by tests
}
// Don't add to ItemBlock if item is excluded
// itemInclusionChecks logs the reason
if !itemBlock.itemBackupper.itemInclusionChecks(log, false, relatedItemMetadata, item, relatedItem.GroupResource) {
continue
sseago marked this conversation as resolved.
Show resolved Hide resolved
}
log.Infof("adding %s %s/%s to ItemBlock", relatedItem.GroupResource, relatedItem.Namespace, relatedItem.Name)
itemBlock.AddUnstructured(relatedItem.GroupResource, item, gvr)
kb.executeItemBlockActions(log, item, relatedItem.GroupResource, relatedItem.Name, relatedItem.Namespace, itemsMap, itemBlock)
Expand All @@ -660,15 +671,11 @@
itemBlock.Log.Debug("Executing pre hooks")
for _, item := range itemBlock.Items {
if item.Gr == kuberesource.Pods {
metadata, key, err := kb.itemMetadataAndKey(item)
key, err := kb.getItemKey(item)
if err != nil {
itemBlock.Log.WithError(errors.WithStack(err)).Error("Error accessing pod metadata")
continue
}
// Don't run hooks if pod is excluded
if !itemBlock.itemBackupper.itemInclusionChecks(itemBlock.Log, false, metadata, item.Item, item.Gr) {
continue
}
// Don't run hooks if pod has already been backed up
if !itemBlock.itemBackupper.backupRequest.BackedUpItems.Has(key) {
preHookPods = append(preHookPods, item)
Expand All @@ -679,7 +686,7 @@
for i, pod := range failedPods {
itemBlock.Log.WithError(errs[i]).WithField("name", pod.Item.GetName()).Error("Error running pre hooks for pod")
// if pre hook fails, flag pod as backed-up and move on
_, key, err := kb.itemMetadataAndKey(pod)
key, err := kb.getItemKey(pod)
if err != nil {
itemBlock.Log.WithError(errors.WithStack(err)).Error("Error accessing pod metadata")
continue
Expand All @@ -704,17 +711,17 @@
return grList
}

func (kb *kubernetesBackupper) itemMetadataAndKey(item itemblock.ItemBlockItem) (metav1.Object, itemKey, error) {
func (kb *kubernetesBackupper) getItemKey(item itemblock.ItemBlockItem) (itemKey, error) {
metadata, err := meta.Accessor(item.Item)
if err != nil {
return nil, itemKey{}, err
return itemKey{}, err

Check warning on line 717 in pkg/backup/backup.go

View check run for this annotation

Codecov / codecov/patch

pkg/backup/backup.go#L717 

Added line #L717 was not covered by tests
}
key := itemKey{
resource: resourceKey(item.Item),
namespace: metadata.GetNamespace(),
name: metadata.GetName(),
}
return metadata, key, nil
return key, nil
}

func (kb *kubernetesBackupper) handleItemBlockPreHooks(itemBlock BackupItemBlock, hookPods []itemblock.ItemBlockItem) ([]itemblock.ItemBlockItem, []itemblock.ItemBlockItem, []error) {
Expand Down
4 changes: 3 additions & 1 deletion pkg/backup/item_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,9 @@ type kubernetesResource struct {
preferredGVR schema.GroupVersionResource
namespace, name, path string
orderedResource bool
inItemBlock bool // set to true during backup processing when added to an ItemBlock
// set to true during backup processing when added to an ItemBlock
// or if the item is excluded from backup.
inItemBlockOrExcluded bool
}

// getItemsFromResourceIdentifiers get the kubernetesResources
Expand Down
19 changes: 16 additions & 3 deletions pkg/backup/itemblock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

"github.com/vmware-tanzu/velero/pkg/itemblock"
Expand All @@ -41,12 +42,12 @@
}

func (b *BackupItemBlock) addKubernetesResource(item *kubernetesResource, log logrus.FieldLogger) *unstructured.Unstructured {
// no-op if item is already in a block
if item.inItemBlock {
// no-op if item has already been processed (in a block or previously excluded)
if item.inItemBlockOrExcluded {
return nil
}
var unstructured unstructured.Unstructured
item.inItemBlock = true
item.inItemBlockOrExcluded = true

f, err := os.Open(item.path)
if err != nil {
Expand All @@ -60,6 +61,18 @@
log.WithError(errors.WithStack(err)).Error("Error decoding JSON from file")
return nil
}

metadata, err := meta.Accessor(&unstructured)
if err != nil {
log.WithError(errors.WithStack(err)).Warn("Error accessing item metadata")
return nil
}

Check warning on line 69 in pkg/backup/itemblock.go

View check run for this annotation

Codecov / codecov/patch

pkg/backup/itemblock.go#L67-L69 

Added lines #L67 - L69 were not covered by tests
// Don't add to ItemBlock if item is excluded
// itemInclusionChecks logs the reason
if !b.itemBackupper.itemInclusionChecks(log, false, metadata, &unstructured, item.groupResource) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To me, it seems this change should already work.
It seems no need to add the same check in the pkg/backup/backup.go.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@blackpiglet When we add related items to the ItemBlock, we first attempt to find it in the list from the item collector -- in that case we already have the item content and don't need to make an APIServer call. That's the code path that adds to the ItemBlock here, so we need the inclusion check to skip excluded items.

In cases where the item is not already in the list of items from the item collector (PVs returned from PVC plugin when includeClusterResources==nil), then we have to pull the item via the APIServer -- that's the code path that makes the check in pkg/backup/backup.go

Note that this itemInclusionChecks call happens within itemblock.addKubernetesResource -- and whenever addKubernetesResource is called from backup.go, there's a continue call afterwards (line 613) , and the second itemInclusionChecks call is after this continue, so this check won't be performed twice.

return nil
}

log.Infof("adding %s %s/%s to ItemBlock", item.groupResource, item.namespace, item.name)
b.AddUnstructured(item.groupResource, &unstructured, item.preferredGVR)
return &unstructured
Expand Down
Loading