Skip to content

Commit

Permalink
feat: added goroutines to aws fetcher
Browse files Browse the repository at this point in the history
  • Loading branch information
AvineshTripathi committed Sep 27, 2023
1 parent 5473f76 commit db99e6a
Showing 1 changed file with 23 additions and 16 deletions.
39 changes: 23 additions & 16 deletions providers/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws
import (
"context"
"strings"
"sync"

log "github.com/sirupsen/logrus"

Expand Down Expand Up @@ -100,28 +101,34 @@ func FetchResources(ctx context.Context, client providers.ProviderClient, region
log.Infof("Komiser will fetch resources from the following regions: %s", strings.Join(regions, ","))
listOfSupportedRegions = regions
}

var sncResource sync.WaitGroup
for _, region := range listOfSupportedRegions {
client.AWSClient.Region = region

for _, fetchResources := range listOfSupportedServices() {
resources, err := fetchResources(ctx, client)
if err != nil {
log.Warnf("[%s][AWS] %s", client.Name, err)
} else {
for _, resource := range resources {
_, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
if err != nil {
log.WithError(err).Errorf("db trigger failed")
sncResource.Add(1)
go func(fetchResources providers.FetchDataFunction) {
resources, err := fetchResources(ctx, client)
if err != nil {
log.Warnf("[%s][AWS] %s", client.Name, err)
} else {
for _, resource := range resources {
_, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
if err != nil {
log.WithError(err).Errorf("db trigger failed")
}
}
if telemetry {
analytics.TrackEvent("discovered_resources", map[string]interface{}{
"provider": "AWS",
"resources": len(resources),
})
}
}
if telemetry {
analytics.TrackEvent("discovered_resources", map[string]interface{}{
"provider": "AWS",
"resources": len(resources),
})
}
}
sncResource.Done()
}(fetchResources)
}
sncResource.Wait()
}
}

Expand Down

0 comments on commit db99e6a

Please sign in to comment.