Skip to content

Commit

Permalink
Data gatherers file (#83)
Browse files Browse the repository at this point in the history 
* Add method for AKS data gatherer to load data from file rather than using AKS API

Signed-off-by: wwwil <wwwil.squires@gmail.com>

* Add local data gatherer for loading data from local files

Signed-off-by: wwwil <wwwil.squires@gmail.com>

* If any data gatherer specifies a data-path then use a LocalDataGatherer to impersonate it

Signed-off-by: wwwil <wwwil.squires@gmail.com>

* Remove data-path handling from AKS data gatherer

Signed-off-by: wwwil <wwwil.squires@gmail.com>

* Add an example fake data for AKS and GKE with config for using fake data

Signed-off-by: wwwil <wwwil.squires@gmail.com>

* Fake -> stubbed

As discussed on standup this morning

Signed-off-by: Charlie Egan <charlieegan3@users.noreply.github.com>

Co-authored-by: William Squires <williamsquires@spatialos.com>
  • Loading branch information
@charlieegan3
charlieegan3 and William Squires authored Feb 5, 2020
1 parent 33cfa96 commit 9d4a340
Show file tree
Hide file tree
Showing 7 changed files with 261 additions and 0 deletions.
21 changes: 21 additions & 0 deletions cmd/check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/jetstack/preflight/pkg/datagatherer/eks"
"github.com/jetstack/preflight/pkg/datagatherer/gke"
"github.com/jetstack/preflight/pkg/datagatherer/k8s"
localdatagatherer "github.com/jetstack/preflight/pkg/datagatherer/local"
"github.com/jetstack/preflight/pkg/output"
"github.com/jetstack/preflight/pkg/output/azblob"
"github.com/jetstack/preflight/pkg/output/gcs"
Expand Down Expand Up @@ -130,6 +131,19 @@ func check() {
for name, config := range gatherersConfig {
// TODO: create gatherer from config in a more clever way. We need to read gatherer config from here and its schema depends on the gatherer itself.
var dg datagatherer.DataGatherer
dataGathererConfig, ok := config.(map[string]interface{})
if !ok {
log.Fatalf("Cannot parse %s data gatherer config.", name)
}
// Check if this data gatherer's config specifies a data-path.
// If it does create a LocalDataGatherer to load this data but keep
// the name of the data gatherer it is impersonating so it can
// provide stubbed data.
if dataPath, ok := dataGathererConfig["data-path"].(string); ok && dataPath != "" {
dg = localdatagatherer.NewLocalDataGatherer(dataPath)
gatherers[name] = dg
continue
}
if name == "eks" {
eksConfig, ok := config.(map[string]interface{})
if !ok {
Expand Down Expand Up @@ -207,6 +221,13 @@ func check() {
log.Fatalf("Cannot create k8s client: %+v", err)
}
dg = k8s.NewPodsDataGatherer(k8sClient)
} else if name == "local" {
localConfig, ok := config.(map[string]interface{})
if !ok {
log.Fatal("Cannot parse 'data-gatherers.local' in config.")
}
dataPath, ok := localConfig["data-path"].(string)
dg = localdatagatherer.NewLocalDataGatherer(dataPath)
} else {
log.Fatalf("Found unsupported data-gatherer %q in config.", name)
}
Expand Down
1 change: 1 addition & 0 deletions docs/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ The following data gatherers are available:
- [Google Kubernetes Engine](docs/datagatherers/gke.md)
- [Amazon Elastic Kubernetes Service](docs/datagatherers/eks.md)
- [Microsoft Azure Kubernetes Service](docs/datagatherers/aks.md)
- [Local](docs/datagatheres/local.md)

# Package Sources

Expand Down
6 changes: 6 additions & 0 deletions docs/datagatherers/local.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Local Data Gatherer

This is intended to be used for reading data for evaluation from the local file
system. It can also be used for 'stubbing' remote data sources when testing
other data gatherers. An example of this can be seen
[here](../../examples/stubbed.preflight.yaml).
45 changes: 45 additions & 0 deletions examples/data/aks.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"aks": {
"Cluster": {
"properties": {
"networkProfile": {
"dockerBridgeCidr": "172.17.0.1/16",
"dnsServiceIP": "10.0.0.10",
"serviceCidr": "10.0.0.0/16",
"podCidr": "10.244.0.0/16",
"networkPlugin": "kubenet"
},
"enableRBAC": true,
"nodeResourceGroup": "MC_preflight_preflight-test_uksouth",
"addonProfiles": {
"httpApplicationRouting": {
"config": null,
"enabled": false
}
},
"servicePrincipalProfile": {
"clientId": "***********************"
},
"agentPoolProfiles": [
{
"osType": "Linux",
"maxPods": 110,
"storageProfile": "ManagedDisks",
"osDiskSizeGB": 100,
"vmSize": "Standard_B2s",
"count": 1,
"name": "agentpool"
}
],
"fqdn": "preflight-test-dns-f2de96e5.hcp.uksouth.azmk8s.io",
"dnsPrefix": "preflight-test-dns",
"kubernetesVersion": "1.14.8",
"provisioningState": "Succeeded"
},
"type": "Microsoft.ContainerService/ManagedClusters",
"name": "preflight-test",
"location": "uksouth",
"id": "/subscriptions/*****************/resourcegroups/preflight/providers/Microsoft.ContainerService/managedClusters/preflight-test"
}
}
}
144 changes: 144 additions & 0 deletions examples/data/gke.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
{
"addonsConfig": {
"horizontalPodAutoscaling": {},
"httpLoadBalancing": {},
"istioConfig": {
"disabled": true
},
"kubernetesDashboard": {
"disabled": true
},
"networkPolicyConfig": {
"disabled": true
}
},
"authenticatorGroupsConfig": {},
"clusterIpv4Cidr": "10.4.0.0/14",
"clusterTelemetry": {
"type": "ENABLED"
},
"createTime": "2020-01-15T18:55:55+00:00",
"currentMasterVersion": "1.14.8-gke.12",
"currentNodeCount": 3,
"currentNodeVersion": "1.14.8-gke.12",
"databaseEncryption": {
"state": "DECRYPTED"
},
"defaultMaxPodsConstraint": {
"maxPodsPerNode": "110"
},
"endpoint": "104.155.177.255",
"initialClusterVersion": "1.14.8-gke.12",
"instanceGroupUrls": [
"https://www.googleapis.com/compute/v1/projects/jetstack-wil/zones/us-central1-a/instanceGroupManagers/gke-standard-cluster-2-default-pool-83d58bed-grp"
],
"ipAllocationPolicy": {
"clusterIpv4Cidr": "10.4.0.0/14",
"clusterIpv4CidrBlock": "10.4.0.0/14",
"clusterSecondaryRangeName": "gke-standard-cluster-2-pods-fddd6f46",
"servicesIpv4Cidr": "10.70.0.0/20",
"servicesIpv4CidrBlock": "10.70.0.0/20",
"servicesSecondaryRangeName": "gke-standard-cluster-2-services-fddd6f46",
"useIpAliases": true
},
"labelFingerprint": "a9dc16a7",
"legacyAbac": {},
"location": "us-central1-a",
"locations": [
"us-central1-a"
],
"loggingService": "logging.googleapis.com/kubernetes",
"maintenancePolicy": {
"resourceVersion": "e3b0c442"
},
"masterAuth": {
"clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURERENDQWZTZ0F3SUJBZ0lSQUtQT2NXc0JGTzFiZTJrbWZxMFZZNzh3RFFZSktvWklodmNOQVFFTEJRQXcKTHpFdE1Dc0dBMVVFQXhNa1pXSmtPVGxqTlRNdE5XRTRZaTAwT1dRMUxUZzJOV1V0TUdVell6a3daakJsWldGaApNQjRYRFRJd01ERXhOVEUzTlRVMU5Wb1hEVEkxTURFeE16RTROVFUxTlZvd0x6RXRNQ3NHQTFVRUF4TWtaV0prCk9UbGpOVE10TldFNFlpMDBPV1ExTFRnMk5XVXRNR1V6WXprd1pqQmxaV0ZoTUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBM0xiSkJ0Vkoxa2gxNERXWmhKMlRZMkxtTzFXSk51NjNLZzhBazVEegpWb3hzTml4VXBZQkZwaUkxN0J1Qm12bXkvQmN1czIrcTFIWmxtLzZNc29wZ1JWaHFFaUdnaDhEanhsaHBhckkwCjNxekJjRTVkaXE1dGd3V3pGeDF0S20wRmdmUGt3ZzRUTHZ2d25VaERMU21ranhuS2RBRVdCbnR2WDZxNkJmZ3AKbEczazJtd0gyYUNiYVNMSm1kRXJUR3RNMVFWN1N0REVYR2N3TlVjOUZheDNQbzk1elFLNjQ4N1U3N291ZVExeQpTQU9JVWZ4aDU2MFpVU1E1OXRYdkVTTWN2SUdKaVUvSWNCVHAvZ1VIUm5xb0RRRitvQmVKRm1abENla2hPeWthCnVlK052a09GSnI1TW1Rdys0ck5uV2NvbTdlZzljRWh2cWhpSFBHYU1wem9xSndJREFRQUJveU13SVRBT0JnTlYKSFE4QkFmOEVCQU1DQWdRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQQoxRDJiT1p4b0lpMG9EOXVGQTY1WXowRFlyclMzWUo3K3lkWXNjVFR0VEg5Ry9PWkhZcU5sTTM1SXJmbE1SYUFaCmNFL3VBUThSaGorcWtuQ01OUHhLL3pqYVYyVGhxMzlYYzZlSTlqeWl5RDVRb3lmSHBPeWF1aGpFTXVoajU1L3oKVmlleHE2VGZ6cXhBakJhWTJhMmpQVk81c1ZyZVZqUnNzUFlncEh4YVk2a25WQlZ4Rzg0WHB6bWJiUFk2cjlBNwphZ2ROWldvQms4cUxtRmxxVnBoYVRsNnpoeXczS2dhL09VQUNlQkZZN0lXQzM2bTVkV2xOWlVXbHRWS3EvaE5NCkV1VlRKMXNUblhFVXplTlBnZ1FaRy9qbzA3bWI0SnFHWmZNaFBZTE9mZExhdUxPcTJhNjAwcnlkUTk4dFB5L0YKS2FSaG0xVWp3M1oxZTNZeno0cUQ4UT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
},
"masterAuthorizedNetworksConfig": {},
"monitoringService": "monitoring.googleapis.com/kubernetes",
"name": "standard-cluster-2",
"network": "default",
"networkConfig": {
"network": "projects/jetstack-wil/global/networks/default",
"subnetwork": "projects/jetstack-wil/regions/us-central1/subnetworks/default"
},
"networkPolicy": {},
"nodeConfig": {
"diskSizeGb": 100,
"diskType": "pd-standard",
"imageType": "COS",
"machineType": "n1-standard-1",
"metadata": {
"disable-legacy-endpoints": "true"
},
"oauthScopes": [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/trace.append"
],
"serviceAccount": "default",
"shieldedInstanceConfig": {
"enableIntegrityMonitoring": true
}
},
"nodePools": [
{
"autoscaling": {},
"config": {
"diskSizeGb": 100,
"diskType": "pd-standard",
"imageType": "COS",
"machineType": "n1-standard-1",
"metadata": {
"disable-legacy-endpoints": "true"
},
"oauthScopes": [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/trace.append"
],
"serviceAccount": "default",
"shieldedInstanceConfig": {
"enableIntegrityMonitoring": true
}
},
"initialNodeCount": 3,
"instanceGroupUrls": [
"https://www.googleapis.com/compute/v1/projects/jetstack-wil/zones/us-central1-a/instanceGroupManagers/gke-standard-cluster-2-default-pool-83d58bed-grp"
],
"locations": [
"us-central1-a"
],
"management": {
"autoRepair": true,
"autoUpgrade": true
},
"maxPodsConstraint": {
"maxPodsPerNode": "110"
},
"name": "default-pool",
"podIpv4CidrSize": 24,
"selfLink": "https://container.googleapis.com/v1beta1/projects/jetstack-wil/zones/us-central1-a/clusters/standard-cluster-2/nodePools/default-pool",
"status": "RUNNING",
"version": "1.14.8-gke.12"
}
],
"releaseChannel": {
"channel": "REGULAR"
},
"selfLink": "https://container.googleapis.com/v1beta1/projects/jetstack-wil/zones/us-central1-a/clusters/standard-cluster-2",
"servicesIpv4Cidr": "10.70.0.0/20",
"shieldedNodes": {},
"status": "RUNNING",
"subnetwork": "default",
"tierSettings": {
"tier": "STANDARD"
},
"zone": "us-central1-a"
}
21 changes: 21 additions & 0 deletions examples/stubbed.preflight.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
cluster-name: my-stubbed-cluster

data-gatherers:
aks:
# Stubbed data for AKS.
data-path: ./examples/data/aks.json
gke:
# Stubbed data for GKE.
data-path: ./examples/data/gke.json

package-sources:
- type: local
dir: preflight-packages/

enabled-packages:
- "examples.jetstack.io/aks_basic"
- "examples.jetstack.io/gke_basic"

outputs:
# Write the report to the CLI.
- type: cli
23 changes: 23 additions & 0 deletions pkg/datagatherer/local/local.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package local

import "io/ioutil"

type LocalDataGatherer struct {
dataPath string
}

// NewLocalDataGatherer returns a LocalDatagatherer with the dataPath provided.
func NewLocalDataGatherer(dataPath string) *LocalDataGatherer {
return &LocalDataGatherer{
dataPath: dataPath,
}
}

// Fetch loads and returns the data from the LocalDatagatherer's dataPath
func (g *LocalDataGatherer) Fetch() (interface{}, error) {
dataBytes, err := ioutil.ReadFile(g.dataPath)
if err != nil {
return nil, err
}
return dataBytes, nil
}

0 comments on commit 9d4a340

Please sign in to comment.