From d6e50638a3b50cc0395802d7c6712bcd7590dba4 Mon Sep 17 00:00:00 2001 From: Guy Margalit Date: Thu, 11 Jul 2019 16:51:09 +0300 Subject: [PATCH] CLI + noobaa v5 --- .dockerignore | 8 +- .travis.yml | 33 +- ...deploy_minikube.sh => install-minikube.sh} | 4 +- .travis/install-operator-sdk.sh | 7 + .travis/install-python.sh | 4 + Makefile | 190 +++-- PROJECT | 1 + README.md | 33 +- build/Dockerfile | 5 +- cmd/cli/bundle/.gitignore | 1 - cmd/cli/cli.go | 169 ---- cmd/manager/main.go | 120 +-- cmd/version/main.go | 11 + .../crds/noobaa_v1alpha1_backingstore_cr.yaml | 9 + .../noobaa_v1alpha1_backingstore_crd.yaml | 43 + .../crds/noobaa_v1alpha1_bucketclass_cr.yaml | 12 + .../crds/noobaa_v1alpha1_bucketclass_crd.yaml | 38 + deploy/crds/noobaa_v1alpha1_noobaa_cr.yaml | 6 +- deploy/crds/noobaa_v1alpha1_noobaa_crd.yaml | 38 + deploy/internal/secret-admin.yaml | 12 + deploy/internal/secret-operator.yaml | 11 + deploy/internal/secret-server.yaml | 10 + deploy/internal/service-mgmt.yaml | 31 + deploy/internal/service-s3.yaml | 17 + deploy/internal/statefulset-core.yaml | 127 +++ deploy/namespace.yaml | 4 + .../{ => manual}/catalog-source-config.yaml | 0 .../{ => manual}/operator-group.yaml | 6 +- .../{ => manual}/operator-source.yaml | 0 .../{ => manual}/operator-subscription.yaml | 4 +- .../noobaa-operator.package.yaml | 2 +- ...operator.v0.1.0.clusterserviceversion.yaml | 93 ++- ...operator.v1.0.0.clusterserviceversion.yaml | 301 +++++++ deploy/operator.yaml | 4 +- deploy/role.yaml | 3 + devenv.sh | 3 +- doc/about-noobaa.md | 17 + doc/backing-store-crd.md | 82 ++ doc/bucket-class-crd.md | 74 ++ doc/noobaa-crd.md | 116 +++ doc/obc-provisioner.md | 56 ++ doc/s3-account.md | 61 ++ go.mod | 23 +- go.sum | 80 ++ .../noobaa/v1alpha1/backingstore_types.go | 49 ++ pkg/apis/noobaa/v1alpha1/bucketclass_types.go | 42 + pkg/apis/noobaa/v1alpha1/noobaa_types.go | 50 +- .../noobaa/v1alpha1/zz_generated.deepcopy.go | 224 ++++- .../noobaa/v1alpha1/zz_generated.openapi.go | 188 ++++- {cmd/cli/gen => pkg/bundle}/bundle.go | 15 +- pkg/cli/bucket.go | 66 ++ pkg/cli/cli.go | 438 ++++++++++ pkg/cli/crd.go | 113 +++ pkg/cli/hub.go | 54 ++ pkg/cli/operator.go | 151 ++++ pkg/cli/system.go | 145 ++++ pkg/controller/add_backingstore.go | 10 + pkg/controller/add_bucketclass.go | 10 + .../backingstore/backingstore_controller.go | 39 + .../bucketclass/bucketclass_controller.go | 39 + pkg/controller/manager.go | 101 +++ pkg/controller/noobaa/noobaa_controller.go | 783 +----------------- pkg/nb/api.go | 162 +++- pkg/nb/rpc.go | 13 +- pkg/system/system.go | 710 ++++++++++++++++ pkg/util/fastmapper.go | 136 +++ pkg/util/util.go | 142 ++++ version/version.go | 10 +- 68 files changed, 4322 insertions(+), 1237 deletions(-) rename .travis/{deploy_minikube.sh => install-minikube.sh} (97%) create mode 100644 .travis/install-operator-sdk.sh create mode 100644 .travis/install-python.sh create mode 100644 PROJECT delete mode 100644 cmd/cli/bundle/.gitignore delete mode 100644 cmd/cli/cli.go create mode 100644 cmd/version/main.go create mode 100644 deploy/crds/noobaa_v1alpha1_backingstore_cr.yaml create mode 100644 deploy/crds/noobaa_v1alpha1_backingstore_crd.yaml create mode 100644 deploy/crds/noobaa_v1alpha1_bucketclass_cr.yaml create mode 100644 deploy/crds/noobaa_v1alpha1_bucketclass_crd.yaml create mode 100644 deploy/internal/secret-admin.yaml create mode 100644 deploy/internal/secret-operator.yaml create mode 100644 deploy/internal/secret-server.yaml create mode 100644 deploy/internal/service-mgmt.yaml create mode 100644 deploy/internal/service-s3.yaml create mode 100644 deploy/internal/statefulset-core.yaml create mode 100644 deploy/namespace.yaml rename deploy/olm-catalog/{ => manual}/catalog-source-config.yaml (100%) rename deploy/olm-catalog/{ => manual}/operator-group.yaml (54%) rename deploy/olm-catalog/{ => manual}/operator-source.yaml (100%) rename deploy/olm-catalog/{ => manual}/operator-subscription.yaml (73%) rename deploy/olm-catalog/{noobaa-operator => package}/noobaa-operator.package.yaml (66%) rename deploy/olm-catalog/{noobaa-operator/0.1.0 => package}/noobaa-operator.v0.1.0.clusterserviceversion.yaml (87%) create mode 100644 deploy/olm-catalog/package/noobaa-operator.v1.0.0.clusterserviceversion.yaml create mode 100644 doc/about-noobaa.md create mode 100644 doc/backing-store-crd.md create mode 100644 doc/bucket-class-crd.md create mode 100644 doc/noobaa-crd.md create mode 100644 doc/obc-provisioner.md create mode 100644 doc/s3-account.md create mode 100644 pkg/apis/noobaa/v1alpha1/backingstore_types.go create mode 100644 pkg/apis/noobaa/v1alpha1/bucketclass_types.go rename {cmd/cli/gen => pkg/bundle}/bundle.go (79%) create mode 100644 pkg/cli/bucket.go create mode 100644 pkg/cli/cli.go create mode 100644 pkg/cli/crd.go create mode 100644 pkg/cli/hub.go create mode 100644 pkg/cli/operator.go create mode 100644 pkg/cli/system.go create mode 100644 pkg/controller/add_backingstore.go create mode 100644 pkg/controller/add_bucketclass.go create mode 100644 pkg/controller/backingstore/backingstore_controller.go create mode 100644 pkg/controller/bucketclass/bucketclass_controller.go create mode 100644 pkg/controller/manager.go create mode 100644 pkg/system/system.go create mode 100644 pkg/util/fastmapper.go create mode 100644 pkg/util/util.go diff --git a/.dockerignore b/.dockerignore index da7be4242..aef3227a0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,7 @@ # we only want to take build/ into the image # so we ignore everything else here -cmd/ -deploy/ -pkg/ +* +!build + +# make sure we ignore vendor explicitly vendor/ -version/ diff --git a/.travis.yml b/.travis.yml index b96575c8c..ec7ecb920 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,10 @@ language: go sudo: required dist: xenial - +sevices: + - docker go: -- 1.12.5 - -stages: - - name: build - - name: deploy +- 1.12.7 env: global: @@ -17,21 +14,21 @@ env: - MINIKUBE_HOME=$HOME - CHANGE_MINIKUBE_NONE_USER=true - KUBECONFIG=$HOME/.kube/config + - GO111MODULE=on + - CGO_ENABLED=0 + +stages: + - name: build jobs: include: - stage: build - name: "Build and unit test" - script: - - make all - - make test - - - stage: build - name: "Run smoke tests" + name: "Build and Test" install: - - make install-sdk - - "./.travis/deploy_minikube.sh" + - bash .travis/install-operator-sdk.sh + - bash .travis/install-python.sh + - bash .travis/install-minikube.sh script: - - make test-e2e - env: - - TEST_GROUP=smoke + - make build + - make test + - make test-integ diff --git a/.travis/deploy_minikube.sh b/.travis/install-minikube.sh similarity index 97% rename from .travis/deploy_minikube.sh rename to .travis/install-minikube.sh index d0e1c2c7e..33d1303d2 100755 --- a/.travis/deploy_minikube.sh +++ b/.travis/install-minikube.sh @@ -15,8 +15,8 @@ set -x # socat is needed for port forwarding sudo apt-get update && sudo apt-get install socat -export MINIKUBE_VERSION=v1.0.0 -export KUBERNETES_VERSION=v1.14.0 +export MINIKUBE_VERSION=v1.2.0 +export KUBERNETES_VERSION=v1.15.0 sudo mount --make-rshared / sudo mount --make-rshared /proc diff --git a/.travis/install-operator-sdk.sh b/.travis/install-operator-sdk.sh new file mode 100644 index 000000000..f3e7fc0cd --- /dev/null +++ b/.travis/install-operator-sdk.sh @@ -0,0 +1,7 @@ +SDK_VERSION="v0.9.0" +SDK_RELEASE="https://github.com/operator-framework/operator-sdk/releases/download/${SDK_VERSION}/operator-sdk-${SDK_VERSION}-x86_64-linux-gnu" + +echo "install-sdk - installing version ${SDK_VERSION}" +curl "${SDK_RELEASE}" -Lo $GOPATH/bin/operator-sdk +chmod +x $GOPATH/bin/operator-sdk +$GOPATH/bin/operator-sdk version diff --git a/.travis/install-python.sh b/.travis/install-python.sh new file mode 100644 index 000000000..eac95f6d2 --- /dev/null +++ b/.travis/install-python.sh @@ -0,0 +1,4 @@ +sudo apt-get update +sudo apt-get install python3-venv +curl -O https://bootstrap.pypa.io/get-pip.py +sudo python3 get-pip.py diff --git a/Makefile b/Makefile index 1c1e7f877..cc65b223c 100644 --- a/Makefile +++ b/Makefile @@ -1,97 +1,143 @@ -VERSION ?= 0.1.0 +# Required build-tools: +# - go +# - git +# - python3 +# - minikube +# - operator-sdk + +VERSION ?= $(shell go run cmd/version/main.go) IMAGE ?= noobaa/noobaa-operator:$(VERSION) REPO ?= github.com/noobaa/noobaa-operator -GO_FLAGS ?= CGO_ENABLED=0 GO111MODULE=on -GO_LINUX_FLAGS ?= GOOS=linux GOARCH=amd64 +GO ?= CGO_ENABLED=0 GO111MODULE=on go +GO_LINUX ?= GOOS=linux GOARCH=amd64 $(GO) +GO_LIST ?= ./cmd/... ./pkg/... ./test/... ./version/... +GOHOSTOS ?= $(shell go env GOHOSTOS) -OUTPUT = build/_output -BIN = $(OUTPUT)/bin -BUNDLE = $(OUTPUT)/bundle -VENV = $(OUTPUT)/venv +OUTPUT ?= build/_output +BIN ?= $(OUTPUT)/bin +OLM ?= $(OUTPUT)/olm +VENV ?= $(OUTPUT)/venv +BUNDLE ?= $(OUTPUT)/bundle -# Default tasks: -all: build - @echo "@@@ All Done." -.PHONY: all - -# Developer tasks: +#------------# +#- Building -# +#------------# -lint: - @echo Linting... - go get -u golang.org/x/lint/golint - golint -set_exit_status=1 $(shell go list ./cmd/... ./pkg/...) -.PHONY: lint +all: local build + @echo "all - done." +.PHONY: all -build: cli image - @echo "@@@ Build Done." +build: image olm + @echo "build - done." .PHONY: build -cli: vendor - ${GO_FLAGS} go generate cmd/cli/cli.go - ${GO_FLAGS} go build -mod=vendor -o $(BIN)/kubectl-noobaa $(REPO)/cmd/cli -.PHONY: cli - -dev: operator - WATCH_NAMESPACE=noobaa OPERATOR_NAME=noobaa-operator $(BIN)/noobaa-operator-local -.PHONY: dev - -operator: vendor - go build -mod=vendor -o $(BIN)/noobaa-operator-local $(REPO)/cmd/manager -.PHONY: operator +image: gen + operator-sdk build $(IMAGE) +.PHONY: image -gen: vendor - operator-sdk generate k8s - operator-sdk generate openapi +gen: vendor $(BUNDLE)/deploy.go .PHONY: gen +$(BUNDLE)/deploy.go: pkg/bundle/bundle.go $(shell find deploy/ -type f) + mkdir -p $(BUNDLE) + $(GO) run pkg/bundle/bundle.go deploy/ $(BUNDLE)/deploy.go + vendor: - ${GO_FLAGS} go mod vendor + $(GO) mod vendor .PHONY: vendor -image: - ${GO_FLAGS} ${GO_LINUX_FLAGS} go build -mod=vendor -o $(BIN)/noobaa-operator $(REPO)/cmd/manager - docker build -f build/Dockerfile -t $(IMAGE) . -.PHONY: image +olm: + rm -rf $(OLM) + mkdir -p $(OLM) + cp deploy/olm-catalog/package/* $(OLM)/ + cp deploy/crds/*crd.yaml $(OLM)/ + python3 -m venv $(VENV) + ( \ + . $(VENV)/bin/activate && \ + pip3 install --upgrade pip && \ + pip3 install operator-courier && \ + operator-courier verify --ui_validate_io $(OLM) \ + ) + @echo "olm - ready at $(OLM)" +.PHONY: olm + + +#-----------# +#- Testing -# +#-----------# + +test: lint unittest + @echo "test - done." +.PHONY: test -bundle: - mkdir -p $(BUNDLE) - cp deploy/olm-catalog/noobaa-operator/*.yaml $(BUNDLE)/ - cp deploy/olm-catalog/noobaa-operator/$(VERSION)/*.yaml $(BUNDLE)/ - cp deploy/crds/*crd.yaml $(BUNDLE)/ - ( python3 -m venv $(VENV) && . $(VENV)/bin/activate && pip install operator-courier >/dev/null ) - ( . $(VENV)/bin/activate && operator-courier verify --ui_validate_io $(BUNDLE) ) -.PHONY: bundle - -push: - @echo TODO: To which tag do we want to push here? +test-integ: scorecard test-e2e + @echo "test-integ - done." +.PHONY: test-integ + +lint: gen + go get -u golang.org/x/lint/golint + golint -set_exit_status=0 ./cmd/... ./pkg/... ./test/... ./version/... +.PHONY: lint + +unittest: gen + go test ./cmd/... ./pkg/... ./version/... +.PHONY: unittest + +test-e2e: gen + # TODO fix test-e2e ! + # operator-sdk test local ./test/e2e \ + # --global-manifest deploy/cluster_role_binding.yaml \ + # --debug \ + # --go-test-flags "-v -parallel=1" +.PHONY: test-e2e + +scorecard: gen + kubectl create ns test-noobaa-operator-scorecard + operator-sdk scorecard \ + --cr-manifest deploy/crds/noobaa_v1alpha1_noobaa_cr.yaml \ + --cr-manifest deploy/crds/noobaa_v1alpha1_backingstore_cr.yaml \ + --cr-manifest deploy/crds/noobaa_v1alpha1_bucketclass_cr.yaml \ + --csv-path deploy/olm-catalog/package/noobaa-operator.v$(VERSION).clusterserviceversion.yaml \ + --namespace test-noobaa-operator-scorecard +.PHONY: scorecard + + +#-------------# +#- Releasing -# +#-------------# + +push-image: + @echo "push-image - It is too risky to push like this, because it easily overrides. @echo version from version/version.go? master? git describe? # docker push $(IMAGE) -.PHONY: push +.PHONY: push-image -test: vendor - go test ./cli/... ./pkg/... -.PHONY: test -test-e2e: vendor - operator-sdk test local ./test/e2e \ - --global-manifest deploy/cluster_role_binding.yaml \ - --debug \ - --go-test-flags "-v -parallel=1" -.PHONY: test +#--------------# +#- Developing -# +#--------------# -clean: - rm $(BIN)/* - rm -rf vendor/ -.PHONY: clean +local: gen + operator-sdk up local &>/dev/null +.PHONY: local + +run: gen + operator-sdk up local --operator-flags "operator run" +.PHONY: run -# Deps +gen-api: gen + operator-sdk generate k8s + operator-sdk generate openapi +.PHONY: gen-api -install-sdk: - @echo Installing SDK ${SDK_VERSION} - curl https://github.com/operator-framework/operator-sdk/releases/download/${SDK_VERSION}/operator-sdk-${SDK_VERSION}-x86_64-linux-gnu -sLo ${GOPATH}/bin/operator-sdk - chmod +x ${GOPATH}/bin/operator-sdk -.PHONY: install-sdk +ensure-gen-api-is-noop: gen-api + git diff -s --exit-code pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go || (echo "Build failed: API has been changed but the deep copy functions aren't up to date. Run 'make gen-api' and update your PR." && exit 1) + git diff -s --exit-code pkg/apis/noobaa/v1alpha1/zz_generated.openapi.go || (echo "Build failed: API has been changed but the deep copy functions aren't up to date. Run 'make gen-api' and update your PR." && exit 1) +.PHONY: ensure-gen-api-is-noop -#TODO scorecard +clean: + rm -rf $(OUTPUT) + rm -rf vendor/ +.PHONY: clean diff --git a/PROJECT b/PROJECT new file mode 100644 index 000000000..093fd52b1 --- /dev/null +++ b/PROJECT @@ -0,0 +1 @@ +repo = github.com/noobaa/noobaa-operator diff --git a/README.md b/README.md index 48723503f..be648a507 100644 --- a/README.md +++ b/README.md @@ -1 +1,32 @@ -# noobaa-operator \ No newline at end of file +# NooBaa Operator + +NooBaa is an object data service for hybrid and multi cloud environments. NooBaa runs on kubernetes, provides an S3 object store service (and Lambda with bucket triggers) to clients both inside and outside the cluster, and uses storage resources from within or outside the cluster, with flexible placement policies to automate data use cases. + +[About NooBaa](doc/about-noobaa.md) + +# Using the operator as CLI + +- Download the compiled operator binary from the [releases page](/releases) +- Run: `noobaa --help` for CLI usage +- Install the operator and noobaa with: `noobaa install` + +# Operator Design + +CRDs +- [NooBaa](doc/noobaa-crd.md) - The basic CRD to deploy a NooBaa system. +- [BackingStore](doc/backing-store-crd.md) - Connection to cloud or local storage to use in policies. +- [BucketClass](doc/bucket-class-crd.md) - Policies applied to a class of buckets. + +Applications +- [S3 Account](doc/s3-account.md) - Method to obtain S3 account credentials for native S3 applications. +- [OBC Provisioner](doc/obc-provisioner.md) - Method to claim a new/existing bucket. + +# Developing + +- Fork and clone the repo: `git clone https://github.com//noobaa-operator` +- Use minikube: `minikube start` +- Source the devenv into your shell: `. devenv.sh` +- Build the project: `make` +- Test with the alias `nb` that runs the local operator from `build/_output/bin` (alias created by devenv) +- `nb install` +- ... diff --git a/build/Dockerfile b/build/Dockerfile index c373d186d..b991090df 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -7,9 +7,10 @@ ENV OPERATOR=/usr/local/bin/noobaa-operator \ # install operator binary COPY build/_output/bin/noobaa-operator ${OPERATOR} +# copy scripts COPY build/bin /usr/local/bin RUN /usr/local/bin/user_setup +USER ${USER_UID} ENTRYPOINT ["/usr/local/bin/entrypoint"] - -USER ${USER_UID} +CMD ["operator", "run"] diff --git a/cmd/cli/bundle/.gitignore b/cmd/cli/bundle/.gitignore deleted file mode 100644 index b0b5171e1..000000000 --- a/cmd/cli/bundle/.gitignore +++ /dev/null @@ -1 +0,0 @@ -bundle.go \ No newline at end of file diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go deleted file mode 100644 index 1476a9c25..000000000 --- a/cmd/cli/cli.go +++ /dev/null @@ -1,169 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/noobaa/noobaa-operator/cmd/cli/bundle" - "github.com/noobaa/noobaa-operator/pkg/apis" - - "github.com/spf13/cobra" - rbacv1 "k8s.io/api/rbac/v1" - apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" -) - -//go:generate go run gen/bundle.go - -// ASCIILogo is noobaa's logo ascii art -const ASCIILogo = ` - /~~\\__~__//~~\ -| | - \~\\_ _//~/ - \\ // - | | - \~~~/ -` - -var namespace = "noobaa" -var ctx = context.TODO() - -// CLI is the top command for noobaa CLI -var CLI = &cobra.Command{ - Use: "noobaa", - Long: "\n NooBaa CLI \n" + ASCIILogo, -} - -// InstallCommand installs to kubernetes -var InstallCommand = &cobra.Command{ - Use: "install", - Short: "Install to kubernetes", - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Installing namespace:%s\n", namespace) - c, err := client.New(config.GetConfigOrDie(), client.Options{}) - fatal(err) - createNamespace(c) - createRBAC(c) - createCRDs(c) - createOperator(c) - }, -} - -// UninstallCommand uninstalls from kubernetes -var UninstallCommand = &cobra.Command{ - Use: "uninstall", - Short: "Uninstall from kubernetes", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("TODO: Uninstalling ...") - }, -} - -// CreateCommand uninstalls from kubernetes -var CreateCommand = &cobra.Command{ - Use: "create", - Short: "Create a NooBaa system", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Creating ...") - c, err := client.New(config.GetConfigOrDie(), client.Options{}) - fatal(err) - createSystem(c) - }, -} - -// DeleteCommand uninstalls from kubernetes -var DeleteCommand = &cobra.Command{ - Use: "delete", - Short: "Delete a NooBaa system", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("TODO: Deleting ...") - }, -} - -func main() { - apiextv1beta1.AddToScheme(scheme.Scheme) - apis.AddToScheme(scheme.Scheme) - CLI.PersistentFlags().StringVarP(&namespace, "namespace", "n", namespace, "Target namespace") - CLI.AddCommand(InstallCommand) - CLI.AddCommand(UninstallCommand) - CLI.AddCommand(CreateCommand) - CLI.AddCommand(DeleteCommand) - fatal(CLI.Execute()) -} - -func fatal(err error) { - if err != nil { - panic(err) - } -} - -func kubeObject(ns string, text string) runtime.Object { - // Decode text (yaml/json) to kube api object - deserializer := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer() - obj, group, err := deserializer.Decode([]byte(text), nil, nil) - // obj, group, err := scheme.Codecs.UniversalDecoder().Decode([]byte(text), nil, nil) - fatal(err) - // not sure if really needed, but set it anyway - obj.GetObjectKind().SetGroupVersionKind(*group) - metaObj := obj.(metav1.Object) - if ns != "" { - metaObj.SetNamespace(ns) - } - return obj -} - -func kubeApply(c client.Client, obj runtime.Object) { - err := c.Create(ctx, obj) - if errors.IsAlreadyExists(err) { - err = c.Update(ctx, obj) - } - fatal(err) -} - -func kubeCreateSkipExisting(c client.Client, obj runtime.Object) { - err := c.Create(ctx, obj) - if errors.IsAlreadyExists(err) { - return - } - fatal(err) -} - -func createNamespace(c client.Client) { - kubeApply(c, kubeObject("", fmt.Sprintf(` -apiVersion: v1 -kind: Namespace -metadata: - name: %s -`, namespace))) -} - -func createRBAC(c client.Client) { - kubeApply(c, kubeObject(namespace, bundle.File_deploy_service_account_yaml)) - kubeApply(c, kubeObject(namespace, bundle.File_deploy_role_yaml)) - kubeApply(c, kubeObject(namespace, bundle.File_deploy_role_binding_yaml)) - kubeApply(c, kubeObject(namespace, bundle.File_deploy_cluster_role_yaml)) - - crb := kubeObject(namespace, bundle.File_deploy_cluster_role_binding_yaml).(*rbacv1.ClusterRoleBinding) - crb.Name = crb.Name + "-" + namespace - for i := range crb.Subjects { - crb.Subjects[i].Namespace = namespace - } - kubeApply(c, crb) -} - -func createCRDs(c client.Client) { - kubeCreateSkipExisting(c, kubeObject("", bundle.File_deploy_crds_noobaa_v1alpha1_noobaa_crd_yaml)) -} - -func createOperator(c client.Client) { - kubeApply(c, kubeObject(namespace, bundle.File_deploy_operator_yaml)) -} - -func createSystem(c client.Client) { - kubeApply(c, kubeObject(namespace, bundle.File_deploy_crds_noobaa_v1alpha1_noobaa_cr_yaml)) -} diff --git a/cmd/manager/main.go b/cmd/manager/main.go index b582c8cd2..55c551871 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -1,125 +1,19 @@ package main import ( - "context" - "flag" - "fmt" - "os" - "runtime" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) _ "k8s.io/client-go/plugin/pkg/client/auth" "github.com/noobaa/noobaa-operator/pkg/apis" - "github.com/noobaa/noobaa-operator/pkg/controller" - "github.com/operator-framework/operator-sdk/pkg/k8sutil" - "github.com/operator-framework/operator-sdk/pkg/leader" - "github.com/operator-framework/operator-sdk/pkg/log/zap" - "github.com/operator-framework/operator-sdk/pkg/metrics" - "github.com/operator-framework/operator-sdk/pkg/restmapper" - sdkVersion "github.com/operator-framework/operator-sdk/version" - "github.com/spf13/pflag" - "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/manager" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/signals" -) + "github.com/noobaa/noobaa-operator/pkg/cli" + "github.com/noobaa/noobaa-operator/pkg/util" -// Change below variables to serve metrics on different host or port. -var ( - metricsHost = "0.0.0.0" - metricsPort int32 = 8383 + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/client-go/kubernetes/scheme" ) -var log = logf.Log.WithName("noobaa").WithName("operator") - -func printVersion() { - log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) - log.Info(fmt.Sprintf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)) - log.Info(fmt.Sprintf("Version of operator-sdk: %v", sdkVersion.Version)) -} - func main() { - // Add the zap logger flag set to the CLI. The flag set must - // be added before calling pflag.Parse(). - pflag.CommandLine.AddFlagSet(zap.FlagSet()) - - // Add flags registered by imported packages (e.g. glog and - // controller-runtime) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - - pflag.Parse() - - // Use a zap logr.Logger implementation. If none of the zap - // flags are configured (or if the zap flag set is not being - // used), this defaults to a production zap logger. - // - // The logger instantiated here can be changed to any logger - // implementing the logr.Logger interface. This logger will - // be propagated through the whole operator, generating - // uniform and structured logs. - logf.SetLogger(zap.Logger()) - - printVersion() - - namespace, err := k8sutil.GetWatchNamespace() - if err != nil { - log.Error(err, "Failed to get watch namespace") - os.Exit(1) - } - - // Get a config to talk to the apiserver - cfg, err := config.GetConfig() - if err != nil { - log.Error(err, "") - os.Exit(1) - } - - ctx := context.TODO() - - // Become the leader before proceeding - err = leader.Become(ctx, "noobaa-operator-lock") - if err != nil { - log.Error(err, "") - os.Exit(1) - } - - // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{ - Namespace: namespace, - MapperProvider: restmapper.NewDynamicRESTMapper, - MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), - }) - if err != nil { - log.Error(err, "") - os.Exit(1) - } - - log.Info("Registering Components.") - - // Setup Scheme for all resources - if err := apis.AddToScheme(mgr.GetScheme()); err != nil { - log.Error(err, "") - os.Exit(1) - } - - // Setup all Controllers - if err := controller.AddToManager(mgr); err != nil { - log.Error(err, "") - os.Exit(1) - } - - // Create Service object to expose the metrics port. - _, err = metrics.ExposeMetricsPort(ctx, metricsPort) - if err != nil { - log.Info(err.Error()) - } - - log.Info("Starting the Cmd.") - - // Start the Cmd - if err := mgr.Start(signals.SetupSignalHandler()); err != nil { - log.Error(err, "Manager exited non-zero") - os.Exit(1) - } + util.Fatal(apiextv1beta1.AddToScheme(scheme.Scheme)) + util.Fatal(apis.AddToScheme(scheme.Scheme)) + cli.New().Run() } diff --git a/cmd/version/main.go b/cmd/version/main.go new file mode 100644 index 000000000..614ce9d65 --- /dev/null +++ b/cmd/version/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" + + "github.com/noobaa/noobaa-operator/version" +) + +func main() { + fmt.Print(version.Version) +} diff --git a/deploy/crds/noobaa_v1alpha1_backingstore_cr.yaml b/deploy/crds/noobaa_v1alpha1_backingstore_cr.yaml new file mode 100644 index 000000000..9aa02d048 --- /dev/null +++ b/deploy/crds/noobaa_v1alpha1_backingstore_cr.yaml @@ -0,0 +1,9 @@ +apiVersion: noobaa.io/v1alpha1 +kind: BackingStore +metadata: + name: default-store +spec: + type: aws-s3 + region: us-east-1 + bucketName: noobaa1-aws-backing-store + credentialsSecretName: aws-credentials-secret diff --git a/deploy/crds/noobaa_v1alpha1_backingstore_crd.yaml b/deploy/crds/noobaa_v1alpha1_backingstore_crd.yaml new file mode 100644 index 000000000..aa574bf8f --- /dev/null +++ b/deploy/crds/noobaa_v1alpha1_backingstore_crd.yaml @@ -0,0 +1,43 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: backingstores.noobaa.io +spec: + group: noobaa.io + names: + kind: BackingStore + listKind: BackingStoreList + plural: backingstores + singular: backingstore + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + type: + type: string + required: + - type + type: object + status: + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true diff --git a/deploy/crds/noobaa_v1alpha1_bucketclass_cr.yaml b/deploy/crds/noobaa_v1alpha1_bucketclass_cr.yaml new file mode 100644 index 000000000..ada93b8c0 --- /dev/null +++ b/deploy/crds/noobaa_v1alpha1_bucketclass_cr.yaml @@ -0,0 +1,12 @@ +apiVersion: noobaa.io/v1alpha1 +kind: BucketClass +metadata: + name: default-class +spec: + placementPolicy: + tiers: + - tier: + mirrors: + - mirror: + spread: + - default-store diff --git a/deploy/crds/noobaa_v1alpha1_bucketclass_crd.yaml b/deploy/crds/noobaa_v1alpha1_bucketclass_crd.yaml new file mode 100644 index 000000000..abcd5921e --- /dev/null +++ b/deploy/crds/noobaa_v1alpha1_bucketclass_crd.yaml @@ -0,0 +1,38 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bucketclasses.noobaa.io +spec: + group: noobaa.io + names: + kind: BucketClass + listKind: BucketClassList + plural: bucketclasses + singular: bucketclass + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + type: object + status: + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true diff --git a/deploy/crds/noobaa_v1alpha1_noobaa_cr.yaml b/deploy/crds/noobaa_v1alpha1_noobaa_cr.yaml index 84999426e..8244f79ad 100644 --- a/deploy/crds/noobaa_v1alpha1_noobaa_cr.yaml +++ b/deploy/crds/noobaa_v1alpha1_noobaa_cr.yaml @@ -2,8 +2,4 @@ apiVersion: noobaa.io/v1alpha1 kind: NooBaa metadata: name: noobaa -spec: { - # Image is the container image to use for the deployment. - # The oeprator will update spec.image to default if missing. - # image: noobaa/noobaa-core:4.0 -} +spec: {} diff --git a/deploy/crds/noobaa_v1alpha1_noobaa_crd.yaml b/deploy/crds/noobaa_v1alpha1_noobaa_crd.yaml index e82c75936..154663cf7 100644 --- a/deploy/crds/noobaa_v1alpha1_noobaa_crd.yaml +++ b/deploy/crds/noobaa_v1alpha1_noobaa_crd.yaml @@ -55,6 +55,10 @@ spec: description: Image (optional) overrides the default image for server pods type: string + imagePullSecret: + description: ImagePullSecret (optional) sets a pull secret for the system + image + type: object storageClassName: description: StorageClassName (optional) overrides the default StorageClass for the PVC that the operator creates, this affects where the system @@ -81,6 +85,40 @@ spec: description: ActualImage is set to report which image the operator is using type: string + conditions: + description: 'Current service state of the noobaa system. Based on: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions + +patchMergeKey=type +patchStrategy=merge' + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about last + transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: Status is the status of the condition. + type: string + type: + description: Type is the type of the condition. + type: string + required: + - type + - status + type: object + type: array observedGeneration: description: ObservedGeneration is the most recent generation observed for this noobaa system. It corresponds to the CR generation, which diff --git a/deploy/internal/secret-admin.yaml b/deploy/internal/secret-admin.yaml new file mode 100644 index 000000000..9aeb50d5c --- /dev/null +++ b/deploy/internal/secret-admin.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: SYSNAME-admin + labels: + app: noobaa +type: Opaque +data: + email: "" + password: "" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" diff --git a/deploy/internal/secret-operator.yaml b/deploy/internal/secret-operator.yaml new file mode 100644 index 000000000..d8f7234ae --- /dev/null +++ b/deploy/internal/secret-operator.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: SYSNAME-operator + labels: + app: noobaa +type: Opaque +data: + email: "" + password: "" + auth_token: "" diff --git a/deploy/internal/secret-server.yaml b/deploy/internal/secret-server.yaml new file mode 100644 index 000000000..31f59a044 --- /dev/null +++ b/deploy/internal/secret-server.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: SYSNAME-operator + labels: + app: noobaa +type: Opaque +data: + jwt: "" + server_secret: "" diff --git a/deploy/internal/service-mgmt.yaml b/deploy/internal/service-mgmt.yaml new file mode 100644 index 000000000..52e8921e7 --- /dev/null +++ b/deploy/internal/service-mgmt.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: SYSNAME-mgmt + labels: + app: noobaa + annotations: + prometheus.io/scrape: "true" + prometheus.io/scheme: http + prometheus.io/port: "8080" +spec: + type: LoadBalancer + selector: + noobaa-mgmt: SYSNAME + ports: + - port: 8080 + name: mgmt + - port: 8443 + name: mgmt-https + - port: 8444 + name: md-https + - port: 8445 + name: bg-https + - port: 8446 + name: hosted-agents-https + - port: 80 + targetPort: 6001 + name: s3 + - port: 443 + targetPort: 6443 + name: s3-https diff --git a/deploy/internal/service-s3.yaml b/deploy/internal/service-s3.yaml new file mode 100644 index 000000000..e407797b7 --- /dev/null +++ b/deploy/internal/service-s3.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: s3 + labels: + app: noobaa +spec: + type: LoadBalancer + selector: + noobaa-s3: SYSNAME + ports: + - port: 80 + targetPort: 6001 + name: s3 + - port: 443 + targetPort: 6443 + name: s3-https diff --git a/deploy/internal/statefulset-core.yaml b/deploy/internal/statefulset-core.yaml new file mode 100644 index 000000000..56bebaea2 --- /dev/null +++ b/deploy/internal/statefulset-core.yaml @@ -0,0 +1,127 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: SYSNAME-core + labels: + app: noobaa +spec: + replicas: 1 + selector: + matchLabels: + noobaa-core: SYSNAME + serviceName: SYSNAME-mgmt + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: noobaa + noobaa-core: SYSNAME + noobaa-mgmt: SYSNAME + noobaa-s3: SYSNAME + spec: + serviceAccountName: noobaa-operator # TODO do we use the same SA? + initContainers: + - name: init-mongo + image: NOOBAA_IMAGE + command: + - "/noobaa_init_files/noobaa_init.sh" + - "init_mongo" + volumeMounts: + - mountPath: /mongo_data + name: mongo-datadir + containers: + - name: mongodb + image: centos/mongodb-36-centos7 + imagePullPolicy: IfNotPresent + command: ['/bin/bash', '-c', '/opt/rh/rh-mongodb36/root/usr/bin/mongod --port 27017 --bind_ip 127.0.0.1 --dbpath /data/mongo/cluster/shard1'] + resources: + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "8Gi" + volumeMounts: + - mountPath: /data + name: mongo-datadir + - name: noobaa-server + image: NOOBAA_IMAGE + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /log + name: logdir + readinessProbe: + # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes + # ready when s3 port is open + tcpSocket: + port: 6001 + timeoutSeconds: 5 + resources: + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "4Gi" + ports: + - containerPort: 80 + - containerPort: 443 + - containerPort: 8080 + - containerPort: 8443 + - containerPort: 8444 + - containerPort: 8445 + - containerPort: 8446 + - containerPort: 60100 + env: + - name: CONTAINER_PLATFORM + value: KUBERNETES + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: noobaa-server + key: jwt + - name: SERVER_SECRET + valueFrom: + secretKeyRef: + name: noobaa-server + key: server_secret + - name: OAUTH_AUTHORIZATION_ENDPOINT + valueFrom: + configMapKeyRef: + name: noobaa-config-map + key: oauth_authorization_endpoint + optional: true + - name: OAUTH_TOKEN_ENDPOINT + valueFrom: + configMapKeyRef: + name: noobaa-config-map + key: oauth_token_endpoint + optional: true + - name: container_dbg + value: "" # replacing the empty value with any value will set the container to dbg mode + volumeClaimTemplates: + # this will provision a dynamic persistent volume (volume is automatically provisioned by a provisioner) + # in minikube it is provisioned as hostPath volume under hosts /tmp which is not persistent between + # minikube restarts. if we want it to be persistent between restarts we need to statically provision a + # volume according to this https://kubernetes.io/docs/setup/minikube/#persistent-volumes + - metadata: + name: logdir + labels: + app: noobaa + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi + - metadata: + name: mongo-datadir + labels: + app: noobaa + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi diff --git a/deploy/namespace.yaml b/deploy/namespace.yaml new file mode 100644 index 000000000..3dda4cdf2 --- /dev/null +++ b/deploy/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: noobaa diff --git a/deploy/olm-catalog/catalog-source-config.yaml b/deploy/olm-catalog/manual/catalog-source-config.yaml similarity index 100% rename from deploy/olm-catalog/catalog-source-config.yaml rename to deploy/olm-catalog/manual/catalog-source-config.yaml diff --git a/deploy/olm-catalog/operator-group.yaml b/deploy/olm-catalog/manual/operator-group.yaml similarity index 54% rename from deploy/olm-catalog/operator-group.yaml rename to deploy/olm-catalog/manual/operator-group.yaml index 6ae52c3d8..0dd2d52f6 100644 --- a/deploy/olm-catalog/operator-group.yaml +++ b/deploy/olm-catalog/manual/operator-group.yaml @@ -1,8 +1,8 @@ apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: - name: noobaa-operatorgroup - namespace: noobaa + name: operatorgroup + namespace: my-noobaa-operator spec: targetNamespaces: - - noobaa + - my-noobaa-operator diff --git a/deploy/olm-catalog/operator-source.yaml b/deploy/olm-catalog/manual/operator-source.yaml similarity index 100% rename from deploy/olm-catalog/operator-source.yaml rename to deploy/olm-catalog/manual/operator-source.yaml diff --git a/deploy/olm-catalog/operator-subscription.yaml b/deploy/olm-catalog/manual/operator-subscription.yaml similarity index 73% rename from deploy/olm-catalog/operator-subscription.yaml rename to deploy/olm-catalog/manual/operator-subscription.yaml index 94edb2aa0..e123c1902 100644 --- a/deploy/olm-catalog/operator-subscription.yaml +++ b/deploy/olm-catalog/manual/operator-subscription.yaml @@ -1,8 +1,8 @@ apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: - name: noobaa-operator-subscription - namespace: noobaa + name: my-noobaa-operator + namespace: my-noobaa-operator spec: channel: alpha name: noobaa-operator diff --git a/deploy/olm-catalog/noobaa-operator/noobaa-operator.package.yaml b/deploy/olm-catalog/package/noobaa-operator.package.yaml similarity index 66% rename from deploy/olm-catalog/noobaa-operator/noobaa-operator.package.yaml rename to deploy/olm-catalog/package/noobaa-operator.package.yaml index 8606def45..7905bfb74 100644 --- a/deploy/olm-catalog/noobaa-operator/noobaa-operator.package.yaml +++ b/deploy/olm-catalog/package/noobaa-operator.package.yaml @@ -1,5 +1,5 @@ packageName: noobaa-operator channels: - name: alpha - currentCSV: noobaa-operator.v0.1.0 + currentCSV: noobaa-operator.v1.0.0 defaultChannel: alpha diff --git a/deploy/olm-catalog/noobaa-operator/0.1.0/noobaa-operator.v0.1.0.clusterserviceversion.yaml b/deploy/olm-catalog/package/noobaa-operator.v0.1.0.clusterserviceversion.yaml similarity index 87% rename from deploy/olm-catalog/noobaa-operator/0.1.0/noobaa-operator.v0.1.0.clusterserviceversion.yaml rename to deploy/olm-catalog/package/noobaa-operator.v0.1.0.clusterserviceversion.yaml index ed556eb51..69269a975 100644 --- a/deploy/olm-catalog/noobaa-operator/0.1.0/noobaa-operator.v0.1.0.clusterserviceversion.yaml +++ b/deploy/olm-catalog/package/noobaa-operator.v0.1.0.clusterserviceversion.yaml @@ -5,7 +5,7 @@ metadata: categories: Storage,Big Data capabilities: Basic Install repository: https://github.com/noobaa/noobaa-operator - containerImage: noobaa/noobaa-operator@sha256:dec4bb34afae86aa32f2cde9a0859eb75130c83ac342be84cccc4689843b6c30 + containerImage: noobaa/noobaa-operator:0.1.0@sha256:dec4bb34afae86aa32f2cde9a0859eb75130c83ac342be84cccc4689843b6c30 createdAt: 2019-07-08T13:10:20.940Z certified: "false" description: NooBaa is an object data service for hybrid and multi cloud environments. @@ -17,7 +17,7 @@ metadata: "kind": "NooBaa", "metadata": { "name": "noobaa", - "namespace": "noobaa" + "namespace": "my-noobaa-operator" }, "spec":{} } @@ -26,15 +26,6 @@ metadata: namespace: placeholder spec: displayName: NooBaa Operator - description: | - NooBaa Operator will manage a NooBaa system in your kubernetes cluster. - ## Getting Started - - Install the operator in a namespace (typically `noobaa`) - - Create a NooBaa system - - The operator will deploy two services - Mgmt and S3, both require credentials. - - Use `kubectl describe noobaa` and `kubectl get noobaa` to get the endpoints and credentials information. - - Feel free to email us or open github issues on any question. - - Enjoy and tell us how it goes. version: 0.1.0 maturity: alpha provider: @@ -44,6 +35,8 @@ spec: url: https://github.com/noobaa/noobaa-core - name: Website url: https://www.noobaa.io + - name: Articles + url: https://noobaa.desk.com maintainers: - email: gmargali@redhat.com name: Guy Margalit @@ -83,13 +76,14 @@ spec: installModes: - supported: true type: OwnNamespace - - supported: true + - supported: false type: SingleNamespace - supported: false type: MultiNamespace - - supported: true + - supported: false type: AllNamespaces install: + strategy: deployment spec: clusterPermissions: - serviceAccountName: noobaa-operator @@ -181,7 +175,78 @@ spec: - deployments/finalizers verbs: - update - strategy: deployment + description: | + The noobaa operator will create and reconcile a NooBaa system in your kubernetes cluster. + + NooBaa provides and S3 object store service abstraction and data placement policies to create hybrid and multi cloud data solutions. + + For more information on using NooBaa refer to [Github](https://github.com/noobaa/noobaa-core) / [Website](https://www.noobaa.io) / [Articles](https://noobaa.desk.com). + + ## How does it work? + + - The operator deploys the noobaa core pod and two services - Mgmt (UI/API) and S3 (object-store). + - Both services require credentials which you will get from a secret that the operator creates - see describe to locate it. + - The service addresses will also appear in the describe output - pick the one that is suitable for your client: + - minikube - use the NodePort address. + - remote cluster - probably need one of the External addresses. + - connect an application on the same cluster - any address should work, but prefer Internal addresses + (e.g https://s3.my-noobaa-operator and https://noobaa-mgmt.my-noobaa-operator) + - Feel free to email us or open github issues on any question. + + ## Getting Started + + ### Notes: + - The following instructions are for **minikube** but it works on any Kubernetes/Openshift clusters. + - This will setup noobaa in the **my-noobaa-operator** namespace. + - You will need **jq**, **curl**, **kubectl** or **oc**, **aws-cli**. + + ### 1. Install OLM (if you don't have it already): + ``` + curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.10.0/install.sh | bash -s 0.10.0 + ``` + + ### 2. Install noobaa-operator and wait for it: + ``` + kubectl create -f https://operatorhub.io/install/noobaa-operator.yaml + kubectl wait pod -n my-noobaa-operator -l noobaa-operator --for=condition=ready + ``` + + ### 3. Create noobaa system and wait for it: + ``` + curl -sL https://operatorhub.io/api/operator?packageName=noobaa-operator | + jq '.operator.customResourceDefinitions[0].yamlExample | .metadata.namespace="my-noobaa-operator"' | + kubectl create -f - + kubectl wait pod -n my-noobaa-operator -l noobaa-core --for=condition=ready + kubectl get noobaa -n my-noobaa-operator -w + # NAME PHASE MGMT-ENDPOINTS S3-ENDPOINTS IMAGE AGE + # noobaa **Ready** [https://192.168.64.12:31121] [https://192.168.64.12:32557] noobaa/noobaa-core:4.0 19m + ``` + + ### 4. Get system information to your shell: + ``` + NOOBAA_SECRET=$(kubectl get noobaa noobaa -n my-noobaa-operator -o json | jq -r '.status.accounts.admin.secretRef.name' ) + NOOBAA_MGMT=$(kubectl get noobaa noobaa -n my-noobaa-operator -o json | jq -r '.status.services.serviceMgmt.nodePorts[0]' ) + NOOBAA_S3=$(kubectl get noobaa noobaa -n my-noobaa-operator -o json | jq -r '.status.services.serviceS3.nodePorts[0]' ) + NOOBAA_ACCESS_KEY=$(kubectl get secret $NOOBAA_SECRET -n my-noobaa-operator -o json | jq -r '.data.AWS_ACCESS_KEY_ID|@base64d') + NOOBAA_SECRET_KEY=$(kubectl get secret $NOOBAA_SECRET -n my-noobaa-operator -o json | jq -r '.data.AWS_SECRET_ACCESS_KEY|@base64d') + ``` + + ### 5. Connect to Mgmt UI: + ``` + # show email/password from the secret: + kubectl get secret $NOOBAA_SECRET -n my-noobaa-operator -o json | jq '.data|map_values(@base64d)' + + # open mgmt UI login: + open $NOOBAA_MGMT + ``` + + ### 6. Connect to S3 with aws-cli: + ``` + alias s3='AWS_ACCESS_KEY_ID=$NOOBAA_ACCESS_KEY AWS_SECRET_ACCESS_KEY=$NOOBAA_SECRET_KEY aws --endpoint $NOOBAA_S3 --no-verify-ssl s3' + s3 ls + s3 sync /var/log/ s3://first.bucket + s3 ls s3://first.bucket + ``` icon: - mediatype: image/png base64data: | diff --git a/deploy/olm-catalog/package/noobaa-operator.v1.0.0.clusterserviceversion.yaml b/deploy/olm-catalog/package/noobaa-operator.v1.0.0.clusterserviceversion.yaml new file mode 100644 index 000000000..cb0101254 --- /dev/null +++ b/deploy/olm-catalog/package/noobaa-operator.v1.0.0.clusterserviceversion.yaml @@ -0,0 +1,301 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + categories: Storage,Big Data + capabilities: Basic Install + repository: https://github.com/noobaa/noobaa-operator + containerImage: noobaa/noobaa-operator:1.0.0@sha256:c926dc93e235516160ea1209c662900747f3d7669c7a58d7d089643936e1cf4c + createdAt: 2019-07-08T13:10:20.940Z + certified: "false" + description: NooBaa is an object data service for hybrid and multi cloud environments. + support: Red Hat + alm-examples: |- + [ + { + "apiVersion": "noobaa.io/v1alpha1", + "kind": "NooBaa", + "metadata": { + "name": "noobaa", + "namespace": "my-noobaa-operator" + }, + "spec":{} + }, + { + "apiVersion": "noobaa.io/v1alpha1", + "kind": "BackingStore", + "metadata": { + "name": "default-store", + "namespace": "my-noobaa-operator" + }, + "spec": { + "type": "aws-s3", + "region": "us-east-1", + "bucketName": "noobaa1-aws-backing-store", + "credentialsSecretName": "aws-credentials-secret" + } + }, + { + "apiVersion": "noobaa.io/v1alpha1", + "kind": "BucketClass", + "metadata": { + "name": "default-class", + "namespace": "my-noobaa-operator" + }, + "spec": { + "placementPolicy": { + "tiers": [{ + "tier": { + "mirrors": [{ + "mirror": { + "spread": [ + "default-store" + ] + } + }] + } + }] + } + } + } + ] + name: noobaa-operator.v1.0.0 + namespace: placeholder +spec: + displayName: NooBaa Operator + version: "1.0.0" + replaces: noobaa-operator.v0.1.0 + maturity: alpha + provider: + name: NooBaa + links: + - name: Github + url: https://github.com/noobaa/noobaa-core + - name: Website + url: https://www.noobaa.io + - name: Articles + url: https://noobaa.desk.com + maintainers: + - email: gmargali@redhat.com + name: Guy Margalit + - email: etamir@redhat.com + name: Eran Tamir + - email: nbecker@redhat.com + name: Nimrod Becker + keywords: + - noobaa + - kubernetes + - openshift + - cloud + - hybrid + - multi + - data + - storage + - s3 + - tiering + - mirroring + labels: + app: noobaa + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - kind: NooBaa + name: noobaas.noobaa.io + version: v1alpha1 + displayName: NooBaa + description: NooBaa provides a flexible S3 data service backed by any storage resource for hybrid and multi-cloud environments + resources: + - kind: Service + version: v1 + - kind: StatefulSet + version: v1 + - kind: Secret + version: v1 + - kind: BackingStore + name: backingstores.noobaa.io + version: v1alpha1 + displayName: Backing Store + description: Add backing stores to customize the data placement locations. + - kind: BucketClass + name: bucketclasses.noobaa.io + version: v1alpha1 + displayName: Bucket Class + description: Add bucket classes to customize the data placement locations. + installModes: + - supported: true + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: false + type: AllNamespaces + install: + strategy: deployment + spec: + clusterPermissions: + - serviceAccountName: noobaa-operator + rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + deployments: + - name: noobaa-operator + spec: + replicas: 1 + selector: + matchLabels: + noobaa-operator: deployment + strategy: {} + template: + metadata: + labels: + app: noobaa + noobaa-operator: deployment + spec: + containers: + - env: + - name: OPERATOR_NAME + value: noobaa-operator + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: noobaa/noobaa-operator:1.0.0 + imagePullPolicy: IfNotPresent + name: noobaa-operator + resources: + limits: + cpu: 250m + memory: 256Mi + serviceAccountName: noobaa-operator + permissions: + - serviceAccountName: noobaa-operator + rules: + - apiGroups: + - noobaa.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + - services + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - apps + resourceNames: + - noobaa-operator + resources: + - deployments/finalizers + verbs: + - update + description: | + The noobaa operator will create and reconcile a NooBaa system in your kubernetes cluster. + + NooBaa provides and S3 object store service abstraction and data placement policies to create hybrid and multi cloud data solutions. + + For more information on using NooBaa refer to [Github](https://github.com/noobaa/noobaa-core) / [Website](https://www.noobaa.io) / [Articles](https://noobaa.desk.com). + + ## How does it work? + + - The operator deploys the noobaa core pod and two services - Mgmt (UI/API) and S3 (object-store). + - Both services require credentials which you will get from a secret that the operator creates - see describe to locate it. + - The service addresses will also appear in the describe output - pick the one that is suitable for your client: + - minikube - use the NodePort address. + - remote cluster - probably need one of the External addresses. + - connect an application on the same cluster - any address should work, but prefer Internal addresses + (e.g https://s3.my-noobaa-operator and https://noobaa-mgmt.my-noobaa-operator) + - Feel free to email us or open github issues on any question. + + ## Getting Started + + ### Notes: + - The following instructions are for **minikube** but it works on any Kubernetes/Openshift clusters. + - This will setup noobaa in the **my-noobaa-operator** namespace. + - You will need **jq**, **curl**, **kubectl** or **oc**, **aws-cli**. + + ### 1. Install OLM (if you don't have it already): + ``` + curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.10.0/install.sh | bash -s 0.10.0 + ``` + + ### 2. Install noobaa-operator and wait for it: + ``` + kubectl create -f https://operatorhub.io/install/noobaa-operator.yaml + kubectl wait pod -n my-noobaa-operator -l noobaa-operator --for=condition=ready + ``` + + ### 3. Create noobaa system and wait for it: + ``` + curl -sL https://operatorhub.io/api/operator?packageName=noobaa-operator | + jq '.operator.customResourceDefinitions[0].yamlExample | .metadata.namespace="my-noobaa-operator"' | + kubectl create -f - + kubectl wait pod -n my-noobaa-operator -l noobaa-core --for=condition=ready + kubectl get noobaa -n my-noobaa-operator -w + # NAME PHASE MGMT-ENDPOINTS S3-ENDPOINTS IMAGE AGE + # noobaa **Ready** [https://192.168.64.12:31121] [https://192.168.64.12:32557] noobaa/noobaa-core:4.0 19m + ``` + + ### 4. Get system information to your shell: + ``` + NOOBAA_SECRET=$(kubectl get noobaa noobaa -n my-noobaa-operator -o json | jq -r '.status.accounts.admin.secretRef.name' ) + NOOBAA_MGMT=$(kubectl get noobaa noobaa -n my-noobaa-operator -o json | jq -r '.status.services.serviceMgmt.nodePorts[0]' ) + NOOBAA_S3=$(kubectl get noobaa noobaa -n my-noobaa-operator -o json | jq -r '.status.services.serviceS3.nodePorts[0]' ) + NOOBAA_ACCESS_KEY=$(kubectl get secret $NOOBAA_SECRET -n my-noobaa-operator -o json | jq -r '.data.AWS_ACCESS_KEY_ID|@base64d') + NOOBAA_SECRET_KEY=$(kubectl get secret $NOOBAA_SECRET -n my-noobaa-operator -o json | jq -r '.data.AWS_SECRET_ACCESS_KEY|@base64d') + ``` + + ### 5. Connect to Mgmt UI: + ``` + # show email/password from the secret: + kubectl get secret $NOOBAA_SECRET -n my-noobaa-operator -o json | jq '.data|map_values(@base64d)' + + # open mgmt UI login: + open $NOOBAA_MGMT + ``` + + ### 6. Connect to S3 with aws-cli: + ``` + alias s3='AWS_ACCESS_KEY_ID=$NOOBAA_ACCESS_KEY AWS_SECRET_ACCESS_KEY=$NOOBAA_SECRET_KEY aws --endpoint $NOOBAA_S3 --no-verify-ssl s3' + s3 ls + s3 sync /var/log/ s3://first.bucket + s3 ls s3://first.bucket + ``` + icon: + - mediatype: image/png + base64data: | + iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAIAAAD2HxkiAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA25pVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNS1jMDIxIDc5LjE1NTc3MiwgMjAxNC8wMS8xMy0xOTo0NDowMCAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpBNjQyRDdGQUIxMDkxMUU0QURFMENEMjA1QUJCMENEMyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoxOTA3OEQwNDAyRjAxMUU1QjdFQkI4RTFBMzY3NkQxRiIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDoxOTA3OEQwMzAyRjAxMUU1QjdFQkI4RTFBMzY3NkQxRiIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ0MgMjAxNCAoV2luZG93cykiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo5NWU4ZDg3YS1mNGU4LTRlMTYtOGIwYi1hZGIzYzY2OThkOGUiIHN0UmVmOmRvY3VtZW50SUQ9InhtcC5kaWQ6QTY0MkQ3RkFCMTA5MTFFNEFERTBDRDIwNUFCQjBDRDMiLz4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz6weHBPAABCm0lEQVR42uy9CXAkWXoelndmXai7UDduoLunu+fagyOSO94lJQ734movLklZYQdFaYNhKyg7QjJtS3IEw7IVorgMUxsh7fIIhuUN7YqkxA1tmKJomhKP5VIzs8dc3dM46wJQ95VVeVRm6s9MAN2Now6gCpVZeP9UYFDoQiHr5fve/33vPx7+Ez/xUxgyZMimZwQaAmTIEAiRIUMgRHbGGvV6rVJB4zAW6/B8pVRC44BAOIJJknRQKLTbLTQUYzFZlg/2C81GHQ3FuUahIThl+/lcPpdTFcXpcqHRGIuRJMm32w/eeisUiSRSaZZl0ZggEJ5v1Uo5l8kI3e4RSSAQTRizlYvFarkcTyZjiSSO42hAEAgfG8+3AX6NWg0NxaRNVVUY6nKxlEingqEwGhAEQqzX6+WzmcP9fTQVrtMEobv17rvgGJPpBZfbjUB4cw2wl89le7KMUDEVa9Tr8JiPxhKpFEXTCIQ3y+q1GjhAvt1GSJj+UniwXy6XAIfRWByB8EZYt9PJZTMoBmgpU3q9zM5O+bCYSKf9gQAC4ezeaUUp5LL7+Tya9Na0Tod/9OAdAGEilb5R8aGbAsLS4SE4QFmS0Fy3uNWqVXhE4/FEMkVSFALhLFiz0chl9totlP5iJzsoFColEIrpSDSKQGhjEwUBvB/KWrSpybK8u71VKh4m0wtenw+B0GamaVohlwMFCN+g2Wxr49vth2+/FQyFk+k0y3EIhPawSrmUz2QEQUAzeJbuacUIY8QSydlLJ5wpEILwy2UyKFt/Vi2fzZaLehgjFI4gEFrOJEnKZzOlw0M0U2fbRFHcfvSodFgEduqZm0MgtIrt50H+5RRFQXP0hlir2XjnzTdCkUgylWbsXxhlbxDWKpVcZq97XHyE7EbZcWFUKp5MIhBOwTodPreXqdeqaC7eZDMKo/bKJZ2dBoIhBMJrsl6vV8hlDwoFNAWRmSZ0u5sPH3r9xVR6wY75bjYDISo+QnaRNWo1eMzHYomkzQqjKBsNcQ4VHyEbYpmulEpxWxVG2QCEQDaA91dR8RGyoQWLXhill+2nff4AAuFVZXc+mwUCiiYWslGtw/PvvvOOPxAEKDqcTgTCy1jp8DCfzUio+AjZFaxWrcAjGk8kUimSJBEIhzVUfIRsvHZQyOupp8mUNQujrAVCSRRz2QyweTRvkI3XZEna3d4yIooLc14vAuE5pmnafj5fyGVBB6IZg2xCBvTqwVtvBsNhgKJ1GoFbAoSVchnkn4Cyz5Bdz3wrlY7y3RIJ3AKFUVMGIaxMAL9GHRUfIbtu5gUTD9hpIpWaemHU1EAoy1I+ky0eHqAJgWxaJgrC9qNHZkTR7Zm7WSDcL+QL2SwqPkJmBWs2Gm+/8UZ4fj6RSjMMM/sgrFUruUym2+mge4/MUlY6PDw5MWpmQdjpdPKZvVoVFR8hs6gBNcvu7RkdNBYCweBMgVDp9fK53EEBtb5GZgPr6oVRD3x+f/K6CqMmDsLi4UE+k5Vl+2WfEQQ6xXJcI2m//mj1Wg0e0Vg8nkpRE24EPsF3b9Tr+WzGftlnOK6pqiRJaN9ojAbjSRIERdP26gR7sF/Qwxjp9Hw0Nrm/Qt67d3/sbyp0u7s727m9PdulX+M4LgpdWZJDkQjfbvd6Pc/cHDrY+SpWrVT2trfcHo8oit1Oh2YYe42nqqoN8IrVGsOynMNhAxAaxUeZzXcf2m7/E2YGQK7D8zBdnnvPe5ZW12ARKR0e1mpVmqadTheC06gGq9jO5mYhl+V5/u5zz6/futVqNWvVqqqptjsSFPRUpVzqdDpOp5Me98WPE4SlYnHzwTt1u538DvADjgQzhiKplY2N5158TyyZ5FvtzO6OqijwT7VKpdVswSo4A931rmvKytm93d2tLdHog96T5XBkPr28HInGXC43DGar0SBI0rK1RX0oXvHgADwNrNRjFLrjAWGr2dx+9O7h/r7NdJTBi8Bpgw9MLizcf/HFxeUVVdNg6kiiWMjnFKVHEPpEEUWhVDwEmupyu203da5fR209fAhT4jE/UpT5WMzhdMKoRqLRaCJBUxRwPHAsFEXZbtum3WpVSkWSpGAyWGJjBoY1n83CBLXdXNHlnwhgk0D+rW5sxJMpWOHMbST6grSJ4uFBpVJOJFPReByB7awBCcpl9oDSXzTgMMIATiAUzzz7XCyR3Hr3YS6TgX8yKt9hQbTNno0kSTtbmyUj3+3qhVFX8oSFXA7kn+2aL8FsAI8Nlw33/tbde/eee94XCOj+UJJM3wi+TpYko6v3kSc8MQ1ker1eq1QBqI7JyHQ7Wrfb3dveAkTJ5zXCU3RPGPf6/cBLdSgqCkxil8eTSKW9Pn+326nrKRya7YSiJInlUlEURJ0fXSGMccnfrFbKMOK2Kz4yF+MOyD+GAe+3vLYOyxjATzyJoxgb6Jph5m+c+z6PD3ZOLzit3b9k0jbkIeTa0aA+9nUw7HA7YokEMJHMzvb2o0fgJGFds10YA3B4cmLU5TZ+R/aE5pbXfj4HOsp2MwZWDXBxsWTy/vMvLK2tw5ABdzp7y0mClGXJ0IRKH8ViynR4jdszN3vndQ1jpcPDRw8fDKxEMzxhDJze2YaxIGdg2sK/gqskSQJcIjhVmqZtN57NRgOgeLmN9BE8ITCNfDYD086O8g/4jygIgVBodX0jnkrpq4nh/a4eszooFMqlUjKVikwynmvBOTeWPkCmNAAfyHLcvedfiCVTIBRBCBA4ztmN7cME23r3XfPEKLfHM35PeFDIg/xrP7HlZRv+acg/lmXX7zxz7/nnA8EgrLX6Anwx/HRNOIQnPDGguGaWEwtTaUZPk33s/wVhb2c7u7s7fCZGH0/4GIo9kOSSxzMXT6Xn5uaA8DdqNfj5pFPGxg9FYyNdF71Db6QPBmGtWt18+KBSKmm26v6iY0zTOp0OsM2F5ZX7L7wArB0mgX6E0yDJAcAD1rqfyw8JwiOmIEkwSvD+QEhst8cwjGlGG1jwVBftf/YDYTQ25/PJg84vMLMFgbCAVoR1E/ytEb8FfWCzsBAMEXB1mITDHKLYD4Q8397b3gYKar+zH3Bc7HZFSYJ7D95vZX0d1iRT/g3DPw1PKO/nRwPhyX5D8fBA01QQirOU71YuFYEK1S9ViXbsCX0DJ5K5dAJPgWEHlRiJzutuoFYDpkfZTSjCZGs26tVKhWHY/hvp/UC4s/nIjukvZvbZnNd7597923fvAiuApzAPhofEVUBoGogc8IpApex4SNApA+G3vfkIpK9y2a244UH4WESoqiiKIAvjyZQ/EARFD1MRlIXt2Cl85Ea9Hu9bKNzvI9mLA5gSHxwR6LJbz9xdXFlxOJ3wdCT4jVUbiDB3QR4k0ws2Pdh5un2AjGR6PXUpGA77Q6F8Zm/70bv1ag3uL8MwNgpjDFw4+v0zjtmJTYG7g9uWWlxcXluDtRPuH7gjE36j3rDjiNYYbjNcwztvvhGORBJ2O9jZbAM7rjxEfUBV7RLIMdN6gZukF5ci81HQR7vbW+CcYYWFH9oCigN9gO3PrDfXS9lIEV5eXwfaA/OmPabww7isVCxWjvuXWF8oWu0QcpOdwj2lGebW3bvRRHz70aNCNiuqqmMmMiUoW8MPCLcgdD1e3+3V1VR6kaTIrrEdasGrNQ52zpj9S4Ihix7szPPtfCZjzY0A83a3Zdnl9jz/nveBVgR2ClSZpmggqPZKspkFEB5ln/FthmHXbt1eXFl1ulywcgMgLe5nBD2e+7BsCMVx5eCPa/8gn8se7u9P8o+MBydCV7/L4fl5WMtgXdvZfNRo1DnOQdst383eIOx2dXcXT6Z1+RcMSeLl5d+FAgZmzMTg3KjX4RGJxhKpJE0zUx/PiR9Cjp+TO3rFG2QKxYXllUg0Cioxs7PDt1sOh5OwiVC0Kwj17DNRlCQpEAotra7FEgnjZlhL/g1vxYP9annKBzvrh5BnMsBCbcqG2q0mwzC3796DMdzefLSfy+E2zHej7DLivV5P6HZAD6zdvp1cWIShB/kHt8HWAfHHBzun0r7AtR7sbBxCnqlWyrbe0oC7Lxs25/WCUIwlksBOK6USDUKFZe3iEinrj7Ih/3iKosD7gfxze+ZA+7VbrTHyz3PoqAr/uyZ46wc7P7i+g51hPAu5bCF3nYeQ63xUHR8dPWcMOx2CIKLxeDAczhmdNdrNJud0wrSxPhStDkJYsM18i6W1tVA4IksSMBDcMGy2zDzYORZPxCd5sHOpeJjPzOAh5I8bBVHU8tr6fCy+u7WZ3dsTBQHWNcCnlaFIWXZMJb36SASSBt4vnkziGG6W8M92A8L9Qt44rys99oOdb8Ih5GbWFHxG1uigEY0ndrY2DwoFkiBYC1e3UNYcR6MWwbm8upZaWuJYrtudWvbZ9RsonN3tLTPfzevzXf0NwRvkshlQStjNsJMNPFjBnw+87yCfByjWKhWQidbMd6MsNXYwQCCQCJJYWFxaWlnxeL0wgY74J4Zh1zZ8+h9Sp9t3CNz+w7ffCobCiXSK4xyX/RwaaL9CPjf9MjQ96KNh1wkATevCXCJIoFGhcBio6d7OtjXz3SjrIFAQBKXXC8/PL62swlf43tx9uX4HeJzFLxjf4lO8YZVyqVopxxJJmEmj1nPA74L8E4zOn9M18Eu9Xm8q91HTVF0o0vTqxsZ8LAZCMZ/NmkJxunf2SetXylStVIAHXsNIwR3qdjouj2fjzp2N28+4PG6go8o0btvjcSEpmL7AYQCNUy+f0QujyuXhC6OM4iPQQvnp9gEyE83gzvqDwfTiIiBBnUZbWtw8XMQojAKV6PP7gao2arVru7PwwQH/FgXhUfjB6AAL3u+Ze/fD4YgIbN5o/jNFg6sCxhKNxz1z3g7fNtNxpgtFWJJq1SpcCefg+lRj6K2vd3dBUhpufJrwMwNLDMusrK/fuXef5TgQaVNcVc21HhYF0DixeMLhcvHgIptN4BeT7uZsYRAa3V/AYvH4M/efTabTsFyBA7QIQ4ALg2kUCAbnY3GaYdqNBm/EKqdb3A0rVKlYlMTz+5fofYAePpj2/udRU3O4m8n0AsAvnkyZ/WMssq8mSxJw1FAoHIlGaYoGvjNpqmxdEBrMBL/9zN27zz0H3A+WJUulv5hXoh/oRZJwt+CWweyq12rgpWmanu51gocpHhw82b/kcR+gqS5hxrakACtFOBK5fe/+0uoqLFt6YtNwXUWuUShqcJ0Oh2NhZQXTsJLRQHByV2hhEGqavmXMsbAUMQwN3ka1XiMpk1bB2sk6HLFE3OvzS7IuJ7RpC0Wjf0mjWi6DZwYHmMtMuQ+Q2S4NlIXb41m/fWfjzjPgq4VuVxelFgsswdDBwgrqGi7v4dtvAXuHqThRgjMQhNQUbxsYcCe9WWC1Eo7Mz/l8MLllSzaVMvos4IFQyB8IFHK53e3tZrPOstx0y2eAve9sbVpgnVI6fJfl2NWNjfTikt5VpNtVrQc/82rh8oCFZXZ39uAmNhogVqdeA0VNd1ECfwLLUrvVBsXl8/lDkYjL5eoZhltvBQUeCFebXFiA68zu7er1B+0253CQNiyfGcuM1mNxnQ58k0ilFpaWfYEAcNGjtF7r3T5YNCmaKh0W97b1s1xg7rndbm0y6cf2AeHx4kQzepfOarXaarWCoVAwHOaMzTSrzWyTnQLwgDyv3b4zH43t7mzv53OgK7gbdjiMmZUCtAXYwaJR1GeODGa9vEKYReDrwOO1Go29t7fzuRy4bjPYY5HpZaGMGQCeoigHhUKjXgdlD8sqYWSQWnD+gQDr6X0W3Peeez6qh4C39Dge3Gn7lM9c6ePrZWVd+PirG7eS6TRJUfDUgmVlpvyD9RHWi613H2Z2d7sdnnM44WcoY+aCrRpVJYxEWwBedm8PtCJAEYQ+/Nyah88IggDTLhSZ9weChVx2b2cHmBjccluUz1yaCHQ6PDiWpZXV1NKS2+UWhK45DpgFeN3pZd2gJ/ks3Jrteq3KMqzL7dEmWVFlexAa7EC/l/rGI0XBhNaFot8fCocdDofc6ynWFYpUemkZ0AhCMZ/VC9X1PgvWLp+5zKID7k5To7E4fNhAMChLcrvdsmBZmWZsvMNKUS2Xd7e3S4cHcC9cLrcFl4npg/C8EVEx7GizmDYS3muVCqARbjk8wEnKFhWK+pkzDMNs3HkGhCKsu4f7+/BzdiYOhzHLymDk/YEAwA/ot7n0WFP+wQrOOhx8q7X54EE+l4WFG/whThAYqie86PaeOhwCBurJ2wr/BEuaLhT395uNBrhEs7THgmGMkz4LHq/3/vMvHsb2AYqwgsBSYq920ac+lKKA/NPTnZdX10D+0VbtKqIZsT64Tr0QbHMzs7tjblwfqXRrjz9lheF7+incYOKUtgYTBSGbyTTq9WA47PJ4lB5g04pCsdvtwmyIxGIBvXN7Bggq325xnIO0l1A0kp4F47OkF5dA/nnm5uCp2G6b4QerfRLwfnBhB/uFzPZ2rVoFLur2WFH+WVUTnnJ/BvCedo/6U/OwsWazCYrLFzhipz1ZVmBVtpj3MNka8KLF1bVwNJrZ2d7P5czjTewiFCVBgEUuHJlPLy8HwmFFlqdVVjZwBTe5Rr1aBepRNFTAUfjBRodVWGE3BjudW6hddFg8Y2S3VUrFdrsZCAR9/gDLMMBALCgUYRJ3eL3Pwu179+dj8czOVvGwSBA4y1pXKJqkGhA45/Onl5ai8QRO4F2ryj8gF7CuweW9u7VZ0BvnwDJny/0wSwTrz47auc7Q/B5GmWFYRQahWGg2G8FgyDPnhRdYNN/NMK/ff8/3Hrjg7M42MGpYSmiLCUVz1QDCyXHcyq1byfQCEA0z+mfNxcLMPoPxBPnXbuqLndPltgv/tB4dPQ91Z53hqRfoZWAUK3S6uU4GQBgIhpxOpy4TrRfGwIzSHrjgWCIJgjaf2cvt7Vkt3w2uENezz9LgAD1erySKvCn/LGZ69hnHwbiVDw8BfpVyiaJol9uu8LMWCM8b7qd2aM4lspSRemseqgzU1B8IwIpoTXZqCEW9z8Ly2np4Pprd3dnP50W9oMY53Ssze08EQyEj1BnRWXSb14/Fs2T2GcNyzXoN4HdYKMBPnE6byT+rg/C8s6yf8ofnclQ9jHEiFFtNcIlzXi+4SYOdWk8o9nqdXg+o1J37z0aisczOdqVUAm2jF8tf70wyss9kURA9c3OphcVoMkmQpGAWVVvM/51kn8HlbT58AFRCFIRZypufchXF6LA8H6S6UGRZWNFBd7WaDX8g6PJ4NKvmu+mFUTgO64UvEDjI57N7u61GA3w4dS01NWb2GfBPhmWWVteSCwv6/BYEtdu1ZvERXJ6qaflMBhxgCxQ1x9mdf1rLE/bZgBneGZ48haXRLOXudjsgFIGgOhwOwKEFe5bq7LSjF0Yl0mlDKGby2QzwVY5zTPpcIcE4/TMaT6QWF71+vyRK/HH0z2pDBAsr3NBquazLv2JRL8b1uM0TnrAZsmnTUc34bzRYPv0GqoYTT73+SCjW6x2e9/p8gEXdSVpSKB4VRtH0ysZGaH4ehGLROCFwEvluT2SfBVNLi6BLNaMXE2bN7DO9JIUDfbG5u3tQyKuKAuupUcGIzZ5NF4T4GVd3OW95zuvNfhnlUqndasG009uxGCVIFqRbZhEzUCxdKBp7NrVKhdI3IcZTGGX+CeDATrdLP1IumQTYW7b46Cj7TJJ2Nx/lMntCp8OC/JvpGjGrpK31hdmAcMUpZ3hiRkSRgdt5uL/Pt1pzfr/T6YSZp0yj++UwQtEsjPIFggeFnBnGAJd4lcKoI/nX7dAknV5aSiwsuFxu+EO8KFqz+Ag+r559Vshnd3cbtRrcPudsyT+LgvBcZ3jG+w0IV/TxlqTRkYnn+W636/Z4QAXpSeG9ngXD0EdCkaJSi0vByHx+b3c/lwPGeOl8N0EQNFUB5glv6A8EYD0y5Z8Vi48YhqKZeq0KRKB8eIgThO2yz2wMQjOGpo3YFe/06wdxWrM5mi4UO505nw/YKVAyIGlWFIrKUWHU2u074flYdm+ndHAAk5K9uOfv2TeR9UOtJFhxUguLEesXH7Fch2/vbD7az+dBMnBGNjZ2Y8wyccJJOkNzA4jACVMoVotFYKden09PtDeaW1tQKB4XRs09c//Z4nw0B/SsXqP1cHW/fDcz+8wMo60uLcdTKXi9peWfsX29t7ud39uDZQLoKDjAm9Y1y0rB+guk3bA7NINgrGoq4JAwlKIsiuBeOu02QBF0v2ooRQveHsHId5uPRgOBAHgJM4zBcufFqY97nxFG2COZXoAlRhTFjsk/McxqNXUcy8KqWjo4BFdfq1VhdXHdAPlnQRBq/f2hNnR1xQWvv/AFR0Kx3QYv4Z6bm/N6LctOTwqjFpaWQ+FILrt3kC/ox5twHHZSMI5jkigoihoMhVOLi4FgED4LwA+zZvGRUX3UqNdzmd2i0f16NrLPbAlCI+h6mfBDP2eI9YOxpp9D//ipGVGE2dDleY/XC2gkCUKxJBT1lM4Oz3Ds+u1nwkYzm0qpZPbFAhEFHg9UbnJhMRqLgbPXe4GavtFi+COM7LNup7v16GEhl5NsVWM5myA08HBm/Ed0hleEsR5RpGmY4pVymed5cIlOp55Rbc18N+MAWr0was7nK+7vZzN7zXod5vHSyirIP/CNevaZVYuPTNqfy+zpTZNbTUYvPnLdTP5pQU3YLzHtPBQNrq4Y3hke7RCQJDwkQSgJAsyMOZ+XZTiQiVac0NpRYVQ0kQA0gkr0B4KhiH6kHCwiFi0+YlmCpKrlUjazWytXgFo7XG4Mwc86IDRA8hSJPJuJdpWE0rNm7tCc/bnZBgaklCgI7jmP2+0BvqpYWyiurK339BJ+i4YfjO1ctt1sgPcrHuzDTxwG0cAQ/CzmCbGrBglHlY5n/OHJC8xmNpqqAs0Dh+PxzDndLhIne5YMYxjnO6qYBfMpDXIBehWWs92tRyD/4BuzGBd5P0uC0NhbP+3cruYMR81HPX1FBEHhJEzwarXS6fAARdBd8CuWzHez4pzWE9A1DbCXz2TADdJI/lnfE56Piv7SbvTqiiGd4WOhaBgs4SC3nE6X2+NhGAZwiGZSn5uoZ59RdK1ayWX29OMTScLhutHhBzuB8DIwO9OhdBIwNoUiz7cFoQsq0eV2gxJDUDx77yijPwAo6p2tzcNCAYZIj2TepOyzWQDhpcIPo+3QnJtDM/D1ulCkKFVVG426IAhut9vcXbBmNcZU1CnQ9Z4sZ3d3gIKClgY6ytyAA6psD0IzRGjOcu0C5zZEQuloOzTn4b4fKX3yAkx2KstytVpzdDouj4dlWRWu+GZDUU8rx/HiwUEhm9WzWxkGyT9beUITfn2j9WPIoRkrjAkCnhHgD0VJcjgcwE7NWP9Nm3Nm62v47I1aLZ/NlEslwugFiuSf/ejomTyzkTPRxphQOjyMj4Uir+/ZuFxOp/MGCUVNI/TiIxZoZ2ZnG+Sf3OtxRjEugpMtQYifh6HpJpRetENzCvYnQrHZaIiCAC6R1csC9Er2UcOeNvJ+ZvYZLDd6Z6pctsvzevaZ04m8n803ZkBWAc3r79wGJ5RiV8mhGfz60xdwpF2PhKIk1Ws1vRbO6dSboMLvWzKB84qkhWEZnCAqpVIhm4HPS9G0Hn5A8m8WQHge7EZH0aDN8EEJpdhRQvlQ+zen3pEgScxoEiOLog5Eh34QmukSZ8MBUkb1UavRAO9XKR7CYKPss1kE4VWrJa5aXTHA2Z7DaU/D3szJMlNPdSQ69ROC7B7GMKulBD37bPMA5J8kwVNUfDSbIDzPGQ4IV5xltVesrjjlDIeA/Tkc2PSBzWZTEEWXy6Uf04thdgxj6PLPiMHs53P7uVy71QL55zDkH0Lg7IJwUGx9cCZa/zYzI77+MtLx2HuA9SSpIcscxzkcDqBzNmKnZutrcOz1SiWfy9WrFfgeyb8ZBeFJsP741uo7NP284+B4BDbaPucA7TdEXviFUU2zmz0QOUmSAIicUUNgcSgeZZ8xbLvdAu9XAvmnqnDxSP7NuCd8alKes8HSL3aPq1iDEhVcCyiO42kyWqx/oDe+nDM8YXTmv3Z4XhJFcIkMCCojjGFN/gl4A9WX2d0G+SeKAstat/gI1koZA5avchilIRCO18BTEENECwgMF/Benm7eFSNujP0LLhdWXHMqq1y9umLk2P2APHJdKJJ6YVQLZJXRVYUx2hZaCoog/+CTFff3QQG2mk299bXTotlnuE6X8CLeVjHNidEZrJ3QvBRGqLY9p4K8d+/+Rf9WrVS63c7kPCAQNtrYt8DPW5WffOlT3s/4L0M1RaL3k637Xyh/+Keaz5Ia8W1uP081PRpLaacPGDoLS7zvCwa8Xhvx9ccukTC6SAE7VVWF1M0SNEQPPrBss17f294qZDO9nl79AJdq0fmKEW1M3CMqKS3wBekTf1v+YIVo/wm5I2CyF+MwzIo5EhRNz8dilgYhduYsivOeHn2BJbBE8gWq9bKw+I9LP/L52vd7NJLE8A/wtz4kpNqE9CqXbxHinMpdBWYapo0Is2FfD0DUeVRPNkr1dQFGnLQtvHb8ERTlMIrfc7u7mZ0dvfcu8E/Kooc3w63vYeoWUQYc/ne9l78sfvb71Nsx1fdp5f4tNfyQLL5F5AmMdGGMZjGXaAcQDudMKJwEdO1Q9RU58A9qH/xHlR9ekuYbdAOAJ+I9gRQWpOBH23fuSuEM3fgOuw+c1qUxl3aGRw730t6yz1Pg0rgeZ9OhKPfgxyRFE0/sTl2D9zOifw5gxAeF/O7WZr1aNcPx1oSfQWzwHF4v4/ynlGe/LH32v5FfdmNEGa93cF7F1ReU9Z9UnvVg3Ktkdo8ouzSWxUgNgXBUEPaZtYQuwdVtuspp9N9uvvRPKz/6Ume9S3YbJH9yh+AbnhB7uPyMkPps+xbowzfZw0dM1akyJ/cDnyhHHfH1uO4UCeClkiyrSg8njtjpNUwdhtPdXbVc2t3c1I9DxHHWqunXJvep4nyeqL6gpn5Z+uT/Jn00rgVqeKWDS4Rx6xVM5fG2R+Ne7t3/uLIhYuo3yV2AK7BT8JkaAuFVQKjDz9jwyJHNJil8mr/3y5WPfKr1IodrFaqh4OppT4XBjzSe7DIa+X5+7aOdFaCpr3H7hxTvUVlKv5unt12nDks9nmj0a5IlEfwSYMPMgJuU/DPOPGy3mpntrVxmT5aP0l+sKv/wDtbbJcohzf33pVf+ufiZ++pqG683cf5k5T259QIud3E+qQU/1nvhA2o6TzReJTMSpsxhHALhCCB8cprixshWyG6ear5fTP1i7Ud/rv5fRRR3la53cZm4WH/Db0m40iU784r7Q+1bPygk6qTwmqPQJXoelTmVUnOGco6IojHB2PwqywY7NdLfxpsXBm8E78g5HJIk5TOZve2tdqsF8o+ysPxTMG0Hryi49nn5+78kfe7DvfdquFjDa+oFKb4mLHlckHFhQ1n4a70XF9Xg28T+A3KfwmjnVIWihUFoVE7A9WFP98M2l0Ael3eoWlLx/i/1D/5i7ZU1KdEkGy2ye2oJvIjDwGs6oBVJYVmM/Fj7zm0psE3Xvscdgj90abQ2Rud2ekpol4OxuXcKwOvJMjhGvU5K32/Cj4+awC+90hnZZ+ANtMP9/b2tzVq5RBrVgJaVf4DAAt4s4q2PKnf/hfTZvyn/kA9jKkQFlD+BEQN/HcaLxzswcO9Vbv815R6DMf+ZzGTxKihGekpC0dogJAjgR0/VrcMSiGsAP5iAn2+9/1eqH325c0cmhTrZ0nAMH2UD+kgokoKKK3e7C59pb/hU7nvs4TZTd6ngf4+F4iBnOOoOzalGxiPB2Ixk6Hs2ehhDBfdFmuz0cl4R+CfLwgyoVSq7RvMlvRm2w0FYtdYRJFwd72aJyh019gXpE/9I+rGUGq4TVfBvwyy+Tw6pjKsdvOXTnB/sPfdhZRUEJAjFOi6AUMSvPYxhdRBSFGOOiTnKBbJZJTs/1rnzy7WP/lT7+9wYUaHqEq4Slx043EA1T3YcGvVSe/3DnUUZU193FIDozqmsIdy1cTvDqytJwmzsCwRVr+IfnZ0a2Wc06L0Oz2d3d3J7u5Iomr13Lcs/RUzZIcoejft78l/+F+Jn36Pc4olmE29jGHYJ2JhsqItLAt5Z0CKf6L34HjWew2uvkxkFwzwYi0D4BAhpGgaLxPEq0d0j68/J8f+z/sr/3PihWM9Xo2qwgOHjaJoHbyIaQjHW8/6V1q33CtESxb/mOAD16FHZs0kwU3SGJ5zWfE2v19PZ6ShCkTDknyL3CrlcZmer1WwwLEsD7bcq/EDm7eE1AMxf773/16Qf/6T8Eo33ygTIP+2KXuuIDelCUbyrLv/Xvedimve7RP4RcchojAOjr0coWh2ELMN1MGmLqoY058+3Xv6l+ofvi4ststUiOyMxkCGXRhCKEimtC9FPtW8tyt5HbPUttshopHE/JrhDM/qGzQk5xU2hqBlH7eo7mRdHFOEFjNFio3R4sLe9VSkV9V701pV/OgIP8NYhUf8hdeOL4qf/jvQjQcxZJSrG3hsxvj8EQlHj8TaJE9/Xu/sZ5RkcI/+c3C0QjTmNoyYfxrAuCAmDduW4dg9Xf5p/8f+qffyVzrNA5mtkU8O1CRH3I+FOCvAnnueX/mp7zaPR3+YOMlTDo7H0E/fjyuEHbVwwNqGoHyXc68FXwshE1c7wT731NU036rW97e39fB5eqZ+FZOHwQxMXM0R5WQv9H9LHvyB+clmNN4laC++Od/F98tZLmNLB2/Oa56/0XvgRZbmKd79F7PK4NGmhaF0QwuJUobof0ta+WP/4T7d/0K8xFaoGpJGYvG42hKIKQtGrst/fXv/hzoKAy686Cw1SAqF4cj+s4AxPKUVVUfR8NyPlBTfz3YzWb3rxe7cL2i+3u3ty9Ipl+aeeekFUWIz6H+QPfVn83A/07glEq463tEvJv5HZEC6JeGdZi3+29+KzWnSTKD4ginMYR9xAEBaI5secz3+t9N8me8E6VeMJEb/enul6hJeQu6SQkn0/2rrzohAp0K3vOA5Airg1RjuDCm1U5zY+Z/jk5ilmdP42KzAAigzHAVM9yOdB/jXrdRr8oYWzz+BrRo+2d3+i98KXxM/9pPyDTgwrE1WAJXFdm5bHQrGr4vJ9ZfWne++r+OTX8Ryl4BNaAiwKQrfHc2vtlisdfi1Sdoi9YJfFp5H/bsIe8C8T8oYQ/0xzI9HzPGDLD9kKp1GsSg7YsBlHdcWoMH4sFBUFQNhuNXc3N8vFQ/CMVs4+IzGihPP7RO371WWQf39P/nBUmwP5d5x9NoVbr2Darqvy1Y3cw1UhFUnKPblrHPM4+yAE6ZJeXFpcXnGxeruuOiv+RaxSdAnxttMlU1OaIqBB9Xw3AsdeaC1/or1Ca9S3HfsFum3mu2lDO0PsstUVF8P4/I3Wk9qow3yuXq+ZTaWsKv+INibtEeWE5vsF6SO/In5qXU03iXoL70xI/g1jMqH+v0uF37y7nfE0MSOjKBAIen1+UdQP4ZplEMYSidWNW+AGT1NTd/ePkyUV15YablLDpwVFPcJLdQKK4wOt9Zc7ySYpvuoodMjenMo8lal4jXne55PSE31Fkt0OL4mSNRXgcfFRBS74Z3s/+Kvij/9Q7/ke0a3iDWzC8q+/fStW/rV7W2+E66fmGsOyoUiEZfX46hjb5FkFhP5AcG3jVjAcvmjBhuHY9LdejVbBHybazmmxJj3CS8gCKSyKwY81b92Wgnt047uOIvz8JN/tulHXp76EJPlWq9vtWg2EpovL4/UKwX+id/9L0md/Wv6gByNB/kmYQkwPfjve9v/9zM7/nz7s0hdizOlyzUej8CHardaMgNDpdC2triZSqWHixTA0343UNv3tcIfzi8y0JpAu3ElRIXp3O6lPt9ZDPefbjuImW3OqtJnvZhFnaEEQGvIPr+CdPFF5Xk39kvRXf0H6WFIL1vAKj4vE9M4rbLDy76xnv7axV3EMZpswwHNebzAc6slyt9OxMQhhZqQWF5fX1o7adQ1tMEzfjJdrnJxuOTmFnBYUjwujqPe1Vj7SXoIJ9Jpzv0h1PApDPl0YNcQ+J3a1HJrzX281EJ4UHwU19/8qvfLPpc88q6zyRONs8dF1Giyaf7Cw/6v3Nne9o+27UBQdCIbcnjnRaJlnPxDCX127dRuWk0tfes7T+ZNECebbSsMzreXTLIwCoTjfc3+wufED3XiF7AIURbzn0Tj8yigaxRmeA2PrgNDIPtO28aqMK3+z99KXpZ/4SO+9GCZWibo6Vfn37Uj1V+9vvjZfVS6bGMNxXHh+nmYYnucv18R5CiD0+f2rG7fCkfmr79fBwL0baL46X/VKdIx3TAmHTxRGCZFPNDbWJP8WU3ubK1EY4VBpDR8SZuN3hlYAoZl9to83D/HWK8pt8H4/K/1lP8YOWXw0Oct4+P/nzu7vL+7zdO/q7+ZyuyPzIBS1SwjFawWhw+FYXFlJphfoscaLO3Tv2/M14BLzPOeVpisUBZVQ7/PpTzZXgZS+6SjvsQ2XStMaaXR4u5IzxEbPoaFIsj1VEB4XH1Vvq7FflD7+j6VPLKjzlyg+Gq+16d7vrmW/cnu35BxnsAGcitfnC4TCsiQJ3a7lQAhLciKVXt3YODqsZwIGA/qniRJPKwstF6MQ04KiXhhFdV0a/VJz7UfaaRlXX3cdVKnunMIOgboxO0OSIPj2dEB4qvgIHOD7lNsdotm4bPHRuOyPUoe/fnfrkb81ofenaToYCrlc7m6nI8uyVUAIzHNtY8MXCFzDEO95+T+PlymVWGy6p0fAjMIoqhOT5364sf5id75E8687DxRc1Quj8P47NGPtFkWSnWsHoQmwXbzG4+Jf773316XPfVJ+icGUClFTMG2K8HszVAf4wfSQyIl3VeYcjkg0RpIU324PbOI8WRDOeb3La+vwB66zWSUM8dvBxvfCdZ/IRDrclHCoz0XeLIzqzn+ysZ6UPY/Y6ttcmdUoTqMu7qChjbHRMEHomlC4LhAet75uHRD1l9W1L4qf/h+lV4KYa+zFR6PaoUv4V7d2/91KvsnK1/l33R5PKDIPIAQoTgGEDMsuLC+nF5emVa4Gw/1qtJr3dBNth1umpwTFk8Io7IXW4sdbK06V+q7zMM+03CpzUhh15ajgKXb3xJkzAML2NYHwpPhoUQv979JHf0X85Kqammjx0ZArMmDvN+5uHbiE6ahikvT5/T5/QBJFURCuCYQwA+LJFPBPl3tqhPDJJfA/JYtwJ5Ybninmu+mFUVTXp3I/0Fz7EJ/q4r3XnPtNvTDqON9tHM2gzr7gekB4XHxUZjHq78gf+pL44x/o3RcIvo43tanKvz+Ll758fxNo0dTnIcMwoXDE4XR2Oh2jt/okQRgMhddu3QoEg5bK1t/2tf8sXuYUMt1yTVEo6oVRlJAS/a80Nl4QwuAPwStquOZS6LOHl14tbHgE40mD0KwyyeL1GtH9nPzil/Xiow/AEFeut/jorD0MNH/zme0/ThZFykKH6gAII9GoUdrSerL7wdVAWK2c5OwA/V1eW4slEtZsVgnOEHT526FGUGBDXXZaOMSNfDe9MKoT+1R9fV52P3BUHnFVTqVOneA1joRSvXXw5EAI/LOM8wWi9n3q4helT/28/JGo5q0S5XE1/rmclRzib21k/s1ats5JFpyHcJs8c3MgFBVF6RwXRl0JhMWDQ+C54GpTi4uLyyssx2HWtjor64VRTjHRdrp60y2MEgAWL7aWPtZagp9kuZZMaqdyNq5OSs1g/SRAaDSf7/g15z+QXvmi+OnbylJLLz6aZvaZQmi/t1QA+ZfzdCw+D+F2+AMBr88nCCJoRVVVQcFdEoT1WtXhdG3cuQPgxuxjBXf3TxJFUIjLTTcxPaEo4Wqb5sOy7+XaSylB+Qb3EGfIJ4XrFcMV8GxyICzh/F9SFr/Z/bmXeu/vEfWpFx+9Gq382r3N754pPrKyMSwbjkRIkgKVGJ6fvyQIfYFAn+IjKxvcqkf+1mvRyrQKo1RMYzAyoIZVQvqS4/e/0PsPPNXzOudONUq7ojMkdU3YngQIcSMXtEi0VzA2oM4zmCLg8lRAuONt/8s7O//fwkGHVjAbGui4/ggcAEIcxzE7W2cahVFGK0s8pPlYzP371Hd+hvvqP6P/Yw0TY44AxznU0yDErpJDQ5BEZzIg5DC6jne/Tn3v35JvYbj8fnXBrQV6uNjD1GuDYpORf2ct+9VbQxUf2dr6gXA2zCyMquuFUa5JF0aB9whqLqcWeIfM/F3md/4n5t9l8fqSFnKoNOPgWIdDNdqHjrgB0ydYT4AnFLvdsZ/lBEsJjZEhzV3BO/+W+vYfkJsRjL2nLjkxrosL2uQzY/5g4eDL90cuPkIgtLRl9cKoIkjElYZnQvBzYIxXDdWJ5j9hfu9n2d/6M3I7qflCmAt8H2CPc+h2HghHdIZPzP/JgfAYipgLYwKa6wFR/Ar12gNyf10NLKppp35ij4RNZo/0u+Har97fAhGoEBp2M+ymgBAzttceBpqv6YVRTHR8hVEAPwojAlqQxvDfoP8U+OfXqNdcGJfU/GZKDXYMQs4E4fiqKyYNwse7A5jDjbF/Sm5/lfpOHW/cU+MRdZ7AZREbp1DMeTpfub37e0uFNtPDbpLdIBCaxjO91+erwHMAh17pSvlu5kkGIW2O0zx/RL31efZrv8T8YReTF7Qgi9HqE+ccPAlCbHwdSq8NhJoRtAhr7h6u/nvqra9Tb7G49j5lwYX5JFxQriwU23Tv66u5f3lnp+gUsJtnNw6EppmFUR1aSV+2MErFVJ/mdGuhXWL/59nf/bvM1x8R5SUtOIdx6pljRk6BcFzVFdcGwpNFh8XIoOY5xFu/TX37j4mdqOZ6Rll0YowhFC8Zw/iPyeKv39t8N9DCbqrdUBCatmsURtEKudh0jQI/jcNonxoWceEL7H/4PPuv/5B6GNW885hbu+DQ+dMgPAdXo1dX6I3wrxWEJ17Rg7F+zfUGsf9V6vVdorihhVNqyoFpZjLN8G/1TrDxG/e2vxkvXUPxEQKhdU0vjAo13gw3/AIT7nIDXQGwsqAaZDD6X9Pf+hn2q79J/zmNUSktQBzLP+xiELIOh3YhCIf1fo+fAgwp8vpBiB0vNH7MCaz7j8hHv019r0N0XlSTPjWC4ZKEKQOhWHIKX93YAwrauN7iIwRC6xpMhf8crey7u3H+/MIoY9ppwMQcmO9b5MP/nv2tX6D/fQ3vLmpBx9Py7/xZq6os5+A47krB+rOV9SQxFRCejAmJEWHMAw7wG9Qb3yDfcWH4e9RFp+YxhOL5YQxY9b6xrBcfwWijiYdAeNoOjgqjtOWnG4EDxuY0zqOFcmTpHzJf/zn233yP2Af4+THHkKdMHoEQPGFfEI4KS4IgO/zUQHjCDjiMCmruDFH/Gv36t4i9BcyzrgtFsoOLp8IY3zSLj0INDEfTDYHwYtv2tb8ZLzt6ZKrlMsIPZEALaXjvn9F/+LfYr/0e9WZI88QwD4aNcMqrCUJ4nD7f82oHqoEm7PD8dEF44hXnMM6nOV4js1+hXi8Q5TvafFxNUFjPKLrHH/lbv/nM9n9KWav4CIHQuiaS6hvh+jvBZrzrXOr6v0F+72+w/+pL9J/CtE9rAQojRp1HJ57wnGD9eWdlDwlLHYRtXhSmD0ITh7ier+eCZesPqYe/Tb6h4dLzWqrLqV9Z3/6d9WzNksVHCISWtjonfTNWeps4+Bnh1w/xzpIWdGKMeqlTzh/T0TMgxK7QlO3IE1oDhCdQhEUqrHmauPC71Dcrfvz3X6hn5ng0nfoYgYagv/1FsBTR/AuaXzuOzl91mp49cV4b9AK7mYKpfs2ZUlOve0sSoaBZhEB4JZNk0Yc5VGycwDgFs/NQp434esuZuWFD9dAODALhGAYIVvKJ7yVo6mgwswUOVUxDEEQgtLCjGBFmM8BRkSEQWoGgacM7w4EREARLBEJko0Lw9L7OEM5Q7f96hEIEQmRXpaCnOac6ovdDzhCBENnkpeDAPSGEQwRCZMMbjhu6sC8p1QbCEqEOgRDZ9TpD5OsQCJFNHIfIGSIQIrt2EA6uPzzrDBEOEQiRXaczxGYwoRQZAuFUDceNHRntSWeojZwginCIQIhsks7wXNraH6UIlAiEyK6Kuqs6QwRDBEJk45eCp17ft7pCM2GIShUQCJFNFJaj59AgQyBENhLqsAGoQ9sxCITIJm7qQOd2WgkiZ4hAiGyszhAbfYdGQ1oQgRDZpKXg6defl0ODUIhAiOx6YYlqehEIkV0NY6dBM3CHZiBlRShEIEQ2oqlXa+509uA1BEMEQmTDG34ehTzrDEfaodGOcIm0IQIhsssKv0u8AClBBEJkVwfiANRp9m+/jQyBEDlDZAiEyPqDSB01X/S8p0gJIhAiG6M/HDWhdGCHUmQIhMgGUcird3NCCaQIhMhGdn6jaL9L9b9AhkCIbCQYXt0ZIhwiECK7EiMd5vCJ/s4QMVIEQmSj+8LRWhsOEc9Ag4pAiGxow4eE5RU5KjIEQmRjdoYoNo9AiGzSOBzo3NRBp4UiQyBENnFYaugsCgRCZJMlpZcLV6DMNQRCZKMBr/+/XiJcgdwhAiGyIQ0/wsxo0m6YrvjIFyIQIhsFh1dOyz6nDT4aWQRCZEOjEGCID+zbO1p1hYZgiECIbERneFYbjrpDo6JcNQRCZFe0q1fWow0ZBEJkY8ahNvbqCmQIhMjOouhp5Iza9P5cWKKdUQRCZMOZqijwlSTJp1E0KFwxOKEU4XBGjEJDMEEHqKqqptEM43S5aJpWDDQ+jTP8SZjh+IVPkTJEIEQ2Mv9UVYUkKbfTyTmdBI6bCBwJZqdc3VmUovgEAiGy8w3whhM453A6nS6KoYGOKqo6DM5GdYaqwVERJUUgRPYEKnT6qTEs63A4GYaB7xVZxvArcE5VAzxf/Hp0NBMCIbKn+KdKURT4P8Cg6Q/NfxpAOUeF5UASiwyB8CY6QJ1/Ek6Xi+M4giDA/z3pnwbB7PQm5+nXP+0MkSEQIjvNP+ErC+BzOMANwlPDAeJPwWxQKAGcKI73jRX13aFBhkB4U/mnqgfUAXig/0D+4U/wz+NNy5Gk4JV2aJAhEN44+QcIJAF+HEezrAk/vH/44YwzvKIURLBEILzR/BNmP3g/QCBBksA+tQsp59PObZC0GwCz084VGQLhjeSf8JVmGIAfuEHN0H9jdG4DtaOm19E/9f5GlBDhEoFw1g1wYqZfkyTJcg6aobXjdFDsFAU9g6FR9zkHO0P8nN+x+OihKYRAeOUBoiiAHwXgo2mYUqrar6ZW1TSiv3MbsM85yBmegrHlI/UIhMMYqqLoZx2ez+7t6nswLIfjhJ4QM/CQ3dOOatTmTqNVV1jcquVypVxCEwl5wstYr9crZLMH+wUTCUqvx7AsOERsiLr4/uEH/XBPYoTX23pfVJKkrXffLR0Wk+m02+NB8wqBcFg7PNjPZzKAwxNOBd8rSg9IKcMAFMknfdRZVJzilKdR9PT+ynAw7gdq61uzUX/7jXpkPppIp2iaQRMMgbCfNeq1XCbDt9unWTuhOy9ZknqyDDhkjPS0E304cix+QFr2cM7QbmqreHgA1DSeSsXiCTTTnjTy3r37aBTAhG53d3s7l9kDpPXfZtC9Yk+vjaAoytw7PbsDoZ35Sf+nI79AwwDG8J8oCHDB5hphfYOxatbr1UoFGIXD4USzDoHwWKMpSj6X3Xz3YbfbGfhi3DBNVWVZNvO2zb4V53izMzA6jaIRcXj6KcCQsBkIj/S2LFfLZZ7nHU694QAC4U2no+ViMZfNSKI40m/B9AdAAA7BK+o6keNIvXhC6yvtBlRLnKf8sP6k1NYjX69W4RGNxxPJFElRCIQ30VrNJpBP+Hrpd9D9j6ZJgqALRZaFx5NCERs9+H6JnBu734WDQqFSKiVS6Ug0iujoDTJRFDM7O5ndnVEd4Pn01PCBgEN4wDPKiOmfwOMMpcROIW8QBe33el0TigI4ZHvR0dNyQFXrtRo8WL0kjEMgnHEDbBRyua13H/J8e4xvawpF1RCKiqIAJJ7kVwOlHYZfUhniuibs2h2EpoGyBZfY7Xb1xjw3TCjeIDpaKZdymYwoCBN6/6fCGCx7UmaB9U1dQ0VMT1q1XIZHPJmMJ1MzsLIgED42vt0G+deo16/hbxntLTShqzso9lgoahcLxbPicNSE0tkzYCtlQyiGIxFER21vgITs3t7u1ubkHOAFOlGHInhFpdeDJ/QTEcVROWefF+BmiGIm6OgpA1Zfr1abjSaoRFjLEAjtageFAsi/q+x/XlUoEgRMJh2KqqqXYlwsFC+3Q0PMLghNk0SxXCzCV5fbc+ocAURHrW61ajWfzXR4fupXYsLDDKnri7rDAZNJ0duTPt3c6crhihm2UrFY0YViKpZIzOQgzJon7HQ6QD7z2Sz4B+tclQlFydizARdmstMzWnB0Uqp7QlGWpZnfw9Dz3RqNaqVMM7TD6UQgtK6KyGX2th89ErpdC16eyR5VVQVyBUIRvjdST4nhheLZnxzHCaUbspHY6/WqlUq71TIbnCM6ai3Ti4+y2Z6VvF8flygbxhrd8imaNpvnY0NUV5y3cXrj2uA36nV4RKJRIKizAUXbe0LgeG9+9zuVUql/4wmreUUTipIkAogAhyAUL06y6Resl8RZ3pjpY3y7fVAouNwezuGw+2ex/c2D6RtPJJ0ul70uG7BkbvfxfBvWdVEQCD3RhhzirGwVQ2b0/gFPOBv6kJoBEIbn5+EBjLRgsf2YIaGoKL1Ws2Ge5UQbZzkNOgD0pp/ROx+LAQJnpgxqdkIU89FYMBQ+aQxjJzZihjFEUdLDGA5dKDLUiVDEULjiCfP6/clU2uV2z9KHmqkQBcxmuEn+QFCSJWvukQ6Eoq4TTaHI0CRxoVDEjpMBbo4mhNVpcWU1lV6YpX3RGQShacDowCWCSux2Oj27sVMz0VSUBFmS4Yl+1MxxufDZHZobAkL4dMl0enV9Y/YihLNGR08Z+EN4HBTy+WxWebpfvdWhSBAkRvR6vVa9LnIcrCYgfvSTsU+X8Go3gZKC2k+k0rPn/WbcEz5pbs8c3EVVVc42ULO+VwTTwxiCAAjUG4Gf6QEBrlASxVn1hHNe78r6Bkj9Gc4avREgxIztU58/AA+Yr9dZSzEWIJ7ku0lGDzj6VESRwGcShCzHLSwtpReXmFmvn7gpIDQN+EwoHOE4R6fDK8ddfW0kFPV8N6OZDUGStHEmqb5HqoNQmiUQwoeNJ1Mg/2Zs/xOB8LGBvgJ6g+NYG9iprbokmVCE5UPsduErSdNmMxtphkqZgqHQ6satQCh00wIwN67RE9xgEBuhUFjpKeAVbXbxADYQipKk82pNA7bWM5iq3UHo9niW19ZiiSR1I9uQ3tDmvxRF+YNBz5xXEAQ9Lmc3l4iZNYqyrBpnd9vXdYDKTS8tLS6vsDeyz9qNBuHJBkB4fp5h2A7ftlkY45idqnZGYCyeWL11y+OZw262oQNh9EgU6JBCLrefz9nryu3LQv2BYDKdntXgOwLhpfgASaYWFsKRSC6TqVbKaEAmZ06XC+Dn8wfQUCA6et6CRNPgEt1uj9GwUEIDMnYdnlxYWF5dm4EKQOQJJ2tevx8eh/tATrP2Sj21skWi0UQqjc5gQiAcweZjsWAoBDgENKLRuNKi5vMl0ws3KviO6OjYjNDz3fy+QECWJMFe+W7WML34aHkltbA42+nXCIQTN5hAwbAtC6OmuX4RBJDP1Y0NtP+J6OjYzCyM2s/nCzmbFUZdv4UjkcQslt4iEFrCYolEKBLOZTKlw0M0GmfNMzcH8g++oqFAdHSS40WS/kDA6/OLoiCKIhoQ01iWXVhahgd7M4qPEAgtIBRZNhSxZWHU2M0oPkqubtxC+5+Ijk7BguGwke+WLeRyM3B8/CUMPn4ylUbBdwTCKfuBRCodCuv5bpVy6eZ8cPB7IP+8Ph+aA4iOWmMlo6iAXhg1ZxRGzXi+G0XT6cWlpZVV7gYXHyEQWtTsWxg1vEXjibVbt9D+J6Kjljb7Fkb1N38gAPwTBd8RCG3CLo4Lo7KZvVqlYveP43Q6E+kFACG6s4iO2k87BUMhl9vd7XRle+a7wWqSXFhYWVt3oP1P5Anta2bL04P9QiGb7dkqojhLp3AiECLDorF4KBS2S2GU1+dLpNJujwfdOERHZ8qOCqP8AclsWGhJYznuqPgIZZ8hEM6qGY3Aww6nq9PhLcVOcYJIpvTW17Y78xjRUWSXsUAwCI/9fC6fy6kWiCiGIpFkKo28HwLhjbNYIhkMR/LZaRZGoeIjREdv/OhPrzAK/F4aFR8hECI7wQMQQpbleJ6/nnw3cMIg/9D+J6KjyE4LM7Mwaj+fn1xhFGjRRAq1vkYgRHaBGYezL4QMoVgpj7kRuFF8lAbqi8YZ0VFkg9ZFmg4EQ2MsjKIoKrW4aBQfoewzBEJkQ5tZGEUzDN/mVfXyQjEaixvFR140pIiOIruMReajwVC4kM3uF/Kj/q7PHwD+iYLvCITIrsxVSBLIZMg4MapWHaowyuF0Avz8gSAaPURHkY3NaKMwyulyC51On8IovfgovbCyvu5woP1P5AmRTcD8gQA8DgqFfC57ttUiaMhEKo2KjxAIkU3covF4MBzKZ7LFwwPzJ3NeLzhAFHxHIER2neyUWVxZAaF4sF+Ym/NGolE0JgiEyKZg4PpWPRtoHOxuBBoCZMgQCJEhQyBEhgzZ9Oy/CDAAp2qeCvi0dTEAAAAASUVORK5CYII= diff --git a/deploy/operator.yaml b/deploy/operator.yaml index cec282351..58175abc2 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -16,8 +16,8 @@ spec: serviceAccountName: noobaa-operator containers: - name: noobaa-operator - image: noobaa/noobaa-operator:0.1.0 - imagePullPolicy: IfNotPresent + image: noobaa/noobaa-operator:1.0.0 + imagePullPolicy: Always resources: limits: cpu: "250m" diff --git a/deploy/role.yaml b/deploy/role.yaml index 1e69b13d9..e433b8406 100644 --- a/deploy/role.yaml +++ b/deploy/role.yaml @@ -8,6 +8,9 @@ rules: - noobaa.io resources: - '*' + - noobaas + - backingstores + - bucketclasses verbs: - '*' - apiGroups: diff --git a/devenv.sh b/devenv.sh index 2d8638eba..444d4edbd 100644 --- a/devenv.sh +++ b/devenv.sh @@ -1,5 +1,6 @@ # source me to your dev shell export GOROOT=$(go env GOROOT) export GO111MODULE=on -export PATH=$PATH:build/_output/bin +[[ ":${PATH}:" =~ ":build/_output/bin:" ]] || PATH=$PATH:build/_output/bin eval $(minikube docker-env) +alias nb='noobaa-operator-local' diff --git a/doc/about-noobaa.md b/doc/about-noobaa.md new file mode 100644 index 000000000..8c6f0c3a9 --- /dev/null +++ b/doc/about-noobaa.md @@ -0,0 +1,17 @@ +[NooBaa Operator](../README.md) / +# About NooBaa + +- [NooBaa Github Project](https://github.com/noobaa/noobaa-core) +- [NooBaa On Ceph Tech Talk (youtube)](https://www.youtube.com/watch?v=uW6NvsYFX-s) +- [NooBaa From Zero To Multi Cloud (youtube)](https://www.youtube.com/watch?v=fuTKXBMwOes) +- [OBC Design Document](https://github.com/kube-object-storage/lib-bucket-provisioner/blob/master/doc/design/object-bucket-lib.md) + + +# Terminology: + +| Term | a.k.a | Description | +|------|-------|-------------| +| NooBaa Core | "The Brain" | | +| NooBaa Endpoint | "S3 server" | stateless | +| NooBaa Agent | "The Daemon" or "Storage Node" | stateful for local FS volume, embeds endpoint | + diff --git a/doc/backing-store-crd.md b/doc/backing-store-crd.md new file mode 100644 index 000000000..d2183eaa1 --- /dev/null +++ b/doc/backing-store-crd.md @@ -0,0 +1,82 @@ +[NooBaa Operator](../README.md) / +# BackingStore CRD + +BackingStore CRD represents a storage target to be used as underlying storage for the data in NooBaa buckets. +These storage targets are used to store deduped+compressed+encrypted chunks of data (encryption keys are stored separatly). +Backing-stores are referred to by name when defining [BucketClass](bucket-class-crd.md). + +Multiple types of backing-stores are supported: aws-s3, s3-compatible, google-cloud-storage, azure-blob, obc, pvc. +Adding support for a new type of backing-store is rather easy as it requires just GET/PUT key-value store, see [Backing-stores supported by NooBaa](https://github.com/noobaa/noobaa-core/tree/master/src/agent/block_store_services). + + +# Definitions + +- CRD: [noobaa_v1alpha1_bucketclass_crd.yaml](../deploy/crds/noobaa_v1alpha1_backingstore_crd.yaml) +- CR: [noobaa_v1alpha1_bucketclass_cr.yaml](../deploy/crds/noobaa_v1alpha1_backingstore_cr.yaml) + + +# Reconcile + +#### OBC type + +The operator will create a claim and the appropriate provisioner will create a new bucket or connect to existing one depending on the obc options. Once the claim is ready its details will be used to configure a cloud resource in NooBaa. + +#### PVC type + +Create a NooBaa storage agent StatefulSet with PVC mounted in each pod. Each agent will connect to the NooBaa brain and provide the PV filesystem storage to be used for storing encrypted chunks of data. + +#### Credentials change + +In case the credentials of a backing-store need to be updated due to a periodic security policy or concern, the appropriate secret should be updated by the user, and the operator will be responsible for watching changes in those secrets and propagating the new credential update to the NooBaa system server. + + +# Read Status + +Here is an example healthy status (see below example of non-healthy status): + +```yaml +apiVersion: noobaa.io/v1alpha1 +kind: BackingStore +metadata: + name: aws-s3 + namespace: noobaa +spec: + ... +status: + health: OK + issues: [] +``` + + +# Delete + +Backing-stores are used for data persistency, therefore there is a cleanup process before they can be deleted. +The operator will use the `finalizer` pattern as explained in the link below, and set a finalizer on every backing-store to mark that external cleanup is needed before it can be delete: + +https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#finalizers + +After marking a backing-store for deletion, the operator will notify the NooBaa server on the deletion which will enter a *decommissioning* state, in which NooBaa will attempt to rebuild the data to a new backing-store location. Once the decomissioning process completes the operator will remove the finalizer and allow the CR to be deleted. + +There are cases where the decommissioning cannot complete due to inability to read the data from the backing-store that is already not serving - for example if the target bucket was already deleted or the credentials were invalidated or there is no network from the system to the backing-store service. In such cases the system status will be used to report these issues and suggest manual resolution for example: + +```yaml +apiVersion: noobaa.io/v1alpha1 +kind: BackingStore +metadata: + name: aws-s3 + namespace: noobaa + finalizers: + - finalizer.noobaa.io +spec: + ... +status: + health: WARNING + issues: + - title: Backing-Store "aws" - Target bucket is missing / access denied + createTime: "2019-06-04T13:05:35.473Z" + lastTime: "2019-06-04T13:05:35.473Z" + - title: Backing-Store "aws" - Cannot remove `finalizer.noobaa.io` to complete deletion until the data rebuild process completes + createTime: "2019-06-04T13:05:35.473Z" + lastTime: "2019-06-04T13:05:35.473Z" + troubleshooting: "https://github.com/noobaa/noobaa-core/wiki/Backing-store-finalizer-troubleshooting" +``` diff --git a/doc/bucket-class-crd.md b/doc/bucket-class-crd.md new file mode 100644 index 000000000..0d7bb1871 --- /dev/null +++ b/doc/bucket-class-crd.md @@ -0,0 +1,74 @@ +[NooBaa Operator](../README.md) / +# BucketClass CRD + +NooBaaBucket CRD represents a class for buckets that defines policies for data placement and more. + +Data placement capabilities are built as a multi-layer structure, here are the layers bottom-up: +- Spread Layer - list of backing-stores, aggregates the storage of multiple stores. +- Mirroring Layer - list of spread-layers, async-mirroring to all mirrors, with locality optimization. +- Tiering Layer - list of mirroring-layers, push cold data to next tier. + +For more information on using bucket-classes from S3 see [S3 Account](s3-account.md). + +# Definitions + +- CRD: [noobaa_v1alpha1_bucketclass_crd.yaml](../deploy/crds/noobaa_v1alpha1_bucketclass_crd.yaml) +- CR: [noobaa_v1alpha1_bucketclass_cr.yaml](../deploy/crds/noobaa_v1alpha1_bucketclass_cr.yaml) + + +# Reconcile + +- The operator will verify that bucket-class is valid - i.e. that the backing-stores exist and can be used. +- Changes to a bucket-class spec will be propagated to buckets that were instantiated from it. +- Other than that the bucket-class is passive, just waiting there for new buckets to use it. + +# Read Status + +Here is an example of healthy status: + +```yaml +apiVersion: noobaa.io/v1alpha1 +kind: BucketClass +metadata: + name: noobaa-default-class + namespace: noobaa +spec: + ... +status: + health: OK + buckets: 31 + issues: [] +``` + +# Delete + +Deleting a bucket-class should not be possible as long as there are buckets referring to it. +Since CRD's do not offer this level of control, the operator will use the `finalizer` pattern as explained in the link below, and set a finalizer on every bucket-class to mark that external cleanup is needed before it can be delete: + +https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#finalizers + +The status of the bucket-class will show the remaining buckets that prevent if from being deleted: + +```yaml +apiVersion: noobaa.io/v1alpha1 +kind: BucketClass +metadata: + name: noobaa-default-class + namespace: noobaa + finalizers: + - finalizer.noobaa.io +spec: + ... +status: + health: WARNING + buckets: 3 + issues: + - title: Bucket-class "noobaa-default-class" - Cannot remove `finalizer.noobaa.io` to complete deletion while it has buckets + buckets: + - bucket1 + - bucket2 + - bucket3 + createTime: "2019-06-04T13:05:35.473Z" + lastTime: "2019-06-04T13:05:35.473Z" + troubleshooting: "https://github.com/noobaa/noobaa-core/wiki/Bucket-class-finalizer-troubleshooting" +``` diff --git a/doc/noobaa-crd.md b/doc/noobaa-crd.md new file mode 100644 index 000000000..62fd4913e --- /dev/null +++ b/doc/noobaa-crd.md @@ -0,0 +1,116 @@ +[NooBaa Operator](../README.md) / +# NooBaa CRD + +NooBaa CRD represents a single installation of NooBaa that includes a set of sub-resources (backing-stores, bucket-classes, and buckets) and has a lifecycle as a single integrated system. + + +# Definitions + +- CRD: [noobaa_v1alpha1_noobaa_crd.yaml](../deploy/crds/noobaa_v1alpha1_noobaa_crd.yaml) +- CR: [noobaa_v1alpha1_noobaa_cr.yaml](../deploy/crds/noobaa_v1alpha1_noobaa_cr.yaml) + + +# Reconcile + +The operator watches for NooBaaSystem changes and reconcile them to apply the following deployment: + +- Kubernetes Resources + - Core server: + - StatefulSet + - Service (mgmt) + - Service (s3) + - Secrets + - Route/Ingress (?) + - Endpoints: + - Deployment + - HorizontalPodAutoscaler +- NooBaa Setup + - Admin Account + - Once the server is up and running the operator will call an API to setup a new system in the server which returns a secret token: + - `noobaa-admin-mgmt-secret` will be created to contain the admin account token. It will be used for next API calls on this system. + - `noobaa-admin-s3-secret` will be created with the admin S3 credentials to allow easy onboarding for the admin. + - Internal backing-store + - In order to reduce the friction of setting up backing-stores right after deployment, we create an internal backing-store with the following characteristics: + - The internal store has limited size + - It is using the core server PV for storage + - It should not be used for production workloads. + - Default bucket-class + - This is a simple class that uses just the internal backing-store. + - Once backing-stores are added the default class should be updated and existing data will automatically move from internal store to the new stores. + - first.bucket + - The operator will create a `first.bucket` using the default bucket-class. + + +# Status + +The operator will set the status of the NooBaaSystem to represent the current state of reconciling to the desired state.\ +Here is the example status structure as would be returned by a `kubectl get noobaa -n noobaa -o yaml`: + +```yaml +apiVersion: noobaa.io/v1alpha1 +kind: NooBaa +metadata: + name: noobaa + namespace: noobaa +spec: + # ... +status: + + health: + backingStores: OK + bucketClasses: OK + buckets: OK + accounts: OK + issues: [] + + counters: + backingStores: 2 + bucketClasses: 3 + buckets: 33 + accounts: 3 + + readme: | + +Welcome to NooBaa + +S3 Endpoint +----------- +- Access key : export AWS_ACCESS_KEY_ID=$(kubectl get secret noobaa-admin-s3-secret -n noobaa -o json | jq -r '.data.AWS_ACCESS_KEY_ID|@base64d') +- Secret key : export AWS_SECRET_ACCESS_KEY=$(kubectl get secret noobaa-admin-s3-secret -n noobaa -o json | jq -r '.data.AWS_SECRET_ACCESS_KEY|@base64d') +- External address : https://222.222.222.222:8443 +- ClusterIP address : https://s3.noobaa +- NodePort address : http://192.168.99.100:30361 +- Port forwarding : kubectl port-forward -n noobaa service/s3 10443:443 # then open https://localhost:10443 +- aws-cli : alias s3="aws --endpoint https://localhost:10443 s3" + +Management +------------- +- Username/password : kubectl get secret noobaa-admin-mgmt-secret -n noobaa -o json | jq '.data|map_values(@base64d)' +- External address : https://111.111.111.111:8443 +- ClusterIP address : https://noobaa-mgmt.noobaa:8443 +- Node port address : http://192.168.99.100:30785 +- Port forwarding : kubectl port-forward -n noobaa service/noobaa-mgmt 11443:8443 # then open https://localhost:11443 +``` + +Example health status when there is an issue with the availability of a backing-store: +```yaml +status: + health: + backingStores: WARNING + bucketClasses: OK + buckets: OK + accounts: OK + issues: + - title: backingStore "aws" is not accessible + createTime: "2019-06-04T13:05:35.473Z" + lastTime: "2019-06-04T13:05:35.473Z" +``` + + +# Delete + +The operator will detect deletion of a system CR, and will followup by deleting all the owned resources. + +This is done by connecting owner references and letting Garbage Collection do the rest as described here: + +https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/ diff --git a/doc/obc-provisioner.md b/doc/obc-provisioner.md new file mode 100644 index 000000000..288913a49 --- /dev/null +++ b/doc/obc-provisioner.md @@ -0,0 +1,56 @@ +[NooBaa Operator](../README.md) / +# OBC Provisioner + +Kubernetes natively supports dynamic provisioning for many types of file and block storage, but lacks support for object bucket provisioning. +In order to provide a native provisioning of object storage buckets, the concept of Object Bucket Claim (OBC/OB) was introduced in a similar manner to Persistent Volume Claim (PVC/PV) + +The `lib-bucket-provisioner` repo provides a library implementation and design to unify the implementations: + +[OBC Design Document](https://github.com/kube-object-storage/lib-bucket-provisioner/blob/master/doc/design/object-bucket-lib.md) + + +# StorageClass + +The administrator will create StorageClasses and control its visibility to app-owners using RBAC rules. + +See https://github.com/kube-object-storage/lib-bucket-provisioner/blob/master/deploy/storageClass.yaml + +Example: + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: noobaa-default-class +provisioner: noobaa.io/bucket +parameters: + reclaimPolicy: Delete + backingStore: noobaa-aws-resource +``` + +# OBC + +Applications that require a bucket will create an OBC and refer to a storage class name. + +See https://github.com/kube-object-storage/lib-bucket-provisioner/blob/master/deploy/example-claim.yaml + +The operator will watch for OBC's and fulfill the claims by create/find existing bucket in NooBaa, and will share a config map and a secret with the application in order to give it all the needed details to work with the bucket. + +Example: + +```yaml +apiVersion: objectbucket.io/v1alpha1 +kind: ObjectBucketClaim +metadata: + name: my-bucket-claim +spec: + generateBucketName: "my-bucket-" + storageClassName: noobaa-default-class + SSL: false +``` + +# Bucket Permissions and Sharing + +The scope of bucket permissions will be at the namespace scope - this means that all the OBC's from the same namespace will receive S3 credentials that has permission to use any other bucket provisioned by that namespace. Notice that also listing buckets with these S3 credentials will return only the subset of buckets claimed by that namespace. + +While there are cases that this namespace scope is not enough, it provides a simple model for sharing and privacy for the initial release. diff --git a/doc/s3-account.md b/doc/s3-account.md new file mode 100644 index 000000000..d136b5c0f --- /dev/null +++ b/doc/s3-account.md @@ -0,0 +1,61 @@ +[NooBaa Operator](../README.md) / +# S3 Account + +Native S3 applications need credentials (access-key, secret-key) and an endpoint in order to use the S3 API from the application. + +In order to provide applications these credentials the application will set annotations `s3-account.noobaa.io/*` as shown in the examples below, to request that the operator provide a secret with S3 credentials. + +The annotations will be added to the service-account in order to create a matching identity in NooBaa and provide a secret with S3 credentials in the application namespace. The operator will fulfill the request by creating a NooBaa account and put the access-key, secret-key, and endpoint in a secret for the application. + +Once the service-account is removed or the annotations are removed, the operator will delete the NooBaa account and the secret. + +# Example + +The annotations on the service-account: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: app-account + namespace: app-namespace + # These annotations are requesting the operator to create a secret called `s3-credentials` + # such that `noobaa-default-class` is used for new buckets. + annotations: + s3-account.noobaa.io/secret-name: s3-credentials + s3-account.noobaa.io/bucket-class: noobaa-default-class +``` + +The operator will create a secret like this: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials + namespace: app-namespace +type: Opaque +data: + AWS_ACCESS_KEY_ID: XXXXX + AWS_SECRET_ACCESS_KEY: YYYYY + S3_ENDPOINT: ZZZZZ +``` + +And the application deployments can load it to pods like this: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app-deployment +spec: + replicas: 42 + template: + spec: + containers: + - name: app-container + # Here we map the secret to the container env + envFrom: + - secretRef: + name: s3-credentials +``` diff --git a/go.mod b/go.mod index 3af811c65..1ef681e26 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,9 @@ require ( github.com/coreos/prometheus-operator v0.26.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/docker/distribution v2.7.1+incompatible - github.com/emicklei/go-restful v2.8.1+incompatible // indirect - github.com/go-logr/logr v0.1.0 + github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect + github.com/go-logr/logr v0.1.0 // indirect github.com/go-logr/zapr v0.1.0 // indirect - github.com/go-openapi/spec v0.18.0 github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 // indirect github.com/golang/mock v1.2.0 // indirect github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect @@ -23,13 +22,12 @@ require ( github.com/hashicorp/go-version v1.2.0 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 // indirect github.com/imdario/mergo v0.3.6 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/operator-framework/operator-sdk v0.8.2-0.20190522220659-031d71ef8154 github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/spf13/cobra v0.0.3 - github.com/spf13/pflag v1.0.3 + github.com/sirupsen/logrus v1.4.2 + github.com/spf13/cobra v0.0.4 go.opencensus.io v0.19.2 // indirect go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect @@ -37,18 +35,19 @@ require ( golang.org/x/arch v0.0.0-20190312162104-788fe5ffcd8c // indirect golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect golang.org/x/lint v0.0.0-20190409202823-959b441ac422 // indirect - golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect - golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect - golang.org/x/text v0.3.2 // indirect + golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect + golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect - golang.org/x/tools v0.0.0-20190703212419-2214986f1668 // indirect - k8s.io/api v0.0.0-20190222213804-5cb15d344471 + golang.org/x/tools v0.0.0-20190723021737-8bb11ff117ca // indirect + k8s.io/api v0.0.0-20190718062839-c8a0b81cb10e k8s.io/apiextensions-apiserver v0.0.0-20190228180357-d002e88f6236 - k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 + k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8 + k8s.io/cli-runtime v0.0.0-20190717024643-59adbd30f884 k8s.io/client-go v2.0.0-alpha.0.0.20181126152608-d082d5923d3c+incompatible k8s.io/code-generator v0.0.0-20180823001027-3dcf91f64f63 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6 k8s.io/kube-openapi v0.0.0-20180711000925-0cf8f7e6ed1d + k8s.io/kubectl v0.0.0-20190720024926-08c6807d9aef sigs.k8s.io/controller-runtime v0.1.10 sigs.k8s.io/controller-tools v0.1.10 sigs.k8s.io/testing_frameworks v0.1.0 // indirect diff --git a/go.sum b/go.sum index 5a69333cf..73d183f81 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,18 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT contrib.go.opencensus.io/exporter/ocagent v0.4.9 h1:8ZbMXpyd04/3LILa/9Tzr8N4HzZNj6Vb2xsaSuR4nQI= contrib.go.opencensus.io/exporter/ocagent v0.4.9/go.mod h1:ueLzZcP7LPhPulEBukGn4aLh7Mx9YJwpVJ9nL2FYltw= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v11.5.2+incompatible h1:NTIEargbhAGNWuT7QEXJ2fqLMFvatupHIscb9FYwVOg= github.com/Azure/go-autorest v11.5.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -19,13 +25,18 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 h1:Kn3rqvbUFqSepE2OqVu0Pn1CbDw9IuMlONapol0zuwk= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/prometheus-operator v0.29.0 h1:Moi4klbr1xUVaofWzlaM12mxwCL294GiLW2Qj8ku0sY= github.com/coreos/prometheus-operator v0.29.0/go.mod h1:SO+r5yZUacDFPKHfPoUjI3hMsH+ZUdiuNNhuSq3WoSg= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -33,15 +44,24 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/emicklei/go-restful v2.8.1+incompatible h1:AyDqLHbJ1quqbWr/OWDw+PlIP8ZFoTmYrGYaxzrLbNg= github.com/emicklei/go-restful v2.8.1+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.0.0+incompatible h1:xregGRMLBeuRcwiOTHRCsPPuzCQlqhxUPbqdw+zNkLc= github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -52,12 +72,20 @@ github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonreference v0.17.0 h1:yJW3HCkTHg7NOA+gZ83IPHzUSnUzGXhGmsdiCcMexbA= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/spec v0.18.0 h1:aIjeyG5mo5/FrvDkpKKEGZPmF9MPHahS72mzfVqeQXQ= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2 h1:SStNd1jRcYtfKCN7R0laGNs80WYYvn5CbBjM2sOmCrE= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi880= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= github.com/gobuffalo/envy v1.6.15 h1:OsV5vOpHYUpP7ZLS6sem1y40/lNX1BZj+ynMiRi21lQ= @@ -102,6 +130,7 @@ github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+d github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= @@ -124,16 +153,24 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/markbates/inflect v1.0.4 h1:5fh1gzTFhfae06u3hzHYO9xe3l3v3nW5Pwt3naLTP5g= github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= @@ -152,6 +189,7 @@ github.com/operator-framework/operator-sdk v0.8.1 h1:TDnWWkYpYuuo53ftRVEsAj0xy7I github.com/operator-framework/operator-sdk v0.8.1/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA= github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE= github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -176,19 +214,32 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2 h1:J7U/N7eRtzjhs26d6GqMh2HBuXP8/Z64Densiiieafo= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= +github.com/spf13/cobra v0.0.4/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A= go.opencensus.io v0.19.2 h1:ZZpq6xI6kv/LuE/5s5UQvBU5vMjvRnPb8PvJrIntAnc= go.opencensus.io v0.19.2/go.mod h1:NO/8qkisMZLZ1FCsKNqtJPwc8/TaclWyY0B6wcYNg9M= @@ -201,9 +252,11 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/arch v0.0.0-20190312162104-788fe5ffcd8c h1:Rx/HTKi09myZ25t1SOlDHmHOy/mKxNAcu0hP1oPX9qM= golang.org/x/arch v0.0.0-20190312162104-788fe5ffcd8c/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -228,9 +281,12 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= @@ -248,13 +304,18 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -271,8 +332,15 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK81+LvmbvWWEnJhzk1Pi9E= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190703212419-2214986f1668 h1:3LJOYcj2ObWSZJXX21oGIIPv5SaOoi5JkzQTWnCXRhg= golang.org/x/tools v0.0.0-20190703212419-2214986f1668/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190717194535-128ec6dfca09 h1:4E8vPHnP5LziSZdudGZpi697p5agtemBV2l/jWYmG+M= +golang.org/x/tools v0.0.0-20190717194535-128ec6dfca09/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386 h1:W/t3IYUOQPd8DK2ssOWA8sjulHHMxzTgiQkSx0z5sRQ= +golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190723021737-8bb11ff117ca h1:SqwJrz6xPBlCUltcEHz2/p01HRPR+VGD+aYLikk8uas= +golang.org/x/tools v0.0.0-20190723021737-8bb11ff117ca/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.2.0 h1:B5VXkdjt7K2Gm6fGBC9C9a1OAKJDT95cTqwet+2zib0= google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU= @@ -305,6 +373,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -314,6 +383,10 @@ k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476 h1:Ws9zfxsgV19 k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb23EwaLY85hoAV4SpXfdao= k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/cli-runtime v0.0.0-20190704050804-7ea68ffa02c5 h1:F/pAp9KvuqaKLwVb1uWPXiv2jdZ8fqyXR6+6xQRn700= +k8s.io/cli-runtime v0.0.0-20190704050804-7ea68ffa02c5/go.mod h1:HYxpxDR0QWcp5ciJglNRCgbtZ9hlQLFd61jzH0V6668= +k8s.io/cli-runtime v0.0.0-20190717024643-59adbd30f884 h1:dL0j4i6exYcHUN687wwzpX82l/zt4t7H7Hbw1BbbOzE= +k8s.io/cli-runtime v0.0.0-20190717024643-59adbd30f884/go.mod h1:+ruWDTqAuYQj3fl0lA6wTha/YDI+opreeAT7WvYWgmY= k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 h1:OH3z6khCtxnJBAc0C5CMYWLl1CoK5R5fngX7wrwdN5c= k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b h1:KH0fUlgdFZH8UMxJ/FDCYHpczfSQKefetq5NjL6BVF0= @@ -322,13 +395,20 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6 h1:4s3/R4+OYYYUKptXPhZKjQ04WJ6Eh k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.2.0 h1:0ElL0OHzF3N+OhoJTL0uca20SxtYt4X4+bzHeqrB83c= k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20180711000925-0cf8f7e6ed1d h1:mn2F9UzCk6KGa7M/d2ibLyRtBQm7n6QvbCjDe/cDWSg= k8s.io/kube-openapi v0.0.0-20180711000925-0cf8f7e6ed1d/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kubectl v0.0.0-20190720024926-08c6807d9aef h1:fPxT9B24zf/iYOCEfNoEyY7jTn5TTzB4ShWqcOgHm/g= +k8s.io/kubectl v0.0.0-20190720024926-08c6807d9aef/go.mod h1:dJLUr+4nPKRDoaWDlnVUe0K2KD5YGri1rZBzeeY6TnU= +k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/controller-runtime v0.1.10 h1:amLOmcekVdnsD1uIpmgRqfTbQWJ2qxvQkcdeFhcotn4= sigs.k8s.io/controller-runtime v0.1.10/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= sigs.k8s.io/controller-tools v0.1.11-0.20190411181648-9d55346c2bde h1:ZkaHf5rNYzIB6CB82keKMQNv7xxkqT0ylOBdfJPfi+k= sigs.k8s.io/controller-tools v0.1.11-0.20190411181648-9d55346c2bde/go.mod h1:ATWLRP3WGxuAN9HcT2LaKHReXIH+EZGzRuMHuxjXfhQ= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/testing_frameworks v0.1.0 h1:2hBE1sDhKWALoqvhi2i/mnQOFZVfWtQFtsfH0QBTI0U= sigs.k8s.io/testing_frameworks v0.1.0/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/pkg/apis/noobaa/v1alpha1/backingstore_types.go b/pkg/apis/noobaa/v1alpha1/backingstore_types.go new file mode 100644 index 000000000..f7c571c60 --- /dev/null +++ b/pkg/apis/noobaa/v1alpha1/backingstore_types.go @@ -0,0 +1,49 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Note 1: Run "operator-sdk generate k8s" to regenerate code after modifying this file +// Note 2: Add custom validation using kubebuilder tags: https://book.kubebuilder.io/reference/generating-crd.html + +func init() { + SchemeBuilder.Register(&BackingStore{}, &BackingStoreList{}) +} + +// BackingStore is the Schema for the backingstores API +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status +type BackingStore struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BackingStoreSpec `json:"spec,omitempty"` + Status BackingStoreStatus `json:"status,omitempty"` +} + +// BackingStoreList contains a list of BackingStore +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type BackingStoreList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BackingStore `json:"items"` +} + +// BackingStoreSpec defines the desired state of BackingStore +// +k8s:openapi-gen=true +type BackingStoreSpec struct { + Type StoreType `json:"type"` +} + +type StoreType string + +const ( + StoreTypeS3 StoreType = "aws-s3" +) + +// BackingStoreStatus defines the observed state of BackingStore +// +k8s:openapi-gen=true +type BackingStoreStatus struct { +} diff --git a/pkg/apis/noobaa/v1alpha1/bucketclass_types.go b/pkg/apis/noobaa/v1alpha1/bucketclass_types.go new file mode 100644 index 000000000..50abc8e68 --- /dev/null +++ b/pkg/apis/noobaa/v1alpha1/bucketclass_types.go @@ -0,0 +1,42 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Note 1: Run "operator-sdk generate k8s" to regenerate code after modifying this file +// Note 2: Add custom validation using kubebuilder tags: https://book.kubebuilder.io/reference/generating-crd.html + +func init() { + SchemeBuilder.Register(&BucketClass{}, &BucketClassList{}) +} + +// BucketClass is the Schema for the bucketclasses API +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status +type BucketClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BucketClassSpec `json:"spec,omitempty"` + Status BucketClassStatus `json:"status,omitempty"` +} + +// BucketClassList contains a list of BucketClass +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type BucketClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BucketClass `json:"items"` +} + +// BucketClassSpec defines the desired state of BucketClass +// +k8s:openapi-gen=true +type BucketClassSpec struct { +} + +// BucketClassStatus defines the observed state of BucketClass +// +k8s:openapi-gen=true +type BucketClassStatus struct { +} diff --git a/pkg/apis/noobaa/v1alpha1/noobaa_types.go b/pkg/apis/noobaa/v1alpha1/noobaa_types.go index 5f1744533..02f78a93a 100644 --- a/pkg/apis/noobaa/v1alpha1/noobaa_types.go +++ b/pkg/apis/noobaa/v1alpha1/noobaa_types.go @@ -1,13 +1,13 @@ package v1alpha1 -// Note 1: Run "operator-sdk generate k8s" to regenerate code after modifying this file -// Note 2: Add custom validation using kubebuilder tags: https://book.kubebuilder.io/reference/generating-crd.html - import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// Note 1: Run "operator-sdk generate k8s" to regenerate code after modifying this file +// Note 2: Add custom validation using kubebuilder tags: https://book.kubebuilder.io/reference/generating-crd.html + func init() { SchemeBuilder.Register(&NooBaa{}, &NooBaaList{}) } @@ -60,7 +60,11 @@ type NooBaaSpec struct { // Image (optional) overrides the default image for server pods // +optional - Image string `json:"image,omitempty"` + Image *string `json:"image,omitempty"` + + // ImagePullSecret (optional) sets a pull secret for the system image + // +optional + ImagePullSecret *corev1.LocalObjectReference `json:"imagePullSecret,omitempty"` // StorageClassName (optional) overrides the default StorageClass // for the PVC that the operator creates, this affects where the @@ -81,6 +85,13 @@ type NooBaaStatus struct { // Phase is a simple, high-level summary of where the System is in its lifecycle Phase SystemPhase `json:"phase"` + // Current service state of the noobaa system. + // Based on: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + Conditions []SystemCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + // ActualImage is set to report which image the operator is using ActualImage string `json:"actualImage"` @@ -109,6 +120,9 @@ const ( // SystemPhaseCreating means the operator is creating the resources on the cluster SystemPhaseCreating SystemPhase = "Creating" + // SystemPhaseWaitingToConnect means the operator is waiting to connect to the pods and services it created + SystemPhaseWaitingToConnect SystemPhase = "WaitingToConnect" + // SystemPhaseConfiguring means the operator is configuring the as requested SystemPhaseConfiguring SystemPhase = "Configuring" @@ -116,6 +130,34 @@ const ( SystemPhaseReady SystemPhase = "Ready" ) +// SystemCondition contains details for the current condition of this system. +type SystemCondition struct { + // Type is the type of the condition. + Type SystemConditionType `json:"type"` + // Status is the status of the condition. + Status SystemConditionStatus `json:"status"` + // Last time we probed the condition. + // +optional + LastProbeTime metav1.Time `json:"lastProbeTime,omitempty"` + // Last time the condition transitioned from one status to another. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // Unique, one-word, CamelCase reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty"` + // Human-readable message indicating details about last transition. + // +optional + Message string `json:"message,omitempty"` +} + +type SystemConditionType string +type SystemConditionStatus SystemPhase + +// These are the valid conditions types and statuses: +const ( + SystemPhaseCond SystemConditionType = "Phase" +) + // AccountsStatus is the status info of admin account type AccountsStatus struct { Admin UserStatus `json:"admin"` diff --git a/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go index fb6cc7e6a..3c3cb51ea 100644 --- a/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go @@ -1,10 +1,11 @@ // +build !ignore_autogenerated -// Code generated by deepcopy-gen. DO NOT EDIT. +// Code generated by operator-sdk. DO NOT EDIT. package v1alpha1 import ( + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -25,6 +26,192 @@ func (in *AccountsStatus) DeepCopy() *AccountsStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackingStore) DeepCopyInto(out *BackingStore) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackingStore. +func (in *BackingStore) DeepCopy() *BackingStore { + if in == nil { + return nil + } + out := new(BackingStore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BackingStore) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackingStoreList) DeepCopyInto(out *BackingStoreList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BackingStore, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackingStoreList. +func (in *BackingStoreList) DeepCopy() *BackingStoreList { + if in == nil { + return nil + } + out := new(BackingStoreList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BackingStoreList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackingStoreSpec) DeepCopyInto(out *BackingStoreSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackingStoreSpec. +func (in *BackingStoreSpec) DeepCopy() *BackingStoreSpec { + if in == nil { + return nil + } + out := new(BackingStoreSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackingStoreStatus) DeepCopyInto(out *BackingStoreStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackingStoreStatus. +func (in *BackingStoreStatus) DeepCopy() *BackingStoreStatus { + if in == nil { + return nil + } + out := new(BackingStoreStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketClass) DeepCopyInto(out *BucketClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketClass. +func (in *BucketClass) DeepCopy() *BucketClass { + if in == nil { + return nil + } + out := new(BucketClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BucketClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketClassList) DeepCopyInto(out *BucketClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BucketClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketClassList. +func (in *BucketClassList) DeepCopy() *BucketClassList { + if in == nil { + return nil + } + out := new(BucketClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BucketClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketClassSpec) DeepCopyInto(out *BucketClassSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketClassSpec. +func (in *BucketClassSpec) DeepCopy() *BucketClassSpec { + if in == nil { + return nil + } + out := new(BucketClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketClassStatus) DeepCopyInto(out *BucketClassStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketClassStatus. +func (in *BucketClassStatus) DeepCopy() *BucketClassStatus { + if in == nil { + return nil + } + out := new(BucketClassStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NooBaa) DeepCopyInto(out *NooBaa) { *out = *in @@ -89,6 +276,16 @@ func (in *NooBaaList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NooBaaSpec) DeepCopyInto(out *NooBaaSpec) { *out = *in + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(string) + **out = **in + } + if in.ImagePullSecret != nil { + in, out := &in.ImagePullSecret, &out.ImagePullSecret + *out = new(v1.LocalObjectReference) + **out = **in + } if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName *out = new(string) @@ -110,6 +307,13 @@ func (in *NooBaaSpec) DeepCopy() *NooBaaSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NooBaaStatus) DeepCopyInto(out *NooBaaStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]SystemCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } out.Accounts = in.Accounts in.Services.DeepCopyInto(&out.Services) return @@ -189,6 +393,24 @@ func (in *ServicesStatus) DeepCopy() *ServicesStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SystemCondition) DeepCopyInto(out *SystemCondition) { + *out = *in + in.LastProbeTime.DeepCopyInto(&out.LastProbeTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SystemCondition. +func (in *SystemCondition) DeepCopy() *SystemCondition { + if in == nil { + return nil + } + out := new(SystemCondition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserStatus) DeepCopyInto(out *UserStatus) { *out = *in diff --git a/pkg/apis/noobaa/v1alpha1/zz_generated.openapi.go b/pkg/apis/noobaa/v1alpha1/zz_generated.openapi.go index d077d8742..884eeab0b 100644 --- a/pkg/apis/noobaa/v1alpha1/zz_generated.openapi.go +++ b/pkg/apis/noobaa/v1alpha1/zz_generated.openapi.go @@ -1,6 +1,4 @@ -// +build !ignore_autogenerated - -// Code generated by openapi-gen. DO NOT EDIT. +// +build ! // This file was autogenerated by openapi-gen. Do not edit it manually! @@ -13,9 +11,157 @@ import ( func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.NooBaa": schema_pkg_apis_noobaa_v1alpha1_NooBaa(ref), - "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.NooBaaSpec": schema_pkg_apis_noobaa_v1alpha1_NooBaaSpec(ref), - "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.NooBaaStatus": schema_pkg_apis_noobaa_v1alpha1_NooBaaStatus(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BackingStore": schema_pkg_apis_noobaa_v1alpha1_BackingStore(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BackingStoreSpec": schema_pkg_apis_noobaa_v1alpha1_BackingStoreSpec(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BackingStoreStatus": schema_pkg_apis_noobaa_v1alpha1_BackingStoreStatus(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BucketClass": schema_pkg_apis_noobaa_v1alpha1_BucketClass(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BucketClassSpec": schema_pkg_apis_noobaa_v1alpha1_BucketClassSpec(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BucketClassStatus": schema_pkg_apis_noobaa_v1alpha1_BucketClassStatus(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.NooBaa": schema_pkg_apis_noobaa_v1alpha1_NooBaa(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.NooBaaSpec": schema_pkg_apis_noobaa_v1alpha1_NooBaaSpec(ref), + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.NooBaaStatus": schema_pkg_apis_noobaa_v1alpha1_NooBaaStatus(ref), + } +} + +func schema_pkg_apis_noobaa_v1alpha1_BackingStore(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BackingStore is the Schema for the backingstores API", + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BackingStoreSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BackingStoreStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BackingStoreSpec", "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BackingStoreStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_noobaa_v1alpha1_BackingStoreSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BackingStoreSpec defines the desired state of BackingStore", + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"type"}, + }, + }, + Dependencies: []string{}, + } +} + +func schema_pkg_apis_noobaa_v1alpha1_BackingStoreStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BackingStoreStatus defines the observed state of BackingStore", + Properties: map[string]spec.Schema{}, + }, + }, + Dependencies: []string{}, + } +} + +func schema_pkg_apis_noobaa_v1alpha1_BucketClass(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BucketClass is the Schema for the bucketclasses API", + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BucketClassSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BucketClassStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BucketClassSpec", "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.BucketClassStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_noobaa_v1alpha1_BucketClassSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BucketClassSpec defines the desired state of BucketClass", + Properties: map[string]spec.Schema{}, + }, + }, + Dependencies: []string{}, + } +} + +func schema_pkg_apis_noobaa_v1alpha1_BucketClassStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "BucketClassStatus defines the observed state of BucketClass", + Properties: map[string]spec.Schema{}, + }, + }, + Dependencies: []string{}, } } @@ -78,6 +224,12 @@ func schema_pkg_apis_noobaa_v1alpha1_NooBaaSpec(ref common.ReferenceCallback) co Format: "", }, }, + "imagePullSecret": { + SchemaProps: spec.SchemaProps{ + Description: "ImagePullSecret (optional) sets a pull secret for the system image", + Ref: ref("k8s.io/api/core/v1.LocalObjectReference"), + }, + }, "storageClassName": { SchemaProps: spec.SchemaProps{ Description: "StorageClassName (optional) overrides the default StorageClass for the PVC that the operator creates, this affects where the system stores its database which contains system config, buckets, objects meta-data and mapping file parts to storage locations.", @@ -88,7 +240,8 @@ func schema_pkg_apis_noobaa_v1alpha1_NooBaaSpec(ref common.ReferenceCallback) co }, }, }, - Dependencies: []string{}, + Dependencies: []string{ + "k8s.io/api/core/v1.LocalObjectReference"}, } } @@ -112,6 +265,25 @@ func schema_pkg_apis_noobaa_v1alpha1_NooBaaStatus(ref common.ReferenceCallback) Format: "", }, }, + "conditions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-patch-merge-key": "type", + "x-kubernetes-patch-strategy": "merge", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Current service state of the noobaa system. Based on: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.SystemCondition"), + }, + }, + }, + }, + }, "actualImage": { SchemaProps: spec.SchemaProps{ Description: "ActualImage is set to report which image the operator is using", @@ -141,6 +313,6 @@ func schema_pkg_apis_noobaa_v1alpha1_NooBaaStatus(ref common.ReferenceCallback) }, }, Dependencies: []string{ - "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.AccountsStatus", "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.ServicesStatus"}, + "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.AccountsStatus", "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.ServicesStatus", "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1.SystemCondition"}, } } diff --git a/cmd/cli/gen/bundle.go b/pkg/bundle/bundle.go similarity index 79% rename from cmd/cli/gen/bundle.go rename to pkg/bundle/bundle.go index b24d5d231..6eeebd16e 100644 --- a/cmd/cli/gen/bundle.go +++ b/pkg/bundle/bundle.go @@ -19,17 +19,18 @@ const ( backtick = "`" backtickReplace = "` + \"`\" + `" nameRE = `[\\.,/?:;'"|\-+=~!@#$%^&*()<>{}\[\]]` - root = "../../" ) var compiledNameRE = regexp.MustCompile(nameRE) func main() { - log.Printf("CLI GEN BUNDLE: Starting ...\n") + src := os.Args[1] + out := os.Args[2] + log.Printf("GEN: Start src=%s out=%s\n", src, out) files := []string{} - err := filepath.Walk(root+"deploy/", + err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { if err == nil && !info.IsDir() { files = append(files, path) @@ -38,18 +39,18 @@ func main() { }, ) - w, err := os.Create("bundle/bundle.go") + w, err := os.Create(out) fatal(err) write(w, "package bundle\n\n") writef(w, "const Version = \"%s\"\n\n", version.Version) for _, path := range files { - name := compiledNameRE.ReplaceAllString(strings.TrimPrefix(path, root), "_") + name := compiledNameRE.ReplaceAllString(path, "_") bytes, err := ioutil.ReadFile(path) fatal(err) sha256Bytes := sha256.Sum256(bytes) sha256Hex := hex.EncodeToString(sha256Bytes[:]) - log.Printf("CLI GEN BUNDLE: Adding name:%s size:%d sha256:%s\n", + log.Printf("GEN: Adding name:%s size:%d sha256:%s\n", name, len(bytes), sha256Hex) writef(w, "const Sha256_%s = \"%s\"\n\n", name, sha256Hex) writef(w, "const File_%s = `", name) @@ -59,7 +60,7 @@ func main() { err = w.Close() fatal(err) - log.Printf("CLI GEN BUNDLE: Done.\n") + log.Printf("GEN: Done.\n") } func fatal(err error) { diff --git a/pkg/cli/bucket.go b/pkg/cli/bucket.go new file mode 100644 index 000000000..10fae8616 --- /dev/null +++ b/pkg/cli/bucket.go @@ -0,0 +1,66 @@ +package cli + +import ( + "fmt" + "os" + "strings" + + "github.com/noobaa/noobaa-operator/pkg/nb" + "github.com/noobaa/noobaa-operator/pkg/system" + "github.com/noobaa/noobaa-operator/pkg/util" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" +) + +func (cli *CLI) BucketCreate(args []string) { + if len(args) < 1 || args[0] == "" { + util.Fatal(fmt.Errorf("Expected 1st argument: bucket-name")) + } + bucketName := args[0] + nbClient := cli.GetNBClient() + _, err := nbClient.CreateBucketAPI(nb.CreateBucketParams{Name: bucketName}) + util.Fatal(err) +} + +func (cli *CLI) BucketDelete(args []string) { + if len(args) < 1 || args[0] == "" { + util.Fatal(fmt.Errorf("Expected 1st argument: bucket-name")) + } + bucketName := args[0] + nbClient := cli.GetNBClient() + _, err := nbClient.DeleteBucketAPI(nb.DeleteBucketParams{Name: bucketName}) + util.Fatal(err) +} + +func (cli *CLI) BucketList() { + nbClient := cli.GetNBClient() + list, err := nbClient.ListBucketsAPI() + util.Fatal(err) + for _, bucket := range list.Buckets { + cli.Log.Println(bucket.Name) + } +} + +func (cli *CLI) GetNBClient() nb.Client { + s := system.New(types.NamespacedName{Namespace: cli.Namespace, Name: cli.SystemName}, cli.Client, scheme.Scheme, nil) + s.Load() + + mgmtStatus := s.NooBaa.Status.Services.ServiceMgmt + if len(mgmtStatus.NodePorts) == 0 { + fmt.Println("❌ Mgmt service not ready") + os.Exit(1) + } + if s.SecretOp.StringData["auth_token"] == "" { + fmt.Println("❌ Auth token not ready") + os.Exit(1) + } + + nodePort := mgmtStatus.NodePorts[0] + nodeIP := nodePort[strings.Index(nodePort, "://")+3 : strings.LastIndex(nodePort, ":")] + nbClient := nb.NewClient(&nb.APIRouterNodePort{ + ServiceMgmt: s.ServiceMgmt, + NodeIP: nodeIP, + }) + nbClient.SetAuthToken(s.SecretOp.StringData["auth_token"]) + return nbClient +} diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go new file mode 100644 index 000000000..b98835576 --- /dev/null +++ b/pkg/cli/cli.go @@ -0,0 +1,438 @@ +package cli + +import ( + "context" + "flag" + "fmt" + "math/rand" + "time" + + "github.com/noobaa/noobaa-operator/pkg/controller" + "github.com/noobaa/noobaa-operator/pkg/system" + "github.com/noobaa/noobaa-operator/pkg/util" + "github.com/noobaa/noobaa-operator/version" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + clientcmd "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubectl/pkg/util/templates" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ASCIILogo1 is an ascii logo of noobaa +const ASCIILogo1 = ` + _ _ ______ +| \ | | | ___ \ +| \| | ___ ___ | |_/ / __ _ __ _ +| . \ |/ _ \ / _ \| ___ \/ _\ |/ _\ | +| |\ | (_) | (_) | |_/ / (_| | (_| | +\_| \_/\___/ \___/\____/ \__,_|\__,_| +` + +// ASCIILogo2 is an ascii logo of noobaa +const ASCIILogo2 = ` +# # +# /~~\___~___/~~\ # +# | | # +# \~~\__ __/~~/ # +# \\ // # +# | | # +# \~~~/ # +# # +# N O O B A A # +` + +type CLI struct { + Client client.Client + Ctx context.Context + Log *logrus.Entry + + Namespace string + SystemName string + StorageClassName string + NooBaaImage string + OperatorImage string + ImagePullSecret string + + // Commands + Cmd *cobra.Command + CmdOptions *cobra.Command + CmdVersion *cobra.Command + CmdInstall *cobra.Command + CmdUninstall *cobra.Command + CmdStatus *cobra.Command + CmdBucket *cobra.Command + CmdCrd *cobra.Command + CmdOlmHub *cobra.Command + CmdOlmLocal *cobra.Command + CmdOperator *cobra.Command + CmdSystem *cobra.Command +} + +func New() *CLI { + + logrus.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + }) + logrus.SetLevel(logrus.DebugLevel) + + rand.Seed(time.Now().UTC().UnixNano()) + + logo := ASCIILogo1 + if rand.Intn(2) == 0 { // 50% chance + logo = ASCIILogo2 + } + + cli := &CLI{ + Client: util.KubeClient(), + Ctx: context.TODO(), + Log: logrus.WithField("mod", "cli"), + + Namespace: "noobaa", //CurrentNamespace(), + SystemName: "noobaa", + StorageClassName: "", + NooBaaImage: system.ContainerImage, + OperatorImage: "noobaa/noobaa-operator:" + version.Version, + ImagePullSecret: "", + + // Root command + Cmd: &cobra.Command{ + Use: "noobaa", + Short: logo, + }, + + // Install Commands: + CmdInstall: &cobra.Command{ + Use: "install", + Short: "Install the operator and create the noobaa system", + }, + CmdUninstall: &cobra.Command{ + Use: "uninstall", + Short: "Uninstall the operator and delete the system", + }, + CmdStatus: &cobra.Command{ + Use: "status", + Short: "Status of the operator and the system", + }, + + // Manage Commands: + CmdBucket: &cobra.Command{ + Use: "bucket", + Short: "Manage noobaa buckets", + }, + + // Advanced Commands: + + CmdCrd: &cobra.Command{ + Use: "crd", + Short: "Deployment of CRDs", + }, + CmdOlmHub: &cobra.Command{ + Use: "olm-hub", + Short: "Deployment using operatorhub.io", + }, + CmdOlmLocal: &cobra.Command{ + Use: "olm-local", + Short: "Deployment using OLM", + }, + CmdOperator: &cobra.Command{ + Use: "operator", + Short: "Deployment using operator", + }, + CmdSystem: &cobra.Command{ + Use: "system", + Short: "Manage noobaa systems (create delete etc.)", + }, + + // Other Commands: + CmdOptions: &cobra.Command{ + Use: "options", + Short: "Print the list of flags inherited by all commands", + }, + CmdVersion: &cobra.Command{ + Use: "version", + Short: "Show version", + }, + } + + cli.CmdVersion.Run = ToRunnable(cli.Version) + cli.CmdInstall.Run = ToRunnable(cli.Install) + cli.CmdUninstall.Run = ToRunnable(cli.Uninstall) + cli.CmdStatus.Run = ToRunnable(cli.Status) + cli.CmdOptions.Run = ToRunnable(func() { + cli.CmdOptions.Usage() + }) + + cli.CmdBucket.AddCommand( + &cobra.Command{ + Use: "create", + Short: "Create a NooBaa bucket", + Run: ToRunnableArgs(cli.BucketCreate), + }, + &cobra.Command{ + Use: "delete", + Short: "Delete a NooBaa bucket", + Run: ToRunnableArgs(cli.BucketDelete), + }, + &cobra.Command{ + Use: "list", + Short: "List NooBaa buckets", + Run: ToRunnable(cli.BucketList), + }, + ) + + cli.CmdCrd.AddCommand( + &cobra.Command{ + Use: "create", + Short: "Create noobaa CRDs", + Run: ToRunnable(cli.CrdsCreate), + }, + &cobra.Command{ + Use: "delete", + Short: "Delete noobaa CRDs", + Run: ToRunnable(cli.CrdsDelete), + }, + &cobra.Command{ + Use: "status", + Short: "Status of noobaa CRDs", + Run: ToRunnable(cli.CrdsStatus), + }, + &cobra.Command{ + Use: "yaml", + Short: "Show bundled CRDs", + Run: ToRunnable(cli.CrdsYaml), + }, + ) + + cli.CmdOlmHub.AddCommand( + &cobra.Command{ + Use: "install", + Short: "Install noobaa-operator from operatorhub.io", + Run: ToRunnable(cli.HubInstall), + }, + &cobra.Command{ + Use: "uninstall", + Short: "Uninstall noobaa-operator from operatorhub.io", + Run: ToRunnable(cli.HubUninstall), + }, + &cobra.Command{ + Use: "status", + Short: "Status of noobaa-operator from operatorhub.io", + Run: ToRunnable(cli.HubStatus), + }, + ) + + cli.CmdOlmLocal.AddCommand( + &cobra.Command{ + Use: "install", + Short: "Install noobaa-operator", + Run: ToRunnable(cli.OperatorInstall), + }, + &cobra.Command{ + Use: "uninstall", + Short: "Uninstall noobaa-operator", + Run: ToRunnable(cli.OperatorUninstall), + }, + ) + + cli.CmdOperator.AddCommand( + &cobra.Command{ + Use: "install-local", + Short: "Install the resources needed for local operator", + Run: ToRunnable(cli.OperatorLocalInstall), + }, + &cobra.Command{ + Use: "uninstall-local", + Short: "Uninstall the resources needed for local operator", + Run: ToRunnable(cli.OperatorLocalUninstall), + }, + &cobra.Command{ + Use: "reconcile-local", + Short: "Runs a reconcile attempt like noobaa-operator", + Run: ToRunnable(cli.OperatorLocalReconcile), + }, + &cobra.Command{ + Use: "install", + Short: "Install noobaa-operator", + Run: ToRunnable(cli.OperatorInstall), + }, + &cobra.Command{ + Use: "uninstall", + Short: "Uninstall noobaa-operator", + Run: ToRunnable(cli.OperatorUninstall), + }, + &cobra.Command{ + Use: "status", + Short: "Status of a noobaa-operator", + Run: ToRunnable(cli.OperatorStatus), + }, + &cobra.Command{ + Use: "run", + Short: "Runs the noobaa-operator", + Run: ToRunnable(controller.OperatorMain), + }, + &cobra.Command{ + Use: "yaml", + Short: "Show bundled noobaa-operator yaml", + Run: ToRunnable(cli.OperatorYamls), + }, + ) + + cli.CmdSystem.AddCommand( + &cobra.Command{ + Use: "create", + Short: "Create a noobaa system", + Run: ToRunnable(cli.SystemCreate), + }, + &cobra.Command{ + Use: "delete", + Short: "Delete a noobaa system", + Run: ToRunnable(cli.SystemDelete), + }, + &cobra.Command{ + Use: "list", + Short: "List noobaa systems", + Run: ToRunnable(cli.SystemList), + }, + &cobra.Command{ + Use: "status", + Short: "Status of a noobaa system", + Run: ToRunnable(cli.SystemStatus), + }, + &cobra.Command{ + Use: "yaml", + Short: "Show bundled noobaa yaml", + Run: ToRunnable(cli.SystemYaml), + }, + ) + + flagset := cli.Cmd.PersistentFlags() + // flagset.AddFlagSet(zap.FlagSet()) + flagset.AddGoFlagSet(flag.CommandLine) + flagset.StringVarP( + &cli.Namespace, "namespace", "n", + cli.Namespace, "Target namespace", + ) + flagset.StringVarP( + &cli.SystemName, "system-name", "N", + cli.SystemName, "NooBaa system name", + ) + flagset.StringVar( + &cli.StorageClassName, "storage-class", + cli.StorageClassName, "Storage class name", + ) + flagset.StringVar( + &cli.NooBaaImage, "noobaa-image", + cli.NooBaaImage, "NooBaa image", + ) + flagset.StringVar( + &cli.OperatorImage, "operator-image", + cli.OperatorImage, "Operator image", + ) + flagset.StringVar( + &cli.ImagePullSecret, "image-pull-secret", + cli.ImagePullSecret, "Image pull secret (must be in same namespace)", + ) + + groups := templates.CommandGroups{ + { + Message: "Install:", + Commands: []*cobra.Command{ + cli.CmdInstall, + cli.CmdStatus, + cli.CmdUninstall, + }, + }, + { + Message: "Manage:", + Commands: []*cobra.Command{ + cli.CmdBucket, + }, + }, + { + Message: "Advanced:", + Commands: []*cobra.Command{ + cli.CmdCrd, + cli.CmdOlmHub, + cli.CmdOlmLocal, + cli.CmdOperator, + cli.CmdSystem, + }, + }, + } + + groups.Add(cli.Cmd) + cli.Cmd.AddCommand( + cli.CmdVersion, + cli.CmdOptions, + ) + templates.ActsAsRootCommand(cli.Cmd, []string{}, groups...) + templates.UseOptionsTemplates(cli.CmdOptions) + return cli +} + +type Runnable func(cmd *cobra.Command, args []string) + +func ToRunnable(f func()) Runnable { + return func(cmd *cobra.Command, args []string) { + f() + } +} + +func ToRunnableArgs(f func(args []string)) Runnable { + return func(cmd *cobra.Command, args []string) { + f(args) + } +} + +func ForEachCommand(cmd *cobra.Command, handler func(c *cobra.Command)) { + for _, c := range cmd.Commands() { + handler(c) + ForEachCommand(c, handler) + } +} + +func (cli *CLI) Run() { + cli.Cmd.Execute() +} + +func (cli *CLI) Version() { + fmt.Printf("version: %s\n", version.Version) + fmt.Printf("noobaa-image: %s\n", cli.NooBaaImage) + fmt.Printf("operator-image: %s\n", cli.OperatorImage) +} + +func (cli *CLI) Install() { + cli.Log.Infof("Namespace: %s", cli.Namespace) + cli.CrdsCreate() + cli.CrdsWaitReady() + cli.OperatorInstall() + cli.SystemCreate() + cli.SystemWaitReady() + cli.Status() +} + +func (cli *CLI) Uninstall() { + cli.Log.Infof("Namespace: %s", cli.Namespace) + cli.SystemDelete() + cli.OperatorUninstall() +} + +func (cli *CLI) Status() { + cli.Log.Infof("Namespace: %s", cli.Namespace) + cli.Log.Info("CRD Status:") + cli.CrdsStatus() + cli.Log.Println("Operator Status:") + cli.OperatorStatus() + cli.Log.Println("System Status:") + cli.SystemStatus() +} + +func CurrentNamespace() string { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + ns, _, err := kubeConfig.Namespace() + util.Fatal(err) + return ns +} diff --git a/pkg/cli/crd.go b/pkg/cli/crd.go new file mode 100644 index 000000000..fab45a022 --- /dev/null +++ b/pkg/cli/crd.go @@ -0,0 +1,113 @@ +package cli + +import ( + "fmt" + "os" + "time" + + "github.com/noobaa/noobaa-operator/build/_output/bundle" + "github.com/noobaa/noobaa-operator/pkg/util" + "sigs.k8s.io/controller-runtime/pkg/client" + + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/cli-runtime/pkg/printers" +) + +func (cli *CLI) CrdsCreate() { + crds := cli.loadCrds() + util.KubeCreateSkipExisting(cli.Client, crds.NooBaa) + util.KubeCreateSkipExisting(cli.Client, crds.BackingStore) + util.KubeCreateSkipExisting(cli.Client, crds.BucketClass) +} + +func (cli *CLI) CrdsDelete() { + crds := cli.loadCrds() + util.KubeDelete(cli.Client, crds.NooBaa) + util.KubeDelete(cli.Client, crds.BackingStore) + util.KubeDelete(cli.Client, crds.BucketClass) +} + +func (cli *CLI) CrdsStatus() { + crds := cli.loadCrds() + util.KubeCheck(cli.Client, crds.NooBaa) + util.KubeCheck(cli.Client, crds.BackingStore) + util.KubeCheck(cli.Client, crds.BucketClass) +} + +func (cli *CLI) CrdsWaitReady() { + crds := cli.loadCrds() + intervalSec := time.Duration(3) + util.Fatal(wait.PollImmediateInfinite(intervalSec*time.Second, func() (bool, error) { + ready, err := cli.CrdWaitReady(crds.NooBaa) + if err != nil { + return false, err + } + if !ready { + return false, nil + } + ready, err = cli.CrdWaitReady(crds.BackingStore) + if err != nil { + return false, err + } + if !ready { + return false, nil + } + ready, err = cli.CrdWaitReady(crds.BucketClass) + if err != nil { + return false, err + } + if !ready { + return false, nil + } + return true, nil + })) +} + +func (cli *CLI) CrdWaitReady(crd *apiextv1beta1.CustomResourceDefinition) (bool, error) { + err := cli.Client.Get(cli.Ctx, client.ObjectKey{Name: crd.Name}, crd) + if err != nil { + return false, err + } + for _, cond := range crd.Status.Conditions { + switch cond.Type { + case apiextv1beta1.Established: + if cond.Status == apiextv1beta1.ConditionTrue { + return true, nil + } + case apiextv1beta1.NamesAccepted: + if cond.Status == apiextv1beta1.ConditionFalse { + return false, fmt.Errorf("Name conflict: %v\n", cond.Reason) + } + } + } + cli.Log.Printf("CRD not ready") + return false, nil +} + +func (cli *CLI) CrdsYaml() { + crds := cli.loadCrds() + p := printers.YAMLPrinter{} + p.PrintObj(crds.NooBaa, os.Stdout) + fmt.Println("---") + p.PrintObj(crds.BackingStore, os.Stdout) + fmt.Println("---") + p.PrintObj(crds.BucketClass, os.Stdout) +} + +type Crds struct { + NooBaa *apiextv1beta1.CustomResourceDefinition + BackingStore *apiextv1beta1.CustomResourceDefinition + BucketClass *apiextv1beta1.CustomResourceDefinition +} + +func (cli *CLI) loadCrds() *Crds { + crds := &Crds{} + o := util.KubeObject(bundle.File_deploy_crds_noobaa_v1alpha1_noobaa_crd_yaml) + crds.NooBaa = o.(*apiextv1beta1.CustomResourceDefinition) + o = util.KubeObject(bundle.File_deploy_crds_noobaa_v1alpha1_backingstore_crd_yaml) + crds.BackingStore = o.(*apiextv1beta1.CustomResourceDefinition) + o = util.KubeObject(bundle.File_deploy_crds_noobaa_v1alpha1_bucketclass_crd_yaml) + crds.BucketClass = o.(*apiextv1beta1.CustomResourceDefinition) + return crds +} diff --git a/pkg/cli/hub.go b/pkg/cli/hub.go new file mode 100644 index 000000000..7f1109422 --- /dev/null +++ b/pkg/cli/hub.go @@ -0,0 +1,54 @@ +package cli + +import ( + "io" + "net/http" + + "github.com/noobaa/noobaa-operator/pkg/util" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" +) + +func (cli *CLI) HubInstall() { + hub := cli.loadHubConf() + for _, obj := range hub.Objects { + util.KubeCreateSkipExisting(cli.Client, obj) + } +} + +func (cli *CLI) HubUninstall() { + hub := cli.loadHubConf() + for _, obj := range hub.Objects { + util.KubeDelete(cli.Client, obj) + } +} + +func (cli *CLI) HubStatus() { + hub := cli.loadHubConf() + for _, obj := range hub.Objects { + util.KubeCheck(cli.Client, obj) + } +} + +type HubConf struct { + Objects []*unstructured.Unstructured +} + +func (cli *CLI) loadHubConf() *HubConf { + hub := &HubConf{} + req, err := http.Get("https://operatorhub.io/install/noobaa-operator.yaml") + util.Fatal(err) + decoder := yaml.NewYAMLOrJSONDecoder(req.Body, 16*1024) + for { + obj := &unstructured.Unstructured{} + err = decoder.Decode(obj) + if err == io.EOF { + break + } + util.Fatal(err) + hub.Objects = append(hub.Objects, obj) + // p := printers.YAMLPrinter{} + // p.PrintObj(obj, os.Stdout) + } + return hub +} diff --git a/pkg/cli/operator.go b/pkg/cli/operator.go new file mode 100644 index 000000000..be21122c4 --- /dev/null +++ b/pkg/cli/operator.go @@ -0,0 +1,151 @@ +package cli + +import ( + "fmt" + "os" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" + + "github.com/noobaa/noobaa-operator/build/_output/bundle" + "github.com/noobaa/noobaa-operator/pkg/system" + "github.com/noobaa/noobaa-operator/pkg/util" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/cli-runtime/pkg/printers" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func (cli *CLI) OperatorInstall() { + c := cli.loadOperatorConf() + util.KubeCreateSkipExisting(cli.Client, c.NS) + util.KubeCreateSkipExisting(cli.Client, c.SA) + util.KubeCreateSkipExisting(cli.Client, c.Role) + util.KubeCreateSkipExisting(cli.Client, c.RoleBinding) + util.KubeCreateSkipExisting(cli.Client, c.ClusterRole) + util.KubeCreateSkipExisting(cli.Client, c.ClusterRoleBinding) + util.KubeCreateSkipExisting(cli.Client, c.Deployment) +} + +func (cli *CLI) OperatorLocalInstall() { + c := cli.loadOperatorConf() + util.KubeCreateSkipExisting(cli.Client, c.NS) + util.KubeCreateSkipExisting(cli.Client, c.SA) + util.KubeCreateSkipExisting(cli.Client, c.Role) + util.KubeCreateSkipExisting(cli.Client, c.RoleBinding) + util.KubeCreateSkipExisting(cli.Client, c.ClusterRole) + util.KubeCreateSkipExisting(cli.Client, c.ClusterRoleBinding) +} + +func (cli *CLI) OperatorUninstall() { + c := cli.loadOperatorConf() + util.KubeDelete(cli.Client, c.NS) + util.KubeDelete(cli.Client, c.SA) + util.KubeDelete(cli.Client, c.Role) + util.KubeDelete(cli.Client, c.RoleBinding) + util.KubeDelete(cli.Client, c.ClusterRole) + util.KubeDelete(cli.Client, c.ClusterRoleBinding) + util.KubeDelete(cli.Client, c.Deployment) +} + +func (cli *CLI) OperatorLocalUninstall() { + c := cli.loadOperatorConf() + util.KubeDelete(cli.Client, c.NS) + util.KubeDelete(cli.Client, c.SA) + util.KubeDelete(cli.Client, c.Role) + util.KubeDelete(cli.Client, c.RoleBinding) + util.KubeDelete(cli.Client, c.ClusterRole) + util.KubeDelete(cli.Client, c.ClusterRoleBinding) +} + +func (cli *CLI) OperatorStatus() { + c := cli.loadOperatorConf() + util.KubeCheck(cli.Client, c.NS) + util.KubeCheck(cli.Client, c.SA) + util.KubeCheck(cli.Client, c.Role) + util.KubeCheck(cli.Client, c.RoleBinding) + util.KubeCheck(cli.Client, c.ClusterRole) + util.KubeCheck(cli.Client, c.ClusterRoleBinding) + util.KubeCheck(cli.Client, c.Deployment) +} + +func (cli *CLI) OperatorLocalReconcile() { + intervalSec := time.Duration(3) + util.Fatal(wait.PollImmediateInfinite(intervalSec*time.Second, func() (bool, error) { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: cli.Namespace, + Name: cli.SystemName, + }, + } + res, err := system.New(req.NamespacedName, cli.Client, scheme.Scheme, nil).Reconcile() + if err != nil { + return false, err + } + if res.Requeue || res.RequeueAfter != 0 { + cli.Log.Printf("\nRetrying in %d seconds\n", intervalSec) + return false, nil + } + return true, nil + })) +} + +func (cli *CLI) OperatorYamls() { + c := cli.loadOperatorConf() + p := printers.YAMLPrinter{} + p.PrintObj(c.NS, os.Stdout) + fmt.Println("---") + p.PrintObj(c.SA, os.Stdout) + fmt.Println("---") + p.PrintObj(c.Role, os.Stdout) + fmt.Println("---") + p.PrintObj(c.RoleBinding, os.Stdout) + fmt.Println("---") + p.PrintObj(c.ClusterRole, os.Stdout) + fmt.Println("---") + p.PrintObj(c.ClusterRoleBinding, os.Stdout) + fmt.Println("---") + p.PrintObj(c.Deployment, os.Stdout) +} + +type OperatorConf struct { + NS *corev1.Namespace + SA *corev1.ServiceAccount + Role *rbacv1.Role + RoleBinding *rbacv1.RoleBinding + ClusterRole *rbacv1.ClusterRole + ClusterRoleBinding *rbacv1.ClusterRoleBinding + Deployment *appsv1.Deployment +} + +func (cli *CLI) loadOperatorConf() *OperatorConf { + c := &OperatorConf{} + c.NS = util.KubeObject(bundle.File_deploy_namespace_yaml).(*corev1.Namespace) + c.SA = util.KubeObject(bundle.File_deploy_service_account_yaml).(*corev1.ServiceAccount) + c.Role = util.KubeObject(bundle.File_deploy_role_yaml).(*rbacv1.Role) + c.RoleBinding = util.KubeObject(bundle.File_deploy_role_binding_yaml).(*rbacv1.RoleBinding) + c.ClusterRole = util.KubeObject(bundle.File_deploy_cluster_role_yaml).(*rbacv1.ClusterRole) + c.ClusterRoleBinding = util.KubeObject(bundle.File_deploy_cluster_role_binding_yaml).(*rbacv1.ClusterRoleBinding) + c.Deployment = util.KubeObject(bundle.File_deploy_operator_yaml).(*appsv1.Deployment) + + c.NS.Name = cli.Namespace + c.SA.Namespace = cli.Namespace + c.Role.Namespace = cli.Namespace + c.RoleBinding.Namespace = cli.Namespace + c.ClusterRole.Namespace = cli.Namespace + c.ClusterRoleBinding.Name = c.ClusterRoleBinding.Name + "-" + cli.Namespace + for i := range c.ClusterRoleBinding.Subjects { + c.ClusterRoleBinding.Subjects[i].Namespace = cli.Namespace + } + c.Deployment.Namespace = cli.Namespace + c.Deployment.Spec.Template.Spec.Containers[0].Image = cli.OperatorImage + if cli.ImagePullSecret != "" { + c.Deployment.Spec.Template.Spec.ImagePullSecrets = + []corev1.LocalObjectReference{{Name: cli.ImagePullSecret}} + } + return c +} diff --git a/pkg/cli/system.go b/pkg/cli/system.go new file mode 100644 index 000000000..cd90221e2 --- /dev/null +++ b/pkg/cli/system.go @@ -0,0 +1,145 @@ +package cli + +import ( + "fmt" + "os" + "time" + + "github.com/noobaa/noobaa-operator/build/_output/bundle" + nbv1 "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1" + "github.com/noobaa/noobaa-operator/pkg/system" + "github.com/noobaa/noobaa-operator/pkg/util" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (cli *CLI) SystemCreate() { + sys := cli.loadSystemDefaults() + util.KubeCreateSkipExisting(cli.Client, sys) +} + +func (cli *CLI) SystemDelete() { + cli.Log.Println("Delete: Starting") + sys := &nbv1.NooBaa{ + ObjectMeta: metav1.ObjectMeta{ + Name: cli.SystemName, + Namespace: cli.Namespace, + }, + } + util.KubeDelete(cli.Client, sys) + cli.Log.Println("Delete: Done") +} + +func (cli *CLI) SystemList() { + list := nbv1.NooBaaList{} + util.Fatal(cli.Client.List(cli.Ctx, nil, &list)) + cli.Log.Println("Namespace, Name, Image") + for _, n := range list.Items { + cli.Log.Printf("%s, %s, %s\n", n.Namespace, n.Name, n.Status.Phase) + } +} + +func (cli *CLI) SystemStatus() { + s := system.New(types.NamespacedName{Namespace: cli.Namespace, Name: cli.SystemName}, cli.Client, scheme.Scheme, nil) + s.Load() + + // sys := cli.loadSystemDefaults() + // util.KubeCheck(cli.Client, sys) + cli.Log.Printf("System Phase = \"%s\"\n", s.NooBaa.Status.Phase) + if s.NooBaa.Status.Phase == nbv1.SystemPhaseReady { + secretRef := s.NooBaa.Status.Accounts.Admin.SecretRef + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretRef.Name, + Namespace: secretRef.Namespace, + }, + } + util.KubeCheck(cli.Client, secret) + + cli.Log.Println("") + cli.Log.Println("#------------------#") + cli.Log.Println("#- Mgmt Addresses -#") + cli.Log.Println("#------------------#") + cli.Log.Println("") + + cli.Log.Println("ExternalDNS :", s.NooBaa.Status.Services.ServiceMgmt.ExternalDNS) + cli.Log.Println("ExternalIP :", s.NooBaa.Status.Services.ServiceMgmt.ExternalIP) + cli.Log.Println("NodePorts :", s.NooBaa.Status.Services.ServiceMgmt.NodePorts) + cli.Log.Println("InternalDNS :", s.NooBaa.Status.Services.ServiceMgmt.InternalDNS) + cli.Log.Println("InternalIP :", s.NooBaa.Status.Services.ServiceMgmt.InternalIP) + cli.Log.Println("PodPorts :", s.NooBaa.Status.Services.ServiceMgmt.PodPorts) + + cli.Log.Println("") + cli.Log.Println("#----------------#") + cli.Log.Println("#- S3 Addresses -#") + cli.Log.Println("#----------------#") + cli.Log.Println("") + + cli.Log.Println("ExternalDNS :", s.NooBaa.Status.Services.ServiceS3.ExternalDNS) + cli.Log.Println("ExternalIP :", s.NooBaa.Status.Services.ServiceS3.ExternalIP) + cli.Log.Println("NodePorts :", s.NooBaa.Status.Services.ServiceS3.NodePorts) + cli.Log.Println("InternalDNS :", s.NooBaa.Status.Services.ServiceS3.InternalDNS) + cli.Log.Println("InternalIP :", s.NooBaa.Status.Services.ServiceS3.InternalIP) + cli.Log.Println("PodPorts :", s.NooBaa.Status.Services.ServiceS3.PodPorts) + + cli.Log.Println("") + cli.Log.Println("#---------------#") + cli.Log.Println("#- Credentials -#") + cli.Log.Println("#---------------#") + cli.Log.Println("") + for key, value := range secret.Data { + cli.Log.Printf("%s: %s\n", key, string(value)) + } + cli.Log.Println("") + } +} + +func (cli *CLI) SystemYaml() { + sys := cli.loadSystemDefaults() + p := printers.YAMLPrinter{} + p.PrintObj(sys, os.Stdout) +} + +func (cli *CLI) loadSystemDefaults() *nbv1.NooBaa { + sys := util.KubeObject(bundle.File_deploy_crds_noobaa_v1alpha1_noobaa_cr_yaml).(*nbv1.NooBaa) + sys.Namespace = cli.Namespace + sys.Name = cli.SystemName + if cli.NooBaaImage != "" { + image := cli.NooBaaImage + sys.Spec.Image = &image + } + if cli.ImagePullSecret != "" { + sys.Spec.ImagePullSecret = &corev1.LocalObjectReference{Name: cli.ImagePullSecret} + } + if cli.StorageClassName != "" { + sc := cli.StorageClassName + sys.Spec.StorageClassName = &sc + } + return sys +} + +func (cli *CLI) SystemWaitReady() { + intervalSec := time.Duration(3) + util.Fatal(wait.PollImmediateInfinite(intervalSec*time.Second, func() (bool, error) { + sys := &nbv1.NooBaa{} + err := cli.Client.Get(cli.Ctx, client.ObjectKey{Namespace: cli.Namespace, Name: cli.SystemName}, sys) + if err != nil { + return false, err + } + if sys.Status.Phase == nbv1.SystemPhaseReady { + cli.Log.Printf("✅ System Phase is \"%s\".\n", sys.Status.Phase) + return true, nil + } + if sys.Status.Phase == nbv1.SystemPhaseRejected { + return false, fmt.Errorf("❌ System Phase is \"%s\". describe noobaa for more information", sys.Status.Phase) + } + cli.Log.Printf("⏳ System Phase is \"%s\". Waiting for it to be ready ...\n", sys.Status.Phase) + return false, nil + })) +} diff --git a/pkg/controller/add_backingstore.go b/pkg/controller/add_backingstore.go new file mode 100644 index 000000000..94901c5b7 --- /dev/null +++ b/pkg/controller/add_backingstore.go @@ -0,0 +1,10 @@ +package controller + +import ( + "github.com/noobaa/noobaa-operator/pkg/controller/backingstore" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, backingstore.Add) +} diff --git a/pkg/controller/add_bucketclass.go b/pkg/controller/add_bucketclass.go new file mode 100644 index 000000000..7f50d16a0 --- /dev/null +++ b/pkg/controller/add_bucketclass.go @@ -0,0 +1,10 @@ +package controller + +import ( + "github.com/noobaa/noobaa-operator/pkg/controller/bucketclass" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, bucketclass.Add) +} diff --git a/pkg/controller/backingstore/backingstore_controller.go b/pkg/controller/backingstore/backingstore_controller.go new file mode 100644 index 000000000..d4f95d957 --- /dev/null +++ b/pkg/controller/backingstore/backingstore_controller.go @@ -0,0 +1,39 @@ +package backingstore + +import ( + nbv1 "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// Add creates a Controller and adds it to the Manager. +// The Manager will set fields on the Controller and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + + // Create a controller that runs reconcile on noobaa backing store + + c, err := controller.New("noobaa-controller", mgr, controller.Options{ + MaxConcurrentReconciles: 1, + Reconciler: reconcile.Func( + func(req reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + }), + }) + if err != nil { + return err + } + + // Watch for changes on resources to trigger reconcile + + primaryHandler := &handler.EnqueueRequestForObject{} + + err = c.Watch(&source.Kind{Type: &nbv1.BackingStore{}}, primaryHandler) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/controller/bucketclass/bucketclass_controller.go b/pkg/controller/bucketclass/bucketclass_controller.go new file mode 100644 index 000000000..0e9a64b45 --- /dev/null +++ b/pkg/controller/bucketclass/bucketclass_controller.go @@ -0,0 +1,39 @@ +package bucketclass + +import ( + nbv1 "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// Add creates a Controller and adds it to the Manager. +// The Manager will set fields on the Controller and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + + // Create a controller that runs reconcile on noobaa bucket class + + c, err := controller.New("noobaa-controller", mgr, controller.Options{ + MaxConcurrentReconciles: 1, + Reconciler: reconcile.Func( + func(req reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + }), + }) + if err != nil { + return err + } + + // Watch for changes on resources to trigger reconcile + + primaryHandler := &handler.EnqueueRequestForObject{} + + err = c.Watch(&source.Kind{Type: &nbv1.BucketClass{}}, primaryHandler) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/controller/manager.go b/pkg/controller/manager.go new file mode 100644 index 000000000..ff1a38ebb --- /dev/null +++ b/pkg/controller/manager.go @@ -0,0 +1,101 @@ +package controller + +import ( + "context" + "fmt" + "os" + "runtime" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "github.com/noobaa/noobaa-operator/pkg/apis" + + "github.com/operator-framework/operator-sdk/pkg/k8sutil" + "github.com/operator-framework/operator-sdk/pkg/leader" + "github.com/operator-framework/operator-sdk/pkg/metrics" + "github.com/operator-framework/operator-sdk/pkg/restmapper" + sdkVersion "github.com/operator-framework/operator-sdk/version" + "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/signals" +) + +// Change below variables to serve metrics on different host or port. +var ( + metricsHost = "0.0.0.0" + metricsPort int32 = 8383 +) + +func printVersion() { + logrus.Infof("Go Version: %s", runtime.Version()) + logrus.Infof("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH) + logrus.Infof("Version of operator-sdk: %v", sdkVersion.Version) +} + +func OperatorMain() { + + printVersion() + + namespace, err := k8sutil.GetWatchNamespace() + if err != nil { + logrus.WithError(err).Errorln("Failed to get watch namespace") + os.Exit(1) + } + + // Get a config to talk to the apiserver + cfg, err := config.GetConfig() + if err != nil { + logrus.WithError(err).Errorln("Failed to get config") + os.Exit(1) + } + + ctx := context.TODO() + + // Become the leader before proceeding + err = leader.Become(ctx, "noobaa-operator-lock") + if err != nil { + logrus.WithError(err).Errorln("Failed to become leader") + os.Exit(1) + } + + // Create a new Cmd to provide shared dependencies and start components + mgr, err := manager.New(cfg, manager.Options{ + Namespace: namespace, + MapperProvider: restmapper.NewDynamicRESTMapper, + MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), + }) + if err != nil { + logrus.WithError(err).Errorln("Failed to create manager") + os.Exit(1) + } + + logrus.Info("Registering Components.") + + // Setup Scheme for all resources + if err := apis.AddToScheme(mgr.GetScheme()); err != nil { + logrus.WithError(err).Errorln("Failed AddToScheme") + os.Exit(1) + } + + // Setup all Controllers + if err := AddToManager(mgr); err != nil { + logrus.WithError(err).Errorln("Failed AddToManager") + os.Exit(1) + } + + // Create Service object to expose the metrics port. + _, err = metrics.ExposeMetricsPort(ctx, metricsPort) + if err != nil { + logrus.WithError(err).Warningln("Failed ExposeMetricsPort") + } + + logrus.Info("Starting the Operator.") + + // Start the manager + if err := mgr.Start(signals.SetupSignalHandler()); err != nil { + logrus.WithError(err).Errorln("Manager exited non-zero") + os.Exit(1) + } +} diff --git a/pkg/controller/noobaa/noobaa_controller.go b/pkg/controller/noobaa/noobaa_controller.go index e6e411b53..8054a4781 100644 --- a/pkg/controller/noobaa/noobaa_controller.go +++ b/pkg/controller/noobaa/noobaa_controller.go @@ -1,812 +1,61 @@ package noobaa import ( - "bytes" - "context" - "crypto/rand" - "encoding/base64" - "fmt" - "strings" - "text/template" - "time" + "github.com/noobaa/noobaa-operator/pkg/system" - "k8s.io/apimachinery/pkg/labels" - - dockerref "github.com/docker/distribution/reference" - "github.com/go-logr/logr" - semver "github.com/hashicorp/go-version" nbv1 "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1" - "github.com/noobaa/noobaa-operator/pkg/nb" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/source" ) -const ( - // ContainerImageOrg is the org of the default image url - ContainerImageOrg = "noobaa" - // ContainerImageRepo is the repo of the default image url - ContainerImageRepo = "noobaa-core" - // ContainerImageTag is the tag of the default image url - ContainerImageTag = "4.0" - // ContainerImageConstraintSemver is the contraints of supported image versions - ContainerImageConstraintSemver = ">=4, <5" - // ContainerImageName is the default image name without the tag/version - ContainerImageName = ContainerImageOrg + "/" + ContainerImageRepo - // ContainerImage is the full default image url - ContainerImage = ContainerImageName + ":" + ContainerImageTag - - // AdminAccountEmail is the default email used for admin account - AdminAccountEmail = "admin@noobaa.io" -) - -var ( - // ContainerImageConstraint is the instantiated semver contraints used for image verification - ContainerImageConstraint, _ = semver.NewConstraint(ContainerImageConstraintSemver) - // NooBaaType is and empty noobaa struct used for passing the object type - NooBaaType = &nbv1.NooBaa{} - - logger = logf.Log.WithName("noobaa") -) - // Add creates a Controller and adds it to the Manager. // The Manager will set fields on the Controller and Start it when the Manager is Started. func Add(mgr manager.Manager) error { - // Create a new controller + // Create a controller that runs reconcile on noobaa system + c, err := controller.New("noobaa-controller", mgr, controller.Options{ MaxConcurrentReconciles: 1, Reconciler: reconcile.Func( func(req reconcile.Request) (reconcile.Result, error) { - return NewReconciler(mgr, req).Reconcile() + return system.New( + req.NamespacedName, + mgr.GetClient(), + mgr.GetScheme(), + mgr.GetRecorder("noobaa-operator"), + ).Reconcile() }), }) if err != nil { return err } - // Watch for changes to primary resource - err = c.Watch(&source.Kind{Type: NooBaaType}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } - - // Watch for changes to secondary resources and requeue the owner resource - ownerHandler := &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: NooBaaType, - } - err = c.Watch(&source.Kind{Type: &appsv1.StatefulSet{}}, ownerHandler) - if err != nil { - return err - } - err = c.Watch(&source.Kind{Type: &corev1.Service{}}, ownerHandler) - if err != nil { - return err - } - err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, ownerHandler) - if err != nil { - return err - } - - return nil -} - -// Reconciler is the context for reconciling noobaa. -// It is created per every reconcile call and keeps state between calls of the reconcile flow. -type Reconciler struct { - Mgr manager.Manager - Request reconcile.Request - Ctx context.Context - Logger logr.Logger - Recorder record.EventRecorder - - NooBaa *nbv1.NooBaa - CoreApp *appsv1.StatefulSet - ServiceMgmt *corev1.Service - ServiceS3 *corev1.Service - SecretOp *corev1.Secret - SecretAdmin *corev1.Secret - NBClient nb.Client -} - -// NewReconciler initializes a new noobaa reconciler -func NewReconciler(mgr manager.Manager, req reconcile.Request) *Reconciler { - return &Reconciler{ - Mgr: mgr, - Request: req, - Ctx: context.TODO(), - Logger: logger.WithValues("name", req.Namespace+"/"+req.Name), - Recorder: mgr.GetRecorder("noobaa-operator"), - NooBaa: &nbv1.NooBaa{}, - } -} - -// Reconcile reads that state of the cluster for a System object, -// and makes changes based on the state read and what is in the System.Spec. -// The Controller will requeue the Request to be processed again if the returned error is non-nil or -// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. -func (r *Reconciler) Reconcile() (reconcile.Result, error) { + // Watch for changes on resources to trigger reconcile - log := r.Logger.WithName("Reconcile") - log.Info("Start ...") + primaryHandler := &handler.EnqueueRequestForObject{} + secondaryHandler := &handler.EnqueueRequestForOwner{IsController: true, OwnerType: &nbv1.NooBaa{}} - err := r.Mgr.GetClient().Get(r.Ctx, r.Request.NamespacedName, r.NooBaa) - if errors.IsNotFound(err) { - log.Info("Ignoring request on deleted noobaa") - return reconcile.Result{}, nil - } + err = c.Watch(&source.Kind{Type: &nbv1.NooBaa{}}, primaryHandler) if err != nil { - log.Error(err, "Failed getting noobaa") - return reconcile.Result{}, err - } - - err = CombineErrors( - r.ReconcileSystem(), - r.UpdateSystemStatus(), - ) - if err == nil { - log.Info("✅ Done") - return reconcile.Result{}, nil - } - if !IsPersistentError(err) { - log.Error(err, "⏳ Temporary Error") - return reconcile.Result{RequeueAfter: 2 * time.Second}, nil - } - log.Error(err, "❌ Persistent Error") - return reconcile.Result{}, nil -} - -// ReconcileSystem runs the reconcile flow and populates System.Status. -func (r *Reconciler) ReconcileSystem() error { - - r.NooBaa.Status.Phase = nbv1.SystemPhaseVerifying - - if err := r.VerifySpecImage(); err != nil { - return err - } - - r.NooBaa.Status.Phase = nbv1.SystemPhaseCreating - - if err := CombineErrors( - r.ReconcileServiceMgmt(), - r.ReconcileServiceS3(), - r.ReconcileCoreApp(), - r.UpdateServiceStatus(r.ServiceMgmt, &r.NooBaa.Status.Services.ServiceMgmt, "mgmt-https"), - r.UpdateServiceStatus(r.ServiceS3, &r.NooBaa.Status.Services.ServiceS3, "s3-https"), - ); err != nil { - return err - } - - r.NooBaa.Status.Phase = nbv1.SystemPhaseConfiguring - - if err := r.SetupNooBaaClient(); err != nil { - return err - } - - if err := r.SetupNooBaaSystem(); err != nil { return err } - - if err := r.SetupAdminAccount(); err != nil { - return err - } - - r.NooBaa.Status.Phase = nbv1.SystemPhaseReady - - return r.Complete() -} - -// UpdateSystemStatus will update the System.Status field to match the observed status -func (r *Reconciler) UpdateSystemStatus() error { - log := r.Logger.WithName("UpdateSystemStatus") - log.Info("Updating noobaa status") - r.NooBaa.Status.ObservedGeneration = r.NooBaa.Generation - return r.Mgr.GetClient().Status().Update(r.Ctx, r.NooBaa) -} - -// VerifySpecImage checks the System.Spec.Image property, -// and sets System.Status.ActualImage -func (r *Reconciler) VerifySpecImage() error { - - log := r.Logger.WithName("VerifySpecImage") - - specImage := ContainerImage - if r.NooBaa.Spec.Image != "" { - specImage = r.NooBaa.Spec.Image - } - - // Parse the image spec as a docker image url - imageRef, err := dockerref.Parse(specImage) - - // If the image cannot be parsed log the incident and mark as persistent error - // since we don't need to retry until the spec is updated. - if err != nil { - log.Error(err, "Invalid image", "image", specImage) - r.Recorder.Eventf(r.NooBaa, corev1.EventTypeWarning, - "BadImage", `Invalid image requested "%s"`, specImage) - r.NooBaa.Status.Phase = nbv1.SystemPhaseRejected - return NewPersistentError(err) - } - - // Get the image name and tag - imageName := "" - imageTag := "" - switch image := imageRef.(type) { - case dockerref.NamedTagged: - log.Info("Parsed image (NamedTagged)", "image", image) - imageName = image.Name() - imageTag = image.Tag() - case dockerref.Tagged: - log.Info("Parsed image (Tagged)", "image", image) - imageTag = image.Tag() - case dockerref.Named: - log.Info("Parsed image (Named)", "image", image) - imageName = image.Name() - default: - log.Info("Parsed image (unstructured)", "image", image) - } - - if imageName == ContainerImageName { - version, err := semver.NewVersion(imageTag) - if err == nil { - log.Info("Parsed version from image tag", "tag", imageTag, "version", version) - if !ContainerImageConstraint.Check(version) { - log.Error(nil, "Unsupported image version", - "image", imageRef, "contraints", ContainerImageConstraint) - r.Recorder.Eventf(r.NooBaa, corev1.EventTypeWarning, - "BadImage", `Unsupported image version requested "%s" not matching constraints "%s"`, - imageRef, ContainerImageConstraint) - r.NooBaa.Status.Phase = nbv1.SystemPhaseRejected - return NewPersistentError(fmt.Errorf(`Unsupported image version "%+v"`, imageRef)) - } - } else { - log.Info("Using custom image version", "image", imageRef, "contraints", ContainerImageConstraint) - r.Recorder.Eventf(r.NooBaa, corev1.EventTypeNormal, - "CustomImage", `Custom image version requested "%s", I hope you know what you're doing ...`, imageRef) - } - } else { - log.Info("Using custom image name", "image", imageRef, "default", ContainerImageName) - r.Recorder.Eventf(r.NooBaa, corev1.EventTypeNormal, - "CustomImage", `Custom image requested "%s", I hope you know what you're doing ...`, imageRef) - } - - // Set ActualImage to be updated in the noobaa status - r.NooBaa.Status.ActualImage = specImage - return nil -} - -// ReconcileObject is a generic call to reconcile a kubernetes object -// desiredFunc can be passed to modify the object before create/update. -// Currently we ignore enforcing a desired state, but it might be needed on upgrades. -func (r *Reconciler) ReconcileObject(obj metav1.Object, desiredFunc func()) error { - - log := r.Logger.WithName("ReconcileObject") - - r.Own(obj) - - op, err := controllerutil.CreateOrUpdate( - r.Ctx, r.Mgr.GetClient(), obj.(runtime.Object), - func(obj runtime.Object) error { - if desiredFunc != nil { - desiredFunc() - } - return nil - }, - ) + err = c.Watch(&source.Kind{Type: &appsv1.StatefulSet{}}, secondaryHandler) if err != nil { - log.Error(err, "Failed") return err } - - log.Info("Done", "op", op) - return nil -} - -// ReconcileServiceMgmt create or update the mgmt service. -func (r *Reconciler) ReconcileServiceMgmt() error { - - r.ServiceMgmt = &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: r.Request.Namespace, - Name: r.Request.Name + "-mgmt", - Labels: map[string]string{"app": "noobaa"}, - Annotations: map[string]string{ - "prometheus.io/scrape": "true", - "prometheus.io/scheme": "http", - "prometheus.io/port": "8080", - }, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - Selector: map[string]string{"noobaa-mgmt": r.Request.Name}, - Ports: []corev1.ServicePort{ - {Port: 8080, Name: "mgmt"}, - {Port: 8443, Name: "mgmt-https"}, - {Port: 8444, Name: "md-https"}, - {Port: 8445, Name: "bg-https"}, - {Port: 8446, Name: "hosted-agents-https"}, - {Port: 80, TargetPort: intstr.FromInt(6001), Name: "s3"}, - {Port: 443, TargetPort: intstr.FromInt(6443), Name: "s3-https"}, - }, - }, - } - return r.ReconcileObject(r.ServiceMgmt, nil) -} - -// ReconcileServiceS3 create or update the s3 service. -func (r *Reconciler) ReconcileServiceS3() error { - - r.ServiceS3 = &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: r.Request.Namespace, - Name: "s3", // TODO: handle collision in namespace - Labels: map[string]string{"app": "noobaa"}, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - Selector: map[string]string{"noobaa-s3": r.Request.Name}, - Ports: []corev1.ServicePort{ - {Port: 80, TargetPort: intstr.FromInt(6001), Name: "s3"}, - {Port: 443, TargetPort: intstr.FromInt(6443), Name: "s3-https"}, - }, - }, - } - return r.ReconcileObject(r.ServiceS3, nil) -} - -// ReconcileCoreApp create or update the core statefulset. -func (r *Reconciler) ReconcileCoreApp() error { - - ns := r.Request.Namespace - name := r.Request.Name - coreAppName := name + "-core" - serviceAccountName := "noobaa-operator" // TODO do we use the same SA? - replicas := int32(1) - image := r.NooBaa.Status.ActualImage - storageClassName := r.NooBaa.Spec.StorageClassName - - r.CoreApp = &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: coreAppName, - Labels: map[string]string{"app": "noobaa"}, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"noobaa-core": name}, - }, - ServiceName: r.ServiceMgmt.Name, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: coreAppName, - Labels: map[string]string{ - "app": "noobaa", - "noobaa-core": name, - "noobaa-mgmt": name, - "noobaa-s3": name, - }, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: serviceAccountName, - Containers: []corev1.Container{{ - Name: coreAppName, - Image: image, - ImagePullPolicy: corev1.PullIfNotPresent, - VolumeMounts: []corev1.VolumeMount{ - {MountPath: "/data", Name: "datadir"}, - {MountPath: "/log", Name: "logdir"}, - }, - Env: []corev1.EnvVar{ - {Name: "CONTAINER_PLATFORM", Value: "KUBERNETES"}, - }, - Ports: []corev1.ContainerPort{ - {ContainerPort: 80}, - {ContainerPort: 443}, - {ContainerPort: 8080}, - {ContainerPort: 8443}, - {ContainerPort: 8444}, - {ContainerPort: 8445}, - {ContainerPort: 8446}, - {ContainerPort: 60100}, - }, - // # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes - // # ready when s3 port is open - ReadinessProbe: &corev1.Probe{ - TimeoutSeconds: 5, - Handler: corev1.Handler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(6001), - }, - }, - }, - // # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("4"), - corev1.ResourceMemory: resource.MustParse("8Gi"), - }, - }, - }}, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: "datadir", - Labels: map[string]string{"app": "noobaa"}, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - StorageClassName: storageClassName, - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("50Gi"), - }, - }, - }, - }, { - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: "logdir", - Labels: map[string]string{"app": "noobaa"}, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - StorageClassName: storageClassName, - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("10Gi"), - }, - }, - }, - }, - }, - }, - } - return r.ReconcileObject(r.CoreApp, func() { - r.CoreApp.Spec.Template.Spec.Containers[0].Image = image - }) -} - -// UpdateServiceStatus populates the status of a service by detecting all of its addresses -func (r *Reconciler) UpdateServiceStatus(srv *corev1.Service, status *nbv1.ServiceStatus, portName string) error { - - log := r.Logger.WithName("UpdateServiceStatus").WithValues("service", srv.Name) - *status = nbv1.ServiceStatus{} - servicePort := nb.FindPortByName(srv, portName) - proto := "http" - if strings.HasSuffix(portName, "https") { - proto = "https" - } - - // Node IP:Port - // Pod IP:Port - - pods := corev1.PodList{} - podsListOptions := &client.ListOptions{ - Namespace: r.Request.Namespace, - LabelSelector: labels.SelectorFromSet(srv.Spec.Selector), - } - err := r.Mgr.GetClient().List(r.Ctx, podsListOptions, &pods) - if err == nil { - for _, pod := range pods.Items { - if pod.Status.Phase == corev1.PodRunning { - if pod.Status.HostIP != "" { - status.NodePorts = append( - status.NodePorts, - fmt.Sprintf("%s://%s:%d", proto, pod.Status.HostIP, servicePort.NodePort), - ) - } - if pod.Status.PodIP != "" { - status.PodPorts = append( - status.PodPorts, - fmt.Sprintf("%s://%s:%s", proto, pod.Status.PodIP, servicePort.TargetPort.String()), - ) - } - } - } - } - - // Cluster IP:Port (of the service) - - if srv.Spec.ClusterIP != "" { - status.InternalIP = append( - status.InternalIP, - fmt.Sprintf("%s://%s:%d", proto, srv.Spec.ClusterIP, servicePort.Port), - ) - status.InternalDNS = append( - status.InternalDNS, - fmt.Sprintf("%s://%s.%s:%d", proto, srv.Name, srv.Namespace, servicePort.Port), - ) - } - - // LoadBalancer IP:Port (of the service) - - if srv.Status.LoadBalancer.Ingress != nil { - for _, lb := range srv.Status.LoadBalancer.Ingress { - if lb.IP != "" { - status.ExternalIP = append( - status.ExternalIP, - fmt.Sprintf("%s://%s:%d", proto, lb.IP, servicePort.Port), - ) - } - if lb.Hostname != "" { - status.ExternalDNS = append( - status.ExternalDNS, - fmt.Sprintf("%s://%s:%d", proto, lb.Hostname, servicePort.Port), - ) - } - } - } - - // External IP:Port (of the service) - - if srv.Spec.ExternalIPs != nil { - for _, ip := range srv.Spec.ExternalIPs { - status.ExternalIP = append( - status.ExternalIP, - fmt.Sprintf("%s://%s:%d", proto, ip, servicePort.Port), - ) - } - } - - log.Info("Collected addresses", "status", status) - return nil -} - -// SetupNooBaaClient initializes the noobaa client for making calls to the server. -func (r *Reconciler) SetupNooBaaClient() error { - - // log := r.Logger.WithName("SetupNooBaaClient") - - if len(r.NooBaa.Status.Services.ServiceMgmt.NodePorts) != 0 { - nodePort := r.NooBaa.Status.Services.ServiceMgmt.NodePorts[0] - nodeIP := nodePort[strings.Index(nodePort, "://")+3 : strings.LastIndex(nodePort, ":")] - r.NBClient = nb.NewClient(&nb.APIRouterNodePort{ - ServiceMgmt: r.ServiceMgmt, - NodeIP: nodeIP, - }) - } else if len(r.NooBaa.Status.Services.ServiceMgmt.PodPorts) != 0 { - podPort := r.NooBaa.Status.Services.ServiceMgmt.PodPorts[0] - podIP := podPort[strings.Index(podPort, "://")+3 : strings.LastIndex(podPort, ":")] - r.NBClient = nb.NewClient(&nb.APIRouterPodPort{ - ServiceMgmt: r.ServiceMgmt, - PodIP: podIP, - }) - } else { - return fmt.Errorf("core pod port not ready yet") - } - - return nil -} - -// SetupNooBaaSystem creates a new system in the noobaa server if not created yet. -func (r *Reconciler) SetupNooBaaSystem() error { - - log := r.Logger.WithName("SetupNooBaaSystem") - ns := r.Request.Namespace - name := r.Request.Name - secretOpName := name + "-operator" - - r.SecretOp = &corev1.Secret{} - err := r.GetObject(secretOpName, r.SecretOp) - if err == nil { - r.NBClient.SetAuthToken(string(r.SecretOp.Data["auth_token"])) - return nil - } - if !errors.IsNotFound(err) { - log.Error(err, "Failed getting operator secret") - return err - } - - randomBytes := make([]byte, 16) - _, err = rand.Read(randomBytes) - fatal(err) - randomPassword := base64.StdEncoding.EncodeToString(randomBytes) - - res, err := r.NBClient.CreateSystemAPI(nb.CreateSystemParams{ - Name: name, - Email: AdminAccountEmail, - Password: randomPassword, - }) - + err = c.Watch(&source.Kind{Type: &corev1.Service{}}, secondaryHandler) if err != nil { return err } - r.NBClient.SetAuthToken(res.Token) - - r.SecretOp = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: secretOpName, - Labels: map[string]string{"app": "noobaa"}, - }, - Type: corev1.SecretTypeOpaque, - StringData: map[string]string{ - "system": name, - "email": AdminAccountEmail, - "password": randomPassword, - "auth_token": res.Token, - }, - } - - r.Own(r.SecretOp) - return r.Mgr.GetClient().Create(r.Ctx, r.SecretOp) -} - -// SetupAdminAccount creates the admin secret -func (r *Reconciler) SetupAdminAccount() error { - - log := r.Logger.WithName("SetupAdminAccount") - ns := r.Request.Namespace - name := r.Request.Name - secretAdminName := name + "-admin" - - r.SecretAdmin = &corev1.Secret{} - err := r.GetObject(secretAdminName, r.SecretAdmin) - if err == nil { - return nil - } - if !errors.IsNotFound(err) { - log.Error(err, "Failed getting admin secret") - return err - } - - r.SecretAdmin = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: secretAdminName, - Labels: map[string]string{"app": "noobaa"}, - }, - Type: corev1.SecretTypeOpaque, - StringData: map[string]string{ - "system": name, - "email": AdminAccountEmail, - "password": string(r.SecretOp.Data["password"]), - }, - } - - log.Info("listing accounts") - res, err := r.NBClient.ListAccountsAPI() + err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, secondaryHandler) if err != nil { return err } - for _, account := range res.Accounts { - if account.Email == AdminAccountEmail { - if len(account.AccessKeys) > 0 { - r.SecretAdmin.StringData["AWS_ACCESS_KEY_ID"] = account.AccessKeys[0].AccessKey - r.SecretAdmin.StringData["AWS_SECRET_ACCESS_KEY"] = account.AccessKeys[0].SecretKey - } - } - } - - r.Own(r.SecretAdmin) - return r.Mgr.GetClient().Create(r.Ctx, r.SecretAdmin) -} - -var readmeTemplate = template.Must(template.New("NooBaaSystem.Status.Readme").Parse(` - - Welcome to NooBaa! - ----------------- - Lets get started: - - 1. Connect to Management console: - - Read your mgmt console login information (email & password) from secret: "{{.SecretAdmin.Name}}". - - kubectl get secret {{.SecretAdmin.Name}} -n {{.SecretAdmin.Namespace}} -o json | jq '.data|map_values(@base64d)' - - Open the management console service - take External IP/DNS or Node Port or use port forwarding: - - kubectl port-forward -n {{.ServiceMgmt.Namespace}} service/{{.ServiceMgmt.Name}} 11443:8443 & - open https://localhost:11443 - - 2. Test S3 client: - - kubectl port-forward -n {{.ServiceS3.Namespace}} service/{{.ServiceS3.Name}} 10443:443 & - NOOBAA_ACCESS_KEY=$(kubectl get secret {{.SecretAdmin.Name}} -n {{.SecretAdmin.Namespace}} -o json | jq -r '.data.AWS_ACCESS_KEY_ID|@base64d') - NOOBAA_SECRET_KEY=$(kubectl get secret {{.SecretAdmin.Name}} -n {{.SecretAdmin.Namespace}} -o json | jq -r '.data.AWS_SECRET_ACCESS_KEY|@base64d') - alias s3='AWS_ACCESS_KEY_ID=$NOOBAA_ACCESS_KEY AWS_SECRET_ACCESS_KEY=$NOOBAA_SECRET_KEY aws --endpoint https://localhost:10443 --no-verify-ssl s3' - s3 ls - -`)) - -// Complete populates the noobaa status at the end of reconcile. -func (r *Reconciler) Complete() error { - - var readmeBuffer bytes.Buffer - err := readmeTemplate.Execute(&readmeBuffer, r) - if err != nil { - return err - } - r.NooBaa.Status.Readme = readmeBuffer.String() - r.NooBaa.Status.Accounts.Admin.SecretRef.Name = r.SecretAdmin.Name - r.NooBaa.Status.Accounts.Admin.SecretRef.Namespace = r.SecretAdmin.Namespace return nil } - -// GetObject gets an object by name from the request namespace. -func (r *Reconciler) GetObject(name string, obj runtime.Object) error { - return r.Mgr.GetClient().Get(r.Ctx, client.ObjectKey{Namespace: r.Request.Namespace, Name: name}, obj) -} - -// Own marks the object as owned by the noobaa controller, -// so that it will be garbage collected once noobaa is deleted. -func (r *Reconciler) Own(obj metav1.Object) { - fatal(controllerutil.SetControllerReference(r.NooBaa, obj, r.Mgr.GetScheme())) -} - -func fatal(err error) { - if err != nil { - logger.Error(err, "PANIC") - panic(err) - } -} - -// PersistentError is an error type that tells the reconcile to avoid requeueing. -type PersistentError struct { - E error -} - -// Error function makes PersistentError implement error interface -func (e *PersistentError) Error() string { return e.E.Error() } - -// assert implement error interface -var _ error = &PersistentError{} - -// NewPersistentError returns a new persistent error. -func NewPersistentError(err error) *PersistentError { - if err == nil { - panic("NewPersistentError expects non nil error") - } - return &PersistentError{E: err} -} - -// IsPersistentError checks if the provided error is persistent. -func IsPersistentError(err error) bool { - _, persistent := err.(*PersistentError) - return persistent -} - -// CombineErrors takes a list of errors and combines them to one. -// Generally it will return the first non-nil error, -// but if a persistent error is found it will be returned -// instead of non-persistent errors. -func CombineErrors(errs ...error) error { - combined := error(nil) - for _, err := range errs { - if err == nil { - continue - } - if combined == nil { - combined = err - continue - } - if IsPersistentError(err) && !IsPersistentError(combined) { - combined = err - } - } - return combined -} diff --git a/pkg/nb/api.go b/pkg/nb/api.go index df0ddb57d..a93647b6e 100644 --- a/pkg/nb/api.go +++ b/pkg/nb/api.go @@ -4,33 +4,55 @@ package nb // Client is the interface providing typed noobaa API calls type Client interface { SetAuthToken(token string) - CreateSystemAPI(CreateSystemParams) (CreateSystemReply, error) + GetAuthToken() string + + ReadAuthAPI() (ReadAuthReply, error) + ListAccountsAPI() (ListAccountsReply, error) -} + ListBucketsAPI() (ListBucketsReply, error) -// CreateSystemParams is the params of system_api.create_system() -type CreateSystemParams struct { - Name string `json:"name"` - Email string `json:"email"` - Password string `json:"password"` + CreateAuthAPI(CreateAuthParams) (CreateAuthReply, error) + CreateSystemAPI(CreateSystemParams) (CreateSystemReply, error) + CreateBucketAPI(CreateBucketParams) (CreateBucketReply, error) + + DeleteBucketAPI(DeleteBucketParams) (DeleteBucketReply, error) } -// CreateSystemReply is the reply of system_api.create_system() -type CreateSystemReply struct { - Token string `json:"token"` +////////// +// READ // +////////// + +// ReadAuthReply is the reply of auth_api.read_auth() +type ReadAuthReply struct { + Account struct { + Name string `json:"name"` + Email string `json:"email"` + IsSupport bool `json:"is_support"` + MustChangePassword bool `json:"must_change_password"` + } `json:"account"` + System struct { + Name string `json:"name"` + } `json:"system"` + AuthorizedBy string `json:"authorized_by"` + Role string `json:"role"` + Extra map[string]interface{} `json:"extra"` } -// CreateSystemAPI calls system_api.create_system() -func (c *RPCClient) CreateSystemAPI(params CreateSystemParams) (CreateSystemReply, error) { - req := RPCRequest{API: "system_api", Method: "create_system", Params: params} +// CreateAuthAPI calls auth_api.read_auth() +func (c *RPCClient) ReadAuthAPI() (ReadAuthReply, error) { + req := RPCRequest{API: "auth_api", Method: "read_auth"} res := struct { RPCResponse `json:",inline"` - Reply CreateSystemReply `json:"reply"` + Reply ReadAuthReply `json:"reply"` }{} err := c.Call(req, &res) return res.Reply, err } +////////// +// LIST // +////////// + // ListAccountsReply is the reply to account_api.list_accounts() type ListAccountsReply struct { Accounts []struct { @@ -53,3 +75,115 @@ func (c *RPCClient) ListAccountsAPI() (ListAccountsReply, error) { err := c.Call(req, &res) return res.Reply, err } + +type ListBucketsReply struct { + Buckets []struct { + Name string `json:"name"` + } `json:"buckets"` +} + +// ListBucketsAPI calls bucket_api.list_buckets() +func (c *RPCClient) ListBucketsAPI() (ListBucketsReply, error) { + req := RPCRequest{API: "bucket_api", Method: "list_buckets"} + res := struct { + RPCResponse `json:",inline"` + Reply ListBucketsReply `json:"reply"` + }{} + err := c.Call(req, &res) + return res.Reply, err +} + +//////////// +// CREATE // +//////////// + +// CreateAuthParams is the params of auth_api.create_auth() +type CreateAuthParams struct { + System string `json:"system"` + Role string `json:"role"` + Email string `json:"email"` + Password string `json:"password"` +} + +// CreateAuthReply is the reply of auth_api.create_auth() +type CreateAuthReply struct { + Token string `json:"token"` +} + +// CreateAuthAPI calls auth_api.create_auth() +func (c *RPCClient) CreateAuthAPI(params CreateAuthParams) (CreateAuthReply, error) { + req := RPCRequest{API: "auth_api", Method: "create_auth", Params: params} + res := struct { + RPCResponse `json:",inline"` + Reply CreateAuthReply `json:"reply"` + }{} + err := c.Call(req, &res) + return res.Reply, err +} + +// CreateSystemParams is the params of system_api.create_system() +type CreateSystemParams struct { + Name string `json:"name"` + Email string `json:"email"` + Password string `json:"password"` +} + +// CreateSystemReply is the reply of system_api.create_system() +type CreateSystemReply struct { + Token string `json:"token"` +} + +// CreateSystemAPI calls system_api.create_system() +func (c *RPCClient) CreateSystemAPI(params CreateSystemParams) (CreateSystemReply, error) { + req := RPCRequest{API: "system_api", Method: "create_system", Params: params} + res := struct { + RPCResponse `json:",inline"` + Reply CreateSystemReply `json:"reply"` + }{} + err := c.Call(req, &res) + return res.Reply, err +} + +// CreateBucketParams is the params of bucket_api.create_bucket() +type CreateBucketParams struct { + Name string `json:"name"` +} + +// CreateBucketReply is the reply of bucket_api.create_bucket() +type CreateBucketReply struct { +} + +// CreateBucketAPI calls bucket_api.create_bucket() +func (c *RPCClient) CreateBucketAPI(params CreateBucketParams) (CreateBucketReply, error) { + req := RPCRequest{API: "bucket_api", Method: "create_bucket", Params: params} + res := struct { + RPCResponse `json:",inline"` + Reply CreateBucketReply `json:"reply"` + }{} + err := c.Call(req, &res) + return res.Reply, err +} + +//////////// +// DELETE // +//////////// + +// DeleteBucketParams is the params of bucket_api.delete_bucket() +type DeleteBucketParams struct { + Name string `json:"name"` +} + +// DeleteBucketReply is the reply of bucket_api.delete_bucket() +type DeleteBucketReply struct { +} + +// DeleteBucketAPI calls bucket_api.delete_bucket() +func (c *RPCClient) DeleteBucketAPI(params DeleteBucketParams) (DeleteBucketReply, error) { + req := RPCRequest{API: "bucket_api", Method: "delete_bucket", Params: params} + res := struct { + RPCResponse `json:",inline"` + Reply DeleteBucketReply `json:"reply"` + }{} + err := c.Call(req, &res) + return res.Reply, err +} diff --git a/pkg/nb/rpc.go b/pkg/nb/rpc.go index f8e458db8..b84baa4b5 100644 --- a/pkg/nb/rpc.go +++ b/pkg/nb/rpc.go @@ -8,11 +8,9 @@ import ( "net/http" "time" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "github.com/sirupsen/logrus" ) -var logger = logf.Log.WithName("noobaa") - // RPCClient makes API calls to noobaa. // Requests to noobaa are plain http requests with json request and json response. type RPCClient struct { @@ -55,6 +53,9 @@ type RPCResponseIfc interface { // SetAuthToken is setting the client token for next calls func (c *RPCClient) SetAuthToken(token string) { c.AuthToken = token } +// GetAuthToken is getting the client token for next calls +func (c *RPCClient) GetAuthToken() string { return c.AuthToken } + // Response is implementing the RPCResponseIfc interface func (r *RPCResponse) Response() *RPCResponse { return r } @@ -66,7 +67,7 @@ var _ RPCResponseIfc = &RPCResponse{} var _ error = &RPCError{} // NewClient initializes an RPCClient with defaults -func NewClient(router APIRouter) *RPCClient { +func NewClient(router APIRouter) Client { return &RPCClient{ Router: router, HTTPClient: http.Client{ @@ -88,8 +89,8 @@ func (c *RPCClient) Call(req RPCRequest, res RPCResponseIfc) error { req.AuthToken = c.AuthToken } address := c.Router.GetAddress(api) - log := logger.WithName("RPC").WithValues("api", api, "method", method) - log.Info("RPC: %s.%s - Call to %s: %+v", api, method, address, req) + log := logrus.WithFields(logrus.Fields{"mod": "nb", "api": api, "method": method}) + log.Infof("RPC: %s.%s - Call to %s: %+v", api, method, address, req) reqBytes, err := json.Marshal(req) fatal(err) diff --git a/pkg/system/system.go b/pkg/system/system.go new file mode 100644 index 000000000..1e62644fb --- /dev/null +++ b/pkg/system/system.go @@ -0,0 +1,710 @@ +package system + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "strings" + "text/template" + "time" + + "github.com/noobaa/noobaa-operator/build/_output/bundle" + nbv1 "github.com/noobaa/noobaa-operator/pkg/apis/noobaa/v1alpha1" + "github.com/noobaa/noobaa-operator/pkg/nb" + "github.com/noobaa/noobaa-operator/pkg/util" + + dockerref "github.com/docker/distribution/reference" + semver "github.com/hashicorp/go-version" + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + // ContainerImageOrg is the org of the default image url + ContainerImageOrg = "noobaa" + // ContainerImageRepo is the repo of the default image url + ContainerImageRepo = "noobaa-core" + // ContainerImageTag is the tag of the default image url + ContainerImageTag = "5" + // ContainerImageConstraintSemver is the contraints of supported image versions + ContainerImageConstraintSemver = ">=5, <6" + // ContainerImageName is the default image name without the tag/version + ContainerImageName = ContainerImageOrg + "/" + ContainerImageRepo + // ContainerImage is the full default image url + ContainerImage = ContainerImageName + ":" + ContainerImageTag + + // AdminAccountEmail is the default email used for admin account + AdminAccountEmail = "admin@noobaa.io" +) + +var ( + // ContainerImageConstraint is the instantiated semver contraints used for image verification + ContainerImageConstraint, _ = semver.NewConstraint(ContainerImageConstraintSemver) + + // NooBaaType is and empty noobaa struct used for passing the object type + NooBaaType = &nbv1.NooBaa{} +) + +type System struct { + Request types.NamespacedName + Client client.Client + Scheme *runtime.Scheme + Ctx context.Context + Logger *logrus.Entry + Recorder record.EventRecorder + NBClient nb.Client + + NooBaa *nbv1.NooBaa + CoreApp *appsv1.StatefulSet + ServiceMgmt *corev1.Service + ServiceS3 *corev1.Service + SecretServer *corev1.Secret + SecretOp *corev1.Secret + SecretAdmin *corev1.Secret +} + +func New(req types.NamespacedName, client client.Client, scheme *runtime.Scheme, recorder record.EventRecorder) *System { + s := &System{ + Request: req, + Client: client, + Scheme: scheme, + Recorder: recorder, + Ctx: context.TODO(), + Logger: logrus.WithFields(logrus.Fields{"mod": "system", "system": req.Namespace + "/" + req.Name}), + NooBaa: util.KubeObject(bundle.File_deploy_crds_noobaa_v1alpha1_noobaa_cr_yaml).(*nbv1.NooBaa), + CoreApp: util.KubeObject(bundle.File_deploy_internal_statefulset_core_yaml).(*appsv1.StatefulSet), + ServiceMgmt: util.KubeObject(bundle.File_deploy_internal_service_mgmt_yaml).(*corev1.Service), + ServiceS3: util.KubeObject(bundle.File_deploy_internal_service_s3_yaml).(*corev1.Service), + SecretServer: util.KubeObject(bundle.File_deploy_internal_secret_server_yaml).(*corev1.Secret), + SecretOp: util.KubeObject(bundle.File_deploy_internal_secret_operator_yaml).(*corev1.Secret), + SecretAdmin: util.KubeObject(bundle.File_deploy_internal_secret_admin_yaml).(*corev1.Secret), + } + SecretResetStringDataFromData(s.SecretOp) + SecretResetStringDataFromData(s.SecretAdmin) + + // Set Namespace + s.NooBaa.Namespace = s.Request.Namespace + s.CoreApp.Namespace = s.Request.Namespace + s.ServiceMgmt.Namespace = s.Request.Namespace + s.ServiceS3.Namespace = s.Request.Namespace + s.SecretServer.Namespace = s.Request.Namespace + s.SecretOp.Namespace = s.Request.Namespace + s.SecretAdmin.Namespace = s.Request.Namespace + + // Set Names + s.NooBaa.Name = s.Request.Name + s.CoreApp.Name = s.Request.Name + "-core" + s.ServiceMgmt.Name = s.Request.Name + "-mgmt" + s.ServiceS3.Name = "s3" // TODO: handle collision in namespace + s.SecretServer.Name = s.Request.Name + "-server" + s.SecretOp.Name = s.Request.Name + "-operator" + s.SecretAdmin.Name = s.Request.Name + "-admin" + + return s +} + +func (s *System) Load() { + util.KubeCheck(s.Client, s.NooBaa) + util.KubeCheck(s.Client, s.CoreApp) + util.KubeCheck(s.Client, s.ServiceMgmt) + util.KubeCheck(s.Client, s.ServiceS3) + util.KubeCheck(s.Client, s.SecretServer) + util.KubeCheck(s.Client, s.SecretOp) + util.KubeCheck(s.Client, s.SecretAdmin) + SecretResetStringDataFromData(s.SecretOp) + SecretResetStringDataFromData(s.SecretAdmin) +} + +// Reconcile reads that state of the cluster for a System object, +// and makes changes based on the state read and what is in the System.Spec. +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (s *System) Reconcile() (reconcile.Result, error) { + + log := s.Logger.WithField("func", "Reconcile") + log.Info("Start ...") + + util.KubeCheck(s.Client, s.NooBaa) + if s.NooBaa.UID == "" { + log.Info("NooBaa not found or already deleted. Skip reconcile.") + return reconcile.Result{}, nil + } + + err := CombineErrors( + s.ReconcileSystem(), + s.UpdateSystemStatus(), + ) + if err == nil { + log.Info("✅ Done") + return reconcile.Result{}, nil + } + if !IsPersistentError(err) { + log.Error(err, "⏳ Temporary Error") + return reconcile.Result{RequeueAfter: 2 * time.Second}, nil + } + log.Error(err, "❌ Persistent Error") + return reconcile.Result{}, nil +} + +// ReconcileSystem runs the reconcile flow and populates System.Status. +func (s *System) ReconcileSystem() error { + + s.NooBaa.Status.Phase = nbv1.SystemPhaseVerifying + + if err := s.CheckSpecImage(); err != nil { + return err + } + + s.NooBaa.Status.Phase = nbv1.SystemPhaseCreating + + if err := s.ReconcileSecretServer(); err != nil { + return err + } + if err := s.ReconcileObject(s.CoreApp, s.SetDesiredCoreApp); err != nil { + return err + } + if err := s.ReconcileObject(s.ServiceMgmt, s.SetDesiredServiceMgmt); err != nil { + return err + } + if err := s.ReconcileObject(s.ServiceS3, s.SetDesiredServiceS3); err != nil { + return err + } + + s.CheckServiceStatus(s.ServiceMgmt, &s.NooBaa.Status.Services.ServiceMgmt, "mgmt-https") + s.CheckServiceStatus(s.ServiceS3, &s.NooBaa.Status.Services.ServiceS3, "s3-https") + + s.NooBaa.Status.Phase = nbv1.SystemPhaseWaitingToConnect + + if err := s.InitNooBaaClient(); err != nil { + return err + } + + s.NooBaa.Status.Phase = nbv1.SystemPhaseConfiguring + + if err := s.ReconcileSecretOp(); err != nil { + return err + } + + if err := s.ReconcileSecretAdmin(); err != nil { + return err + } + + s.NooBaa.Status.Phase = nbv1.SystemPhaseReady + + return s.Complete() +} + +func (s *System) ReconcileSecretServer() error { + util.KubeCheck(s.Client, s.SecretServer) + SecretResetStringDataFromData(s.SecretServer) + + if s.SecretServer.StringData["jwt"] == "" { + s.SecretServer.StringData["jwt"] = randomString(16) + } + if s.SecretServer.StringData["server_secret"] == "" { + s.SecretServer.StringData["server_secret"] = randomString(16) + } + util.KubeCreateSkipExisting(s.Client, s.SecretServer) + return nil +} + +func (s *System) SetDesiredCoreApp() { + s.CoreApp.Spec.Template.Labels["noobaa-core"] = s.Request.Name + s.CoreApp.Spec.Template.Labels["noobaa-mgmt"] = s.Request.Name + s.CoreApp.Spec.Template.Labels["noobaa-s3"] = s.Request.Name + s.CoreApp.Spec.Selector.MatchLabels["noobaa-core"] = s.Request.Name + s.CoreApp.Spec.ServiceName = s.ServiceMgmt.Name + + podSpec := &s.CoreApp.Spec.Template.Spec + podSpec.ServiceAccountName = "noobaa-operator" // TODO do we use the same SA? + for i := range podSpec.InitContainers { + if podSpec.InitContainers[i].Image == "NOOBAA_IMAGE" { + podSpec.InitContainers[i].Image = s.NooBaa.Status.ActualImage + } + } + for i := range podSpec.Containers { + if podSpec.Containers[i].Image == "NOOBAA_IMAGE" { + podSpec.Containers[i].Image = s.NooBaa.Status.ActualImage + } + } + if s.NooBaa.Spec.ImagePullSecret == nil { + podSpec.ImagePullSecrets = + []corev1.LocalObjectReference{} + } else { + podSpec.ImagePullSecrets = + []corev1.LocalObjectReference{*s.NooBaa.Spec.ImagePullSecret} + } + for i := range s.CoreApp.Spec.VolumeClaimTemplates { + pvc := &s.CoreApp.Spec.VolumeClaimTemplates[i] + pvc.Spec.StorageClassName = s.NooBaa.Spec.StorageClassName + + // TODO we want to own the PVC's by NooBaa system but get errors on openshift: + // Warning FailedCreate 56s statefulset-controller + // create Pod noobaa-core-0 in StatefulSet noobaa-core failed error: + // Failed to create PVC mongo-datadir-noobaa-core-0: + // persistentvolumeclaims "mongo-datadir-noobaa-core-0" is forbidden: + // cannot set blockOwnerDeletion if an ownerReference refers to a resource + // you can't set finalizers on: , , ... + + // s.Own(pvc) + } +} + +func (s *System) SetDesiredServiceMgmt() { + s.ServiceMgmt.Spec.Selector["noobaa-mgmt"] = s.Request.Name +} + +func (s *System) SetDesiredServiceS3() { + s.ServiceS3.Spec.Selector["noobaa-s3"] = s.Request.Name +} + +// CheckSpecImage checks the System.Spec.Image property, +// and sets System.Status.ActualImage +func (s *System) CheckSpecImage() error { + + log := s.Logger.WithField("func", "CheckSpecImage") + + specImage := ContainerImage + if s.NooBaa.Spec.Image != nil { + specImage = *s.NooBaa.Spec.Image + } + + // Parse the image spec as a docker image url + imageRef, err := dockerref.Parse(specImage) + + // If the image cannot be parsed log the incident and mark as persistent error + // since we don't need to retry until the spec is updated. + if err != nil { + log.Error(err, "Invalid image", "image", specImage) + if s.Recorder != nil { + s.Recorder.Eventf(s.NooBaa, corev1.EventTypeWarning, + "BadImage", `Invalid image requested "%s"`, specImage) + } + s.NooBaa.Status.Phase = nbv1.SystemPhaseRejected + return NewPersistentError(err) + } + + // Get the image name and tag + imageName := "" + imageTag := "" + switch image := imageRef.(type) { + case dockerref.NamedTagged: + log.Info("Parsed image (NamedTagged)", "image", image) + imageName = image.Name() + imageTag = image.Tag() + case dockerref.Tagged: + log.Info("Parsed image (Tagged)", "image", image) + imageTag = image.Tag() + case dockerref.Named: + log.Info("Parsed image (Named)", "image", image) + imageName = image.Name() + default: + log.Info("Parsed image (unstructured)", "image", image) + } + + if imageName == ContainerImageName { + version, err := semver.NewVersion(imageTag) + if err == nil { + log.Info("Parsed version from image tag", "tag", imageTag, "version", version) + if !ContainerImageConstraint.Check(version) { + log.Error(nil, "Unsupported image version", + "image", imageRef, "contraints", ContainerImageConstraint) + if s.Recorder != nil { + s.Recorder.Eventf(s.NooBaa, corev1.EventTypeWarning, + "BadImage", `Unsupported image version requested "%s" not matching constraints "%s"`, + imageRef, ContainerImageConstraint) + } + s.NooBaa.Status.Phase = nbv1.SystemPhaseRejected + return NewPersistentError(fmt.Errorf(`Unsupported image version "%+v"`, imageRef)) + } + } else { + log.Info("Using custom image version", "image", imageRef, "contraints", ContainerImageConstraint) + if s.Recorder != nil { + s.Recorder.Eventf(s.NooBaa, corev1.EventTypeNormal, + "CustomImage", `Custom image version requested "%s", I hope you know what you're doing ...`, imageRef) + } + } + } else { + log.Info("Using custom image name", "image", imageRef, "default", ContainerImageName) + if s.Recorder != nil { + s.Recorder.Eventf(s.NooBaa, corev1.EventTypeNormal, + "CustomImage", `Custom image requested "%s", I hope you know what you're doing ...`, imageRef) + } + } + + // Set ActualImage to be updated in the noobaa status + s.NooBaa.Status.ActualImage = specImage + return nil +} + +// CheckServiceStatus populates the status of a service by detecting all of its addresses +func (s *System) CheckServiceStatus(srv *corev1.Service, status *nbv1.ServiceStatus, portName string) { + + log := s.Logger.WithField("func", "CheckServiceStatus").WithField("service", srv.Name) + *status = nbv1.ServiceStatus{} + servicePort := nb.FindPortByName(srv, portName) + proto := "http" + if strings.HasSuffix(portName, "https") { + proto = "https" + } + + // Node IP:Port + // Pod IP:Port + pods := corev1.PodList{} + podsListOptions := &client.ListOptions{ + Namespace: s.Request.Namespace, + LabelSelector: labels.SelectorFromSet(srv.Spec.Selector), + } + err := s.Client.List(s.Ctx, podsListOptions, &pods) + if err == nil { + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + if pod.Status.HostIP != "" { + status.NodePorts = append( + status.NodePorts, + fmt.Sprintf("%s://%s:%d", proto, pod.Status.HostIP, servicePort.NodePort), + ) + } + if pod.Status.PodIP != "" { + status.PodPorts = append( + status.PodPorts, + fmt.Sprintf("%s://%s:%s", proto, pod.Status.PodIP, servicePort.TargetPort.String()), + ) + } + } + } + } + + // Cluster IP:Port (of the service) + if srv.Spec.ClusterIP != "" { + status.InternalIP = append( + status.InternalIP, + fmt.Sprintf("%s://%s:%d", proto, srv.Spec.ClusterIP, servicePort.Port), + ) + status.InternalDNS = append( + status.InternalDNS, + fmt.Sprintf("%s://%s.%s:%d", proto, srv.Name, srv.Namespace, servicePort.Port), + ) + } + + // LoadBalancer IP:Port (of the service) + if srv.Status.LoadBalancer.Ingress != nil { + for _, lb := range srv.Status.LoadBalancer.Ingress { + if lb.IP != "" { + status.ExternalIP = append( + status.ExternalIP, + fmt.Sprintf("%s://%s:%d", proto, lb.IP, servicePort.Port), + ) + } + if lb.Hostname != "" { + status.ExternalDNS = append( + status.ExternalDNS, + fmt.Sprintf("%s://%s:%d", proto, lb.Hostname, servicePort.Port), + ) + } + } + } + + // External IP:Port (of the service) + if srv.Spec.ExternalIPs != nil { + for _, ip := range srv.Spec.ExternalIPs { + status.ExternalIP = append( + status.ExternalIP, + fmt.Sprintf("%s://%s:%d", proto, ip, servicePort.Port), + ) + } + } + + log.Info("Collected addresses", "status", status) +} + +// InitNooBaaClient initializes the noobaa client for making calls to the server. +func (s *System) InitNooBaaClient() error { + + if len(s.NooBaa.Status.Services.ServiceMgmt.NodePorts) == 0 { + return fmt.Errorf("core pod port not ready yet") + } + + nodePort := s.NooBaa.Status.Services.ServiceMgmt.NodePorts[0] + nodeIP := nodePort[strings.Index(nodePort, "://")+3 : strings.LastIndex(nodePort, ":")] + s.NBClient = nb.NewClient(&nb.APIRouterNodePort{ + ServiceMgmt: s.ServiceMgmt, + NodeIP: nodeIP, + }) + s.NBClient.SetAuthToken(s.SecretOp.StringData["auth_token"]) + _, err := s.NBClient.ReadAuthAPI() + return err + + // if len(s.NooBaa.Status.Services.ServiceMgmt.PodPorts) != 0 { + // podPort := s.NooBaa.Status.Services.ServiceMgmt.PodPorts[0] + // podIP := podPort[strings.Index(podPort, "://")+3 : strings.LastIndex(podPort, ":")] + // s.NBClient = nb.NewClient(&nb.APIRouterPodPort{ + // ServiceMgmt: s.ServiceMgmt, + // PodIP: podIP, + // }) + // s.NBClient.SetAuthToken(s.SecretOp.StringData["auth_token"]) + // return nil + // } + +} + +// ReconcileSecretOp creates a new system in the noobaa server if not created yet. +func (s *System) ReconcileSecretOp() error { + + // log := s.Logger.WithName("ReconcileSecretOp") + util.KubeCheck(s.Client, s.SecretOp) + SecretResetStringDataFromData(s.SecretOp) + + if s.SecretOp.StringData["auth_token"] != "" { + return nil + } + + if s.SecretOp.StringData["email"] == "" { + s.SecretOp.StringData["email"] = AdminAccountEmail + } + + if s.SecretOp.StringData["password"] == "" { + s.SecretOp.StringData["password"] = randomString(16) + err := s.Client.Create(s.Ctx, s.SecretOp) + if err != nil { + return err + } + } + + res, err := s.NBClient.CreateAuthAPI(nb.CreateAuthParams{ + System: s.Request.Name, + Role: "admin", + Email: s.SecretOp.StringData["email"], + Password: s.SecretOp.StringData["password"], + }) + if err == nil { + s.SecretOp.StringData["auth_token"] = res.Token + } else { + res, err := s.NBClient.CreateSystemAPI(nb.CreateSystemParams{ + Name: s.Request.Name, + Email: s.SecretOp.StringData["email"], + Password: s.SecretOp.StringData["password"], + }) + if err != nil { + return err + } + s.SecretOp.StringData["auth_token"] = res.Token + } + s.NBClient.SetAuthToken(s.SecretOp.StringData["auth_token"]) + return s.Client.Update(s.Ctx, s.SecretOp) +} + +// ReconcileSecretAdmin creates the admin secret +func (s *System) ReconcileSecretAdmin() error { + + log := s.Logger.WithField("func", "ReconcileSecretAdmin") + + util.KubeCheck(s.Client, s.SecretAdmin) + SecretResetStringDataFromData(s.SecretAdmin) + + ns := s.Request.Namespace + name := s.Request.Name + secretAdminName := name + "-admin" + + s.SecretAdmin = &corev1.Secret{} + err := s.GetObject(secretAdminName, s.SecretAdmin) + if err == nil { + return nil + } + if !errors.IsNotFound(err) { + log.Error(err, "Failed getting admin secret") + return err + } + + s.SecretAdmin = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: secretAdminName, + Labels: map[string]string{"app": "noobaa"}, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{ + "system": name, + "email": AdminAccountEmail, + "password": string(s.SecretOp.Data["password"]), + }, + } + + log.Info("listing accounts") + res, err := s.NBClient.ListAccountsAPI() + if err != nil { + return err + } + for _, account := range res.Accounts { + if account.Email == AdminAccountEmail { + if len(account.AccessKeys) > 0 { + s.SecretAdmin.StringData["AWS_ACCESS_KEY_ID"] = account.AccessKeys[0].AccessKey + s.SecretAdmin.StringData["AWS_SECRET_ACCESS_KEY"] = account.AccessKeys[0].SecretKey + } + } + } + + s.Own(s.SecretAdmin) + return s.Client.Create(s.Ctx, s.SecretAdmin) +} + +var readmeTemplate = template.Must(template.New("NooBaaSystem.Status.Readme").Parse(` + + Welcome to NooBaa! + ----------------- + + Lets get started: + + 1. Connect to Management console: + + Read your mgmt console login information (email & password) from secret: "{{.SecretAdmin.Name}}". + + kubectl get secret {{.SecretAdmin.Name}} -n {{.SecretAdmin.Namespace}} -o json | jq '.data|map_values(@base64d)' + + Open the management console service - take External IP/DNS or Node Port or use port forwarding: + + kubectl port-forward -n {{.ServiceMgmt.Namespace}} service/{{.ServiceMgmt.Name}} 11443:8443 & + open https://localhost:11443 + + 2. Test S3 client: + + kubectl port-forward -n {{.ServiceS3.Namespace}} service/{{.ServiceS3.Name}} 10443:443 & + NOOBAA_ACCESS_KEY=$(kubectl get secret {{.SecretAdmin.Name}} -n {{.SecretAdmin.Namespace}} -o json | jq -r '.data.AWS_ACCESS_KEY_ID|@base64d') + NOOBAA_SECRET_KEY=$(kubectl get secret {{.SecretAdmin.Name}} -n {{.SecretAdmin.Namespace}} -o json | jq -r '.data.AWS_SECRET_ACCESS_KEY|@base64d') + alias s3='AWS_ACCESS_KEY_ID=$NOOBAA_ACCESS_KEY AWS_SECRET_ACCESS_KEY=$NOOBAA_SECRET_KEY aws --endpoint https://localhost:10443 --no-verify-ssl s3' + s3 ls + +`)) + +// Complete populates the noobaa status at the end of reconcile. +func (s *System) Complete() error { + + var readmeBuffer bytes.Buffer + err := readmeTemplate.Execute(&readmeBuffer, s) + if err != nil { + return err + } + s.NooBaa.Status.Readme = readmeBuffer.String() + s.NooBaa.Status.Accounts.Admin.SecretRef.Name = s.SecretAdmin.Name + s.NooBaa.Status.Accounts.Admin.SecretRef.Namespace = s.SecretAdmin.Namespace + return nil +} + +func (s *System) UpdateSystemStatus() error { + log := s.Logger.WithField("func", "UpdateSystemStatus") + log.Info("Updating noobaa status") + s.NooBaa.Status.ObservedGeneration = s.NooBaa.Generation + return s.Client.Status().Update(s.Ctx, s.NooBaa) +} + +func (s *System) Own(obj metav1.Object) { + util.Fatal(controllerutil.SetControllerReference(s.NooBaa, obj, s.Scheme)) +} + +// GetObject gets an object by name from the request namespace. +func (s *System) GetObject(name string, obj runtime.Object) error { + return s.Client.Get(s.Ctx, client.ObjectKey{Namespace: s.Request.Namespace, Name: name}, obj) +} + +// ReconcileObject is a generic call to reconcile a kubernetes object +// desiredFunc can be passed to modify the object before create/update. +// Currently we ignore enforcing a desired state, but it might be needed on upgrades. +func (s *System) ReconcileObject(obj runtime.Object, desiredFunc func()) error { + + kind := obj.GetObjectKind().GroupVersionKind().Kind + objMeta, _ := meta.Accessor(obj) + log := s.Logger.WithField("func", "ReconcileObject").WithField("kind", kind).WithField("name", objMeta.GetName()) + + s.Own(objMeta) + + op, err := controllerutil.CreateOrUpdate( + s.Ctx, s.Client, obj.(runtime.Object), + func(obj runtime.Object) error { + if desiredFunc != nil { + desiredFunc() + } + return nil + }, + ) + if err != nil { + log.Error(err, "Failed") + return err + } + + log.Info("Done", "op", op) + return nil +} + +// PersistentError is an error type that tells the reconcile to avoid requeueing. +type PersistentError struct { + E error +} + +// Error function makes PersistentError implement error interface +func (e *PersistentError) Error() string { return e.E.Error() } + +// assert implement error interface +var _ error = &PersistentError{} + +// NewPersistentError returns a new persistent error. +func NewPersistentError(err error) *PersistentError { + if err == nil { + panic("NewPersistentError expects non nil error") + } + return &PersistentError{E: err} +} + +// IsPersistentError checks if the provided error is persistent. +func IsPersistentError(err error) bool { + _, persistent := err.(*PersistentError) + return persistent +} + +// CombineErrors takes a list of errors and combines them to one. +// Generally it will return the first non-nil error, +// but if a persistent error is found it will be returned +// instead of non-persistent errors. +func CombineErrors(errs ...error) error { + combined := error(nil) + for _, err := range errs { + if err == nil { + continue + } + if combined == nil { + combined = err + continue + } + if IsPersistentError(err) && !IsPersistentError(combined) { + combined = err + } + } + return combined +} + +func SecretResetStringDataFromData(secret *corev1.Secret) { + secret.StringData = map[string]string{} + for key, val := range secret.Data { + secret.StringData[key] = string(val) + } + secret.Data = map[string][]byte{} +} + +func randomString(numBytes int) string { + randomBytes := make([]byte, numBytes) + _, err := rand.Read(randomBytes) + util.Fatal(err) + return base64.StdEncoding.EncodeToString(randomBytes) +} diff --git a/pkg/util/fastmapper.go b/pkg/util/fastmapper.go new file mode 100644 index 000000000..286d1df7d --- /dev/null +++ b/pkg/util/fastmapper.go @@ -0,0 +1,136 @@ +package util + +import ( + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/discovery" + "k8s.io/client-go/restmapper" +) + +type APIGroupFilterFunc func(*metav1.APIGroup) bool + +// FastRESTMapper loads the mapper data from the server with filter and concurrency +// and rediscovers the mapping on meta.NoKindMatchError errors +// See https://github.com/kubernetes-sigs/controller-runtime/issues/537 +type FastRESTMapper struct { + Discovery discovery.DiscoveryInterface + Filter APIGroupFilterFunc + Mapper meta.RESTMapper +} + +func NewFastRESTMapper(dc discovery.DiscoveryInterface, filter APIGroupFilterFunc) meta.RESTMapper { + return &FastRESTMapper{ + Discovery: dc, + Filter: filter, + Mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), + } +} + +func (m *FastRESTMapper) ReDiscover() error { + groups, err := m.Discovery.ServerGroups() + if err != nil { + return err + } + wg := wait.Group{} + totalCount := 0 + filterCount := 0 + var grs []*restmapper.APIGroupResources + for _, group := range groups.Groups { + filtered := m.Filter(&group) + logrus.Tracef("Group: %s %v", group.Name, filtered) + totalCount++ + if !filtered { + continue + } + filterCount++ + gr := &restmapper.APIGroupResources{ + Group: group, + VersionedResources: make(map[string][]metav1.APIResource), + } + grs = append(grs, gr) + wg.Start(func() { m.ReDiscoverGroupResources(gr) }) + } + wg.Wait() + logrus.Tracef("Filtered %d/%d", filterCount, totalCount) + m.Mapper = restmapper.NewDiscoveryRESTMapper(grs) + return nil +} + +func (m *FastRESTMapper) ReDiscoverGroupResources(gr *restmapper.APIGroupResources) error { + var errResult error + for _, version := range gr.Group.Versions { + resources, err := m.Discovery.ServerResourcesForGroupVersion(version.GroupVersion) + if err != nil { + errResult = err + } + gr.VersionedResources[version.Version] = resources.APIResources + } + return errResult +} + +func (m *FastRESTMapper) ReDiscoverOnError(err error) bool { + _, retry := err.(*meta.NoKindMatchError) + if retry { + m.ReDiscover() + } + return retry +} + +func (m *FastRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { + res, err := m.Mapper.KindFor(resource) + if m.ReDiscoverOnError(err) { + res, err = m.Mapper.KindFor(resource) + } + return res, err +} + +func (m *FastRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { + res, err := m.Mapper.KindsFor(resource) + if m.ReDiscoverOnError(err) { + res, err = m.Mapper.KindsFor(resource) + } + return res, err +} + +func (m *FastRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { + res, err := m.Mapper.ResourceFor(input) + if m.ReDiscoverOnError(err) { + res, err = m.Mapper.ResourceFor(input) + } + return res, err +} + +func (m *FastRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { + res, err := m.Mapper.ResourcesFor(input) + if m.ReDiscoverOnError(err) { + res, err = m.Mapper.ResourcesFor(input) + } + return res, err +} + +func (m *FastRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + res, err := m.Mapper.RESTMapping(gk, versions...) + if m.ReDiscoverOnError(err) { + res, err = m.Mapper.RESTMapping(gk, versions...) + } + return res, err +} + +func (m *FastRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { + res, err := m.Mapper.RESTMappings(gk, versions...) + if m.ReDiscoverOnError(err) { + res, err = m.Mapper.RESTMappings(gk, versions...) + } + return res, err +} + +func (m *FastRESTMapper) ResourceSingularizer(resource string) (string, error) { + res, err := m.Mapper.ResourceSingularizer(resource) + if m.ReDiscoverOnError(err) { + res, err = m.Mapper.ResourceSingularizer(resource) + } + return res, err +} diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..987880513 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,142 @@ +package util + +import ( + "context" + "strings" + + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +var ctx = context.TODO() + +func KubeConfig() *rest.Config { + return config.GetConfigOrDie() +} + +func KubeClient() client.Client { + config := KubeConfig() + mapper := meta.NewLazyRESTMapperLoader(func() (meta.RESTMapper, error) { + dc := discovery.NewDiscoveryClientForConfigOrDie(config) + return NewFastRESTMapper(dc, func(g *metav1.APIGroup) bool { + if g.Name == "" || + g.Name == "apps" || + g.Name == "noobaa.io" || + g.Name == "operator.openshift.io" || + g.Name == "cloudcredential.openshift.io" || + strings.HasSuffix(g.Name, ".k8s.io") { + return true + } + return false + }), nil + }) + c, err := client.New(config, client.Options{Mapper: mapper, Scheme: scheme.Scheme}) + Fatal(err) + return c +} + +func KubeObject(text string) runtime.Object { + // Decode text (yaml/json) to kube api object + deserializer := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer() + obj, group, err := deserializer.Decode([]byte(text), nil, nil) + // obj, group, err := scheme.Codecs.UniversalDecoder().Decode([]byte(text), nil, nil) + Fatal(err) + // not sure if really needed, but set it anyway + obj.GetObjectKind().SetGroupVersionKind(*group) + return obj +} + +func KubeApply(c client.Client, obj runtime.Object) bool { + metaObj, _ := meta.Accessor(obj) + objKey := client.ObjectKey{Namespace: metaObj.GetNamespace(), Name: metaObj.GetName()} + gvk := obj.GetObjectKind().GroupVersionKind() + clone := obj.DeepCopyObject() + err := c.Get(ctx, objKey, clone) + if err == nil { + err = c.Update(ctx, obj) + if err == nil { + logrus.Printf("✅ %s %s Updated.\n", gvk.Kind, metaObj.GetName()) + return false + } + } + if errors.IsNotFound(err) { + err = c.Create(ctx, obj) + if err == nil { + logrus.Printf("✅ %s %s Created.\n", gvk.Kind, metaObj.GetName()) + return true + } + } + Fatal(err) + return false +} + +func KubeCreateSkipExisting(c client.Client, obj runtime.Object) bool { + metaObj, _ := meta.Accessor(obj) + objKey := client.ObjectKey{Namespace: metaObj.GetNamespace(), Name: metaObj.GetName()} + gvk := obj.GetObjectKind().GroupVersionKind() + clone := obj.DeepCopyObject() + err := c.Get(ctx, objKey, clone) + if err == nil { + logrus.Printf("✅ %s %s Already exists.\n", gvk.Kind, metaObj.GetName()) + return false + } + if errors.IsNotFound(err) { + err = c.Create(ctx, obj) + if err == nil { + logrus.Printf("✅ %s %s Created.\n", gvk.Kind, metaObj.GetName()) + return true + } + } + Fatal(err) + return false +} + +func KubeDelete(c client.Client, obj runtime.Object) bool { + metaObj, _ := meta.Accessor(obj) + // objKey := client.ObjectKey{Namespace: metaObj.GetNamespace(), Name: metaObj.GetName()} + gvk := obj.GetObjectKind().GroupVersionKind() + err := c.Delete(ctx, obj) + if err == nil { + logrus.Printf("✅ %s %s Deleted.\n", gvk.Kind, metaObj.GetName()) + return true + } + if errors.IsNotFound(err) { + logrus.Printf("❌ %s %s Not found.\n", gvk.Kind, metaObj.GetName()) + return false + } + Fatal(err) + return false +} + +func KubeCheck(c client.Client, obj runtime.Object) bool { + metaObj, _ := meta.Accessor(obj) + objKey := client.ObjectKey{Namespace: metaObj.GetNamespace(), Name: metaObj.GetName()} + gvk := obj.GetObjectKind().GroupVersionKind() + err := c.Get(ctx, objKey, obj) + if err == nil { + logrus.Printf("✅ %s %s Exists.\n", gvk.Kind, metaObj.GetName()) + return true + } + if errors.IsNotFound(err) { + logrus.Printf("❌ %s %s Not found.\n", gvk.Kind, metaObj.GetName()) + return false + } + Fatal(err) + return false +} + +func Fatal(err error) { + if err != nil { + logrus.Println("☠️ Fatal Error", err) + panic(err) + } +} diff --git a/version/version.go b/version/version.go index b42cce124..987bb19be 100644 --- a/version/version.go +++ b/version/version.go @@ -1,12 +1,6 @@ package version -import "fmt" - -var ( +const ( // Version is the noobaa-operator version (semver) - Version = "0.1.0" + Version = "1.0.0" ) - -func main() { - fmt.Println(Version) -}