Merge branch 'master' into fix-example

.gitattributes

@@ -0,0 +1,3 @@
+# Ensure that git would checkout files with lf instead of crlf on windows
+*.formatted text eol=lf
+*.yml text eol=lf

.github/workflows/ci.yaml

@@ -5,27 +5,30 @@ on:
     branches:
       - master
 jobs:
-  build:
+  build-linux:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.14
-        uses: actions/setup-go@v1
+      - uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.15
         id: go
 
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v1
+      - uses: azure/setup-helm@v1
+        with:
+          version: "v3.3.1" # default is latest stable
+        id: helm
+
+      - uses: actions/checkout@v2
 
       - name: Install command dependencies
         run: |
           mkdir -p $HOME/go/bin
           export GOPATH=$HOME/go
-          echo "::set-env name=GOPATH::$HOME/go"
-          echo "::set-env name=GO_VERSION::$(go version | awk '{ print $3}' | sed 's/^go//')"
-          echo "::add-path::$HOME/go/bin"
-          make install-ci install
+          echo "GOPATH=$GOPATH" >> $GITHUB_ENV
+          echo "GO_VERSION=$(go version | awk '{ print $3}' | sed 's/^go//')" >> $GITHUB_ENV
+          echo "$HOME/go/bin" >> $GITHUB_PATH
+          make install-ci create-cluster install
 
       - name: Build
         run: make build
@@ -34,16 +37,44 @@ jobs:
         run: make lint
 
       - name: Test
-        run: TEST_FLAGS="-tags integration" make test
+        run: make test TEST_FLAGS="-tags integration"
 
       - name: Coverage Report
         run: make publish-coverage
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
       - name: goreleaser
-        uses: goreleaser/goreleaser-action@master
+        uses: goreleaser/goreleaser-action@v2
         with:
           args: release --snapshot --skip-publish --rm-dist --release-notes .release-notes.md
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  build-windows:
+    name: Build(Windows)
+    runs-on: windows-latest
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+        id: go
+
+      - uses: azure/setup-helm@v1
+        with:
+          version: "v3.3.1" # default is latest stable
+        id: helm
+
+      - uses: actions/checkout@v2
+
+      - name: Install command dependencies
+        run: |
+          echo "GOPATH=$HOME/go" >> $GITHUB_ENV
+          echo "GO_VERSION=$(go version | awk '{ print $3}' | sed 's/^go//')" >> $GITHUB_ENV
+          echo "$HOME/go/bin" >> $GITHUB_PATH
+
+      - name: Build
+        run: make build
+
+      - name: Test
+        run: make test

.github/workflows/release.yaml

@@ -10,19 +10,17 @@ jobs:
     runs-on: ubuntu-latest
     name: goreleaser
     steps:
-      - name: Set up Go 1.14
-        uses: actions/setup-go@v1
+      - uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.15
         id: go
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Install package dependencies
         run: |
-          echo "::set-env name=GO_VERSION::$(go version | awk '{ print $3}' | sed 's/^go//')"
+          echo "GO_VERSION=$(go version | awk '{ print $3}' | sed 's/^go//')" >> $GITHUB_ENV
           make get
       - name: Release via goreleaser
-        uses: goreleaser/goreleaser-action@master
+        uses: goreleaser/goreleaser-action@v2
         with:
           args: release --rm-dist --release-notes .release-notes.md
         env:

.gitignore

@@ -7,6 +7,8 @@ coverage.txt
 dist/
 helm.tar.gz
 linux-amd64/
+darwin-amd64/
 bin/
 .release-notes.md
 .tools/
+.vscode/

.goreleaser.yaml

@@ -35,7 +35,7 @@ archives:
 
 brews:
   - name: qbec
-    github:
+    tap:
       owner: splunk
       name: homebrew-tap
     url_template: https://github.com/splunk/qbec/releases/download/{{.Tag}}/{{.ArtifactName}}

CHANGELOG.md

@@ -1,6 +1,64 @@
 Changelog
 ---
 
+## v0.13.3 (Dec 23, 2020)
+
+* Add ability to add the component name as a label in addition to the existing annotation. This is opt-in and is activated
+  by setting the `addComponentLabel` property to `true` in qbec.yaml (thanks @korroot and @hudymi).
+
+## v0.13.2 (Dec 3, 2020)
+
+* Allow force options to be set via environment variables (thanks @splkforrest)
+
+## v0.13.1 (Dec 2, 2020)
+
+* Fix a bug where the `alpha fmt` command would stop processing arguments after encountering a directory.
+
+## v0.13.0 (Nov 27, 2020)
+
+* Misc. CI build changes
+* Update jsonnet library to `v0.17.0` and k8s client libs to `v1.17.13`
+* Add json formatter to the `qbec alpha fmt` command
+* Fix diff commands to show skipped updates and deletes based on qbec directives specified for existing objects.
+  This will no longer show spurious diffs for deletes and updates if those have been turned off. 
+* Use per-namespace queries by default when multiple namespaces are present, allow using cluster-scoped queries
+  using an opt-in flag.
+* The `--env-file` option now allows http(s) URLs in addition to local files. In addition, the `envFiles` attribute
+  in `qbec.yaml` can also contain http(s) URLs. (thanks @dan1)
+* String data in secrets is now obfuscated in addition to binary data
+
+### Incompatibilities
+
+This release is incompatible from previous minor versions in the following ways:
+
+* `qbec apply` will now wait on all objects by default. That is, the `--wait-all` now defaults to `true`.
+  To get the previous behavior, you need to add `--wait-all=false` to the `apply` command.
+* `qbec diff` now exits 0 by default even when diffs are found. To restore previous behavior, add `--error-exit`
+  to the command.
+* qbec now defaults to per-namespace queries when multiple namespaces are present. To get the previous behavior
+  of using cluster-scoped queries add `clusterScopedLists: true` under `spec` in `qbec.yaml`
+* The command line syntax of the `qbec alpha fmt` command has changed in incompatible ways. Instead of options like
+ `--jsonnet`, `--yaml` etc. you need to specify options as `--type=jsonnet`, `--type=yaml` and so on.
+* YAML formatter now follows `prettier` conventions requiring arrays to be indented under the parent key.
+* Any corner-case behavior from updating k8s client and jsonnet libraries.  
+
+## v0.12.5 (Oct 2, 2020)
+
+* Add a new `wait-policy` directive to disable waits on specific deployments
+  and daemonsets. The annotation `"directives.qbec.io/wait-policy": "never"`
+  will cause qbec to not wait on the deployment even if it has changed.
+
+## v0.12.4 (Sep 24, 2020)
+
+* Add `--wait-all` flag to the `apply` command to wait on all objects instead of just the ones that were changed in the
+  current run.
+
+## v0.12.3 (Sep 9, 2020)
+
+* Add ability to import a bag of files using a glob pattern (see #153 for details). At this point this should be
+  considered experimental. Do not rely on it yet until the next release when we will have docs for it.
+* Add windows build in CI, thanks to @harsimranmaan
+
 ## v0.12.2 (Aug 30, 2020)
 
 * Fix a bug where under certain circumstances of failed discovery, qbec would delete resources not meant to be deleted.

Makefile

@@ -1,6 +1,6 @@
 include Makefile.tools
 
-VERSION         := 0.12.2
+VERSION         := 0.13.3
 SHORT_COMMIT    := $(shell git rev-parse --short HEAD 2>/dev/null || echo dev)
 GO_VERSION      := $(shell go version | awk '{ print $$3}' | sed 's/^go//')
 
@@ -50,18 +50,26 @@ lint: check-format
 .PHONY: check-format
 check-format:
 	@echo "Running gofmt..."
-	$(eval unformatted=$(shell find . -name '*.go' | grep -v ./.git | grep -v vendor | xargs gofmt -l))
+	$(eval unformatted=$(shell find . -name '*.go' | grep -v ./.git | grep -v vendor | xargs gofmt -s -l))
 	$(if $(strip $(unformatted)),\
 		$(error $(\n) Some files are ill formatted! Run: \
-			$(foreach file,$(unformatted),$(\n)    gofmt -w $(file))$(\n)),\
+			$(foreach file,$(unformatted),$(\n)    gofmt -s -w $(file))$(\n)),\
 		@echo All files are well formatted.\
 	)
+
+
 .PHONY: install-ci
-install-ci: .tools/kind
-	curl -sSL -o helm.tar.gz https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz
-	tar -xvzf helm.tar.gz
-	mv linux-amd64/helm $(GOPATH)/bin/
+install-ci: HELM_VERSION := 3.3.1
+install-ci: HELM_PLATFORM := $(shell uname|  tr '[:upper:]' '[:lower:]')
+install-ci:
+	# Refactor helm install into a separate step
+	# curl -sSL -o helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-${HELM_PLATFORM}-amd64.tar.gz
+	# tar -xvzf helm.tar.gz
+	# mv ${HELM_PLATFORM}-amd64/helm $(GOPATH)/bin/
 	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(GOPATH)/bin v1.21.0
+
+.PHONY: create-cluster
+create-cluster:	.tools/kind
 	.tools/kind create cluster
 
 .PHONY: install

examples/test-app/environments/dev.libsonnet

@@ -1,4 +1,4 @@
-local base = import './base.libsonnet';
+local base = import '_.libsonnet';
 
 base {
     components +: {

examples/test-app/environments/prod.libsonnet

@@ -1,4 +1,4 @@
-local base = import './base.libsonnet';
+local base = import '_.libsonnet';
 
 base {
     components +: {

examples/test-app/lib/globutil.libsonnet

@@ -0,0 +1,61 @@
+local len = std.length;
+local split = std.split;
+local join = std.join;
+
+// keepDirs returns a key mapping function for the number of directories to be retained
+local keepDirs = function(num=0) function(s) (
+  if num < 0
+  then
+    s
+  else (
+    local elems = split(s, '/');
+    local preserveRight = num + 1;
+    if len(elems) <= preserveRight
+    then
+      s
+    else (
+      local remove = len(elems) - preserveRight;
+      join('/', elems[remove:])
+    )
+  )
+);
+
+// stripExtension is a key mapping function that strips the file extension from the key
+local stripExtension = function(s) (
+  local parts = split(s, '/');
+  local dirs = parts[:len(parts) - 1];
+  local file = parts[len(parts) - 1];
+  local fileParts = split(file, '.');
+  local fixed = if len(fileParts) == 1 then file else join('.', fileParts[:len(fileParts) - 1]);
+  join('/', dirs + [fixed])
+);
+
+// compose composes an array of map functions by applying them in sequence
+local compose = function(arr) function(s) std.foldl(function(prev, fn) fn(prev), arr, s);
+
+// transform transforms an object, mapping keys using the key mapper and values using the valueMapper.
+// It ensures that the key mapping does not produce duplicate keys.
+local transform = function(globObject, keyMapper=function(s) s, valueMapper=function(o) o) (
+  local keys = std.objectFields(globObject);
+  std.foldl(function(obj, key) (
+    local mKey = keyMapper(key);
+    local val = globObject[key];
+    if std.objectHas(obj, mKey)
+    then
+      error 'multiple keys map to the same value: %s' % [mKey]
+    else
+      obj { [mKey]: valueMapper(val) }
+  ), keys, {})
+);
+
+// nameOnly is a key mapper that removes all directories and strips extensions from file names,
+// syntax sugar for the common case.
+local nameOnly = compose([keepDirs(0), stripExtension]);
+
+{
+  transform:: transform,
+  keepDirs:: keepDirs,
+  stripExtension:: stripExtension,
+  compose:: compose,
+  nameOnly:: nameOnly,
+}

examples/test-app/lib/objects.libsonnet

@@ -36,14 +36,12 @@
         spec: {
             selector: {
                 matchLabels: {
-                    name: name,
+                    app: name
                 },
             },
             template: {
                 metadata: {
-                    labels: {
-                        name: name,
-                    },
+                    labels: { app: name },
                 },
                 spec: {
                     containers: [

examples/test-app/params.libsonnet

@@ -1,10 +1,5 @@
-local p = {
-    _: import './environments/base.libsonnet',
-    dev: import './environments/dev.libsonnet',
-    prod: import './environments/prod.libsonnet',
-};
-
-local env = std.extVar('qbec.io/env');
-
-if std.objectHas(p, env) then p[env] else error 'Environment ' + env + ' not defined in ' + std.thisFile
+local globutil = import 'globutil.libsonnet';
+local p = globutil.transform(import 'glob-import:environments/*.libsonnet', globutil.nameOnly);
 
+local key = std.extVar('qbec.io/env');
+if std.objectHas(p, key) then p[key] else error 'Environment ' + key + ' not defined in environments/'