v0.1 release candidate (#2)

* Refactor the code to return error always

* Re-factor things a bit to have tests

* Use templates for values

* Get environments working

* Add simple test for step outputs

* Update README

* Implement 'if' condition

* Allow running steps on success, failure and always

* Add job outputs

* Add writing job outputs to a file in JSON format

* Update todo list in README

* Update job binary to handle inputs

* Extract tests to separate yaml files

.github/workflows/test.yml

@@ -17,21 +17,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: go test
+      - name: Run tests
         shell: bash
         run: |
-          cd cmd/ops-run
-          go test
+          make test
 
-      - name: go build
+      - name: Build binary
         shell: bash
         run: |
-          cd cmd/ops-run
+          cd cmd/job
           go build .
-          ./ops-run
-
-      - name: run-job
-        shell: bash
-        run: |
-          cd cmd/ops-run
-          ./ops-run run-job -f ../../sample-files/job.yaml
+          ./job

.gitignore

@@ -19,3 +19,6 @@
 
 # Go workspace file
 go.work
+
+# Compiled binaries
+/cmd/job/job

Makefile

@@ -0,0 +1,9 @@
+.DEFAULT_GOAL := help
+
+.PHONY: help test
+
+test: ## Runs tests
+	cd pkg/runtime/local && go test
+
+help: ## Displays this help
+	@awk 'BEGIN {FS = ":.*##"; printf "$(MAKEFILE_NAME)\n\nUsage:\n  make \033[1;36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[1;36m%-25s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)

README.md

@@ -1,47 +1,49 @@
-# streamline
+# charlotte
 
-#### Latest state
-It's been copied from a different repository so it's probably failing to
-compile.
+### Job
+Simple app that takes a YAML file that contains a set of steps which are bash scripts, wrapped into simple logic, inputs, outputs etc.
+and executes it.
 
-Docker bit is under development so no idea what's there TBH.
+#### Running test suite
 
-#### Building
-Use the following:
+    make test
 
-    cd cmd/ops-run
-    go build .
-
-#### Testing
-Use the following:
-
-    ./ops-run run-job -f ../../sample-files/job.yaml
-
-
-#### Phase 0
-Phase 0 is to get something running simple bash scripts, with some simple
-dependencies between steps, some logic, piping standard output and error.
-And all that should be possible to execute locally, in a specific
-container or on a specified kubernetes cluster.
-
-It should be possible to create job specification in YAML.
+#### Building `job` binary
 
-Once each phase is done, we shall ensure it's all covered with tests, even
-if we have to write 50 or so of these guys.
-
-
-## TODO
-
-### Phase 0.1 - bash script blocks only
-#### 0.1.1 run them locally
-#### 0.1.2 spin a docker container and run inside
-#### 0.1.3 spin a pod in a kubernetes cluster
-#### 0.1.4 pipe stdout and stderr properly
-#### 0.1.5 add outputs
-#### 0.1.6 add inputs
-
-### Phase 0.2 - dependencies and handling logic
-...
-### Phase 0.3 - master <-?-> master (worker) <-> worker
-...
+    cd cmd/job
+    go build .
 
+#### Running `job`
+
+    cd cmd/job
+    ./job run-local -j ../../sample-files/job.yaml -r /tmp/job-result.txt -i ../../sample-files/job-inputs.json --quiet
+    cat /tmp/job-result.txt
+
+Also, there are test files in the `pkg/job/runtime/local/tests` directory that can be used.
+
+#### v0.1
+
+- [x] pipe stdout and stderr to files
+- [x] environment (global and in-step)
+- [x] variables
+- [x] job inputs
+- [x] step outputs
+- [x] `continue_on_error`
+- [x] values using golang templates
+- [x] `if` - conditional steps (value templated, must equal to string `'true'`)
+- [x] running step(s) on success
+- [x] running step(s) on failure
+- [x] running step(s) always
+- [x] tmp directory for step outputs
+- [x] gather job outputs 
+- [x] write job outputs to json file
+- [x] handle input: `--inputs`, `--job`, `--result` without aliases (and `--quiet`)
+- [x] prepare sample yaml files - same as the test ones, so the test would just include them?
+
+#### v0.2
+- [ ] validation
+- [ ] extract steps so that they can be included (include file with inputs) + proper validation for that
+- [ ] ...
+
+### Pipeline
+Layer on top of Jobs.

cmd/job/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"charlotte/pkg/job"
+	jobrun "charlotte/pkg/jobrun"
+	localruntime "charlotte/pkg/runtime/local"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	gocli "github.com/nicholasgasior/go-broccli"
+)
+
+func main() {
+	cli := gocli.NewCLI("job", "Run job locally", "Streamln <hello@streamln.dev>")
+	cmdRun := cli.AddCmd("run-local", "Runs YAML job file", runJobHandler)
+	cmdRun.AddFlag("job", "j", "FILENAME", "Path to filename with a job", gocli.TypePathFile, gocli.IsRequired|gocli.IsExistent|gocli.IsRegularFile)
+	cmdRun.AddFlag("inputs", "i", "FILENAME", "Path to file containing input values", gocli.TypePathFile, gocli.IsRequired|gocli.IsExistent|gocli.IsRegularFile)
+	cmdRun.AddFlag("quiet", "q", "", "Do not print step stdout and stderr", gocli.TypeBool, 0, gocli.OnTrue(func(c *gocli.Cmd) {
+
+	}))
+	cmdRun.AddFlag("result", "r", "FILENAME", "Path to write JSON result", gocli.TypePathFile, 0)
+	_ = cli.AddCmd("version", "Prints version", versionHandler)
+	if len(os.Args) == 2 && (os.Args[1] == "-v" || os.Args[1] == "--version") {
+		os.Args = []string{"App", "version"}
+	}
+	os.Exit(cli.Run())
+}
+
+func versionHandler(c *gocli.CLI) int {
+	fmt.Fprintf(os.Stdout, VERSION+"\n")
+	return 0
+}
+
+func runJobHandler(c *gocli.CLI) int {
+	jobFile := c.Flag("job")
+	j, err := job.NewFromFile(jobFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error parsing job from file %s: %s\n", jobFile, err.Error())
+		return 1
+	}
+
+	inputsFile := c.Flag("inputs")
+	b, err := os.ReadFile(inputsFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error reading inputs file %s: %s\n", inputsFile, err.Error())
+		return 1
+	}
+	var jobRunInputs jobrun.JobRunInputs
+	err = json.Unmarshal(b, &jobRunInputs)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error unmarshalling inputs file %s: %s\n", inputsFile, err.Error())
+		return 1
+	}
+
+	quiet := false
+	if c.Flag("quiet") == "true" {
+		quiet = true
+	}
+	runenv := localruntime.NewLocalRuntime(quiet)
+	jobRunResult := j.Run(runenv, &jobRunInputs)
+	if !quiet && !jobRunResult.Success {
+		fmt.Fprintf(os.Stderr, "Job %s failed locally with: %s\n", jobFile, jobRunResult.Error.Error())
+	}
+
+	resultJson, err := json.Marshal(jobRunResult)
+	if err != nil {
+		if !quiet {
+			fmt.Fprintf(os.Stdout, "Marshalling run result to JSON failed\n")
+		}
+		return 4
+	}
+
+	outputFile := c.Flag("result")
+	if outputFile != "" {
+		err = os.WriteFile(outputFile, resultJson, 0600)
+		if err != nil {
+			if !quiet {
+				fmt.Fprintf(os.Stdout, "Writing JSON run result to file failed\n")
+			}
+			return 5
+		}
+	} else {
+		fmt.Fprintf(os.Stdout, "%s", resultJson)
+	}
+
+	if !jobRunResult.Success {
+		return 1
+	}
+
+	return 0
+}

cmd/job/version.go

@@ -0,0 +1,3 @@
+package main
+
+const VERSION = "0.1.0"

go.mod

@@ -1,10 +1,87 @@
 module charlotte
 
-go 1.20
+go 1.22.0
+
+toolchain go1.22.1
 
 require (
-	github.com/mikogs/go-broccli/v2 v2.0.0
-	github.com/mikogs/go-valifieldator v1.1.1
+	github.com/docker/docker v25.0.3+incompatible
+	github.com/nicholasgasior/go-broccli v2.0.0+incompatible
+	github.com/nicholasgasior/struct-validator v0.3.0
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/api v0.30.1
+	k8s.io/apimachinery v0.30.1
+	k8s.io/kubectl v0.30.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+require (
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/containerd/containerd v1.7.13 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 // indirect
+	go.opentelemetry.io/otel v1.23.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.23.1 // indirect
+	go.opentelemetry.io/otel/metric v1.23.1 // indirect
+	go.opentelemetry.io/otel/sdk v1.23.1 // indirect
+	go.opentelemetry.io/otel/trace v1.23.1 // indirect
+	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.18.0 // indirect
+	gotest.tools/v3 v3.5.1 // indirect
+	k8s.io/client-go v0.30.1
 )