Feature/containerize

.github/dependabot.yml

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Iván SZKIBA
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/release.yml

@@ -20,6 +20,15 @@ jobs:
           fetch-depth: 0
       - name: Set up Go
         uses: actions/setup-go@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: 'amd64,arm64'
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}          
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:

.goreleaser.yaml

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 project_name: k6x
+env:
+  - OWNER=szkiba
 before:
   hooks:
     - go mod tidy
@@ -12,7 +14,7 @@ builds:
     goos:  [ 'darwin', 'linux', 'windows' ]
     goarch: [ 'amd64', 'arm64' ]
     ldflags:
-      - '-s -w -X {{.ModulePath}}/internal/cmd._version={{.Version}} -X {{.ModulePath}}/internal/cmd._appname={{.ProjectName}}'
+      - '-s -w -X {{.ModulePath}}/internal/cmd._version={{.Version}} -X {{.ModulePath}}/internal/cmd._appname={{.ProjectName}} -X {{.ModulePath}}/internal/cmd._owner={{index .Env "GITHUB_REPOSITORY_OWNER"}}'
 source:
   enabled: true
   name_template: '{{ .ProjectName }}_{{ .Version }}_source'
@@ -30,7 +32,7 @@ nfpms:
     description: |-
       Automatic k6 provisioning with extensions.
 
-    license: MIT
+    license: AGPL-3.0-only
     formats: [ 'deb', 'rpm' ]
     umask: 0o022
     overrides:
@@ -61,3 +63,61 @@ changelog:
       - '^chore:'
       - '^docs:'
       - '^test:'
+
+dockers:
+  - id: amd64
+    dockerfile: Dockerfile.goreleaser
+    use: buildx
+    image_templates:
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}-amd64"
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-amd64"
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:latest-amd64"
+
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--pull"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.licenses=AGPL-3.0-only"
+  - id: arm64
+    dockerfile: Dockerfile.goreleaser
+    use: buildx
+    image_templates:
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:{{ .Tag }}-amd64"
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}-amd64"
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-amd64"
+      - "{{ .Env.OWNER }}/{{ .ProjectName }}:latest-amd64"
+
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--pull"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.licenses=AGPL-3.0-only"
+
+docker_manifests:
+- id: tag
+  name_template: "{{ .Env.OWNER }}/{{ .ProjectName }}:{{ .Tag }}"
+  image_templates:
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}:{{ .Tag }}-amd64"
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}-arm64"
+- id: major
+  name_template: "{{ .Env.OWNER }}/{{ .ProjectName }}:{{ .Tag }}"
+  image_templates:
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}-amd64"
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}-arm64"
+- id: major-minor
+  name_template: "{{ .Env.OWNER }}/{{ .ProjectName }}:{{ .Tag }}"
+  image_templates:
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-amd64"
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}-arm64"
+- id: latest
+  name_template: "{{ .Env.OWNER }}/{{ .ProjectName }}:latest"
+  image_templates:
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}:latest"
+  - "{{ .Env.OWNER }}/{{ .ProjectName }}:latest"

Dockerfile.goreleaser

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2023 Iván SZKIBA
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+FROM golang:1.21.1-alpine3.18
+VOLUME /cache
+RUN apk add --no-cache ca-certificates git && \
+    adduser -D -u 10000 -g 10000 -h /home/k6x k6x && \
+    mkdir -p /cache/go-build /cache/go-mod /cache/k6x && \
+    chown -R 10000:10000 /cache
+ENV XDG_CACHE_HOME="/cache" GOCACHE="/cache/go-build" GOMODCACHE="/cache/go-mod"
+COPY k6x /usr/bin/k6x
+
+USER 10000
+WORKDIR /home/k6x
+ENTRYPOINT ["k6x"]

README.md

@@ -52,6 +52,18 @@ If you have a go development environment, the installation can also be done with
 go install github.com/szkiba/k6x@latest
 ```
 
+## Docker
+
+k6x is also available as a docker image on Docker Hub under the name [szkiba/k6x](https://hub.docker.com/r/szkiba/k6x).
+
+This docker image can be used as a replacement for the official k6 docker image to run test scripts that use extensions. The basic use of the image is the same as using the official k6 image.
+
+The image automatically provides  [k6](https://k6.io) with the [extensions](https://k6.io/docs/extensions/) used by the tests. To do this, [k6x](https://github.com/szkiba/k6x) analyzes the test script and creates a list of required extensions (it also parses the command line to detect output extensions). Based on this list, k6x builds (and caches) the k6 binary and runs it.
+
+The build step is done using the go compiler included in the image. The partial results of the go compilation and build steps are saved to the volume in the `/cache` path (this is where the go cache and the go module cache are placed). By making this volume persistent, the time required for the build step can be significantly reduced.
+
+The k6x docker builder (`--builder docker`) also uses this docker image. It creates a local volume called `k6x-cache` and mounts it to the `/cache` path. Thanks to this, the docker build runs almost at the same speed as the native build (apart from the first build).
+
 ## Extras
 
 ### Pragma
@@ -80,24 +92,29 @@ Read the version constraints syntax in the [Version Constraints](#version-constr
 
 Reusable artifacts (k6 binary, HTTP responses) are stored in the subdirectory `k6x` under the directory defined by the `XDG_CACHE_HOME` environment variable. The default of `XDG_CACHE_HOME` depends on the operating system (Windows: `%LOCALAPPDATA%\cache`, Linux: `~/.cache`, macOS: `~/Library/Caches`)
 
-The directory where k6x stores the compiled k6 binary can be specified in the `K6X_BIN_DIR` environment variable. If it is missing, the `.k6x` directory is used if it exists in the current working directory, otherwise the k6 binary is stored in the cache directory described above. In addition, the location of the directory used to store k6 can also be specified using the `--bin-dir` command line option. See the [Options](#options) section for more information.
+The directory where k6x stores the compiled k6 binary can be specified in the `K6X_BIN_DIR` environment variable. If it is missing, the `.k6x` directory is used if it exists in the current working directory, otherwise the k6 binary is stored in the cache directory described above. In addition, the location of the directory used to store k6 can also be specified using the `--bin-dir` command line option. See the [Flags](#flags) section for more information.
 
 The `version` command displays the path of the cached k6 executable after the version number.
 
 > **Note**
 > You can avoid rebuilding the k6 binary in the default k6x cache during development if you create a .k6x directory in the current working directory. In this case, k6x will automatically use this local directory to cache the k6 binary.
 
-### Options
+### Flags
 
-The k6 subcommands are extended with some global command line options related to building and caching the k6 binary.
+The k6 subcommands are extended with some global command line flags related to building and caching the k6 binary.
 
 - `--clean` the cached k6 binary will be deleted and a new binary will be built
 - `--dry` only the cached k6 binary will be updated if necessary, the k6 command will not be executed
 - `--bin-dir path` the directory specified here will be used to cache the k6 binary (it will overwrite the value of `K6X_BIN_DIR`)
   ```
   k6x run --bin-dir ./custom-k6 script.js
   ```
+- `--with dependency`  you can specify additional dependencies and version constraints, the form of the `dependency` is the same as that used in the `"use k6 with"` pragma (practically the same as the string after the `use k6 with`)
 
+  ```
+  k6x run --with k6/x/mock script.js
+  ```
+
 - `--builder list` a comma-separated list of builders (default: `native,docker`), available builders:
   - `native` this builder uses the installed go compiler if available, otherwise the next builder is used without error
   - `docker` this builder uses Docker Engine, which can be local or remote (specified in `DOCKER_HOST` environment variable)
@@ -113,7 +130,7 @@ Some new subcommands will also appear, which are related to building the k6 bina
 - `build` builds k6 based on the dependencies in the test script.
   ```
   Usage:
-    k6x build [flags] script
+    k6x build [flags] [script]
 
   Flags:
     -o, --out name  output extension name
@@ -125,7 +142,7 @@ Some new subcommands will also appear, which are related to building the k6 bina
 - `deps` display of k6 and extension dependencies used in the test script
   ```
   Usage:
-    k6x deps [flags] script
+    k6x deps [flags] [script]
 
   Flags:
     -o, --out name  output extension name
@@ -138,7 +155,7 @@ Some new subcommands will also appear, which are related to building the k6 bina
 
 The new subcommands (`build`, `deps`) display help in the usual way, with the `--help` or `-h` command line option.
 
-The k6 subcommands (`version`, `run` etc) also display help with the `--help` or `-h` command line option, so in this case the new k6x launcher options are displayed before the normal k6 help.
+The k6 subcommands (`version`, `run` etc) also display help with the `--help` or `-h` command line option, so in this case the new k6x launcher flags are displayed before the normal k6 help.
 
 ### Remote Docker
 
@@ -159,13 +176,13 @@ The git repository for each extension is determined based on the [k6 extension r
 
 Taking into account the optional version constraints, the appropriate extension version is selected from the git tags of the extension's git repository. Currently, only GitHub repositories are supported, if required, additional repository managers can be supported (eg GitLab).
 
-If the Go compiler is installed, the k6 binary is created using it. Otherwise the custom k6 binary is created using the [xk6 custom k6 builder docker image](https://hub.docker.com/r/grafana/xk6/). The Docker Engine API is accessed using the [docker go client](https://pkg.go.dev/github.com/docker/docker/client), so there is no need for a docker cli command and even a remote Docker Engine can be used.
+If the Go compiler is installed, the k6 binary is created using it. Otherwise the custom k6 binary is created using the [szkiba/k6x](https://hub.docker.com/r/szkiba/k6x) docker image. The Docker Engine API is accessed using the [docker go client](https://pkg.go.dev/github.com/docker/docker/client), so there is no need for a docker cli command and even a remote Docker Engine can be used.
 
 The compiled k6 binary is stored in the cache. This binary will be used as long as the extensions included in it meet the current requirements, taking into account the optional version constraints. In order to increase the efficiency of the cache, the newly built k6 binary will also include the previously used extensions (if they are still included in the registry).
 
 At this point, the k6 binary is executed from the cache with exactly the same arguments that were used to start the k6x command.
 
-You can read more about the development ideas in the [Future](#future) section.
+You can read more about the development ideas in the [Feature Request](https://github.com/szkiba/k6x/issues?q=is%3Aopen+is%3Aissue+label%3Afeature) list.
 
 ### Limitations
 
@@ -242,6 +259,3 @@ major change is API breaking. For example,
 * `^0.0` is equivalent to `>=0.0.0 <0.1.0`
 * `^0` is equivalent to `>=0.0.0 <1.0.0`
 
-## Future
-
-One possible future development idea is to make it possible to use pre-built k6 binaries. Instead of building, the k6 binary could be downloaded from properly prepared custom k6 GitHub releases. This will allow you to use k6x even without Docker Engine. In addition, the use of pre-built k6 binaries will significantly reduce the cache loading time (from 45-50 seconds to a few seconds).

internal/builder/docker.go

@@ -24,26 +24,29 @@ import (
 	"github.com/docker/cli/cli/connhelper"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
 )
 
-const xk6Image = "grafana/xk6"
+const (
+	builderImage = "szkiba/k6x"
+	cacheVolume  = "k6x-cache"
+	cachePath    = "/cache"
+	workdirPath  = "/home/k6x"
+)
 
-func cmdline(ings resolver.Ingredients) ([]string, []string) {
+func (b *dockerBuilder) cmdline(ings resolver.Ingredients) ([]string, []string) {
 	args := make([]string, 0, 2*len(ings))
 	env := make([]string, 0, 1)
 
 	env = append(env, "GOOS="+runtime.GOOS)
-	args = append(args, "build")
+	env = append(env, "GOARCH="+runtime.GOARCH)
 
-	if ing, ok := ings.K6(); ok {
-		args = append(args, ing.Tag())
-		env = append(env, "K6_VERSION="+ing.Tag())
-	}
+	args = append(args, "build")
 
-	for _, ext := range ings.Extensions() {
-		args = append(args, "--with", ext.Module+"@"+ext.Tag())
+	for _, ing := range ings {
+		args = append(args, "--with", ing.Name+" "+ing.Tag())
 	}
 
 	return args, env
@@ -92,9 +95,9 @@ func (b *dockerBuilder) close() {
 }
 
 func (b *dockerBuilder) pull(ctx context.Context) error {
-	logrus.Debugf("Pulling %s image", xk6Image)
+	logrus.Debugf("Pulling %s image", builderImage)
 
-	reader, err := b.cli.ImagePull(ctx, xk6Image, types.ImagePullOptions{})
+	reader, err := b.cli.ImagePull(ctx, builderImage, types.ImagePullOptions{})
 	if err != nil {
 		return err
 	}
@@ -134,18 +137,28 @@ func (b *dockerBuilder) pull(ctx context.Context) error {
 }
 
 func (b *dockerBuilder) start(ctx context.Context, ings resolver.Ingredients) (string, error) {
-	cmd, env := cmdline(ings)
+	cmd, env := b.cmdline(ings)
 
 	logrus.Debugf("Executing %s", strings.Join(cmd, " "))
 
 	conf := &container.Config{
-		Image: xk6Image,
+		Image: builderImage,
 		Cmd:   cmd,
-		Tty:   true,
+		Tty:   false,
 		Env:   env,
 	}
 
-	resp, err := b.cli.ContainerCreate(ctx, conf, nil, nil, nil, "")
+	hconf := &container.HostConfig{
+		Mounts: []mount.Mount{
+			{
+				Type:   mount.TypeVolume,
+				Source: cacheVolume,
+				Target: cachePath,
+			},
+		},
+	}
+
+	resp, err := b.cli.ContainerCreate(ctx, conf, hconf, nil, nil, "")
 	if err != nil {
 		return "", err
 	}
@@ -178,7 +191,11 @@ func (b *dockerBuilder) log(ctx context.Context, id string) error {
 
 	var out io.ReadCloser
 
-	out, err := b.cli.ContainerLogs(ctx, id, types.ContainerLogsOptions{ShowStdout: true})
+	out, err := b.cli.ContainerLogs(
+		ctx,
+		id,
+		types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true},
+	)
 	if err != nil {
 		return err
 	}
@@ -191,7 +208,7 @@ func (b *dockerBuilder) log(ctx context.Context, id string) error {
 }
 
 func (b *dockerBuilder) copy(ctx context.Context, id string, dir string, afs afero.Fs) error {
-	binary, _, err := b.cli.CopyFromContainer(ctx, id, "/xk6")
+	binary, _, err := b.cli.CopyFromContainer(ctx, id, workdirPath)
 	if err != nil {
 		return err
 	}

internal/cmd/build.go

@@ -71,6 +71,23 @@ func addOptional(ctx context.Context, res resolver.Resolver, deps, opt dependenc
 	}
 }
 
+func addDeps(ctx context.Context, res resolver.Resolver, deps, req dependency.Dependencies) error {
+	if len(req) == 0 {
+		return nil
+	}
+
+	_, err := res.Resolve(ctx, req)
+	if err != nil {
+		return err
+	}
+
+	for name, dep := range req {
+		deps[name] = dep
+	}
+
+	return nil
+}
+
 func collectDependencies(
 	ctx context.Context,
 	res resolver.Resolver,
@@ -102,6 +119,14 @@ func collectDependencies(
 
 	addOptional(ctx, res, deps, opts.dependencies())
 
+	if err := addDeps(ctx, res, deps, opts.with); err != nil {
+		return nil, err
+	}
+
+	if _, has := deps.K6(); !has {
+		deps["k6"] = &dependency.Dependency{Name: "k6"}
+	}
+
 	return deps, nil
 }
 

internal/cmd/cmd_build.go

@@ -35,12 +35,13 @@ func buildCommand(
 const buildUsage = `Build custom k6 binary for a script.
 
 Usage:
-  {{.appname}} build [flags] script
+  {{.appname}} build [flags] [script]
 
 Flags:
-  -o, --out name  output extension name
-  --bin-dir path  folder for custom k6 binary (default: {{.bin}})
-  --builder list  comma separated list of builders (default: native,docker)
-  --no-color      disable colored output  
-  -h, --help      display this help
+  -o, --out name     output extension name
+  --bin-dir path     folder for custom k6 binary (default: {{.bin}})
+  --with dependency  additional dependency and version constraints
+  --builder list     comma separated list of builders (default: native,docker)
+  --no-color         disable colored output  
+  -h, --help         display this help
 `