Feature/storage api

.github/workflows/ci.yml

@@ -70,7 +70,7 @@ jobs:
         with:
           directory: ${{ runner.temp }}
 
-      - name: Make
+      - name: "Run System Tests"
         env:
           RUN_LONG_TESTS: 'yes'
           AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
@@ -82,12 +82,12 @@ jobs:
           sudo mkdir -p /srv ; sudo chown runner /srv
           COVERAGE_DIR=${{ runner.temp }} make all
 
-      - name: Merge code coverage
+      - name: "Merge code coverage"
         run: |
           go install github.com/wadey/gocovmerge@latest
           ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
 
-      - name: Upload code coverage
+      - name: "Upload code coverage"
         uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -185,15 +185,15 @@ jobs:
           APTLY_USER: ${{ secrets.APTLY_USER }}
           APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
         run: |
-          ./upload-artifacts.sh ci ${{ matrix.suite }}
+          .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
 
       - name: "Publish release to aptly"
         if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
         env:
           APTLY_USER: ${{ secrets.APTLY_USER }}
           APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
         run: |
-          ./upload-artifacts.sh release ${{ matrix.suite }}
+          .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
 
   ci-binary-build:
     name: "Build"
@@ -247,7 +247,8 @@ jobs:
         run: |
           make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
 
-      - uses: actions/upload-artifact@v4
+      - name: "Upload Artifacts"
+        uses: actions/upload-artifact@v4
         if: startsWith(github.event.ref, 'refs/tags')
         with:
           name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}

Makefile

@@ -26,7 +26,7 @@ prepare:  ## Install go module dependencies
 	go mod download
 	# install and initialize swagger
 	go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	go mod verify
 	go mod tidy -v
 	go generate
@@ -70,8 +70,8 @@ docker-test: ## Run system tests
 	@rm -f aptly.test
 	go generate
 	# install and initialize swagger
-	go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@latest
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# build coverage binary
 	go test -v -coverpkg="./..." -c -tags testruncli
 	@echo Running python tests ...
@@ -126,7 +126,7 @@ releasetype:  # Print release type (ci/release)
 build:  ## Build aptly
 	# install and initialize swagger
 	unset GOBIN; go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# prepare
 	go mod tidy
 	go generate
@@ -137,15 +137,15 @@ dev-server: prepare  ## Run dev-server
 	go install github.com/air-verse/air@v1.52.3
 	cp debian/aptly.conf /var/lib/aptly/.aptly.conf
 	sed -i /enableSwaggerEndpoint/s/false/true/ /var/lib/aptly/.aptly.conf
-	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142
 
 dpkg:  ## Build debian packages
 	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
 	# go generate
 	GOPATH=$$PWD/.go go generate -v
 	# install and initialize swagger
 	go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# set debian version
 	@if [ "`make -s releasetype`" = "ci" ]; then  \
 		echo CI Build, setting version... ; \
@@ -169,7 +169,7 @@ binaries:  ## Build binary releases (FreeBSD, MacOS, Linux tar)
 	@make version > VERSION
 	# install and initialize swagger
 	GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# build aptly
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
 	# install
@@ -194,7 +194,7 @@ docker-image:  ## Build aptly-dev docker image
 docker-build:  ## Build aptly in docker container
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd make build
 
-docker-aptly:  ## Build and run aptly commands in docker container
+docker-shell:  ## Run aptly and other commands in docker container
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd
 
 docker-deb:  ## Build debian packages in docker container
@@ -222,4 +222,4 @@ clean:  ## remove local build and module cache
 	test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true
 	rm -rf build/ docs/ obj-*-linux-gnu*
 
-.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-aptly clean releasetype dpkg dev-server docker-dev-server
+.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server

api/publish.go

@@ -57,6 +57,7 @@ func parseEscapedPath(path string) string {
 // @Tags Publish
 // @Produce  json
 // @Success 200 {array} deb.PublishedRepo
+// @Failure 500 {object} Error "Internal Error"
 // @Router /api/publish [get]
 func apiPublishList(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()

api/router.go

@@ -125,6 +125,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 			api.GET("/metrics", apiMetricsGet())
 		}
 		api.GET("/version", apiVersion)
+		api.GET("/storage", apiDiskFree)
 
 		isReady := &atomic.Value{}
 		isReady.Store(false)

api/storage.go

@@ -0,0 +1,43 @@
+package api
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/gin-gonic/gin"
+)
+
+type diskFree struct {
+	// Storage size [MiB]
+	Total uint64
+	// Available Storage [MiB]
+	Free uint64
+	// Percentage Full
+	PercentFull float32
+}
+
+// @Summary Get Storage Utilization
+// @Description **Get disk free information of aptly storage**
+// @Tags Status
+// @Produce json
+// @Success 200 {object} diskFree "Storage information"
+// @Failure 400 {object} Error "Internal Error"
+// @Router /api/storage [get]
+func apiDiskFree(c *gin.Context) {
+	var df diskFree
+
+	fs := context.Config().GetRootDir()
+
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(fs, &stat)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
+		return
+	}
+
+	df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
+	df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
+	df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
+
+	c.JSON(200, df)
+}

go.mod

@@ -101,16 +101,18 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.0-rc.0 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.15 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
 	google.golang.org/grpc v1.64.1 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -127,5 +129,5 @@ require (
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.3
-	go.etcd.io/etcd/client/v3 v3.5.0-rc.0
+	go.etcd.io/etcd/client/v3 v3.5.15
 )

go.sum

@@ -365,10 +365,16 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/etcd/api/v3 v3.5.0-rc.0 h1:oPIG9qBFXiGJFM1StoFd5EVllfVIhd7pPjB/mcMCzCg=
 go.etcd.io/etcd/api/v3 v3.5.0-rc.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk=
+go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 h1:7mW2vCG+tthhWwORSD/G0eesRZ6ZkOh19P7CyrT9xfA=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA=
+go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU=
 go.etcd.io/etcd/client/v3 v3.5.0-rc.0 h1:W2gUtoUho3+X5tCffYA3VfA7fH9DokrhGMscvZskuXc=
 go.etcd.io/etcd/client/v3 v3.5.0-rc.0/go.mod h1:5+cKiK2IEMa230cLFrLL3+3sKlPPKASjiLOmG8tZ8vY=
+go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4=
+go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
@@ -529,6 +535,12 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
 google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

system/Dockerfile

@@ -8,6 +8,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg a
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly
+RUN sed -i 's/#force_color_prompt=yes/force_color_prompt=yes/' /var/lib/aptly/.bashrc
 
 RUN mkdir /work
 WORKDIR /work

system/build-deb

@@ -5,6 +5,9 @@ chown -R `stat -c %u /work/src` /var/lib/aptly
 
 su aptly -c 'set -e; cd /work/src;
 GOPATH=$PWD/.go go generate -v
+# install and initialize swagger
+GOPATH=$PWD/.go go install github.com/swaggo/swag/cmd/swag@latest
+PATH=$PWD/.go/bin:$PATH swag init
 git checkout debian/changelog
 DEBEMAIL="CI <runner@github>" dch -v `make version` "CI build"
 dpkg-buildpackage -us -uc -b -d

system/run.py

@@ -11,6 +11,7 @@
 import traceback
 import random
 import subprocess
+import time
 
 from lib import BaseTest
 from s3_lib import S3Test
@@ -76,15 +77,15 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
                 if name in testignore:
                     continue
 
-                testout.clear()
-
                 o = getattr(testModule, name)
-
                 if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and
                         o is not SwiftTest and o is not S3Test and o is not AzureTest and
                         o is not APITest and o is not FileSystemEndpointTest):
                     continue
 
+                testout.clear()
+                start_time = time.time()
+
                 newBase = o.__bases__[0]
                 if lastBase is not None and lastBase is not newBase:
                     lastBase.shutdown_class()
@@ -102,7 +103,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
                     if not matches:
                         continue
 
-                orig_stdout.write("%s: %s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"])))
+                orig_stdout.write("· %-13s ➔ %-48s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"])))
                 orig_stdout.flush()
 
                 t = o()
@@ -118,20 +119,41 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
 
                 numTests += 1
 
+                failed = False
+                t.captureResults = capture_results
+                t.coverage_dir = coverage_dir
+                typ = None
+                val = None
+                tb = None
                 try:
-                    t.captureResults = capture_results
-                    t.coverage_dir = coverage_dir
                     t.test()
                 except Exception:
-                    numFailed += 1
                     typ, val, tb = sys.exc_info()
+                    failed = True
+
+                end_time = time.time()
+                execution_time = int(end_time - start_time) + 1
+                minutes = execution_time // 60
+                seconds = execution_time % 60
+                if minutes > 0:
+                    minutes = f"{minutes}m"
+                    if seconds < 10:
+                        seconds = f"0{seconds}"
+                else:
+                    minutes = "  "
+                    if seconds < 10:
+                        seconds = f" {seconds}"
+                duration = f"{minutes}{seconds}s"
+
+                if failed:
+                    numFailed += 1
                     fails.append((test, t, typ, val, tb, testModule))
-                    orig_stdout.write(colored("\b\b\b\bFAIL\n", color="red", attrs=["bold"]))
+                    orig_stdout.write(colored("\b\b\b\bFAIL", color="red", attrs=["bold"]) + f"   {duration}\n")
 
                     orig_stdout.write(testout.get_contents())
                     traceback.print_exception(typ, val, tb, file=orig_stdout)
                 else:
-                    orig_stdout.write(colored("\b\b\b\bOK \n", color="green", attrs=["bold"]))
+                    orig_stdout.write(colored("\b\b\b\bOK", color="green", attrs=["bold"]) + f"   {duration}\n")
 
                 t.shutdown()
 

system/t04_mirror/edit.py

@@ -112,6 +112,5 @@ class EditMirror10Test(BaseTest):
     """
     edit mirror: change archive url
     """
-    requiresFTP = True
     fixtureCmds = ["aptly mirror create -ignore-signatures mirror10 http://repo.aptly.info/system-tests/ftp.ru.debian.org/debian bookworm main"]
     runCmd = "aptly mirror edit -ignore-signatures -archive-url http://repo.aptly.info/system-tests/ftp.ch.debian.org/debian mirror10"

system/t04_mirror/update.py

@@ -162,7 +162,7 @@ class UpdateMirror10Test(BaseTest):
     outputMatchPrepare = filterOutSignature
 
 
-class UpdateMirror11Test(BaseTest):
+class UpdateMirror11FTPTest(BaseTest):
     """
     update mirrors: update over FTP
     """

system/t06_publish/repo.py

@@ -671,12 +671,12 @@ class PublishRepo26Test(BaseTest):
     """
     publish repo: sign with passphrase
     """
-    skipTest = "Failing on CI"
     fixtureCmds = [
         "aptly repo create local-repo",
         "aptly repo add local-repo ${files}",
+        "gpg --import --batch --passphrase verysecret ${files}/aptly_passphrase.sec"
     ]
-    runCmd = "aptly publish repo -keyring=${files}/aptly_passphrase.pub -secret-keyring=${files}/aptly_passphrase.sec -passphrase=verysecret -distribution=maverick local-repo"
+    runCmd = "aptly publish repo -batch -keyring=${files}/aptly_passphrase.pub -passphrase=verysecret -distribution=maverick local-repo"
     gold_processor = BaseTest.expand_environ
 
     def outputMatchPrepare(_, s):

system/t12_api/storage.py

@@ -0,0 +1,11 @@
+from api_lib import APITest
+
+
+class TaskAPITestSwaggerDocs(APITest):
+    """
+    GET /docs
+    """
+
+    def check(self):
+        resp = self.get("/api/storage")
+        self.check_equal(resp.status_code, 200)