diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1eeb4a5a..b689dd37e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: with: directory: ${{ runner.temp }} - - name: Make + - name: "Run System Tests" env: RUN_LONG_TESTS: 'yes' AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1" @@ -82,12 +82,12 @@ jobs: sudo mkdir -p /srv ; sudo chown runner /srv COVERAGE_DIR=${{ runner.temp }} make all - - name: Merge code coverage + - name: "Merge code coverage" run: | go install github.com/wadey/gocovmerge@latest ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt - - name: Upload code coverage + - name: "Upload code coverage" uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -185,7 +185,7 @@ jobs: APTLY_USER: ${{ secrets.APTLY_USER }} APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }} run: | - ./upload-artifacts.sh ci ${{ matrix.suite }} + .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }} - name: "Publish release to aptly" if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes' @@ -193,7 +193,7 @@ jobs: APTLY_USER: ${{ secrets.APTLY_USER }} APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }} run: | - ./upload-artifacts.sh release ${{ matrix.suite }} + .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }} ci-binary-build: name: "Build" @@ -247,7 +247,8 @@ jobs: run: | make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} - - uses: actions/upload-artifact@v4 + - name: "Upload Artifacts" + uses: actions/upload-artifact@v4 if: startsWith(github.event.ref, 'refs/tags') with: name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }} diff --git a/upload-artifacts.sh b/.github/workflows/scripts/upload-artifacts.sh similarity index 100% rename from upload-artifacts.sh rename to .github/workflows/scripts/upload-artifacts.sh diff --git a/Makefile b/Makefile index 3ef5169ff..cc5d34893 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ prepare: ## Install go module dependencies go mod download # install and initialize swagger go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q go mod verify go mod tidy -v go generate @@ -70,8 +70,8 @@ docker-test: ## Run system tests @rm -f aptly.test go generate # install and initialize swagger - go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@latest + PATH=$(BINPATH)/:$(PATH) swag init -q # build coverage binary go test -v -coverpkg="./..." -c -tags testruncli @echo Running python tests ... @@ -126,7 +126,7 @@ releasetype: # Print release type (ci/release) build: ## Build aptly # install and initialize swagger unset GOBIN; go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q # prepare go mod tidy go generate @@ -137,7 +137,7 @@ dev-server: prepare ## Run dev-server go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf /var/lib/aptly/.aptly.conf sed -i /enableSwaggerEndpoint/s/false/true/ /var/lib/aptly/.aptly.conf - PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142 + PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142 dpkg: ## Build debian packages @test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1) @@ -145,7 +145,7 @@ dpkg: ## Build debian packages GOPATH=$$PWD/.go go generate -v # install and initialize swagger go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q # set debian version @if [ "`make -s releasetype`" = "ci" ]; then \ echo CI Build, setting version... ; \ @@ -169,7 +169,7 @@ binaries: ## Build binary releases (FreeBSD, MacOS, Linux tar) @make version > VERSION # install and initialize swagger GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q # build aptly GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static' # install @@ -194,7 +194,7 @@ docker-image: ## Build aptly-dev docker image docker-build: ## Build aptly in docker container @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd make build -docker-aptly: ## Build and run aptly commands in docker container +docker-shell: ## Run aptly and other commands in docker container @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd docker-deb: ## Build debian packages in docker container @@ -222,4 +222,4 @@ clean: ## remove local build and module cache test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true rm -rf build/ docs/ obj-*-linux-gnu* -.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-aptly clean releasetype dpkg dev-server docker-dev-server +.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server diff --git a/api/publish.go b/api/publish.go index e3de9b4c7..a520878a0 100644 --- a/api/publish.go +++ b/api/publish.go @@ -57,6 +57,7 @@ func parseEscapedPath(path string) string { // @Tags Publish // @Produce json // @Success 200 {array} deb.PublishedRepo +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish [get] func apiPublishList(c *gin.Context) { collectionFactory := context.NewCollectionFactory() diff --git a/api/router.go b/api/router.go index 148b0f4af..faafcbcd3 100644 --- a/api/router.go +++ b/api/router.go @@ -125,6 +125,7 @@ func Router(c *ctx.AptlyContext) http.Handler { api.GET("/metrics", apiMetricsGet()) } api.GET("/version", apiVersion) + api.GET("/storage", apiDiskFree) isReady := &atomic.Value{} isReady.Store(false) diff --git a/api/storage.go b/api/storage.go new file mode 100644 index 000000000..4d306b0b4 --- /dev/null +++ b/api/storage.go @@ -0,0 +1,43 @@ +package api + +import ( + "fmt" + "syscall" + + "github.com/gin-gonic/gin" +) + +type diskFree struct { + // Storage size [MiB] + Total uint64 + // Available Storage [MiB] + Free uint64 + // Percentage Full + PercentFull float32 +} + +// @Summary Get Storage Utilization +// @Description **Get disk free information of aptly storage** +// @Tags Status +// @Produce json +// @Success 200 {object} diskFree "Storage information" +// @Failure 400 {object} Error "Internal Error" +// @Router /api/storage [get] +func apiDiskFree(c *gin.Context) { + var df diskFree + + fs := context.Config().GetRootDir() + + var stat syscall.Statfs_t + err := syscall.Statfs(fs, &stat) + if err != nil { + AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err)) + return + } + + df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576 + df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576 + df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0 + + c.JSON(200, df) +} diff --git a/go.mod b/go.mod index 382bacbf9..7b6568a79 100644 --- a/go.mod +++ b/go.mod @@ -101,8 +101,8 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - go.etcd.io/etcd/api/v3 v3.5.0-rc.0 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect @@ -110,7 +110,9 @@ require ( golang.org/x/sync v0.8.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/grpc v1.64.1 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -127,5 +129,5 @@ require ( github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 - go.etcd.io/etcd/client/v3 v3.5.0-rc.0 + go.etcd.io/etcd/client/v3 v3.5.15 ) diff --git a/go.sum b/go.sum index d25aac4af..f2ec6ca46 100644 --- a/go.sum +++ b/go.sum @@ -365,10 +365,16 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.0-rc.0 h1:oPIG9qBFXiGJFM1StoFd5EVllfVIhd7pPjB/mcMCzCg= go.etcd.io/etcd/api/v3 v3.5.0-rc.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 h1:7mW2vCG+tthhWwORSD/G0eesRZ6ZkOh19P7CyrT9xfA= go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= go.etcd.io/etcd/client/v3 v3.5.0-rc.0 h1:W2gUtoUho3+X5tCffYA3VfA7fH9DokrhGMscvZskuXc= go.etcd.io/etcd/client/v3 v3.5.0-rc.0/go.mod h1:5+cKiK2IEMa230cLFrLL3+3sKlPPKASjiLOmG8tZ8vY= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= @@ -529,6 +535,12 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/system/Dockerfile b/system/Dockerfile index 22bdb6ff4..7663fd8b6 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg a apt-get clean && rm -rf /var/lib/apt/lists/* RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly +RUN sed -i 's/#force_color_prompt=yes/force_color_prompt=yes/' /var/lib/aptly/.bashrc RUN mkdir /work WORKDIR /work diff --git a/system/build-deb b/system/build-deb index d161f3bfb..902d51de6 100755 --- a/system/build-deb +++ b/system/build-deb @@ -5,6 +5,9 @@ chown -R `stat -c %u /work/src` /var/lib/aptly su aptly -c 'set -e; cd /work/src; GOPATH=$PWD/.go go generate -v +# install and initialize swagger +GOPATH=$PWD/.go go install github.com/swaggo/swag/cmd/swag@latest +PATH=$PWD/.go/bin:$PATH swag init git checkout debian/changelog DEBEMAIL="CI " dch -v `make version` "CI build" dpkg-buildpackage -us -uc -b -d diff --git a/system/run.py b/system/run.py index 419dd4d1f..f4a2ef4fa 100755 --- a/system/run.py +++ b/system/run.py @@ -11,6 +11,7 @@ import traceback import random import subprocess +import time from lib import BaseTest from s3_lib import S3Test @@ -76,15 +77,15 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non if name in testignore: continue - testout.clear() - o = getattr(testModule, name) - if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and o is not SwiftTest and o is not S3Test and o is not AzureTest and o is not APITest and o is not FileSystemEndpointTest): continue + testout.clear() + start_time = time.time() + newBase = o.__bases__[0] if lastBase is not None and lastBase is not newBase: lastBase.shutdown_class() @@ -102,7 +103,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non if not matches: continue - orig_stdout.write("%s: %s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"]))) + orig_stdout.write("· %-13s ➔ %-48s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"]))) orig_stdout.flush() t = o() @@ -118,20 +119,41 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non numTests += 1 + failed = False + t.captureResults = capture_results + t.coverage_dir = coverage_dir + typ = None + val = None + tb = None try: - t.captureResults = capture_results - t.coverage_dir = coverage_dir t.test() except Exception: - numFailed += 1 typ, val, tb = sys.exc_info() + failed = True + + end_time = time.time() + execution_time = int(end_time - start_time) + 1 + minutes = execution_time // 60 + seconds = execution_time % 60 + if minutes > 0: + minutes = f"{minutes}m" + if seconds < 10: + seconds = f"0{seconds}" + else: + minutes = " " + if seconds < 10: + seconds = f" {seconds}" + duration = f"{minutes}{seconds}s" + + if failed: + numFailed += 1 fails.append((test, t, typ, val, tb, testModule)) - orig_stdout.write(colored("\b\b\b\bFAIL\n", color="red", attrs=["bold"])) + orig_stdout.write(colored("\b\b\b\bFAIL", color="red", attrs=["bold"]) + f" {duration}\n") orig_stdout.write(testout.get_contents()) traceback.print_exception(typ, val, tb, file=orig_stdout) else: - orig_stdout.write(colored("\b\b\b\bOK \n", color="green", attrs=["bold"])) + orig_stdout.write(colored("\b\b\b\bOK", color="green", attrs=["bold"]) + f" {duration}\n") t.shutdown() diff --git a/system/t04_mirror/UpdateMirror11Test_gold b/system/t04_mirror/UpdateMirror11FTPTest_gold similarity index 100% rename from system/t04_mirror/UpdateMirror11Test_gold rename to system/t04_mirror/UpdateMirror11FTPTest_gold diff --git a/system/t04_mirror/edit.py b/system/t04_mirror/edit.py index 7981c26dd..09a5b18f3 100644 --- a/system/t04_mirror/edit.py +++ b/system/t04_mirror/edit.py @@ -112,6 +112,5 @@ class EditMirror10Test(BaseTest): """ edit mirror: change archive url """ - requiresFTP = True fixtureCmds = ["aptly mirror create -ignore-signatures mirror10 http://repo.aptly.info/system-tests/ftp.ru.debian.org/debian bookworm main"] runCmd = "aptly mirror edit -ignore-signatures -archive-url http://repo.aptly.info/system-tests/ftp.ch.debian.org/debian mirror10" diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index b8ad5c268..e950d9817 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -162,7 +162,7 @@ class UpdateMirror10Test(BaseTest): outputMatchPrepare = filterOutSignature -class UpdateMirror11Test(BaseTest): +class UpdateMirror11FTPTest(BaseTest): """ update mirrors: update over FTP """ diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index fd3409d68..7e4b10c51 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -671,12 +671,12 @@ class PublishRepo26Test(BaseTest): """ publish repo: sign with passphrase """ - skipTest = "Failing on CI" fixtureCmds = [ "aptly repo create local-repo", "aptly repo add local-repo ${files}", + "gpg --import --batch --passphrase verysecret ${files}/aptly_passphrase.sec" ] - runCmd = "aptly publish repo -keyring=${files}/aptly_passphrase.pub -secret-keyring=${files}/aptly_passphrase.sec -passphrase=verysecret -distribution=maverick local-repo" + runCmd = "aptly publish repo -batch -keyring=${files}/aptly_passphrase.pub -passphrase=verysecret -distribution=maverick local-repo" gold_processor = BaseTest.expand_environ def outputMatchPrepare(_, s): diff --git a/system/t12_api/storage.py b/system/t12_api/storage.py new file mode 100644 index 000000000..4f6f3ba03 --- /dev/null +++ b/system/t12_api/storage.py @@ -0,0 +1,11 @@ +from api_lib import APITest + + +class TaskAPITestSwaggerDocs(APITest): + """ + GET /docs + """ + + def check(self): + resp = self.get("/api/storage") + self.check_equal(resp.status_code, 200) diff --git a/system/t13_etcd/mirror_update.py b/system/t13_etcd/mirror_update.py index d03b2c4d5..2e4a04b16 100644 --- a/system/t13_etcd/mirror_update.py +++ b/system/t13_etcd/mirror_update.py @@ -9,7 +9,6 @@ UpdateMirror8Test, \ UpdateMirror9Test, \ UpdateMirror10Test, \ - UpdateMirror11Test, \ UpdateMirror12Test, \ UpdateMirror13Test, \ UpdateMirror14Test, \ @@ -34,7 +33,6 @@ "UpdateMirror8Test", "UpdateMirror9Test", "UpdateMirror10Test", - "UpdateMirror11Test", "UpdateMirror12Test", "UpdateMirror13Test", "UpdateMirror14Test", @@ -130,14 +128,6 @@ class UpdateMirror10TestEtcd(UpdateMirror10Test): databaseUrl = "127.0.0.1:2379" -class UpdateMirror11TestEtcd(UpdateMirror11Test): - """ - update mirrors: update over FTP - """ - databaseType = "etcd" - databaseUrl = "127.0.0.1:2379" - - class UpdateMirror12TestEtcd(UpdateMirror12Test): """ update mirrors: update with udebs