diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bf628999a..6786d70a5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,31 +81,39 @@ jobs: env: LOG_PANIC_ON_INVALIDCHARS: true # check that log lines contains no invalid chars (evidence of format mismatch) steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Go environment - uses: actions/setup-go@v4 + - uses: actions/checkout@v3 + - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha + - uses: actions/setup-go@v4 with: go-version: '1.21' + - run: mkdir -p "$PWD/gocoverage-unit/" - name: Run Go test (and collect code coverage) - if: github.event_name == 'pull_request' || github.ref == 'refs/heads/dev' # quicker, non-race test in case it's a PR or push to dev - run: go test -coverprofile=unit.covdata.txt ./... + # quicker, non-race test in case it's a PR or push to dev + if: github.event_name == 'pull_request' || + github.ref == 'refs/heads/dev' + run: go test ./... + -cover -coverpkg=./... -covermode=count -args -test.gocoverdir="$PWD/gocoverage-unit/" - name: Run Go test -race (and collect code coverage) - if: github.event_name == 'push' && github.ref != 'refs/heads/dev' # this is further limited to selected branches at the beginning of this file + # note that -race can easily make the crypto stuff 10x slower + # this is further limited to selected branches at the beginning of this file + if: github.event_name == 'push' && + github.ref != 'refs/heads/dev' env: GORACE: atexit_sleep_ms=10 # the default of 1000 makes every Go package test sleep for 1s; see https://go.dev/issues/20364 - run: go test -coverprofile=unit.covdata.txt -vet=off -timeout=15m -race ./... # note that -race can easily make the crypto stuff 10x slower - - name: Store code coverage artifact + run: go test ./... + -race -timeout=15m -vet=off + -cover -coverpkg=./... -covermode=atomic -args -test.gocoverdir="$PWD/gocoverage-unit/" + - name: Store code coverage artifact (unit) uses: actions/upload-artifact@v3 with: - name: unit-covdata - path: unit.covdata.txt + name: gocoverage-unit@${{ env.SHA }} + path: gocoverage-unit/ job_compose_test: runs-on: [self-hosted, ci2-1] steps: - - name: Check out the repo - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha - name: Run compose script env: TESTSUITE_BUILD_TAG: ${{ github.sha }} @@ -113,24 +121,23 @@ jobs: COMPOSE_DVOTE_PORT_MAPPING: "9090" # this binds gateway0 to a random available port on docker host (needed for concurrent job runs) COMPOSE_HOST_PATH: ${{ github.workspace }}/dockerfiles/testsuite LOG_PANIC_ON_INVALIDCHARS: true # check that log lines contains no invalid chars (evidence of format mismatch) - GOCOVERDIR: "./gocoverage/" # collect code coverage when running binaries + GOCOVERDIR: "./gocoverage-integration/" # collect code coverage when running binaries CONCURRENT: 1 # run all the start_test.sh tests concurrently + BUILDARGS: "-race" # this makes the integration test only slightly slower (around +10%) unlike the abismal effect in unit test (10x) run: | cd dockerfiles/testsuite && ./start_test.sh - - name: Store code coverage artifact + - name: Store code coverage artifact (integration) uses: actions/upload-artifact@v3 with: - name: integration-covdata - path: dockerfiles/testsuite/gocoverage/covdata.txt + name: gocoverage-integration@${{ env.SHA }} + path: dockerfiles/testsuite/gocoverage-integration/ job_go_build_for_mac: runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release') steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Go environment - uses: actions/setup-go@v4 + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 with: go-version: '1.21' - name: Run go build for Mac @@ -139,31 +146,75 @@ jobs: # It's surprisingly hard with some deps like bazil.org/fuse. GOOS=darwin go build ./... - job_upload_coverage: - name: Publish code coverage - runs-on: ubuntu-latest + call-docker-release: + name: Docker + needs: [job_go_checks, job_go_test, job_compose_test] + if: github.event_name == 'push' # this is limited to selected branches at the beginning of this file + uses: vocdoni/vocdoni-node/.github/workflows/docker-release.yml@main + secrets: inherit + with: + image-tag: ${{ github.ref_name }} + + job_gocoverage_textfmt: + name: Convert coverage (bin->txt) + continue-on-error: true # never mark the whole CI as failed because of this job needs: [job_go_test, job_compose_test] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Fetch coverage (unit) - uses: actions/download-artifact@v3 + - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha + - uses: actions/download-artifact@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + cache: false + - name: Convert gocoverage format + run: | + go tool covdata textfmt -i=gocoverage-unit@${{ env.SHA }}/ \ + -o gocoverage-unit@${{ env.SHA }}.txt + go tool covdata textfmt -i=gocoverage-integration@${{ env.SHA }}/ \ + -o gocoverage-integration@${{ env.SHA }}.txt + - name: Merge both files + run: | + go install github.com/wadey/gocovmerge@latest + # dirty hack since integration is mode atomic and unit mode count, which are perfectly mergeable + # but gocovmerge doesn't recognize this: "cannot merge profiles with different modes" + sed 's/mode: count/mode: atomic/' gocoverage-unit@${{ env.SHA }}.txt \ + > gocoverage-unit@${{ env.SHA }}.tmp + gocovmerge gocoverage-unit@${{ env.SHA }}.tmp \ + gocoverage-integration@${{ env.SHA }}.txt \ + > gocoverage-merged@${{ env.SHA }}.txt + rm -f gocoverage-unit@${{ env.SHA }}.tmp + - name: Store code coverage artifact (all, textfmt) + if: ${{ always() }} + uses: actions/upload-artifact@v3 with: - name: unit-covdata - - name: Fetch coverage (integration) - uses: actions/download-artifact@v3 + name: gocoverage-all-textfmt@${{ env.SHA }} + path: gocoverage-*.txt + + job_gocoverage_coveralls: + name: Publish coverage (Coveralls) + runs-on: ubuntu-latest + needs: [job_gocoverage_textfmt] + continue-on-error: true # never mark the whole CI as failed because of this job + steps: + - uses: actions/checkout@v3 + - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha + - uses: actions/download-artifact@v3 with: - name: integration-covdata - + name: gocoverage-all-textfmt@${{ env.SHA }} - name: Send coverage to coveralls.io (unit) + if: ${{ always() }} uses: shogo82148/actions-goveralls@v1 with: - path-to-profile: unit.covdata.txt + path-to-profile: gocoverage-unit@${{ env.SHA }}.txt flag-name: unit parallel: true - name: Send coverage to coveralls.io (integration) + if: ${{ always() }} uses: shogo82148/actions-goveralls@v1 with: - path-to-profile: covdata.txt + path-to-profile: gocoverage-integration@${{ env.SHA }}.txt flag-name: integration parallel: true - name: Tell coveralls.io we're done @@ -172,11 +223,23 @@ jobs: with: parallel-finished: true - call-docker-release: - name: Docker - needs: [job_go_checks, job_go_test, job_compose_test] - if: github.event_name == 'push' # this is limited to selected branches at the beginning of this file - uses: vocdoni/vocdoni-node/.github/workflows/docker-release.yml@main - secrets: inherit - with: - image-tag: ${{ github.ref_name }} + job_gocoverage_deepsource: + name: Publish coverage (DeepSource) + runs-on: ubuntu-latest + needs: [job_gocoverage_textfmt] + continue-on-error: true # never mark the whole CI as failed because of this job + steps: + - uses: actions/checkout@v3 + - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha + - uses: actions/download-artifact@v3 + with: + name: gocoverage-all-textfmt@${{ env.SHA }} + - name: Send coverage to DeepSource + env: + DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }} + run: | + # Install the CLI + curl https://deepsource.io/cli | sh + + # Send the report to DeepSource + ./bin/deepsource report --analyzer test-coverage --key go --value-file gocoverage-merged@${{ env.SHA }}.txt diff --git a/.gitignore b/.gitignore index ed190126a..fb7763b20 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ zkCircuits # Output of the go coverage tool, specifically when used with LiteIDE *.out +gocoverage* # Docker files and dirs env.local diff --git a/Dockerfile b/Dockerfile index 7b2ab46a2..dd077750f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ COPY --from=builder /go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/pack /go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-amd64/libwasmer.so # Support for go-rapidsnark prover (https://github.com/iden3/go-rapidsnark/tree/main/prover) RUN apt-get update && \ - apt-get install -y libc6-dev libomp-dev openmpi-common libgomp1 curl && \ + apt-get install --no-install-recommends -y libc6-dev libomp-dev openmpi-common libgomp1 curl && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 80a660a9b..d450caaea 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -453,14 +453,12 @@ func accountSetValidator(cli *VocdoniCLI) error { if err != nil { return err } - pubKey := []byte{} + pubKey := cli.getCurrentAccount().PublicKey if pubKeyStr != "" { pubKey, err = hex.DecodeString(pubKeyStr) if err != nil { return err } - } else { - pubKey = cli.getCurrentAccount().PublicKey } hash, err := cli.api.AccountSetValidator(pubKey, name) diff --git a/cmd/end2endtest/account.go b/cmd/end2endtest/account.go index 929a46977..b2f80cd66 100644 --- a/cmd/end2endtest/account.go +++ b/cmd/end2endtest/account.go @@ -190,15 +190,15 @@ func testSendTokens(api *apiclient.HTTPclient, aliceKeys, bobKeys *ethereum.Sign // send a couple of token txs to increase the nonce, without waiting for them to be mined // this tests that the mempool transactions are properly ordered. - wg := sync.WaitGroup{} + var wg sync.WaitGroup wg.Add(1) go func() { log.Warnf("send transactions with nonce+1, should not be mined before the others") // send 1 token to burn address with nonce + 1 (should be mined after the other txs) - if _, err = alice.TransferWithNonce(state.BurnAddress, 1, aliceAcc.Nonce+1); err != nil { + if _, err := alice.TransferWithNonce(state.BurnAddress, 1, aliceAcc.Nonce+1); err != nil { log.Fatalf("cannot burn tokens: %v", err) } - if _, err = bob.TransferWithNonce(state.BurnAddress, 1, bobAcc.Nonce+1); err != nil { + if _, err := bob.TransferWithNonce(state.BurnAddress, 1, bobAcc.Nonce+1); err != nil { log.Fatalf("cannot burn tokens: %v", err) } wg.Done() @@ -208,6 +208,7 @@ func testSendTokens(api *apiclient.HTTPclient, aliceKeys, bobKeys *ethereum.Sign var txhasha, txhashb []byte wg.Add(1) go func() { + var err error txhasha, err = alice.TransferWithNonce(bobKeys.Address(), amountAtoB, aliceAcc.Nonce) if err != nil { log.Fatalf("cannot send tokens: %v", err) diff --git a/cmd/voconed/voconed.go b/cmd/voconed/voconed.go index 33c478462..ec03430f6 100644 --- a/cmd/voconed/voconed.go +++ b/cmd/voconed/voconed.go @@ -19,6 +19,7 @@ import ( "go.vocdoni.io/dvote/crypto/zk/circuit" "go.vocdoni.io/dvote/internal" "go.vocdoni.io/dvote/log" + "go.vocdoni.io/dvote/metrics" "go.vocdoni.io/dvote/vochain/state" "go.vocdoni.io/dvote/vocone" "go.vocdoni.io/proto/build/go/models" @@ -247,6 +248,9 @@ func main() { log.Fatal(err) } + vc.MetricsAgent = metrics.NewAgent("/metrics", + time.Duration(10)*time.Second, vc.Router) + // enable faucet if requested, this will create a new account and attach the faucet API to the vocone API if config.enableFaucetWithAmount > 0 { faucetAccount := ethereum.SignKeys{} diff --git a/data/data.go b/data/data.go index 1993130b7..350cb18d9 100644 --- a/data/data.go +++ b/data/data.go @@ -6,7 +6,6 @@ import ( "fmt" "go.vocdoni.io/dvote/data/ipfs" - "go.vocdoni.io/dvote/metrics" "go.vocdoni.io/dvote/types" ) @@ -19,8 +18,7 @@ type Storage interface { Unpin(ctx context.Context, path string) error ListPins(ctx context.Context) (map[string]string, error) URIprefix() string - Stats(ctx context.Context) map[string]any - CollectMetrics(ctx context.Context, ma *metrics.Agent) error + Stats() map[string]any Stop() error } diff --git a/data/datamocktest.go b/data/datamocktest.go index 1bfe180ef..4c1608d10 100644 --- a/data/datamocktest.go +++ b/data/datamocktest.go @@ -8,7 +8,6 @@ import ( "time" "go.vocdoni.io/dvote/data/ipfs" - "go.vocdoni.io/dvote/metrics" "go.vocdoni.io/dvote/test/testcommon/testutil" "go.vocdoni.io/dvote/types" ) @@ -80,11 +79,7 @@ func (d *DataMockTest) URIprefix() string { return d.prefix } -func (*DataMockTest) Stats(_ context.Context) map[string]any { - return nil -} - -func (*DataMockTest) CollectMetrics(_ context.Context, _ *metrics.Agent) error { +func (*DataMockTest) Stats() map[string]any { return nil } diff --git a/data/ipfs/ipfs.go b/data/ipfs/ipfs.go index 6960a4446..6f5920efa 100644 --- a/data/ipfs/ipfs.go +++ b/data/ipfs/ipfs.go @@ -131,6 +131,9 @@ func (i *Handler) Init(d *types.DataStore) error { return err } + go i.updateStats(time.Minute) + i.registerMetrics() + return nil } @@ -207,20 +210,8 @@ func (i *Handler) Unpin(ctx context.Context, path string) error { } // Stats returns stats about the IPFS node. -func (i *Handler) Stats(ctx context.Context) map[string]any { - peers, err := i.CoreAPI.Swarm().Peers(ctx) - if err != nil { - return map[string]any{"error": err.Error()} - } - addresses, err := i.CoreAPI.Swarm().KnownAddrs(ctx) - if err != nil { - return map[string]any{"error": err.Error()} - } - pins, err := i.countPins(ctx) - if err != nil { - return map[string]any{"error": err.Error()} - } - return map[string]any{"peers": len(peers), "addresses": len(addresses), "pins": pins} +func (i *Handler) Stats() map[string]any { + return map[string]any{"peers": stats.Peers.Load(), "addresses": stats.KnownAddrs.Load(), "pins": stats.Pins.Load()} } func (i *Handler) countPins(ctx context.Context) (int, error) { diff --git a/data/ipfs/metrics.go b/data/ipfs/metrics.go index 6d6599c99..168ca6bad 100644 --- a/data/ipfs/metrics.go +++ b/data/ipfs/metrics.go @@ -2,77 +2,86 @@ package ipfs import ( "context" + "sync" "time" "github.com/prometheus/client_golang/prometheus" + "go.uber.org/atomic" "go.vocdoni.io/dvote/metrics" ) -// File collectors -var ( - // FilePeers ... - FilePeers = prometheus.NewGauge(prometheus.GaugeOpts{ +var stats struct { + Peers atomic.Float64 + KnownAddrs atomic.Float64 + Pins atomic.Float64 +} + +// registerMetrics registers prometheus metrics +func (i *Handler) registerMetrics() { + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "file", Name: "peers", Help: "The number of connected peers", - }) - // FileAddresses ... - FileAddresses = prometheus.NewGauge(prometheus.GaugeOpts{ + }, + stats.Peers.Load)) + + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "file", Name: "addresses", Help: "The number of registered addresses", - }) - // FilePins ... - FilePins = prometheus.NewGauge(prometheus.GaugeOpts{ + }, + stats.KnownAddrs.Load)) + + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "file", Name: "pins", Help: "The number of pinned files", - }) -) - -// RegisterMetrics to initialize the metrics to the agent -func (*Handler) registerMetrics(ma *metrics.Agent) { - ma.Register(FilePeers) - ma.Register(FileAddresses) - ma.Register(FilePins) -} - -// setMetrics to be called as a loop and grab metrics -func (i *Handler) setMetrics(ctx context.Context) error { - peers, err := i.CoreAPI.Swarm().Peers(ctx) - if err != nil { - return err - } - FilePeers.Set(float64(len(peers))) - addresses, err := i.CoreAPI.Swarm().KnownAddrs(ctx) - if err != nil { - return err - } - FileAddresses.Set(float64(len(addresses))) - pins, err := i.countPins(ctx) - if err != nil { - return err - } - FilePins.Set(float64(pins)) - return nil + }, + stats.Pins.Load)) } -// CollectMetrics constantly updates the metric values for prometheus +// updateStats constantly updates the ipfs stats (Peers, KnownAddrs, Pins) // The function is blocking, should be called in a go routine -// If the metrics Agent is nil, do nothing -func (i *Handler) CollectMetrics(ctx context.Context, ma *metrics.Agent) error { - if ma != nil { - i.registerMetrics(ma) - for { - time.Sleep(ma.RefreshInterval) - tctx, cancel := context.WithTimeout(ctx, time.Minute) - err := i.setMetrics(tctx) - cancel() - if err != nil { - return err +func (i *Handler) updateStats(interval time.Duration) { + for { + t := time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), interval) + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + list, err := i.CoreAPI.Swarm().Peers(ctx) + if err == nil { + stats.Peers.Store(float64(len(list))) + } + wg.Done() + }() + + wg.Add(1) + go func() { + list, err := i.CoreAPI.Swarm().KnownAddrs(ctx) + if err == nil { + stats.KnownAddrs.Store(float64(len(list))) } - } + wg.Done() + }() + + wg.Add(1) + go func() { + count, err := i.countPins(ctx) + if err == nil { + stats.Pins.Store(float64(count)) + } + wg.Done() + }() + + wg.Wait() + + cancel() + + time.Sleep(interval - time.Since(t)) } - return nil } diff --git a/dockerfiles/testsuite/start_test.sh b/dockerfiles/testsuite/start_test.sh index 4276d6782..eb9e6c694 100755 --- a/dockerfiles/testsuite/start_test.sh +++ b/dockerfiles/testsuite/start_test.sh @@ -14,7 +14,7 @@ # e2etest_dynamicensuselection: run dynamic census test export COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 COMPOSE_INTERACTIVE_NO_CLI=1 -[ -n "$GOCOVERDIR" ] && export BUILDARGS="-cover" # docker compose build passes this to go 1.20 so that binaries collect code coverage +[ -n "$GOCOVERDIR" ] && export BUILDARGS="$BUILDARGS -cover" # docker compose build passes this to go 1.20 so that binaries collect code coverage COMPOSE_CMD=${COMPOSE_CMD:-"docker compose"} COMPOSE_CMD_RUN="$COMPOSE_CMD run" @@ -172,7 +172,14 @@ log "### Waiting for test suite to be ready ###" for i in {1..20}; do check_gw_is_up && break || sleep 2 done -log "### Test suite ready ###" + +if [ i == 20 ] ; then + log "### Timed out waiting! Abort, don't even try running tests ###" + tests_to_run=() + GOCOVERDIR= +else + log "### Test suite ready ###" +fi # create temp dir results="/tmp/.vocdoni-test$RANDOM" @@ -208,15 +215,20 @@ done if [ -n "$GOCOVERDIR" ] ; then log "### Collect all coverage files in $GOCOVERDIR ###" - mkdir -p $GOCOVERDIR - $COMPOSE_CMD down + rm -rf "$GOCOVERDIR" + mkdir -p "$GOCOVERDIR" + $COMPOSE_CMD stop $COMPOSE_CMD_RUN --user=`id -u`:`id -g` -v $(pwd):/wd/ gocoverage sh -c "\ cp -rf /app/run/gocoverage/. /wd/$GOCOVERDIR go tool covdata textfmt \ -i=\$(find /wd/$GOCOVERDIR/ -type d -printf '%p,'| sed 's/,$//') \ - -o=/wd/$GOCOVERDIR/covdata.txt + -o=/wd/$GOCOVERDIR/gocoverage-integration.txt + go tool covdata merge \ + -i=\$(find /wd/$GOCOVERDIR/ -type d -printf '%p,'| sed 's/,$//') \ + -o=/wd/$GOCOVERDIR/ " - log "### Coverage data in textfmt left in $GOCOVERDIR/covdata.txt ###" + log "### Coverage data in textfmt left in $GOCOVERDIR/gocoverage-integration.txt ###" + log "### Coverage data in binary fmt left in $GOCOVERDIR ###" fi [ $CLEAN -eq 1 ] && { @@ -232,4 +244,6 @@ fi # remove temp dir rm -rf $results +if [ "$RET" == "66" ] ; then log "### Race detected! Look for WARNING: DATA RACE in the logs above ###"; fi + exit $RET diff --git a/go.mod b/go.mod index 89313db7d..4be8c8abf 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,8 @@ require ( github.com/vocdoni/go-snark v0.0.0-20210709152824-f6e4c27d7319 github.com/vocdoni/storage-proofs-eth-go v0.1.6 go.mongodb.org/mongo-driver v1.12.1 - go.vocdoni.io/proto v1.15.4-0.20231016082837-3478af3ba0ed + go.uber.org/atomic v1.11.0 + go.vocdoni.io/proto v1.15.4-0.20231017174559-1d9ea54cd9ad golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/net v0.16.0 @@ -297,7 +298,6 @@ require ( go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/fx v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index bb6b9585b..d39f1a570 100644 --- a/go.sum +++ b/go.sum @@ -1646,16 +1646,10 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= -go.vocdoni.io/proto v1.14.6-0.20230802094125-e07a41fda290 h1:o5MrI+nI5GJDUYMEdzP/JJbMwdiOU1mr1LnrmXaN2q4= -go.vocdoni.io/proto v1.14.6-0.20230802094125-e07a41fda290/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo= -go.vocdoni.io/proto v1.15.4-0.20231011091653-867fca9311dc h1:b5XOLixBqpUrGrFLVNv/PJXHb04OWuHoM0jlkjKnNH0= -go.vocdoni.io/proto v1.15.4-0.20231011091653-867fca9311dc/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo= -go.vocdoni.io/proto v1.15.4-0.20231011092110-b68d0304b69a h1:b9CcMivQKLltLe7CY2Atjf6t64m6QmF+7S7Dojp3HAc= -go.vocdoni.io/proto v1.15.4-0.20231011092110-b68d0304b69a/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo= -go.vocdoni.io/proto v1.15.4-0.20231011153201-c6adb37abbe8 h1:jswlpiefE2m+asSVxtXVqVnivevnce3K1tuJ/DsLD4w= -go.vocdoni.io/proto v1.15.4-0.20231011153201-c6adb37abbe8/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo= go.vocdoni.io/proto v1.15.4-0.20231016082837-3478af3ba0ed h1:kxWgxXo7LZ6ksFRoFAdl2A1AOviEfY8+8aAJaCiiV80= go.vocdoni.io/proto v1.15.4-0.20231016082837-3478af3ba0ed/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo= +go.vocdoni.io/proto v1.15.4-0.20231017174559-1d9ea54cd9ad h1:v+ixFtzq/7nwa/y7lY/LiFcVOPjK7aF9EpyX6RWCS2M= +go.vocdoni.io/proto v1.15.4-0.20231017174559-1d9ea54cd9ad/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= diff --git a/metrics/metrics.go b/metrics/metrics.go index 615d1ec11..6e08e7538 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -23,8 +23,8 @@ func NewAgent(path string, interval time.Duration, router *httprouter.HTTProuter return &ma } -// Register adds a prometheus collector -func (*Agent) Register(c prometheus.Collector) { +// Register the provided prometheus collector, ignoring any error returned (simply logs a Warn) +func Register(c prometheus.Collector) { err := prometheus.Register(c) if err != nil { log.Warnf("cannot register metrics: (%s) (%+v)", err, c) diff --git a/service/ipfs.go b/service/ipfs.go index abfd64bc6..b80459df1 100644 --- a/service/ipfs.go +++ b/service/ipfs.go @@ -1,7 +1,6 @@ package service import ( - "context" "os" "time" @@ -25,14 +24,10 @@ func (vs *VocdoniService) IPFS(ipfsconfig *config.IPFSCfg) (storage data.Storage go func() { for { time.Sleep(time.Second * 120) - tctx, cancel := context.WithTimeout(context.Background(), time.Minute) - log.Monitor("ipfs storage", storage.Stats(tctx)) - cancel() + log.Monitor("ipfs storage", storage.Stats()) } }() - go storage.CollectMetrics(context.Background(), vs.MetricsAgent) - if len(ipfsconfig.ConnectKey) > 0 { log.Infow("starting ipfsconnect service", "key", ipfsconfig.ConnectKey) ipfsconn := ipfsconnect.New( diff --git a/service/vochain.go b/service/vochain.go index 2cd0e3f4f..9d4c3749d 100644 --- a/service/vochain.go +++ b/service/vochain.go @@ -132,9 +132,6 @@ func (vs *VocdoniService) Start() error { if vs.Stats == nil { vs.Stats = vochaininfo.NewVochainInfo(vs.App) go vs.Stats.Start(10) - - // Grab metrics - go vs.Stats.CollectMetrics(vs.MetricsAgent) } if !vs.Config.NoWaitSync { diff --git a/subpub/discovery.go b/subpub/discovery.go index 0f905b856..d6655fa7f 100644 --- a/subpub/discovery.go +++ b/subpub/discovery.go @@ -10,13 +10,23 @@ import ( discrouting "github.com/libp2p/go-libp2p/p2p/discovery/routing" discutil "github.com/libp2p/go-libp2p/p2p/discovery/util" multiaddr "github.com/multiformats/go-multiaddr" + "github.com/prometheus/client_golang/prometheus" "go.vocdoni.io/dvote/log" + "go.vocdoni.io/dvote/metrics" +) + +// Metrics exported via prometheus +var ( + dhtLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "file", + Name: "peers_dht_latency", + Help: "The time it takes FindPeers to discover peers", + }) ) // setupDiscovery creates a DHT discovery service and attaches it to the libp2p Host. // This lets us automatically discover peers and connect to them. func (s *SubPub) setupDiscovery(ctx context.Context) { - // Set a function as stream handler. This function is called when a peer // initiates a connection and starts a stream with this peer. if !s.OnlyDiscover { @@ -29,6 +39,8 @@ func (s *SubPub) setupDiscovery(ctx context.Context) { s.routing = discrouting.NewRoutingDiscovery(s.node.DHT) discutil.Advertise(ctx, s.routing, s.Topic) + metrics.Register(dhtLatency) + // Discover new peers periodically go func() { // this spawns a single background task per instance for { @@ -46,6 +58,7 @@ func (s *SubPub) setupDiscovery(ctx context.Context) { } func (s *SubPub) discover(ctx context.Context) { + dhtLatencyTimer := prometheus.NewTimer(dhtLatency) // Now, look for others who have announced. // This is like your friend telling you the location to meet you. log.Debugf("looking for peers in topic %s", s.Topic) @@ -69,6 +82,8 @@ func (s *SubPub) discover(ctx context.Context) { continue } // new peer; let's connect to it + // first update the latency metrics + dhtLatencyTimer.ObserveDuration() connectCtx, cancel := context.WithTimeout(ctx, time.Second*10) if err := s.node.PeerHost.Connect(connectCtx, peer); err != nil { cancel() diff --git a/test/api_test.go b/test/api_test.go index cb305e8e9..a9c483eb5 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -18,6 +18,7 @@ import ( "go.vocdoni.io/dvote/util" "go.vocdoni.io/dvote/vochain" "go.vocdoni.io/dvote/vochain/state" + "go.vocdoni.io/dvote/vochain/state/electionprice" "go.vocdoni.io/proto/build/go/models" "google.golang.org/protobuf/proto" ) @@ -254,6 +255,107 @@ func TestAPIaccount(t *testing.T) { qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) } +func TestAPIElectionCost(t *testing.T) { + // cheap election + runAPIElectionCostWithParams(t, + electionprice.ElectionParameters{ + MaxCensusSize: 100, + ElectionDuration: 2000, + EncryptedVotes: false, + AnonymousVotes: false, + MaxVoteOverwrite: 1, + }, + 10000, 5000, + 5, 1000, + 6) + + // bigger census size, duration, reduced network capacity, etc + runAPIElectionCostWithParams(t, + electionprice.ElectionParameters{ + MaxCensusSize: 5000, + ElectionDuration: 10000, + EncryptedVotes: false, + AnonymousVotes: false, + MaxVoteOverwrite: 3, + }, + 200000, 6000, + 10, 100, + 762) + + // very expensive election + runAPIElectionCostWithParams(t, + electionprice.ElectionParameters{ + MaxCensusSize: 100000, + ElectionDuration: 1000000, + EncryptedVotes: true, + AnonymousVotes: true, + MaxVoteOverwrite: 10, + }, + 100000, 700000, + 10, 100, + 547026) +} + +func runAPIElectionCostWithParams(t *testing.T, + electionParams electionprice.ElectionParameters, + startBlock uint32, initialBalance uint64, + txCostNewProcess, networkCapacity uint64, + expectedPrice uint64, +) { + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.VoteHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + ) + + token1 := uuid.New() + c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + err := server.VochainAPP.State.SetTxBaseCost(models.TxType_NEW_PROCESS, txCostNewProcess) + qt.Assert(t, err, qt.IsNil) + err = server.VochainAPP.State.SetElectionPriceCalc() + qt.Assert(t, err, qt.IsNil) + server.VochainAPP.State.ElectionPriceCalc.SetCapacity(networkCapacity) + + // Block 1 + server.VochainAPP.AdvanceTestBlock() + + signer := createAccount(t, c, server, initialBalance) + + // Block 2 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 2) + + censusRoot := createCensus(t, c) + + // first check predictedPrice equals the hardcoded expected + + predictedPrice := predictPriceForElection(t, c, electionParams) + qt.Assert(t, predictedPrice, qt.Equals, expectedPrice) + + // now check balance before creating election and then after creating, + // and confirm the balance decreased exactly the expected amount + + qt.Assert(t, requestAccount(t, c, signer.Address().String()).Balance, + qt.Equals, initialBalance) + + createElection(t, c, signer, electionParams, censusRoot, startBlock, server.VochainAPP.ChainID()) + + // Block 3 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 3) + + balance := requestAccount(t, c, signer.Address().String()).Balance + qt.Assert(t, balance, + qt.Equals, initialBalance-predictedPrice, + qt.Commentf("endpoint /elections/price predicted cost %d, "+ + "but actual election creation costed %d", predictedPrice, initialBalance-balance)) +} + func waitUntilHeight(t testing.TB, c *testutil.TestHTTPclient, h uint32) { for { resp, code := c.Request("GET", nil, "chain", "info") @@ -276,3 +378,150 @@ func waitUntilHeight(t testing.TB, c *testutil.TestHTTPclient, h uint32) { time.Sleep(time.Second * 1) } } + +func requestAccount(t testing.TB, c *testutil.TestHTTPclient, address string) api.Account { + resp, code := c.Request("GET", nil, "accounts", address) + qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) + acct := api.Account{} + err := json.Unmarshal(resp, &acct) + qt.Assert(t, err, qt.IsNil) + return acct +} + +func createCensus(t testing.TB, c *testutil.TestHTTPclient) (root types.HexBytes) { + resp, code := c.Request("POST", nil, "censuses", "weighted") + qt.Assert(t, code, qt.Equals, 200) + censusData := &api.Census{} + qt.Assert(t, json.Unmarshal(resp, censusData), qt.IsNil) + + id1 := censusData.CensusID.String() + resp, code = c.Request("POST", nil, "censuses", id1, "publish") + qt.Assert(t, code, qt.Equals, 200) + qt.Assert(t, json.Unmarshal(resp, censusData), qt.IsNil) + qt.Assert(t, censusData.CensusID, qt.IsNotNil) + return censusData.CensusID +} + +func createElection(t testing.TB, c *testutil.TestHTTPclient, + signer *ethereum.SignKeys, + electionParams electionprice.ElectionParameters, + censusRoot types.HexBytes, + startBlock uint32, + chainID string, +) api.ElectionCreate { + metadataBytes, err := json.Marshal( + &api.ElectionMetadata{ + Title: map[string]string{"default": "test election"}, + Description: map[string]string{"default": "test election description"}, + Version: "1.0", + }) + + qt.Assert(t, err, qt.IsNil) + metadataURI := ipfs.CalculateCIDv1json(metadataBytes) + + tx := models.Tx_NewProcess{ + NewProcess: &models.NewProcessTx{ + Txtype: models.TxType_NEW_PROCESS, + Nonce: 0, + Process: &models.Process{ + StartBlock: startBlock, + BlockCount: electionParams.ElectionDuration, + Status: models.ProcessStatus_READY, + CensusRoot: censusRoot, + CensusOrigin: models.CensusOrigin_OFF_CHAIN_TREE_WEIGHTED, + Mode: &models.ProcessMode{AutoStart: true, Interruptible: true}, + VoteOptions: &models.ProcessVoteOptions{ + MaxCount: 1, + MaxValue: 1, + MaxVoteOverwrites: electionParams.MaxVoteOverwrite, + }, + EnvelopeType: &models.EnvelopeType{ + EncryptedVotes: electionParams.EncryptedVotes, + Anonymous: electionParams.AnonymousVotes, + }, + Metadata: &metadataURI, + MaxCensusSize: electionParams.MaxCensusSize, + }, + }, + } + + txb, err := proto.Marshal(&models.Tx{Payload: &tx}) + qt.Assert(t, err, qt.IsNil) + signedTxb, err := signer.SignVocdoniTx(txb, chainID) + qt.Assert(t, err, qt.IsNil) + stx := models.SignedTx{Tx: txb, Signature: signedTxb} + stxb, err := proto.Marshal(&stx) + qt.Assert(t, err, qt.IsNil) + + election := api.ElectionCreate{ + TxPayload: stxb, + Metadata: metadataBytes, + } + resp, code := c.Request("POST", election, "elections") + qt.Assert(t, code, qt.Equals, 200) + err = json.Unmarshal(resp, &election) + qt.Assert(t, err, qt.IsNil) + + return election +} + +func predictPriceForElection(t testing.TB, c *testutil.TestHTTPclient, + electionParams electionprice.ElectionParameters) uint64 { + predicted := struct { + Price uint64 `json:"price"` + }{} + + resp, code := c.Request("POST", electionParams, "elections", "price") + qt.Assert(t, code, qt.Equals, 200) + err := json.Unmarshal(resp, &predicted) + qt.Assert(t, err, qt.IsNil) + + return predicted.Price +} + +func createAccount(t testing.TB, c *testutil.TestHTTPclient, + server testcommon.APIserver, initialBalance uint64) *ethereum.SignKeys { + signer := ethereum.SignKeys{} + qt.Assert(t, signer.Generate(), qt.IsNil) + + // metadata + meta := &api.AccountMetadata{ + Version: "1.0", + } + metaData, err := json.Marshal(meta) + qt.Assert(t, err, qt.IsNil) + + fp, err := vochain.GenerateFaucetPackage(server.Account, signer.Address(), initialBalance) + qt.Assert(t, err, qt.IsNil) + + // transaction + stx := models.SignedTx{} + infoURI := "ipfs://" + ipfs.CalculateCIDv1json(metaData) + sik, err := signer.AccountSIK(nil) + qt.Assert(t, err, qt.IsNil) + stx.Tx, err = proto.Marshal(&models.Tx{Payload: &models.Tx_SetAccount{ + SetAccount: &models.SetAccountTx{ + Txtype: models.TxType_CREATE_ACCOUNT, + Nonce: new(uint32), + InfoURI: &infoURI, + Account: signer.Address().Bytes(), + FaucetPackage: fp, + SIK: sik, + }, + }}) + qt.Assert(t, err, qt.IsNil) + stx.Signature, err = signer.SignVocdoniTx(stx.Tx, server.VochainAPP.ChainID()) + qt.Assert(t, err, qt.IsNil) + stxb, err := proto.Marshal(&stx) + qt.Assert(t, err, qt.IsNil) + + // send the transaction and metadata + accSet := api.AccountSet{ + Metadata: metaData, + TxPayload: stxb, + } + resp, code := c.Request("POST", &accSet, "accounts") + qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) + + return &signer +} diff --git a/test/testcommon/api.go b/test/testcommon/api.go index 5e0d872bd..2c4709809 100644 --- a/test/testcommon/api.go +++ b/test/testcommon/api.go @@ -57,7 +57,7 @@ func (d *APIserver) Start(t testing.TB, apis ...string) { d.VochainAPP = vochain.TestBaseApplication(t) // create and add balance for the pre-created Account - err = d.VochainAPP.State.CreateAccount(d.Account.Address(), "", nil, 100000) + err = d.VochainAPP.State.CreateAccount(d.Account.Address(), "", nil, 1000000) qt.Assert(t, err, qt.IsNil) d.VochainAPP.CommitState() diff --git a/vochain/genesis/genesis.go b/vochain/genesis/genesis.go index 68ff9fa4b..b8d45447e 100644 --- a/vochain/genesis/genesis.go +++ b/vochain/genesis/genesis.go @@ -39,8 +39,8 @@ var Genesis = map[string]Vochain{ } var devGenesis = Doc{ - GenesisTime: time.Date(2023, time.October, 10, 1, 0, 0, 0, time.UTC), - ChainID: "vocdoni-dev-24", + GenesisTime: time.Date(2023, time.October, 17, 1, 0, 0, 0, time.UTC), + ChainID: "vocdoni-dev-25", ConsensusParams: &ConsensusParams{ Block: BlockParams{ MaxBytes: 2097152, diff --git a/vochain/ist/validators.go b/vochain/ist/validators.go index 993d731d3..db38b9947 100644 --- a/vochain/ist/validators.go +++ b/vochain/ist/validators.go @@ -12,6 +12,10 @@ import ( This mechanism is responsible for the management and updating of validator power based on their voting and proposing performance over time. It operates by evaluating the performance of validators in terms of votes on proposals they have accrued. +As a general idea, when a validator does not participate on the block production, their power will decay over time. +On the other hand, if a validator participates in the block production, their power will increase over time until it reaches the maximum power. +When a new validator joins the validator set, it starts with the minimum power (currently 5). + The mechanism is based on the following parameters: 1. `maxPower`: This represents the maximum power that any validator can achieve. Currently set to 100. @@ -50,7 +54,7 @@ const ( maxPower = 100 // maximum power of a validator updatePowerPeriod = 10 // number of blocks to wait before updating validators power positiveScoreThreshold = 80 // if this minimum score is kept, the validator power will be increased - powerDecayRate = 0.05 // 10% decay rate + powerDecayRate = 0.05 // 5% decay rate ) func (c *Controller) updateValidatorScore(voteAddresses [][]byte, proposer []byte) error { @@ -93,15 +97,20 @@ func (c *Controller) updateValidatorScore(voteAddresses [][]byte, proposer []byt validators[idx].Score = newScore } // update or remove the validator - if validators[idx].Power <= 0 && len(validators) > 3 { - if err := c.state.RemoveValidator(validators[idx].Address); err != nil { - return fmt.Errorf("cannot remove validator: %w", err) - } - } else { - if err := c.state.AddValidator(validators[idx]); err != nil { - return fmt.Errorf("cannot update validator score: %w", err) + if validators[idx].Power <= 0 { + if len(validators) <= 3 { + // cannot remove the last 3 validators + validators[idx].Power = 1 + } else { + if err := c.state.RemoveValidator(validators[idx].Address); err != nil { + return fmt.Errorf("cannot remove validator: %w", err) + } + continue } } + if err := c.state.AddValidator(validators[idx]); err != nil { + return fmt.Errorf("cannot update validator score: %w", err) + } } return nil } diff --git a/vochain/start.go b/vochain/start.go index 3a12ecf39..73c525e96 100644 --- a/vochain/start.go +++ b/vochain/start.go @@ -229,11 +229,11 @@ func newTendermint(app *BaseApplication, return nil, fmt.Errorf("cannot create or load node key: %w", err) } } - log.Debugf("tendermint p2p config: %+v", tconfig.P2P) - - log.Infow("tendermint config", + log.Infow("vochain initialized", "db-backend", tconfig.DBBackend, - "pubkey", hex.EncodeToString(pv.Key.PubKey.Bytes()), + "publicKey", hex.EncodeToString(pv.Key.PubKey.Bytes()), + "accountAddr", app.NodeAddress, + "validatorAddr", pv.Key.PubKey.Address(), "external-address", tconfig.P2P.ExternalAddress, "nodeId", nodeKey.ID(), "seed", tconfig.P2P.SeedMode) diff --git a/vochain/transaction/election_tx.go b/vochain/transaction/election_tx.go index 7e8f4ac37..478ff3447 100644 --- a/vochain/transaction/election_tx.go +++ b/vochain/transaction/election_tx.go @@ -250,7 +250,7 @@ func checkRevealProcessKeys(tx *models.AdminTx, process *models.Process) error { func (t *TransactionHandler) txElectionCostFromProcess(process *models.Process) uint64 { return t.state.ElectionPriceCalc.Price(&electionprice.ElectionParameters{ MaxCensusSize: process.GetMaxCensusSize(), - ElectionDuration: process.BlockCount + process.StartBlock, + ElectionDuration: process.BlockCount, EncryptedVotes: process.GetEnvelopeType().EncryptedVotes, AnonymousVotes: process.GetEnvelopeType().Anonymous, MaxVoteOverwrite: process.GetVoteOptions().MaxVoteOverwrites, diff --git a/vochain/vochaininfo/metrics.go b/vochain/vochaininfo/metrics.go index b607c6a09..2f7f32d76 100644 --- a/vochain/vochaininfo/metrics.go +++ b/vochain/vochaininfo/metrics.go @@ -1,113 +1,72 @@ package vochaininfo import ( - "time" - "github.com/prometheus/client_golang/prometheus" "go.vocdoni.io/dvote/metrics" ) -// Vochain collectors -var ( - // VochainHeight ... - VochainHeight = prometheus.NewGauge(prometheus.GaugeOpts{ +// registerMetrics registers each of the vochain prometheus metrics +func (vi *VochainInfo) registerMetrics() { + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "height", Help: "Height of the vochain (last block)", - }) - // VochainMempool ... - VochainMempool = prometheus.NewGauge(prometheus.GaugeOpts{ + }, + func() float64 { return float64(vi.Height()) })) + + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "mempool", Help: "Number of Txs in the mempool", - }) - // VochainAppTree ... - VochainAppTree = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "vochain", - Name: "app_tree", - Help: "Size of the app tree", - }) - // VochainProcessTree ... - VochainProcessTree = prometheus.NewGauge(prometheus.GaugeOpts{ + }, + func() float64 { return float64(vi.MempoolSize()) })) + + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "process_tree", Help: "Size of the process tree", - }) - // VochainVoteTree ... - VochainVoteTree = prometheus.NewGauge(prometheus.GaugeOpts{ + }, + func() float64 { p, _, _ := vi.TreeSizes(); return float64(p) })) + + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "vote_tree", Help: "Size of the vote tree", - }) - // VochainVotesPerMinute ... - VochainVotesPerMinute = prometheus.NewGauge(prometheus.GaugeOpts{ + }, + func() float64 { _, v, _ := vi.TreeSizes(); return float64(v) })) + + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "vote_tree_increase_last_minute", Help: "Number of votes included in the vote tree the last 60 seconds", - }) - // VochainAppTree ... - VochainVoteCache = prometheus.NewGauge(prometheus.GaugeOpts{ + }, + func() float64 { _, _, vxm := vi.TreeSizes(); return float64(vxm) })) + + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "vote_cache", Help: "Size of the current vote cache", - }) + }, + func() float64 { return float64(vi.VoteCacheSize()) })) - VochainAccountTree = prometheus.NewGauge(prometheus.GaugeOpts{ + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "account_tree", Help: "Size of the account tree", - }) + }, + func() float64 { return float64(vi.AccountTreeSize()) })) - VochainSIKTree = prometheus.NewGauge(prometheus.GaugeOpts{ + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "sik_tree", Help: "Size of the SIK tree", - }) + }, + func() float64 { return float64(vi.SIKTreeSize()) })) - VochainTokensBurned = prometheus.NewGauge(prometheus.GaugeOpts{ + metrics.Register(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "vochain", Name: "tokens_burned", Help: "Balance of the burn address", - }) -) - -// registerMetrics registers each of the vochain prometheus metrics -func (*VochainInfo) registerMetrics(ma *metrics.Agent) { - ma.Register(VochainHeight) - ma.Register(VochainMempool) - ma.Register(VochainAppTree) - ma.Register(VochainProcessTree) - ma.Register(VochainVoteTree) - ma.Register(VochainVotesPerMinute) - ma.Register(VochainVoteCache) - ma.Register(VochainAccountTree) - ma.Register(VochainSIKTree) - ma.Register(VochainTokensBurned) -} - -// setMetrics updates the metrics values to the current state -func (vi *VochainInfo) setMetrics() { - VochainHeight.Set(float64(vi.Height())) - VochainMempool.Set(float64(vi.MempoolSize())) - p, v, vxm := vi.TreeSizes() - VochainProcessTree.Set(float64(p)) - VochainVoteTree.Set(float64(v)) - VochainVotesPerMinute.Set(float64(vxm)) - VochainVoteCache.Set(float64(vi.VoteCacheSize())) - VochainAccountTree.Set(float64(vi.AccountTreeSize())) - VochainSIKTree.Set(float64(vi.SIKTreeSize())) - VochainTokensBurned.Set(float64(vi.TokensBurned())) -} - -// CollectMetrics constantly updates the metric values for prometheus -// The function is blocking, should be called in a go routine -// If the metrics Agent is nil, do nothing -func (vi *VochainInfo) CollectMetrics(ma *metrics.Agent) { - if ma != nil { - vi.registerMetrics(ma) - for { - time.Sleep(ma.RefreshInterval) - vi.setMetrics() - } - } + }, + func() float64 { return float64(vi.TokensBurned()) })) } diff --git a/vochain/vochaininfo/vochaininfo.go b/vochain/vochaininfo/vochaininfo.go index 756818817..f30f85356 100644 --- a/vochain/vochaininfo/vochaininfo.go +++ b/vochain/vochaininfo/vochaininfo.go @@ -222,6 +222,7 @@ func (vi *VochainInfo) NPeers() int { // TODO: use time.Duration instead of int64 func (vi *VochainInfo) Start(sleepSecs int64) { log.Infof("starting vochain info service every %d seconds", sleepSecs) + vi.registerMetrics() var duration time.Duration var pheight, height int64 var h1, h10, h60, h360, h1440 int64 diff --git a/vocone/vocone.go b/vocone/vocone.go index b6149b613..12ad96a48 100644 --- a/vocone/vocone.go +++ b/vocone/vocone.go @@ -123,11 +123,11 @@ func NewVocone(dataDir string, keymanager *ethereum.SignKeys, disableIPFS bool, // EnableAPI starts the HTTP API server. It is not enabled by default. func (vc *Vocone) EnableAPI(host string, port int, URLpath string) (*api.API, error) { - var httpRouter httprouter.HTTProuter - if err := httpRouter.Init(host, port); err != nil { + vc.Router = new(httprouter.HTTProuter) + if err := vc.Router.Init(host, port); err != nil { return nil, err } - uAPI, err := api.NewAPI(&httpRouter, URLpath, vc.Config.DataDir, db.TypePebble) + uAPI, err := api.NewAPI(vc.Router, URLpath, vc.Config.DataDir, db.TypePebble) if err != nil { return nil, err } @@ -499,10 +499,15 @@ func vochainPrintInfo(sleepSecs int64, vi *vochaininfo.VochainInfo) { m = vi.MempoolSize() p, v, vxm = vi.TreeSizes() vc = vi.VoteCacheSize() - log.Infof("[vochain info] height:%d mempool:%d "+ - "processes:%d votes:%d vxm:%d voteCache:%d blockTime:{%s}", - h, m, p, v, vxm, vc, b.String(), - ) + log.Monitor("[vochain info]", map[string]any{ + "height": h, + "mempool": m, + "processes": p, + "votes": v, + "vxm": vxm, + "voteCache": vc, + "blockTime": b.String(), + }) time.Sleep(time.Duration(sleepSecs) * time.Second) } }