Merge remote-tracking branch 'origin/main' into validator_update

Signed-off-by: p4u <pau@dabax.net>

.github/workflows/main.yml

@@ -81,56 +81,63 @@ jobs:
     env:
       LOG_PANIC_ON_INVALIDCHARS: true # check that log lines contains no invalid chars (evidence of format mismatch)
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set up Go environment
-        uses: actions/setup-go@v4
+      - uses: actions/checkout@v3
+      - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha
+      - uses: actions/setup-go@v4
         with:
           go-version: '1.21'
+      - run: mkdir -p "$PWD/gocoverage-unit/"
       - name: Run Go test (and collect code coverage)
-        if: github.event_name == 'pull_request' || github.ref == 'refs/heads/dev' # quicker, non-race test in case it's a PR or push to dev
-        run: go test -coverprofile=unit.covdata.txt ./...
+        # quicker, non-race test in case it's a PR or push to dev
+        if: github.event_name == 'pull_request' ||
+            github.ref == 'refs/heads/dev'
+        run: go test ./...
+          -cover -coverpkg=./... -covermode=count -args -test.gocoverdir="$PWD/gocoverage-unit/"
       - name: Run Go test -race (and collect code coverage)
-        if: github.event_name == 'push' && github.ref != 'refs/heads/dev' # this is further limited to selected branches at the beginning of this file
+        # note that -race can easily make the crypto stuff 10x slower
+        # this is further limited to selected branches at the beginning of this file
+        if: github.event_name == 'push' &&
+            github.ref != 'refs/heads/dev'
         env:
           GORACE: atexit_sleep_ms=10 # the default of 1000 makes every Go package test sleep for 1s; see https://go.dev/issues/20364
-        run: go test -coverprofile=unit.covdata.txt -vet=off -timeout=15m -race ./... # note that -race can easily make the crypto stuff 10x slower
-      - name: Store code coverage artifact
+        run: go test ./...
+          -race -timeout=15m -vet=off
+          -cover -coverpkg=./... -covermode=atomic -args -test.gocoverdir="$PWD/gocoverage-unit/"
+      - name: Store code coverage artifact (unit)
         uses: actions/upload-artifact@v3
         with:
-          name: unit-covdata
-          path: unit.covdata.txt
+          name: gocoverage-unit@${{ env.SHA }}
+          path: gocoverage-unit/
 
   job_compose_test:
     runs-on: [self-hosted, ci2-1]
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v3
+      - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha
       - name: Run compose script
         env:
           TESTSUITE_BUILD_TAG: ${{ github.sha }}
           COMPOSE_PROJECT_NAME: testsuite_${{ github.run_id }} # unique name for docker compose (needed for concurrent job runs)
           COMPOSE_DVOTE_PORT_MAPPING: "9090" # this binds gateway0 to a random available port on docker host (needed for concurrent job runs)
           COMPOSE_HOST_PATH: ${{ github.workspace }}/dockerfiles/testsuite
           LOG_PANIC_ON_INVALIDCHARS: true # check that log lines contains no invalid chars (evidence of format mismatch)
-          GOCOVERDIR: "./gocoverage/" # collect code coverage when running binaries
+          GOCOVERDIR: "./gocoverage-integration/" # collect code coverage when running binaries
           CONCURRENT: 1 # run all the start_test.sh tests concurrently
+          BUILDARGS: "-race" # this makes the integration test only slightly slower (around +10%) unlike the abismal effect in unit test (10x)
         run: |
           cd dockerfiles/testsuite && ./start_test.sh
-      - name: Store code coverage artifact
+      - name: Store code coverage artifact (integration)
         uses: actions/upload-artifact@v3
         with:
-          name: integration-covdata
-          path: dockerfiles/testsuite/gocoverage/covdata.txt
+          name: gocoverage-integration@${{ env.SHA }}
+          path: dockerfiles/testsuite/gocoverage-integration/
 
   job_go_build_for_mac:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release')
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set up Go environment
-        uses: actions/setup-go@v4
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: '1.21'
       - name: Run go build for Mac
@@ -139,31 +146,75 @@ jobs:
           # It's surprisingly hard with some deps like bazil.org/fuse.
           GOOS=darwin go build ./...
 
-  job_upload_coverage:
-    name: Publish code coverage
-    runs-on: ubuntu-latest
+  call-docker-release:
+    name: Docker
+    needs: [job_go_checks, job_go_test, job_compose_test]
+    if: github.event_name == 'push' # this is limited to selected branches at the beginning of this file
+    uses: vocdoni/vocdoni-node/.github/workflows/docker-release.yml@main
+    secrets: inherit
+    with:
+      image-tag: ${{ github.ref_name }}
+
+  job_gocoverage_textfmt:
+    name: Convert coverage (bin->txt)
+    continue-on-error: true # never mark the whole CI as failed because of this job
     needs: [job_go_test, job_compose_test]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Fetch coverage (unit)
-        uses: actions/download-artifact@v3
+      - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha
+      - uses: actions/download-artifact@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+          cache: false
+      - name: Convert gocoverage format
+        run: |
+          go tool covdata textfmt -i=gocoverage-unit@${{ env.SHA }}/ \
+                                  -o gocoverage-unit@${{ env.SHA }}.txt
+          go tool covdata textfmt -i=gocoverage-integration@${{ env.SHA }}/ \
+                                  -o gocoverage-integration@${{ env.SHA }}.txt
+      - name: Merge both files
+        run: |
+          go install github.com/wadey/gocovmerge@latest
+          # dirty hack since integration is mode atomic and unit mode count, which are perfectly mergeable
+          # but gocovmerge doesn't recognize this: "cannot merge profiles with different modes"
+          sed 's/mode: count/mode: atomic/' gocoverage-unit@${{ env.SHA }}.txt \
+                                          > gocoverage-unit@${{ env.SHA }}.tmp
+          gocovmerge gocoverage-unit@${{ env.SHA }}.tmp \
+                     gocoverage-integration@${{ env.SHA }}.txt \
+                  >  gocoverage-merged@${{ env.SHA }}.txt
+          rm -f gocoverage-unit@${{ env.SHA }}.tmp
+      - name: Store code coverage artifact (all, textfmt)
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
         with:
-          name: unit-covdata
-      - name: Fetch coverage (integration)
-        uses: actions/download-artifact@v3
+          name: gocoverage-all-textfmt@${{ env.SHA }}
+          path: gocoverage-*.txt
+
+  job_gocoverage_coveralls:
+    name: Publish coverage (Coveralls)
+    runs-on: ubuntu-latest
+    needs: [job_gocoverage_textfmt]
+    continue-on-error: true # never mark the whole CI as failed because of this job
+    steps:
+      - uses: actions/checkout@v3
+      - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha
+      - uses: actions/download-artifact@v3
         with:
-          name: integration-covdata
-
+          name: gocoverage-all-textfmt@${{ env.SHA }}
       - name: Send coverage to coveralls.io (unit)
+        if: ${{ always() }}
         uses: shogo82148/actions-goveralls@v1
         with:
-          path-to-profile: unit.covdata.txt
+          path-to-profile: gocoverage-unit@${{ env.SHA }}.txt
           flag-name: unit
           parallel: true
       - name: Send coverage to coveralls.io (integration)
+        if: ${{ always() }}
         uses: shogo82148/actions-goveralls@v1
         with:
-          path-to-profile: covdata.txt
+          path-to-profile: gocoverage-integration@${{ env.SHA }}.txt
           flag-name: integration
           parallel: true
       - name: Tell coveralls.io we're done
@@ -172,11 +223,23 @@ jobs:
         with:
           parallel-finished: true
 
-  call-docker-release:
-    name: Docker
-    needs: [job_go_checks, job_go_test, job_compose_test]
-    if: github.event_name == 'push' # this is limited to selected branches at the beginning of this file
-    uses: vocdoni/vocdoni-node/.github/workflows/docker-release.yml@main
-    secrets: inherit
-    with:
-      image-tag: ${{ github.ref_name }}
+  job_gocoverage_deepsource:
+    name: Publish coverage (DeepSource)
+    runs-on: ubuntu-latest
+    needs: [job_gocoverage_textfmt]
+    continue-on-error: true # never mark the whole CI as failed because of this job
+    steps:
+      - uses: actions/checkout@v3
+      - uses: benjlevesque/short-sha@v2.2 # sets env.SHA to the first 7 chars of github.sha
+      - uses: actions/download-artifact@v3
+        with:
+          name: gocoverage-all-textfmt@${{ env.SHA }}
+      - name: Send coverage to DeepSource
+        env: 
+          DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
+        run: |
+          # Install the CLI
+          curl https://deepsource.io/cli | sh
+
+          # Send the report to DeepSource
+          ./bin/deepsource report --analyzer test-coverage --key go --value-file gocoverage-merged@${{ env.SHA }}.txt

.gitignore

@@ -16,6 +16,7 @@ zkCircuits
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+gocoverage*
 
 # Docker files and dirs
 env.local

Dockerfile

@@ -28,7 +28,7 @@ COPY --from=builder /go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/pack
                     /go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-amd64/libwasmer.so
 # Support for go-rapidsnark prover (https://github.com/iden3/go-rapidsnark/tree/main/prover)
 RUN apt-get update && \
-	apt-get install -y libc6-dev libomp-dev openmpi-common libgomp1 curl && \
+	apt-get install --no-install-recommends -y libc6-dev libomp-dev openmpi-common libgomp1 curl && \
 	apt-get autoremove -y && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*

cmd/cli/main.go

@@ -453,14 +453,12 @@ func accountSetValidator(cli *VocdoniCLI) error {
 	if err != nil {
 		return err
 	}
-	pubKey := []byte{}
+	pubKey := cli.getCurrentAccount().PublicKey
 	if pubKeyStr != "" {
 		pubKey, err = hex.DecodeString(pubKeyStr)
 		if err != nil {
 			return err
 		}
-	} else {
-		pubKey = cli.getCurrentAccount().PublicKey
 	}
 
 	hash, err := cli.api.AccountSetValidator(pubKey, name)

cmd/end2endtest/account.go

@@ -190,15 +190,15 @@ func testSendTokens(api *apiclient.HTTPclient, aliceKeys, bobKeys *ethereum.Sign
 
 	// send a couple of token txs to increase the nonce, without waiting for them to be mined
 	// this tests that the mempool transactions are properly ordered.
-	wg := sync.WaitGroup{}
+	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
 		log.Warnf("send transactions with nonce+1, should not be mined before the others")
 		// send 1 token to burn address with nonce + 1 (should be mined after the other txs)
-		if _, err = alice.TransferWithNonce(state.BurnAddress, 1, aliceAcc.Nonce+1); err != nil {
+		if _, err := alice.TransferWithNonce(state.BurnAddress, 1, aliceAcc.Nonce+1); err != nil {
 			log.Fatalf("cannot burn tokens: %v", err)
 		}
-		if _, err = bob.TransferWithNonce(state.BurnAddress, 1, bobAcc.Nonce+1); err != nil {
+		if _, err := bob.TransferWithNonce(state.BurnAddress, 1, bobAcc.Nonce+1); err != nil {
 			log.Fatalf("cannot burn tokens: %v", err)
 		}
 		wg.Done()
@@ -208,6 +208,7 @@ func testSendTokens(api *apiclient.HTTPclient, aliceKeys, bobKeys *ethereum.Sign
 	var txhasha, txhashb []byte
 	wg.Add(1)
 	go func() {
+		var err error
 		txhasha, err = alice.TransferWithNonce(bobKeys.Address(), amountAtoB, aliceAcc.Nonce)
 		if err != nil {
 			log.Fatalf("cannot send tokens: %v", err)

cmd/voconed/voconed.go

@@ -19,6 +19,7 @@ import (
 	"go.vocdoni.io/dvote/crypto/zk/circuit"
 	"go.vocdoni.io/dvote/internal"
 	"go.vocdoni.io/dvote/log"
+	"go.vocdoni.io/dvote/metrics"
 	"go.vocdoni.io/dvote/vochain/state"
 	"go.vocdoni.io/dvote/vocone"
 	"go.vocdoni.io/proto/build/go/models"
@@ -247,6 +248,9 @@ func main() {
 		log.Fatal(err)
 	}
 
+	vc.MetricsAgent = metrics.NewAgent("/metrics",
+		time.Duration(10)*time.Second, vc.Router)
+
 	// enable faucet if requested, this will create a new account and attach the faucet API to the vocone API
 	if config.enableFaucetWithAmount > 0 {
 		faucetAccount := ethereum.SignKeys{}

data/data.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 
 	"go.vocdoni.io/dvote/data/ipfs"
-	"go.vocdoni.io/dvote/metrics"
 	"go.vocdoni.io/dvote/types"
 )
 
@@ -19,8 +18,7 @@ type Storage interface {
 	Unpin(ctx context.Context, path string) error
 	ListPins(ctx context.Context) (map[string]string, error)
 	URIprefix() string
-	Stats(ctx context.Context) map[string]any
-	CollectMetrics(ctx context.Context, ma *metrics.Agent) error
+	Stats() map[string]any
 	Stop() error
 }
 

data/datamocktest.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"go.vocdoni.io/dvote/data/ipfs"
-	"go.vocdoni.io/dvote/metrics"
 	"go.vocdoni.io/dvote/test/testcommon/testutil"
 	"go.vocdoni.io/dvote/types"
 )
@@ -80,11 +79,7 @@ func (d *DataMockTest) URIprefix() string {
 	return d.prefix
 }
 
-func (*DataMockTest) Stats(_ context.Context) map[string]any {
-	return nil
-}
-
-func (*DataMockTest) CollectMetrics(_ context.Context, _ *metrics.Agent) error {
+func (*DataMockTest) Stats() map[string]any {
 	return nil
 }
 

data/ipfs/ipfs.go

@@ -131,6 +131,9 @@ func (i *Handler) Init(d *types.DataStore) error {
 		return err
 	}
 
+	go i.updateStats(time.Minute)
+	i.registerMetrics()
+
 	return nil
 }
 
@@ -207,20 +210,8 @@ func (i *Handler) Unpin(ctx context.Context, path string) error {
 }
 
 // Stats returns stats about the IPFS node.
-func (i *Handler) Stats(ctx context.Context) map[string]any {
-	peers, err := i.CoreAPI.Swarm().Peers(ctx)
-	if err != nil {
-		return map[string]any{"error": err.Error()}
-	}
-	addresses, err := i.CoreAPI.Swarm().KnownAddrs(ctx)
-	if err != nil {
-		return map[string]any{"error": err.Error()}
-	}
-	pins, err := i.countPins(ctx)
-	if err != nil {
-		return map[string]any{"error": err.Error()}
-	}
-	return map[string]any{"peers": len(peers), "addresses": len(addresses), "pins": pins}
+func (i *Handler) Stats() map[string]any {
+	return map[string]any{"peers": stats.Peers.Load(), "addresses": stats.KnownAddrs.Load(), "pins": stats.Pins.Load()}
 }
 
 func (i *Handler) countPins(ctx context.Context) (int, error) {