initial

.github/workflows/push-docker.yaml

@@ -0,0 +1,25 @@
+name: Deploy Images to GHCR
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  push-store-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@main
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: 'Build Inventory Image'
+        run: |
+          docker build . --tag ghcr.io/qubic/go-node-fetcher:${{github.ref_name}}
+          docker push ghcr.io/qubic/go-node-fetcher:${{github.ref_name}}

.github/workflows/test.yaml

@@ -0,0 +1,25 @@
+on: [push, pull_request]
+name: Go Test
+jobs:
+  test-nocache:
+    strategy:
+      matrix:
+        go-version: [1.20.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go-version }}
+          cache: false
+      - run: go test ./...
+
+  test-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: 1.20.x
+      - run: go test ./...

.gitignore

@@ -0,0 +1,2 @@
+.idea/
+.DS_Store

Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.21 AS builder
+ENV CGO_ENABLED=0
+
+WORKDIR /src
+COPY . /src
+
+RUN go build -o "/src/bin/go-node-fetcher"
+
+# We don't need golang to run binaries, just use alpine.
+FROM alpine:latest
+COPY --from=builder /src/bin/go-node-fetcher /app/go-node-fetcher
+RUN chmod +x /app/go-node-fetcher
+
+EXPOSE 8080
+
+WORKDIR /app
+
+ENTRYPOINT ["./go-node-fetcher"]

blacklisted.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"sync"
+	"time"
+)
+
+type blacklistedPeers struct {
+	peers map[string]int64 // map of blacklisted peers and the time the peer was blacklisted
+	mux   sync.RWMutex
+}
+
+func newBlacklistedPeers() *blacklistedPeers {
+	return &blacklistedPeers{
+		peers: make(map[string]int64),
+	}
+}
+
+func (bp *blacklistedPeers) add(peer string) {
+	bp.mux.Lock()
+	defer bp.mux.Unlock()
+
+	// if it already exists, then do nothing as it's already blacklisted
+	if _, ok := bp.peers[peer]; ok {
+		return
+	}
+
+	bp.peers[peer] = time.Now().UTC().Unix()
+}
+
+func (bp *blacklistedPeers) remove(peer string) {
+	bp.mux.Lock()
+	defer bp.mux.Unlock()
+
+	delete(bp.peers, peer)
+}
+
+func (bp *blacklistedPeers) isBlacklisted(peer string) bool {
+	bp.mux.RLock()
+	defer bp.mux.RUnlock()
+
+	_, ok := bp.peers[peer]
+
+	return ok
+}
+
+func (bp *blacklistedPeers) get() []string {
+	bp.mux.RLock()
+	defer bp.mux.RUnlock()
+
+	var peers []string
+	for peer := range bp.peers {
+		peers = append(peers, peer)
+	}
+
+	return peers
+}
+
+func (bp *blacklistedPeers) isEmpty() bool {
+	bp.mux.RLock()
+	defer bp.mux.RUnlock()
+
+	return len(bp.peers) == 0
+}
+
+// reset resets blacklisted peers
+func (bp *blacklistedPeers) reset() {
+	bp.mux.Lock()
+	defer bp.mux.Unlock()
+
+	bp.peers = make(map[string]int64)
+}

distinct.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"github.com/pkg/errors"
+	qubic "github.com/qubic/go-node-connector"
+	"log"
+	"math/rand"
+	"sync"
+	"time"
+)
+
+type distinctPeers struct {
+	bp                        *blacklistedPeers
+	peers                     map[string]struct{}
+	maxPeers                  int
+	mux                       sync.RWMutex
+	exchangeConnectionTimeout time.Duration
+}
+
+func newDistinctPeers(startingPeer string, maxPeers int, exchangeConnectionTimeout time.Duration, bp *blacklistedPeers) *distinctPeers {
+	dp := distinctPeers{
+		bp:                        bp,
+		peers:                     make(map[string]struct{}, maxPeers),
+		maxPeers:                  maxPeers,
+		exchangeConnectionTimeout: exchangeConnectionTimeout,
+	}
+	dp.setPeers([]string{startingPeer})
+
+	return &dp
+}
+
+func (p *distinctPeers) build() ([]string, error) {
+	peer := p.getRandomPeer()
+	err := p.exchangePeerList(peer)
+	if err != nil {
+		return nil, errors.Wrap(err, "exchanging peer list")
+	}
+
+	if p.isEmpty() {
+		if p.bp.isEmpty() {
+			log.Println("No distinct peers found, no blacklisted peers found")
+		}
+		return p.bp.get(), nil
+	}
+
+	return p.get(), nil
+}
+
+func (p *distinctPeers) exchangePeerList(peer string) error {
+	if p.isFull() {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), p.exchangeConnectionTimeout)
+	defer cancel()
+
+	qc, err := qubic.NewClient(ctx, peer, qubicPort)
+	if err != nil {
+		return errors.Wrap(err, "creating new connection")
+	}
+	qc.Close()
+
+	unmetPeers := p.getUnmetPeers(qc.Peers)
+	p.setPeers(unmetPeers)
+
+	for _, peer := range unmetPeers {
+		p.exchangePeerList(peer)
+	}
+
+	return nil
+}
+
+func (p *distinctPeers) getUnmetPeers(peers []string) []string {
+	var unmetPeers []string
+	for _, peer := range peers {
+		if _, ok := p.peers[peer]; !ok {
+			unmetPeers = append(unmetPeers, peer)
+		}
+	}
+
+	return unmetPeers
+}
+
+func (p *distinctPeers) get() []string {
+	p.mux.RLock()
+	defer p.mux.RUnlock()
+
+	var peers []string
+	for peer := range p.peers {
+		peers = append(peers, peer)
+	}
+
+	return peers
+}
+
+func (p *distinctPeers) setPeers(peers []string) {
+	p.mux.Lock()
+	defer p.mux.Unlock()
+
+	for _, peer := range peers {
+		p.peers[peer] = struct{}{}
+	}
+}
+
+func (p *distinctPeers) isFull() bool {
+	p.mux.RLock()
+	defer p.mux.RUnlock()
+
+	return len(p.peers) >= p.maxPeers
+}
+
+func (p *distinctPeers) isEmpty() bool {
+	p.mux.RLock()
+	defer p.mux.RUnlock()
+
+	return len(p.peers) == 0
+}
+
+func (p *distinctPeers) getRandomPeer() string {
+	p.mux.RLock()
+	defer p.mux.RUnlock()
+
+	if p.isEmpty() {
+		return ""
+	}
+
+	peers := p.get()
+
+	return peers[rand.Intn(len(p.peers))]
+}

go.mod

@@ -0,0 +1,14 @@
+module github.com/qubic/go-node-fetcher
+
+go 1.21.3
+
+require (
+	github.com/ardanlabs/conf v1.5.0
+	github.com/pkg/errors v0.9.1
+	github.com/qubic/go-node-connector v0.3.0
+)
+
+require (
+	github.com/cloudflare/circl v1.3.6 // indirect
+	golang.org/x/sys v0.3.0 // indirect
+)

go.sum

@@ -0,0 +1,15 @@
+github.com/ardanlabs/conf v1.5.0 h1:5TwP6Wu9Xi07eLFEpiCUF3oQXh9UzHMDVnD3u/I5d5c=
+github.com/ardanlabs/conf v1.5.0/go.mod h1:ILsMo9dMqYzCxDjDXTiwMI0IgxOJd0MOiucbQY2wlJw=
+github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg=
+github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
+github.com/cloudflare/fourq v0.0.0-20170427000316-8ada258cf9c8 h1:748sGeXXbplK0UVPDLbhh53hejCnvv/u6jn2RPBfyI8=
+github.com/cloudflare/fourq v0.0.0-20170427000316-8ada258cf9c8/go.mod h1:13nQglQo5cpucnNY80duyW/6HK+WQ9+dHZ70UzAy6Jw=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/qubic/go-node-connector v0.3.0 h1:jmdge7CK5CGoz7iC7SzKQhN23oLKzo8oLlPzqCMYy9o=
+github.com/qubic/go-node-connector v0.3.0/go.mod h1:y0eMsGPY1DFEzz2JovUgEIwBZ/27zoJ8G81nX9yt8xI=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

handler.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+type Handler struct {
+	rp *Peers
+}
+
+type response struct {
+	Peers       []string `json:"peers"`
+	Length      int      `json:"length"`
+	LastUpdated int64    `json:"last_updated"`
+}
+
+func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
+	p := h.rp.Get()
+	res := response{
+		Peers:       p.Peers,
+		Length:      len(p.Peers),
+		LastUpdated: p.UpdatedAt,
+	}
+	b, err := json.Marshal(res)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	w.Write(b)
+}