Initial implementation

.github/workflows/push-docker-dev.yaml

@@ -0,0 +1,25 @@
+name: Deploy dev images to GHCR
+
+on:
+  push:
+    branches:
+      - 'dev'
+
+jobs:
+  push-store-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@main
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: 'Build Inventory Image'
+        run: |
+          docker build . --tag ghcr.io/qubic/frontend-processor:dev
+          docker push ghcr.io/qubic/frontend-processor:dev

.github/workflows/push-docker.yaml

@@ -0,0 +1,25 @@
+name: Deploy prod images to GHCR
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  push-store-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@main
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: 'Build Inventory Image'
+        run: |
+          docker build . --tag ghcr.io/qubic/frontend-processor:${{github.ref_name}}
+          docker push ghcr.io/qubic/frontend-processor:${{github.ref_name}}

docker-compose.yaml

@@ -6,5 +6,21 @@ services:
     environment:
       MONGO_INITDB_ROOT_USERNAME: user
       MONGO_INITDB_ROOT_PASSWORD: pass
+    networks:
+      - processor
     ports:
-      - "27017:27017"
+      - "27017:27017"
+
+  processor:
+    image: ghcr.io/qubic/frontend-processor:dev
+    container_name: frontend_processor
+    ports:
+      - "8080:8080"
+    networks:
+      - processor
+    depends_on:
+      - mongo
+    restart: unless-stopped
+
+networks:
+  processor:

go.mod

@@ -3,20 +3,20 @@ module github.com/qubic/frontend-service-processor
 go 1.22.4
 
 require (
-	github.com/ardanlabs/conf v1.5.0 // indirect
+	github.com/ardanlabs/conf v1.5.0
+	github.com/pkg/errors v0.9.1
+	go.mongodb.org/mongo-driver v1.16.0
+)
+
+require (
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/spf13/cobra v1.8.1 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.mongodb.org/mongo-driver v1.16.0 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 )

go.sum

@@ -1,22 +1,18 @@
 github.com/ardanlabs/conf v1.5.0 h1:5TwP6Wu9Xi07eLFEpiCUF3oQXh9UzHMDVnD3u/I5d5c=
 github.com/ardanlabs/conf v1.5.0/go.mod h1:ILsMo9dMqYzCxDjDXTiwMI0IgxOJd0MOiucbQY2wlJw=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
-github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
+github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
-github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
@@ -30,8 +26,8 @@ go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4B
 go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -51,11 +47,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -6,10 +6,12 @@ import (
 	"github.com/ardanlabs/conf"
 	"github.com/pkg/errors"
 	"github.com/qubic/frontend-service-processor/src/db"
+	"github.com/qubic/frontend-service-processor/src/service"
 	"github.com/qubic/frontend-service-processor/src/spectrum"
 	"log"
 	"os"
 	"strconv"
+	"time"
 )
 
 const prefix = "QUBIC_FRONTEND_PROCESSOR"
@@ -25,8 +27,12 @@ type Configuration struct {
 		OutputFile   string `conf:"default:spectrumData.json"`
 	}
 	Service struct {
-		CoinGeckoToken string `cong:"default:XXXXXXXXXXXXXXXXXXXXX"`
-		WebPort        string `conf:"default:80"`
+		ArchiverUrl            string `conf:"default:https://testapi.qubic.org"`
+		ArchiverStatusPath     string `conf:"default:/v1/status"`
+		ArchiverLatestTickPath string `conf:"default:/v1/latestTick"`
+
+		CoinGeckoToken     string        `cong:"default:XXXXXXXXXXXXXXXXXXXXX"`
+		DataScrapeInterval time.Duration `conf:"default:1m"`
 	}
 	Mongo struct {
 		Username string `conf:"default:user"`
@@ -37,6 +43,7 @@ type Configuration struct {
 
 		Database           string `conf:"default:qubic_frontend"`
 		SpectrumCollection string `conf:"default:spectrum_data"`
+		DataCollection     string `conf:"default:general_data"`
 	}
 }
 
@@ -107,9 +114,34 @@ func run() error {
 
 	switch config.App.Mode {
 	case "service":
-		println("Web service")
+		println("Processor")
 
-		//var result bson.M
+		mongoConnection := db.Connection{
+			Username:          config.Mongo.Username,
+			Password:          config.Mongo.Password,
+			Hostname:          config.Mongo.Hostname,
+			Port:              config.Mongo.Port,
+			ConnectionOptions: config.Mongo.Options,
+		}
+
+		s := service.Service{
+			CoinGeckoToken:         config.Service.CoinGeckoToken,
+			ArchiverUrl:            config.Service.ArchiverUrl,
+			ArchiverStatusPath:     config.Service.ArchiverStatusPath,
+			ArchiverLatestTickPath: config.Service.ArchiverLatestTickPath,
+
+			DatabaseConnection: mongoConnection,
+			Database:           config.Mongo.Database,
+			SpectrumCollection: config.Mongo.SpectrumCollection,
+			DataCollection:     config.Mongo.DataCollection,
+
+			ScrapeInterval: config.Service.DataScrapeInterval,
+		}
+
+		err := s.RunService()
+		if err != nil {
+			return errors.Wrap(err, "running the web service")
+		}
 
 		break
 	case "spectrum_parser":

src/archiver/types.go

@@ -0,0 +1,32 @@
+package archiver
+
+type ProcessedTick struct {
+	TickNumber uint32 `json:"tickNumber"`
+	Epoch      uint32 `json:"epoch"`
+}
+
+type SkippedTicksInterval struct {
+	StartTick uint32 `json:"startTick"`
+	EndTick   uint32 `json:"endTick"`
+}
+
+type ProcessedTickIntervals struct {
+	InitialProcessedTick uint32 `json:"initialProcessedTick"`
+	LastProcessedTick    uint32 `json:"lastProcessedTick"`
+}
+
+type ProcessedTickIntervalsPerEpoch struct {
+	Epoch     uint32                   `json:"epoch"`
+	Intervals []ProcessedTickIntervals `json:"intervals"`
+}
+
+type StatusResponse struct {
+	LastProcessedTick              *ProcessedTick                    `json:"lastProcessedTick"`
+	LastProcessedTicksPerEpoch     map[uint32]uint32                 `json:"lastProcessedTicksPerEpoch"`
+	SkippedTicks                   []*SkippedTicksInterval           `json:"skippedTicks"`
+	ProcessedTickIntervalsPerEpoch []*ProcessedTickIntervalsPerEpoch `json:"processedTickIntervalsPerEpoch"`
+}
+
+type LatestTickResponse struct {
+	LatestTick uint32 `json:"latestTick"`
+}