init

.envrc

@@ -0,0 +1 @@
+dotenv

.gitignore

@@ -0,0 +1,6 @@
+.env
+build
+dump.txt
+data.json
+config.*.json
+export

cmds/main.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"fmt"
+	"github.com/gofiber/adaptor/v2"
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/pprof"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.dfds.cloud/grafana-cost-exporter/conf"
+	"go.dfds.cloud/grafana-cost-exporter/internal/loki_client"
+	"go.dfds.cloud/grafana-cost-exporter/internal/metrics"
+	"go.dfds.cloud/grafana-cost-exporter/internal/prom_client"
+	"time"
+)
+
+func main() {
+	app := fiber.New(fiber.Config{DisableStartupMessage: true})
+	app.Use(pprof.New())
+
+	app.Get("/metrics", adaptor.HTTPHandler(promhttp.Handler()))
+
+	go worker()
+	err := app.Listen(":8080")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func worker() {
+	config, err := conf.LoadConfig()
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println(config)
+
+	sleepInterval, err := time.ParseDuration(fmt.Sprintf("%ds", config.WorkerInterval))
+	if err != nil {
+		panic(err)
+	}
+
+	for {
+		fmt.Println("Getting Grafana Cloud cost data")
+
+		lokiClient := loki_client.NewClient(config.Loki.Endpoint, &loki_client.Auth{
+			Username: config.Loki.Auth.Username,
+			Token:    config.Loki.Auth.Token,
+		})
+		promClient := prom_client.NewClient(config.Prometheus.Endpoint)
+		gatherer := metrics.NewGatherer(lokiClient, promClient)
+		//data := gatherer.GetAllMetrics()
+		gatherer.GetAllMetrics()
+
+		//metricsByCaps := metrics.ByCapability(data)
+		//
+		//pricingData := conf.LoadData()
+		//
+		//pricingProd := metrics.Pricing{
+		//	NetworkTransfer: pricingData.Pricing.Prod.NetworkTransfer,
+		//	Storage:         pricingData.Pricing.Prod.Storage,
+		//}
+		//
+		//pricingDev := metrics.Pricing{
+		//	NetworkTransfer: pricingData.Pricing.Dev.NetworkTransfer,
+		//	Storage:         pricingData.Pricing.Dev.Storage,
+		//}
+		//
+		//csvData := metrics.CapabilityResponseToCostCsv(metricsByCaps, pricingProd, pricingDev)
+
+		//serialised, err := json.Marshal(csvData)
+		//if err != nil {
+		//	log.Fatal(err)
+		//}
+		//
+		//fmt.Println(string(serialised))
+
+		fmt.Println("New metrics published")
+		time.Sleep(sleepInterval)
+	}
+}

conf/conf.go

@@ -0,0 +1,63 @@
+package conf
+
+import (
+	"encoding/json"
+	"github.com/kelseyhightower/envconfig"
+	"log"
+	"os"
+)
+
+type Config struct {
+	WorkerInterval int `json:"workerInterval"`
+	Prometheus     struct {
+		Endpoint string `json:"endpoint"`
+	}
+	Loki struct {
+		Endpoint string `json:"endpoint"`
+		Auth     struct {
+			Username string `json:"username"`
+			Token    string `json:"token"`
+		}
+	}
+}
+
+const APP_CONF_PREFIX = "GCE"
+
+func LoadConfig() (Config, error) {
+	var conf Config
+	err := envconfig.Process(APP_CONF_PREFIX, &conf)
+
+	if conf.WorkerInterval == 0 {
+		conf.WorkerInterval = 60
+	}
+
+	return conf, err
+}
+
+type Data struct {
+	Pricing struct {
+		Prod struct {
+			NetworkTransfer float64 `json:"networkTransfer"`
+			Storage         float64 `json:"storage"`
+		} `json:"prod"`
+		Dev struct {
+			NetworkTransfer float64 `json:"networkTransfer"`
+			Storage         float64 `json:"storage"`
+		} `json:"dev"`
+	}
+}
+
+func LoadData() Data {
+	data, err := os.ReadFile("data.json")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var payload Data
+	err = json.Unmarshal(data, &payload)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return payload
+}

go.mod

@@ -0,0 +1,31 @@
+module go.dfds.cloud/grafana-cost-exporter
+
+go 1.21.7
+
+require (
+	github.com/gofiber/adaptor/v2 v2.2.1
+	github.com/gofiber/fiber/v2 v2.52.0
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/prometheus/client_golang v1.18.0
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.51.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+)

go.sum

@@ -0,0 +1,55 @@
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gofiber/adaptor/v2 v2.2.1 h1:givE7iViQWlsTR4Jh7tB4iXzrlKBgiraB/yTdHs9Lv4=
+github.com/gofiber/adaptor/v2 v2.2.1/go.mod h1:AhR16dEqs25W2FY/l8gSj1b51Azg5dtPDmm+pruNOrc=
+github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE=
+github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
+github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=

internal/loki_client/client.go

@@ -0,0 +1,124 @@
+package loki_client
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+type Client struct {
+	endpoint string
+	auth     *Auth
+	http     *http.Client
+}
+
+type Auth struct {
+	Username string `json:"username"`
+	Token    string `json:"token"`
+}
+
+func NewClient(endpoint string, auth *Auth) *Client {
+	return &Client{
+		endpoint: endpoint,
+		http:     http.DefaultClient,
+		auth:     auth,
+	}
+}
+
+func (c *Client) PrepareHttpRequest(req *http.Request) {
+	req.Header.Set("Authorization", fmt.Sprintf("Basic %s", base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", c.auth.Username, c.auth.Token)))))
+}
+
+func (c *Client) LabelValues(label string) (*LabelValuesResponse, error) {
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v1/label/%s/values", c.endpoint, label), nil)
+	if err != nil {
+		return nil, err
+	}
+	c.PrepareHttpRequest(req)
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var payload *LabelValuesResponse
+
+	err = json.Unmarshal(data, &payload)
+
+	return payload, err
+}
+
+func (c *Client) Volume(query string, start int64, end int64) (*VolumeResponse, error) {
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v1/index/volume", c.endpoint), nil)
+	if err != nil {
+		return nil, err
+	}
+	c.PrepareHttpRequest(req)
+
+	queryValues := req.URL.Query()
+	queryValues.Set("query", query)
+	queryValues.Set("start", fmt.Sprintf("%d", start))
+	queryValues.Set("end", fmt.Sprintf("%d", end))
+	req.URL.RawQuery = queryValues.Encode()
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println(resp.StatusCode)
+
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var payload *VolumeResponse
+
+	err = json.Unmarshal(data, &payload)
+
+	return payload, err
+}
+
+func (c *Client) VolumeRange(query string, start int64, end int64) (*VolumeRangeResponse, error) {
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v1/index/volume_range", c.endpoint), nil)
+	if err != nil {
+		return nil, err
+	}
+	c.PrepareHttpRequest(req)
+
+	queryValues := req.URL.Query()
+	queryValues.Set("query", query)
+	queryValues.Set("start", fmt.Sprintf("%d", start))
+	queryValues.Set("end", fmt.Sprintf("%d", end))
+	req.URL.RawQuery = queryValues.Encode()
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var payload *VolumeRangeResponse
+
+	err = json.Unmarshal(data, &payload)
+
+	return payload, err
+}