Introduce Paid Traffic Ratio chart

Makefile

@@ -37,8 +37,5 @@ web-validate:
 	docker run -it --rm -v `pwd`/web:/web --platform linux/amd64 ghcr.io/validator/validator:23.4.11 \
 		vnu --skip-non-html --Werror /web
 
-clean-vm:
-	docker compose -f moonlet/compose.yaml down -v
-
 clean-duckdb:
 	docker compose -f duckdb/compose.yaml down -v

README.md

@@ -2,7 +2,7 @@
 
 The goal of this project is to increase the transparency of 🪐 [Filecoin Saturn](https://saturn.tech) network – the fastest growing, community-run distributed content delivery network (CDN) for Web3.
 
-We use historical data about the network gathered by 🌖 [Saturn Moonlet](https://github.com/31z4/saturn-moonlet) to prepare and publish analytical reports on Saturn Observatory [website](https://observatory.cronian.tech) every month. Raw data that is used to generate the reports [is available](https://gateway.ipfs.io/ipfs/bafybeiewsxrzbdzas7ta42ewnvsig7to7njsicubfykn3w5f5frhofu7nq) on IPFS and Filecoin.
+We use historical data about the network gathered by 🌖 [Saturn Moonlet](https://github.com/31z4/saturn-moonlet) to prepare and publish analytical reports on Saturn Observatory [website](https://observatory.cronian.tech) every month. Raw data that is used to generate the reports [is available](https://gateway.ipfs.io/ipfs/bafybeifmkotlmwf4c55kynudl5uxljmuyyx54gvpjzgxhr2ld564matzdu) on IPFS and Filecoin.
 
 Saturn Observatory compliments official tools like [Saturn Node Dashboard](https://dashboard.saturn.tech) and [Saturn Explorer](https://explorer.saturn.tech), aiming to provide better insights into the network state and performance. To highlight a few:
 

analytics.sql

@@ -3,15 +3,40 @@ CREATE TABLE IF NOT EXISTS saturn_node_creation AS FROM '/inputs/saturn_node_cre
 CREATE TABLE IF NOT EXISTS saturn_node_estimated_earnings AS FROM '/inputs/saturn_node_estimated_earnings.csv.gz';
 CREATE TABLE IF NOT EXISTS saturn_node_bandwidth_served AS FROM '/inputs/saturn_node_bandwidth_served.csv.gz';
 CREATE TABLE IF NOT EXISTS saturn_node_retrievals AS FROM '/inputs/saturn_node_retrievals.csv.gz';
+CREATE TABLE IF NOT EXISTS saturn_node_sent_bytes AS FROM '/inputs/saturn_node_sent_bytes.csv.gz';
 
 
--- Returns network traffic over time.
+-- Calculate paid network traffic over time.
+CREATE TEMP VIEW paid_traffic AS
+SELECT
+    observed_at,
+    sum(bandwidth_served_bytes) as paid_bytes
+FROM saturn_node_bandwidth_served
+GROUP BY observed_at;
+
+
+-- Returns paid network traffic ratio over time.
 COPY (
+    WITH total_traffic AS (
+        SELECT
+            observed_at,
+            sum(sent_bytes) as total_bytes
+        FROM saturn_node_sent_bytes
+        GROUP BY observed_at
+    )
     SELECT
         observed_at,
-        sum(bandwidth_served_bytes)
-    FROM saturn_node_bandwidth_served
-    GROUP BY observed_at
+        paid_bytes / total_bytes
+    FROM total_traffic
+    JOIN paid_traffic USING (observed_at)
+    ORDER BY observed_at  -- Ordering is required for deterministic results.
+) TO '/outputs/saturn_traffic_ratio.csv';
+
+
+-- Returns network traffic over time.
+COPY (
+    SELECT *
+    FROM paid_traffic
     ORDER BY observed_at -- Ordering is required for deterministic results.
 ) TO '/outputs/saturn_traffic.csv';
 

moonlet/export-csv.sh

@@ -73,3 +73,7 @@ python_export 'saturn_node_retrievals' \
 python_export 'saturn_node_estimated_earnings' \
     'increase(saturn_node_estimated_earnings_fil_total)' \
     'observed_at,node_id,estimated_earnings_fil'
+
+python_export 'saturn_node_sent_bytes' \
+    'increase(saturn_node_sent_bytes_total)' \
+    'observed_at,node_id,sent_bytes'

web/index.html

@@ -34,7 +34,7 @@ <h1>🔭 Saturn Observatory</h1>
     <p>
         Raw CSV data that we use to generate the plots are available on IPFS and Filecoin under CID
         <a
-            href="https://gateway.ipfs.io/ipfs/bafybeiewsxrzbdzas7ta42ewnvsig7to7njsicubfykn3w5f5frhofu7nq">bafybeiewsxrzbdzas7ta42ewnvsig7to7njsicubfykn3w5f5frhofu7nq</a>.
+            href="https://gateway.ipfs.io/ipfs/bafybeifmkotlmwf4c55kynudl5uxljmuyyx54gvpjzgxhr2ld564matzdu">bafybeifmkotlmwf4c55kynudl5uxljmuyyx54gvpjzgxhr2ld564matzdu</a>.
     </p>
 
     <p>To learn more about how data is gathered and analyzed, check the project's <a
@@ -129,12 +129,27 @@ <h2 id="nodes-without-traffic">Nodes Without Traffic</h2>
         The result is the ratio between nodes without traffic and the active nodes.
     </p>
     <p>
-        Gaps on the chart might be caused by several reasons. Including but not limited to issues with our API scrapper,
-        issues with the orchestrator or the lambda API, or issues with the Saturn network itself.
+        Gaps and spikes on the chart might be caused by several reasons. Including but not limited to issues with our
+        API scrapper, issues with the orchestrator or the lambda API, or issues with the Saturn network itself.
     </p>
 
     <div id="saturn-active-node-without-traffic"></div>
 
+    <h2 id="traffic-ratio">Paid Traffic Ratio</h2>
+    <p>
+        This chart shows the ratio between outgoing Saturn network traffic (considered as "paid") and the total outgoing
+        traffic served by a node. Total outgoing traffic is estimated using network interface stats reported by the <a
+            href="https://orchestrator.strn.pl/stats">orchestrator API</a>. Paid traffic is estimated using the <a
+            href="https://uc2x7t32m6qmbscsljxoauwoae0yeipw.lambda-url.us-west-2.on.aws">lambda
+            API</a> that is used by <a href="https://dashboard.saturn.tech/address/all">Saturn Dashboard</a>.
+    </p>
+    <p>
+        Gaps and spikes on the chart might be caused by several reasons. Including but not limited to issues with our
+        API scrapper, issues with the orchestrator or the lambda API, or issues with the Saturn network itself.
+    </p>
+
+    <div id="saturn-traffic-ratio"></div>
+
     <h2 id="earnings-and-traffic-distribution">Earnings & Traffic Distribution</h2>
     <p>
         To construct this chart, first, we calculate earnings and traffic for every node that was reported as "active"

web/main.js

@@ -12,7 +12,7 @@ if (month === null) {
 }
 
 function dataUrl(file) {
-    return `https://gateway.ipfs.io/ipfs/bafybeibd2uxwsdycccx7r5tdemjg7gkw5bnvctld3lbvavmfwa57lcqmju/year=${year}/month=${month}/${file}`;
+    return `https://gateway.ipfs.io/ipfs/bafybeiak2c6se7bmttyjwhtsaswmu34a72rrz4gublnc3grnjnqs567wja/year=${year}/month=${month}/${file}`;
 }
 
 const PLOTLY_CONF = {
@@ -108,13 +108,11 @@ function parseRetrievals(text) {
     });
 }
 
-function parseResponseDuration(text) {
+function parseTrafficRatio(text) {
     return d3.csvParseRows(text, (d, i) => {
         return {
             date: new Date(d[0]),
-            p5: +d[2],
-            p50: +d[3],
-            p95: +d[4],
+            ratio: +d[1],
         };
     });
 }
@@ -186,7 +184,6 @@ function plotRetrievals(retrievals_data) {
         x: x,
         y: y,
         type: 'bar',
-        name: "Retrievals"
     }];
 
     const layout = {
@@ -209,6 +206,40 @@ function plotRetrievals(retrievals_data) {
     Plotly.newPlot(element, traces, layout, PLOTLY_CONF);
 }
 
+// Plot paid network traffic ration over time.
+function plotTrafficRatio(traffic_ratio_data) {
+    const x = [], y = [];
+    traffic_ratio_data.forEach((e) => {
+        x.push(e.date);
+        y.push(e.ratio);
+    });
+
+    const traces = [{
+        x: x,
+        y: y,
+    }];
+
+    const layout = {
+        yaxis: {
+            fixedrange: true,
+            title: {
+                text: "Traffic ratio",
+            },
+            tickformat: ".0%",
+        },
+        xaxis: {
+            fixedrange: true
+        },
+        hovermode: 'x unified',
+        legend: {
+            orientation: 'h',
+        },
+    };
+
+    const element = document.getElementById("saturn-traffic-ratio");
+    Plotly.newPlot(element, traces, layout, PLOTLY_CONF);
+}
+
 // Plot the number of active Saturn nodes and network traffic by country over time.
 function plotActiveNodeByCountry(node_data, traffic_data, earnings_data) {
     let contries = new Map();
@@ -714,6 +745,7 @@ const text = await Promise.all([
     d3.text(dataUrl("/saturn_traffic_by_country.csv")),
     d3.text(dataUrl("/saturn_earnings_by_country.csv")),
     d3.text(dataUrl("/saturn_retrievals.csv")),
+    d3.text(dataUrl("/saturn_traffic_ratio.csv")),
 ]);
 
 const active_node_data = parseActiveNode(text[0]);
@@ -724,6 +756,7 @@ const active_node_by_country_data = parseActiveNodeByCountry(text[4]);
 const traffic_by_country_data = parseTrafficByCountry(text[5]);
 const earnings_by_country_data = parseEarningsByCountry(text[6]);
 const retrievals_data = parseRetrievals(text[7]);
+const traffic_ratio_data = parseTrafficRatio(text[8]);
 
 plotActiveNodeAndTraffic(active_node_data, traffic_data);
 plotRetrievals(retrievals_data);
@@ -733,6 +766,7 @@ plotCountryStats(country_stats_data);
 plotActiveNodeByCountry(active_node_by_country_data, traffic_by_country_data, earnings_by_country_data);
 
 plotActiveNodeWithoutTraffic(active_node_data);
+plotTrafficRatio(traffic_ratio_data);
 plotActiveNodeDistribution(active_node_stats_data);
 
 plotActiveNodeAge(active_node_stats_data);