Skip to content

Commit

Permalink
Merge pull request #117 from api3dao/116-clean-up-metrics-endpoint-fu…
Browse files Browse the repository at this point in the history 
…rther

Further reduce complexity of metrics endpoint
  • Loading branch information
@aquarat
aquarat authored Dec 23, 2024
2 parents 9fa0ebb + 21b6a03 commit 3ba898e
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 191 deletions.
172 changes: 0 additions & 172 deletions pages/api/metrics.ts
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,13 @@
import { ethers } from "ethers";
import type { NextApiRequest, NextApiResponse } from "next";

// import { Votings, Wallets, Supply, Treasuries } from "../../services/api";

import prisma from "./../../services/db";

// This function is used for comparison of the chain state sync with cloudflare
const getHeadBlock = async (): Promise<bigint> => {
const jsonRpc = new ethers.providers.JsonRpcProvider(
process.env.API3TRACKER_EXTERNAL_ENDPOINT || "https://cloudflare-eth.com/",
);
const result = await ethers.utils.fetchJson(
jsonRpc.connection,
` {"id":"head","method":"eth_blockNumber","params":[],"jsonrpc":"2.0"} `,
);
return BigInt(result.result);
};

export default async function handler(
_req: NextApiRequest,
res: NextApiResponse<string>,
) {
const out = ["# HELP up Server is running", "# TYPE up gauge", "up 1", ""];

// 1. Block numbers: external, max downloaded block, max porcessed block
const headBlock = await getHeadBlock();
out.push(
"# HELP head_block_number: latest block number at cloudflare or other external JSON+RPC provider",
);
out.push("# TYPE head_block_number gauge");
out.push("head_block_number " + headBlock);

const status = await prisma.syncStatus.findMany({
where: { id: 1 },
});
Expand All @@ -46,154 +23,5 @@ export default async function handler(
out.push("processed_block_number " + status[0].processed);
}

// Multiple queries below are commented as they slow down the response of this endpoint, and we don't every use the outputs.
// 2. treasuries balances
// const names = await Treasuries.fetchList();
// if (names.length > 0) out.push("");
// for (const ttype of names) {
// const tokens = await Treasuries.fetch(ttype);
// const valueAPI3 =
// tokens
// .filter((x: any) => x.token === "API3")
// .map((x: any) => x.value)[0] || 0;
// const valueUSDC =
// tokens
// .filter((x: any) => x.token === "USDC")
// .map((x: any) => x.value)[0] || 0;
// const valueETH =
// tokens
// .filter((x: any) => x.token === "ETH")
// .map((x: any) => x.value)[0] || 0;
// out.push("# HELP treasury_api3: API3 tokens in the treasury");
// out.push("# TYPE treasury_api3: gauge");
// out.push('treasury_api3{name="' + ttype + '"} ' + valueAPI3);
// out.push("# HELP treasury_usdc: USDC tokens in the treasury");
// out.push("# TYPE treasury_usdc: gauge");
// out.push('treasury_usdc{name="' + ttype + '"} ' + valueUSDC);
// out.push("# HELP treasury_eth: ETH balance in the treasury");
// out.push("# TYPE treasury_eth: gauge");
// out.push('treasury_eth{name="' + ttype + '"} ' + valueETH);
// }

// 3. supply
// const supply = await Supply.fetch();
// if (supply) {
// out.push("# HELP circulating_supply Circulating Supply");
// out.push("# TYPE circulating_supply gauge");
// out.push("circulating_supply " + supply.circulatingSupply.toString());
//
// out.push("# HELP total_staked Total Staked Tokens");
// out.push("# TYPE total_staked gauge");
// out.push("total_staked " + supply.totalStaked.toString());
//
// out.push("# HELP staking_target Staking Target");
// out.push("# TYPE staking_target gauge");
// out.push("staking_target " + supply.stakingTarget.toString());
//
// out.push("# HELP staking_target_pct Reaching staking target, in percents");
// out.push("# TYPE staking_target_pct gauge");
// out.push(
// "staking_target_pct " +
// supply.totalStaked.mul(100).div(supply.stakingTarget).toString(),
// );
//
// // out.push("total_locked " + supply.totalLocked.toString());
// out.push("# HELP locked number of locked tokens");
// out.push("# TYPE locked gauge");
// out.push(
// 'locked{where="governance"} ' + supply.lockedByGovernance.toString(),
// );
// out.push('locked{where="vestings"} ' + supply.lockedVestings.toString());
// out.push('locked{where="rewards"} ' + supply.lockedRewards.toString());
// out.push('locked{where="time"} ' + supply.timeLocked.toString());
// }

// 4. Number of members (per tag)
// out.push("");
// const total = await Wallets.fetchCount("");
// out.push("# TYPE members_total gauge");
// out.push("members_total " + total);
// const vested = await Wallets.fetchCount("vested");
// out.push("# TYPE members_total_vested gauge");
// out.push("members_total_vested " + vested);
// const voters = await Wallets.fetchCount("voter");
// out.push("# TYPE members_total_voters gauge");
// out.push("members_total_voters " + voters);
// const supports = await Wallets.fetchCount("supports");
// out.push("# TYPE members_total_supports gauge");
// out.push("members_total_supports " + supports);
// const withdrawn = await Wallets.fetchCount("withdrawn");
// out.push("# TYPE members_total_withdrawn gauge");
// out.push("members_total_withdrawn " + withdrawn);

// 5. Number of votes (per status)
// out.push("");
// out.push("# HELP votings number of votings");
// out.push("# TYPE votings gauge");
// const counts = await Votings.fetchCounts();
// for (const v of counts) {
// out.push('votings{status="' + v.status + '"} ' + v.total);
// }

// 6. Status estimates - of downloaded user state history
// out.push("");
// const blocksWithStatus = (
// await prisma.memberEvent.groupBy({
// where: { eventName: "Status" },
// by: ["blockNumber"],
// })
// ).reduce((map: any, obj: any) => {
// map[obj.blockNumber] = 1;
// return map;
// }, {});
//
// const blocksWithoutStatusTotal = (
// await prisma.memberEvent.groupBy({
// where: { eventName: { not: "Status" } },
// by: ["blockNumber"],
// })
// )
// .map((x: any) => 1 - (blocksWithStatus[x.blockNumber] || 0))
// .reduce((partialSum, a) => partialSum + a, 0);
//
// out.push(
// "# HELP blocks_without_shares returns the estimate amount of history to be downloaded",
// );
// out.push("# TYPE blocks_without_shares gauge");
// out.push("blocks_without_status " + blocksWithoutStatusTotal);
// out.push(
// "# HELP blocks_with_status returns the estimate amount of history blocks that were already downloaded",
// );
// out.push("# TYPE blocks_with_status gauge");
// out.push("blocks_with_status " + Object.keys(blocksWithStatus).length);

// const addressWithStatus = (
// await prisma.memberEvent.groupBy({
// where: { eventName: "Status" },
// by: ["address"],
// })
// ).reduce((map: any, obj: any) => {
// map[obj.address] = 1;
// return map;
// }, {});
// const addressWithRewardsTotal = (
// await prisma.memberEvent.groupBy({
// where: { eventName: { not: "Status" } },
// by: ["address"],
// })
// )
// .map((x: any) => 1 - (addressWithStatus[x.address] || 0))
// .reduce((partialSum, a) => partialSum + a, 0);
// out.push(
// "# HELP address_without_status returns the amount of members that have no any shares history checked",
// );
// out.push("# TYPE address_without_status gauge");
// out.push("address_without_status " + addressWithRewardsTotal);
// out.push(
// "# HELP address_with_status returns the estimate amount of addresses that were already downloaded",
// );
// out.push("# TYPE address_with_status gauge");
// out.push("address_with_status " + Object.keys(addressWithStatus).length);

res.status(200).send(out.join("\n"));
}
45 changes: 26 additions & 19 deletions services/sync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
type Log,
type Provider,
} from "@ethersproject/abstract-provider";
import { Prisma } from "@prisma/client";
import { Prisma, type PrismaPromise } from "@prisma/client";
import { BigNumber, ethers } from "ethers";
import { uniq } from "lodash";

Expand All @@ -26,6 +26,8 @@ import { VotingReader } from "./../services/voting";
import prisma from "./db";
import { fetchWebconfig } from "./webconfig";

import BatchPayload = Prisma.BatchPayload;

const rewardsPct = (newApr: number): number => {
const epochLength = 604_800;
const week = epochLength / 3600 / 24;
Expand Down Expand Up @@ -223,9 +225,8 @@ export const Sync = {
});
return status.length > 0;
},
saveBlock: async (b: BlockFullInfo) => {
saveBlock: async (b: BlockFullInfo, tx: PrismaPromise<BatchPayload>[]) => {
const blockHash = Hash.asBuffer(b.block.hash);
const tx = [];
// 1. save block
tx.push(
prisma.cacheBlock.createMany({
Expand Down Expand Up @@ -284,18 +285,6 @@ export const Sync = {
}),
);
}
// 5. update syncing status
tx.push(
prisma.syncStatus.updateMany({
where: { id: 1 },
data: {
updatedAt: new Date().toISOString(),
downloaded: b.block.number,
},
}),
);

await prisma.$transaction(tx);
},
updateProcessed: async (blockNumber: number) => {
// update block number
Expand Down Expand Up @@ -1333,12 +1322,12 @@ export const Events = {
const jsonRpc = new ethers.providers.JsonRpcProvider(endpoint);
// get the head block
const block = await jsonRpc.getBlock("latest");
const headBlockNumber = block.number;

console.log(
"Contracts",
addresses,
"Head Block",
headBlockNumber,
block.number,
"Last block in DB",
lastDownloadedBlock,
);
Expand All @@ -1348,7 +1337,7 @@ export const Events = {
minBlock,
batchSize,
lastDownloadedBlock,
headBlockNumber,
block.number,
);
console.log(
"Expecting",
Expand All @@ -1357,6 +1346,19 @@ export const Events = {
"Batch size",
batchSize,
);

// If the DB transaction fails, this will roll back, so it can be at the start of the tx
const tx: PrismaPromise<BatchPayload>[] = [
// Update syncing status
prisma.syncStatus.updateMany({
where: { id: 1 },
data: {
updatedAt: new Date().toISOString(),
downloaded: block.number,
},
}),
];

for (const cMap of batches) {
const blockMap = new Map<number, Array<Log>>();
for (const [_address, filter] of cMap.entries()) {
Expand All @@ -1383,10 +1385,15 @@ export const Events = {
logs,
priceReader,
);
Sync.saveBlock(fullInfo);
// Updates are accumulated in `tx`
await Sync.saveBlock(fullInfo, tx);
}
}
}

// Finally, execute pending updates
await prisma.$transaction(tx);

return total;
},
};

0 comments on commit 3ba898e

Please sign in to comment.