Skip to content

Commit

Permalink
Add predictoor tables (#56)
Browse files Browse the repository at this point in the history
* Add predictoor data table structure

* Better test

* Add predictoorRewards table structure

* Parse predictoor csvs

* Add predictoor router

* Add predictoor rewards into rewardsSummary

* Update test
  • Loading branch information
trizin authored Jun 22, 2023
1 parent dd8d73e commit 2fc1f6e
Show file tree
Hide file tree
Showing 10 changed files with 146 additions and 66 deletions.
37 changes: 29 additions & 8 deletions db/structure.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// NOTE: When adding or modifying structures
// make sure to update test/validate_db.test.js

var allocationsTable = `CREATE TABLE allocations(
const allocationsTable = `CREATE TABLE allocations(
chainID INT NOT NULL
,nft_addr VARCHAR(94) NOT NULL
,LP_addr VARCHAR(94) NOT NULL
Expand All @@ -11,15 +11,15 @@ var allocationsTable = `CREATE TABLE allocations(
,round INT NOT NULL
,PRIMARY KEY(chainID, nft_addr, LP_addr, round) );`

var nftVolsTable = `CREATE TABLE nft_vols(
const nftVolsTable = `CREATE TABLE nft_vols(
chainID INT NOT NULL
,basetoken_addr VARCHAR(94) NOT NULL
,nft_addr VARCHAR(94) NOT NULL
,vol_amt FLOAT(94,10) NOT NULL
,round INT NOT NULL
,PRIMARY KEY(chainID, nft_addr, basetoken_addr, round) );`

var nftinfoTable = `CREATE TABLE nft_info(
const nftinfoTable = `CREATE TABLE nft_info(
chainID INT NOT NULL
,nft_addr VARCHAR(94) NOT NULL
,did VARCHAR(300) NOT NULL
Expand All @@ -42,15 +42,15 @@ var nftinfoTable = `CREATE TABLE nft_info(
,round INT NOT NULL
,PRIMARY KEY(chainID, nft_addr, round) );`

var vebalsTable = `CREATE TABLE vebals(
const vebalsTable = `CREATE TABLE vebals(
LP_addr VARCHAR(94) NOT NULL
,balance FLOAT(94,10) NOT NULL
,locked_amt FLOAT(94,10) NOT NULL
,unlock_time INT NOT NULL
,round INT NOT NULL
,PRIMARY KEY(LP_addr, round) )`

var rewardsInfo = `CREATE TABLE rewards_info(
const rewardsInfo = `CREATE TABLE rewards_info(
chainID INT NOT NULL,
LP_addr VARCHAR(94) NOT NULL,
nft_addr VARCHAR(94) NOT NULL,
Expand All @@ -59,17 +59,18 @@ var rewardsInfo = `CREATE TABLE rewards_info(
round INT NOT NULL,
PRIMARY KEY(chainID, nft_addr, LP_addr, token, round))`

var passiveRewardsInfo = `CREATE TABLE passive_rewards_info(
const passiveRewardsInfo = `CREATE TABLE passive_rewards_info(
LP_addr VARCHAR(94) NOT NULL,
balance FLOAT(94,10) NOT NULL,
reward FLOAT(94,10) NOT NULL,
round INT NOT NULL,
PRIMARY KEY(LP_addr, round))`

var rewardsSummary = `CREATE TABLE rewards_summary(
const rewardsSummary = `CREATE TABLE rewards_summary(
LP_addr VARCHAR(94) NOT NULL,
passive_amt FLOAT(94, 10) NOT NULL,
curating_amt FLOAT(94, 10) NOT NULL,
predictoor_amt FLOAT(94, 10) NOT NULL,
round INT NOT NULL,
PRIMARY KEY(LP_addr, round))`

Expand All @@ -80,6 +81,24 @@ const ownersInfo = `CREATE TABLE owners_info(
PRIMARY KEY(chainID, nft_addr)
)`

const predictoorData = `CREATE TABLE predictoor_data(
chainID INT NOT NULL,
predictoor_addr VARCHAR(42) NOT NULL,
accuracy FLOAT(94, 10) NOT NULL,
n_preds INT NOT NULL,
n_correct_preds INT NOT NULL,
round INT NOT NULL,
PRIMARY KEY(chainID, predictoor_addr, round)
)`

const predictoorRewards = `CREATE TABLE predictoor_rewards(
chainID INT NOT NULL,
predictoor_addr VARCHAR(42) NOT NULL,
OCEAN_amt FLOAT(94, 10) NOT NULL,
round INT NOT NULL,
PRIMARY KEY(predictoor_addr, round)
)`

module.exports = {
allocationsTable,
nftVolsTable,
Expand All @@ -88,5 +107,7 @@ module.exports = {
passiveRewardsInfo,
nftinfoTable,
rewardsSummary,
ownersInfo
ownersInfo,
predictoorData,
predictoorRewards
}
6 changes: 6 additions & 0 deletions src/comps/fs/dir.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ function readDataDir(dataDir) {

let rates = []
let symbols = []
let predictoor_data = []
let predictoor_rewards = []

let files = fs.readdirSync(dataDir)
let hashsum = ""
Expand Down Expand Up @@ -42,6 +44,10 @@ function readDataDir(dataDir) {
rates.push(...parseCsv(`${dataDir}${file}`))
} else if (file.includes("symbols-")) {
symbols.push(...parseCsv(`${dataDir}${file}`))
} else if (file.includes("predictoor_data")) {
predictoor_data.push(...parseCsv(`${dataDir}${file}`))
} else if (file.includes("predictoor_rewards")) {
predictoor_rewards.push(...parseCsv(`${dataDir}${file}`))
} else continue

let hash = crypto.createHash("sha256")
Expand Down
15 changes: 14 additions & 1 deletion src/comps/update/batch.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ const batchUpdateRound = async ({
passiveRewardsInfo,
nftinfo,
ownerInfo,
roundNumber
roundNumber,
predictoor_data,
predictoor_rewards
}) => {
await dropTable("allocations", roundNumber)
await updateDb(allocations, "allocations", roundNumber)
Expand All @@ -33,10 +35,21 @@ const batchUpdateRound = async ({

await dropTable("rewards_summary", roundNumber)
await updateRewardsSummary(roundNumber)

if (ownerInfo) {
await dropTable("owners_info", roundNumber)
await updateDb(ownerInfo, "owners_info", roundNumber)
}

if (predictoor_data) {
await dropTable("predictoor_data", roundNumber)
await updateDb(predictoor_data, "predictoor_data", roundNumber)
}

if (predictoor_rewards) {
await dropTable("predictoor_rewards", roundNumber)
await updateDb(predictoor_rewards, "predictoor_rewards", roundNumber)
}
}

module.exports = { batchUpdateRound }
19 changes: 11 additions & 8 deletions src/comps/update/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -35,20 +35,23 @@ async function updateDb(data, dbname, round) {
}

async function updateRewardsSummary(round) {
// aggregates the sum of "passive" and "curating" rewards for each "LP_addr" for a given "round" by getting the sum of `amt` from `rewards_info` and `passive_rewards_info` tables
// Aggregates the sum of "passive," "curating," and "predictoor" rewards for each "LP_addr" for a given "round" by getting the sum of `amt` from `rewards_info`, `passive_rewards_info`, and `predictoor_rewards` tables
await db.promise().query(
`
INSERT INTO rewards_summary(LP_addr,passive_amt,curating_amt,round)
select LP_addr,sum(passive) as passive_amt,sum(curating) as curating_amt,? FROM
INSERT INTO rewards_summary(LP_addr, passive_amt, curating_amt, predictoor_amt, round)
SELECT LP_addr, SUM(passive) AS passive_amt, SUM(curating) AS curating_amt, SUM(OCEAN_amt) AS predictoor_amt, ? FROM
(
select LP_addr,sum(reward) as passive,0 as curating from passive_rewards_info WHERE round=? group by LP_addr
SELECT LP_addr, SUM(reward) AS passive, 0 AS curating, 0 AS OCEAN_amt FROM passive_rewards_info WHERE round = ? GROUP BY LP_addr
UNION
select LP_addr,0 as passive,sum(amt) as curating from rewards_info WHERE round=? group by LP_addr
) as foo group by LP_addr`,
[round, round, round]
)
SELECT LP_addr, 0 AS passive, SUM(amt) AS curating, 0 AS OCEAN_amt FROM rewards_info WHERE round = ? GROUP BY LP_addr
UNION
SELECT predictoor_addr AS LP_addr, 0 AS passive, 0 AS curating, OCEAN_amt FROM predictoor_rewards WHERE round = ?
) AS foo GROUP BY LP_addr`,
[round, round, round, round]
);
}


module.exports = {
updateDb,
dropTable,
Expand Down
6 changes: 5 additions & 1 deletion src/comps/update/sync.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ async function sync(dataDir, roundNumber) {
nftinfo,
rates,
symbols,
hashsum
hashsum,
predictoor_data,
predictoor_rewards
} = readDataDir(dataDir)

if (round_hash_map[roundNumber] == hashsum) {
Expand Down Expand Up @@ -87,6 +89,8 @@ async function sync(dataDir, roundNumber) {
rewardsInfo,
passiveRewardsInfo,
nftinfo,
predictoor_data,
predictoor_rewards,
roundNumber: roundNumber
})
}
Expand Down
2 changes: 2 additions & 0 deletions src/routes/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const rewardsInfoRouter = require("./rewards_info")
const rewardsSummaryRouter = require("./rewards_summary")
const nftInfoRouter = require("./nft_info")
const apyRouter = require("./apy")
const predictoorRouter = require("./predictoor")

app.use(cors())
app.use(express.json({ limit: "100mb" }))
Expand All @@ -19,6 +20,7 @@ app.use(nftvolsRouter)
app.use(rewardsInfoRouter)
app.use(rewardsSummaryRouter)
app.use(nftInfoRouter)
app.use("/predictoor", predictoorRouter)
app.use("/apy", apyRouter)

app.listen(6234)
15 changes: 15 additions & 0 deletions src/routes/predictoor/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
const express = require("express")
const { getPredictoorData, getPredictoorRewards } = require("../../services/rewards_info")
const router = express.Router()

router.post("/data", async (req, res) => {
let data = await getPredictoorData(req.body)
res.json(data)
})

router.post("/rewards", async (req, res) => {
let data = await getPredictoorRewards(req.body)
res.json(data)
})

module.exports = router
32 changes: 32 additions & 0 deletions src/services/predictoor/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
const { selectQuery } = require("../querier")

const getPredictoorData = ({ query, sort, limit, offset, group, fields, join }) => {
return selectQuery(
query,
sort,
limit,
offset,
"predictoor_data",
group,
fields,
join
)
}

const getPredictoorRewards = ({ query, sort, limit, offset, group, fields, join }) => {
return selectQuery(
query,
sort,
limit,
offset,
"predictoor_rewards",
group,
fields,
join
)
}

module.exports = {
getPredictoorData,
getPredictoorRewards
}
14 changes: 8 additions & 6 deletions test/db_operations.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,16 @@ describe("Test db operations", () => {
await updateRewardsSummary(round)
expect(db.promise().query).toHaveBeenCalledWith(
`
INSERT INTO rewards_summary(LP_addr,passive_amt,curating_amt,round)
select LP_addr,sum(passive) as passive_amt,sum(curating) as curating_amt,? FROM
INSERT INTO rewards_summary(LP_addr, passive_amt, curating_amt, predictoor_amt, round)
SELECT LP_addr, SUM(passive) AS passive_amt, SUM(curating) AS curating_amt, SUM(OCEAN_amt) AS predictoor_amt, ? FROM
(
select LP_addr,sum(reward) as passive,0 as curating from passive_rewards_info WHERE round=? group by LP_addr
SELECT LP_addr, SUM(reward) AS passive, 0 AS curating, 0 AS OCEAN_amt FROM passive_rewards_info WHERE round = ? GROUP BY LP_addr
UNION
select LP_addr,0 as passive,sum(amt) as curating from rewards_info WHERE round=? group by LP_addr
) as foo group by LP_addr`,
[round, round, round]
SELECT LP_addr, 0 AS passive, SUM(amt) AS curating, 0 AS OCEAN_amt FROM rewards_info WHERE round = ? GROUP BY LP_addr
UNION
SELECT predictoor_addr AS LP_addr, 0 AS passive, 0 AS curating, OCEAN_amt FROM predictoor_rewards WHERE round = ?
) AS foo GROUP BY LP_addr`,
[round, round, round, round]
)
})

Expand Down
66 changes: 24 additions & 42 deletions test/validate_db.test.js
Original file line number Diff line number Diff line change
@@ -1,44 +1,26 @@
const { Parser } = require("node-sql-parser")
const parser = new Parser()
const {
allocationsTable,
nftVolsTable,
vebalsTable,
rewardsInfo,
passiveRewardsInfo,
nftinfoTable,
rewardsSummary,
ownersInfo
} = require("../db/structure")
const { Parser } = require("node-sql-parser");
const parser = new Parser();
const dbStructure = require("../db/structure");

describe("SQL Table Structures", () => {
test.each([
["allocationsTable", allocationsTable],
["nftVolsTable", nftVolsTable],
["vebalsTable", vebalsTable],
["rewardsInfo", rewardsInfo],
["passiveRewardsInfo", passiveRewardsInfo],
["nftinfoTable", nftinfoTable],
["rewardsSummary", rewardsSummary],
["ownersInfo", ownersInfo]
])("validates structure of %s", (tableName, sqlQuery) => {
let ast
try {
ast = parser.astify(sqlQuery)
} catch (error) {
throw new Error(
`Error parsing SQL for ${tableName}: ${error.message}`
)
}
if (sqlQuery.endsWith(";")) {
// parser returns an array if the query ends with a ;
ast.forEach((ast) => {
expect(ast.type).toBe("create")
expect(ast.keyword).toBe("table")
})
} else {
expect(ast.type).toBe("create")
expect(ast.keyword).toBe("table")
}
})
})
Object.entries(dbStructure).forEach(([tableName, sqlQuery]) => {
test(`validates structure of ${tableName}`, () => {
let ast;
try {
ast = parser.astify(sqlQuery);
} catch (error) {
throw new Error(`Error parsing SQL for ${tableName}: ${error.message}`);
}
if (sqlQuery.endsWith(";")) {
// parser returns an array if the query ends with a ;
ast.forEach((ast) => {
expect(ast.type).toBe("create");
expect(ast.keyword).toBe("table");
});
} else {
expect(ast.type).toBe("create");
expect(ast.keyword).toBe("table");
}
});
});
});

0 comments on commit 2fc1f6e

Please sign in to comment.