Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Google sheets with cache #434

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions apps/velo-external-db/test/resources/provider_resources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,19 @@ import * as googleSheet from '@wix-velo/external-db-google-sheets'

import { AnyFixMe, ConnectionCleanUp, IDataProvider, ISchemaProvider } from '@wix-velo/velo-external-db-types'

// const googleSheet = require('@wix-velo/external-db-google-sheets')
// const googleSheetTestEnv = require('./engines/google_sheets_resources')

export const env: {
dataProvider: IDataProvider
schemaProvider: ISchemaProvider
cleanup: ConnectionCleanUp
driver: AnyFixMe
dataDriver?: AnyFixMe
} = {
dataProvider: Uninitialized,
schemaProvider: Uninitialized,
cleanup: Uninitialized,
driver: Uninitialized,
dataDriver: Uninitialized
}

const dbInit = async(impl: any) => {
Expand All @@ -48,6 +48,9 @@ const dbInit = async(impl: any) => {
env.dataProvider = new impl.DataProvider(pool, driver.filterParser)
env.schemaProvider = new impl.SchemaProvider(pool, testResources.schemaProviderTestVariables?.() )
env.driver = driver
if (impl.dataDriver) {
env.dataDriver = impl.dataDriver()
}
env.cleanup = cleanup
}

Expand Down
12 changes: 12 additions & 0 deletions apps/velo-external-db/test/storage/data_provider.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,18 @@ describe(`Data API: ${currentDbImplementationName()}`, () => {

await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }])
})

test('reading from cache', async() => {
await givenCollectionWith([ctx.entity], ctx.collectionName, ctx.entityFields)

const res = await env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, 0, 50, ctx.projection)

await env.dataDriver.addItemsToCollection(ctx.collectionName, [ctx.anotherEntity])

await new Promise(resolve => setTimeout(resolve, 1000))

await expect(env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, 0, 50, ctx.projection)).resolves.toEqual(res)
})


const ctx = {
Expand Down
5 changes: 3 additions & 2 deletions libs/external-db-config/src/readers/gcp_config_reader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ export class GcpGoogleSheetsConfigReader implements IConfigReader {
constructor() { }

async readConfig() {
const { CLIENT_EMAIL, SHEET_ID, API_PRIVATE_KEY, SECRET_KEY } = process.env
return { clientEmail: CLIENT_EMAIL, apiPrivateKey: API_PRIVATE_KEY, sheetId: SHEET_ID, secretKey: SECRET_KEY }
const { CLIENT_EMAIL, SHEET_ID, API_PRIVATE_KEY, SECRET_KEY, ENABLE_CACHE, STD_TTL, CHECK_PERIOD } = process.env
return { clientEmail: CLIENT_EMAIL, apiPrivateKey: API_PRIVATE_KEY, sheetId: SHEET_ID, secretKey: SECRET_KEY,
enableCache: ENABLE_CACHE === 'true' ? true : false, stdTtl: STD_TTL, checkPeriod: CHECK_PERIOD }
}

}
Expand Down
2 changes: 1 addition & 1 deletion libs/external-db-google-sheets/src/connection_provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default async(config: GoogleSheetsConfig): Promise<DbProviders<GoogleSpre
await loadSheets(doc)

const databaseOperations = new DatabaseOperations(doc)
const dataProvider = new DataProvider(doc)
const dataProvider = new DataProvider(doc, undefined, config.enableCache, config.stdTtl, config.checkPeriod)
const schemaProvider = new SchemaProvider(doc)

const cleanup = async() => {
Expand Down
123 changes: 113 additions & 10 deletions libs/external-db-google-sheets/src/google_sheet_data_provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,112 @@ import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
import { AdapterFilter, IDataProvider, Item, NotEmptyAdapterFilter } from '@wix-velo/velo-external-db-types'
import { sheetFor, headersFrom, dateFormatColumns } from './google_sheet_utils'
const { include, eq } = AdapterOperators

interface State {
[sheetName: string]: StateItem
}

interface StateItem {
rows: Promise<GoogleSpreadsheetRow[]>,
timestamp: number
}


export default class DataProvider implements IDataProvider {
doc: GoogleSpreadsheet
testSupport: any

constructor(doc: GoogleSpreadsheet, testSupport?: any) {
state: State
enableCache: boolean
// Standard time to live in seconds.
stdTTL: number
// Time in seconds to check all data and delete expired keys
checkPeriod: number

constructor(doc: GoogleSpreadsheet, testSupport?: any, enableCache = true, stdTTL = 15, checkPeriod = 60) {
this.doc = doc
this.testSupport = testSupport
this.enableCache = enableCache
this.state = {}
this.stdTTL = stdTTL
this.checkPeriod = checkPeriod
setInterval(this.checkAndClearState.bind(this), checkPeriod * 1000)
}

// STATE RELATED FUNCTIONS ////////////////////////////////////////////////////////////////////////

private isOlderThanStdTTL(timestamp: number) {
return (Date.now() - timestamp) > this.stdTTL * 1000
}

clearCollectionFromState(key: string) {
delete this.state[key]
}

private checkAndClearState() {
const keys = Object.keys(this.state)
keys.forEach(key => {
if (this.isOlderThanStdTTL(this.state[key].timestamp)) {
this.clearCollectionFromState(key)
}
})
}

private async insertItemToStateCollection(sheetName: string, _items: GoogleSpreadsheetRow[]) {

// If sheet is in cache, add the new items to the existing ones
if (this.state[sheetName]) {
this.state[sheetName].rows = Promise.resolve([...await this.state[sheetName].rows, ..._items])
}
}

private async updateItemInStateCollection(sheetName: string, _item: GoogleSpreadsheetRow) {
// If sheet is in cache, update the item
if(this.state[sheetName]) {
const rows = await this.state[sheetName].rows
const index = rows.findIndex(row => row['_id'] === _item['_id'])
rows[index] = _item
this.state[sheetName].rows = Promise.resolve(rows)
}
}

private async deleteItemFromStateCollection(sheetName: string, id: string) {
// If sheet is in cache, delete the item
if(this.state[sheetName]) {
const rows = await this.state[sheetName].rows
const rowsAfterDeletion = rows.filter(row => row['_id'] !== id)
this.state[sheetName].rows = Promise.resolve(rowsAfterDeletion)
}

}

private async rowsFor(sheet: GoogleSpreadsheetWorksheet, options?: any) {

// If cache is disabled, return rows without caching
if (!this.enableCache) {
return sheet.getRows(options)
}

// Sheet is not in cache
if (!this.state[sheet.title]) {
this.state[sheet.title] = {
rows: sheet.getRows(options),
timestamp: Date.now()
}

return this.state[sheet.title].rows
}

// Sheet is in cache but is older than stdTTL
if(this.isOlderThanStdTTL(this.state[sheet.title].timestamp)) {
this.state[sheet.title] = {
rows: sheet.getRows(options),
timestamp: Date.now()
}
return this.state[sheet.title].rows
}

return this.state[sheet.title].rows

}

// FIND RELATED FUNCTIONS ////////////////////////////////////////////////////////////////////////
Expand All @@ -23,11 +122,11 @@ export default class DataProvider implements IDataProvider {
}

async findRowById(sheet: GoogleSpreadsheetWorksheet, id: string) {
const rows = await sheet.getRows()
const rows = await this.rowsFor(sheet)
return rows.find(r => r['_id'] === id)
}

async find(collectionName: string, _filter: AdapterFilter, sort: any, skip: number, limit: number, _projection: string[]): Promise<Item[]> {
async find(collectionName: string, _filter: AdapterFilter, sort: any, skip: number, limit: number, _projection: string[]): Promise<Item[]> {
const sheet: GoogleSpreadsheetWorksheet = await sheetFor(collectionName, this.doc)
const filter = _filter as NotEmptyAdapterFilter
const projection: string[] = this.testSupport? this.testSupport.projection(_projection) : _projection
Expand All @@ -42,7 +141,7 @@ export default class DataProvider implements IDataProvider {
return row !== undefined ? [this.formatRow(row, projection)] : []
}

const rows: GoogleSpreadsheetRow[] = await sheet.getRows({ offset: skip, limit })
const rows: GoogleSpreadsheetRow[] = await this.rowsFor(sheet, { offset: skip, limit })
return rows.map(r => this.formatRow(r, projection))
}

Expand All @@ -57,15 +156,16 @@ export default class DataProvider implements IDataProvider {
return row !== undefined ? 1 : 0
}

const rows: GoogleSpreadsheetRow[] = await sheet.getRows()
const rows: GoogleSpreadsheetRow[] = await this.rowsFor(sheet)
return rows.length
}

// INSERT RELATED FUNCTIONS ////////////////////////////////////////////////////////////////////////

async insert(collectionName: string, items: Item[]): Promise<number> {
const sheet: GoogleSpreadsheetWorksheet = await sheetFor(collectionName, this.doc)
await sheet.addRows(items)
const res = await sheet.addRows(items)
await this.insertItemToStateCollection(sheet.title, res)
return items.length
}

Expand All @@ -83,8 +183,9 @@ export default class DataProvider implements IDataProvider {
const updatePromises = await Promise.all(
itemsToUpdate.map( async(item) => {
const rowToUpdate = await this.findRowById(sheet, item['_id'] as string)
if (rowToUpdate) {
return await this.updateRow(rowToUpdate, item)
if (rowToUpdate) {
await this.updateRow(rowToUpdate, item)
await this.updateItemInStateCollection(sheet.title, rowToUpdate)
}
})
)
Expand All @@ -102,7 +203,9 @@ export default class DataProvider implements IDataProvider {
const sheet: GoogleSpreadsheetWorksheet = await sheetFor(collectionName, this.doc)
const idsToRemove = ids.slice(0, 3) // google-sheets API can update till 3 items at the same time

const removePromises = Promise.all(idsToRemove.map(async id => this.deleteRow(sheet, id)))
const removePromises = Promise.all(idsToRemove.map(async id => {
this.deleteRow(sheet, id).then(_i => this.deleteItemFromStateCollection(sheet.title, id))
}))

return (await removePromises).length
}
Expand Down
1 change: 1 addition & 0 deletions libs/external-db-google-sheets/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import ConfigValidator from './google_sheet_config_validator'

export const driver = () => require('../tests/drivers/sql_filter_transformer_test_support')
export const opsDriver = () => require('../tests/drivers/db_operations_test_support')
export const dataDriver = () => require('../tests/drivers/data_provider_test_support')

export class GoogleSheetConnector extends DbConnector {
constructor() {
Expand Down
5 changes: 4 additions & 1 deletion libs/external-db-google-sheets/src/types.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
export type GoogleSheetsConfig = {
sheetId: string,
clientEmail?: string,
apiPrivateKey?: string
apiPrivateKey?: string,
enableCache?: boolean,
stdTtl?: number,
checkPeriod?: number,
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@

import axios from 'axios'
import { PORT } from '../e2e-testkit/google_sheets_resources'

const axiosClient = axios.create({
baseURL: `http://localhost:${PORT}/v4/spreadsheets/`
})

export const addItemsToCollection = async(sheetTitle:any, items: any) => {
return await axiosClient.post('/test/add_data', {
sheetTitle,
items
})
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ const { app: mockServer } = require ('../mock_google_sheets_api')
import { Server } from 'http'

export const SHEET_ID = '1rNU4Cr7rebYOn-QKpvwTdOxSo5Qf4VPNEyFqPAzBgFA'
const PORT = 1502
export const PORT = 1502

let _server: Server

Expand Down Expand Up @@ -32,12 +32,22 @@ export const shutdownEnv = async() => {
}

export const setActive = () => {
process.env = { ...process.env, ...enviormentVariables }
process.env = { ...process.env, ...enviormentVariables, ...enableCacheEnvs }
}

export const enableCacheEnvs = {
ENABLE_CACHE: 'true',
STD_TTL: '15',
CHECK_PERIOD: '60'
}

export const disableCacheEnvs = {
ENABLE_CACHE: 'false',
}

export const enviormentVariables = {
TYPE: 'google-sheet',
SHEET_ID: SHEET_ID
SHEET_ID: SHEET_ID,
}

export const name = 'google-sheets'
16 changes: 14 additions & 2 deletions libs/external-db-google-sheets/tests/mock_google_sheets_api.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@ const doc = new GoogleSpreadsheetDoc('spreadsheet-doc-id', 'spreadsheet-doc-titl
app.use(express.json())
app.use('/v4/spreadsheets/', v4SpreadsheetsRouter)


// TEST SUPPORT ROUTES

v4SpreadsheetsRouter.post('/test/add_data', (req, res) => {
const { sheetTitle, items } = req.body
const sheet = doc.getSheet(sheetTitle)

items.forEach(item => {
sheet.addRows([Object.values(item)], 'A1')
})

res.send()
})

// DOC INFO ROUTES
v4SpreadsheetsRouter.get('/:sheetId/', (_, res) => {
res.send(doc.docInfo())
Expand Down Expand Up @@ -71,7 +85,5 @@ v4SpreadsheetsRouter.put('/:sheetId/values/:sheetName_range', (req, res) => {
res.send(actionRes)
})



module.exports = { app, cleanupSheets: doc.cleanupSheets }

1 change: 1 addition & 0 deletions libs/velo-external-db-types/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export enum SchemaOperations {
NotOperator = 'not',
IncludeOperator = 'include',
FilterByEveryField = 'filterByEveryField',
Cache = 'cache',
}

export type FieldWithQueryOperators = ResponseField & { queryOperators: string[] }
Expand Down