From 4b5cfda09c4d85b7236622d7e85eeb3649cf852f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Raphael=20St=C3=A4bler?= Date: Sun, 27 Aug 2023 12:04:46 +0200 Subject: [PATCH] Adapt for digital ocean hosting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit e380094cd9e0b416ccf4c1183631117163b2b1ce Author: Raphael Stäbler Date: Sat Aug 26 09:19:22 2023 +0200 Enable all cronjobs commit ff6957ecd8df2dac9e3cc9a1774c80c6dded3a0b Author: Raphael Stäbler Date: Fri Aug 25 20:08:04 2023 +0200 Fix crontab commit 6e705d34c81e0cac06ee55462dff6c1488998c4c Author: Raphael Stäbler Date: Fri Aug 25 18:01:15 2023 +0200 Add support for env variables commit 779f3003fad2c648ffe3762a0bb2c3358a5ee1c1 Author: Raphael Stäbler Date: Fri Aug 25 17:39:29 2023 +0200 Update crontab commit 86480d2e0c6381dead04ce3e6bf839ee5ee333d3 Author: Raphael Stäbler Date: Wed Aug 23 18:21:43 2023 +0200 Fix new relic and update docker image commit 1cf8e4c9e9bc90e06663c99dc797c755ffcd2997 Author: Raphael Stäbler Date: Wed Aug 23 10:29:50 2023 +0200 Update for digitalocean deployment commit 4bcadafa438859a65b5a72d8d07c83431ea3c3c1 Author: Raphael Stäbler Date: Fri Aug 25 14:01:34 2023 +0200 Add cleanup tootstats command --- Dockerfile | 38 +++++++++++++++++++ README.md | 41 ++++----------------- crontab | 27 ++++++++++++++ entrypoint.sh | 7 ++++ package.json | 9 +++-- src/commands/aggregate/dailyaccountstats.ts | 2 +- src/commands/aggregate/dailytootstats.ts | 2 +- src/commands/cleanup/accountdata.ts | 2 +- src/commands/cleanup/accounts.ts | 2 +- src/commands/cleanup/tootstats.ts | 2 +- src/commands/cleanup/usercredentials.ts | 2 +- src/commands/cleanup/users.ts | 2 +- src/commands/fetch/accountstats.ts | 2 +- src/commands/fetch/initialstats.ts | 8 ++-- src/commands/fetch/tootstats.ts | 2 +- src/commands/hello/index.ts | 8 +++- src/commands/mail/weeklystats.ts | 8 ++-- src/commands/tools/rebuilddailytootstats.ts | 2 +- src/helpers/completeLogging.ts | 2 +- src/helpers/logger.ts | 5 ++- 20 files changed, 113 insertions(+), 60 deletions(-) create mode 100644 Dockerfile create mode 100644 crontab create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f6d8eba --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:22.04 + +ARG NEWRELIC_LICENSE_KEY + +WORKDIR /analytodon-cli + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install -y cron curl ca-certificates + +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && apt-get install -y nodejs + +# Remove package lists for smaller image sizes +RUN rm -rf /var/lib/apt/lists/* \ + && which cron \ + && rm -rf /etc/cron.*/* + +COPY crontab hello-cron +COPY entrypoint.sh entrypoint.sh +COPY package.json package.json +COPY newrelic.js newrelic.js +COPY tsconfig.json tsconfig.json +COPY README.md README.md +COPY bin bin +COPY src src + +RUN npm i typescript +RUN npm run build + +RUN ./bin/run hello analytodon -f "the build process" + +RUN crontab hello-cron +RUN chmod +x entrypoint.sh + +ENTRYPOINT ["/analytodon-cli/entrypoint.sh"] + +# https://manpages.ubuntu.com/manpages/trusty/man8/cron.8.html +# -f | Stay in foreground mode, don't daemonize. +# -L loglevel | Tell cron what to log about jobs (errors are logged regardless of this value) as the sum of the following values: +CMD ["cron","-f", "-L", "2"] \ No newline at end of file diff --git a/README.md b/README.md index b0abd06..4e86aec 100644 --- a/README.md +++ b/README.md @@ -3,52 +3,25 @@ CLI for [Analytodon](https://www.analytodon.com). - -- [Usage](#usage) -- [Commands](#commands) +* [Analytodon CLI](#analytodon-cli) +* [Usage](#usage) +* [Commands](#commands) # Usage - ```sh-session $ npm install -g analytodon-cli $ analytodon-cli COMMAND running command... $ analytodon-cli (--version) -analytodon-cli/0.0.0 darwin-arm64 node-v16.19.0 +analytodon-cli/0.0.0 darwin-arm64 node-v18.17.1 $ analytodon-cli --help [COMMAND] USAGE $ analytodon-cli COMMAND ... ``` - -Recommended cron configuration: - -```crontab -# Analytodon initial stats -* * * * * /path/to/analytodon-cli fetch initialstats - -# Analytodon fetch -35 * * * * /path/to/analytodon-cli fetch accountstats -35 * * * * /path/to/analytodon-cli fetch tootstats - -# Analytodon aggregate -5 * * * * /path/to/analytodon-cli aggregate dailyaccountstats -5 * * * * /path/to/analytodon-cli aggregate dailytootstats - -# Analytodon weekly mail -0 19,20,21,22,23 * * 0 /path/to/analytodon-cli mail weeklystats -0 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17 * * 1 /path/to/analytodon-cli mail weeklystats - -# Analytodon cleanup -45 11 * * * /path/to/analytodon-cli cleanup users -46 11 * * * /path/to/analytodon-cli cleanup usercredentials -47 11 * * * /path/to/analytodon-cli cleanup accounts -48 11 * * * /path/to/analytodon-cli cleanup accountdata -``` - # Commands @@ -325,7 +298,7 @@ DESCRIPTION EXAMPLES $ oex hello friend --from oclif - hello friend from oclif! (./src/commands/hello/index.ts) + hello friend from oclif! ``` _See code: [dist/commands/hello/index.ts](https://github.com/blazer82/analytodon-cli/blob/v0.0.0/dist/commands/hello/index.ts)_ @@ -440,7 +413,7 @@ ALIASES $ analytodon-cli plugins add EXAMPLES - $ analytodon-cli plugins:install myplugin + $ analytodon-cli plugins:install myplugin $ analytodon-cli plugins:install https://github.com/someuser/someplugin @@ -500,7 +473,7 @@ ALIASES $ analytodon-cli plugins add EXAMPLES - $ analytodon-cli plugins:install myplugin + $ analytodon-cli plugins:install myplugin $ analytodon-cli plugins:install https://github.com/someuser/someplugin diff --git a/crontab b/crontab new file mode 100644 index 0000000..637ea63 --- /dev/null +++ b/crontab @@ -0,0 +1,27 @@ +# Analytodon cron jobs + +#*/15 * * * * /analytodon-cli/bin/run hello analytodon -f "the cron tab" + +# analytodon initial stats +* * * * * /analytodon-cli/bin/run fetch initialstats + +# analytodon fetch +35 * * * * /analytodon-cli/bin/run fetch accountstats +35 * * * * /analytodon-cli/bin/run fetch tootstats -a + +# analytodon aggregate +5 * * * * /analytodon-cli/bin/run aggregate dailyaccountstats +5 * * * * /analytodon-cli/bin/run aggregate dailytootstats + +# analytodon weekly mail +0 19,20,21,22,23 * * 0 /analytodon-cli/bin/run mail weeklystats +0 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17 * * 1 /analytodon-cli/bin/run mail weeklystats + +# analytodon cleanup +45 11 * * * /analytodon-cli/bin/run cleanup users +46 11 * * * /analytodon-cli/bin/run cleanup usercredentials +47 11 * * * /analytodon-cli/bin/run cleanup accounts +48 11 * * * /analytodon-cli/bin/run cleanup accountdata +40 2 * * * /analytodon-cli/bin/run cleanup tootstats + +# An empty line is required at the end of this file for a valid cron file. diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..44f42d7 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +env >> /etc/environment + +# execute CMD +echo "$@" +exec "$@" \ No newline at end of file diff --git a/package.json b/package.json index 09ddb14..06d8a30 100644 --- a/package.json +++ b/package.json @@ -17,14 +17,15 @@ "/oclif.manifest.json" ], "dependencies": { + "@newrelic/pino-enricher": "^1.1.1", "@oclif/core": "^1.19.1", "@oclif/plugin-help": "^5", "@oclif/plugin-plugins": "^2.1.7", "axios": "^0.27.2", "megalodon": "^4.1.3", "mongodb": "^4.12.1", - "newrelic": "^9.7.0", - "pino": "^8.7.0", + "newrelic": "^10.6.2", + "pino": "^8.15.0", "timezone-support": "^3.1.0" }, "devDependencies": { @@ -63,11 +64,11 @@ } }, "scripts": { - "build": "shx rm -rf dist && tsc -b && oclif manifest && oclif readme", + "build": "rm -rf dist && tsc -b && oclif manifest && oclif readme", "prettier:check": "npx prettier --check '{src,test}/**/*.ts'", "prettier:write": "npx prettier --write '{src,test}/**/*.ts'", "lint": "eslint . --ext .ts --config .eslintrc && npm run prettier:check", - "postpack": "shx rm -f oclif.manifest.json", + "postpack": "rm -f oclif.manifest.json", "posttest": "npm run lint", "pack": "npm run build && oclif pack tarballs --no-xz -t linux-x64 && npm run postpack", "test": "mocha --forbid-only \"test/**/*.test.ts\"", diff --git a/src/commands/aggregate/dailyaccountstats.ts b/src/commands/aggregate/dailyaccountstats.ts index 737c3be..03b337d 100644 --- a/src/commands/aggregate/dailyaccountstats.ts +++ b/src/commands/aggregate/dailyaccountstats.ts @@ -14,7 +14,7 @@ export default class DailyAccountStats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/aggregate/dailytootstats.ts b/src/commands/aggregate/dailytootstats.ts index 6661b3a..65042ed 100644 --- a/src/commands/aggregate/dailytootstats.ts +++ b/src/commands/aggregate/dailytootstats.ts @@ -14,7 +14,7 @@ export default class DailyTootStats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/cleanup/accountdata.ts b/src/commands/cleanup/accountdata.ts index 66008b2..a9a60ac 100644 --- a/src/commands/cleanup/accountdata.ts +++ b/src/commands/cleanup/accountdata.ts @@ -12,7 +12,7 @@ export default class Accountdata extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/cleanup/accounts.ts b/src/commands/cleanup/accounts.ts index 488fe25..af9b996 100644 --- a/src/commands/cleanup/accounts.ts +++ b/src/commands/cleanup/accounts.ts @@ -15,7 +15,7 @@ export default class Accounts extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/cleanup/tootstats.ts b/src/commands/cleanup/tootstats.ts index c40b22a..90ea25f 100644 --- a/src/commands/cleanup/tootstats.ts +++ b/src/commands/cleanup/tootstats.ts @@ -16,7 +16,7 @@ export default class Tootstats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/cleanup/usercredentials.ts b/src/commands/cleanup/usercredentials.ts index 64a95ce..3217565 100644 --- a/src/commands/cleanup/usercredentials.ts +++ b/src/commands/cleanup/usercredentials.ts @@ -15,7 +15,7 @@ export default class Usercredentials extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/cleanup/users.ts b/src/commands/cleanup/users.ts index 53c9d8c..6b55ddd 100644 --- a/src/commands/cleanup/users.ts +++ b/src/commands/cleanup/users.ts @@ -15,7 +15,7 @@ export default class Users extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/fetch/accountstats.ts b/src/commands/fetch/accountstats.ts index 4729c04..e989987 100644 --- a/src/commands/fetch/accountstats.ts +++ b/src/commands/fetch/accountstats.ts @@ -14,7 +14,7 @@ export default class AccountStats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/fetch/initialstats.ts b/src/commands/fetch/initialstats.ts index 457cc5f..480d84e 100644 --- a/src/commands/fetch/initialstats.ts +++ b/src/commands/fetch/initialstats.ts @@ -16,7 +16,7 @@ export default class InitialStats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', @@ -30,12 +30,12 @@ export default class InitialStats extends Command { host: Flags.string({ char: 'h', description: 'App host URL', - default: 'app.analytodon.com', + default: process.env.APP_URL || 'https://app.analytodon.com', }), authorization: Flags.string({ char: 't', description: 'Authorization header', - required: true, + default: process.env.EMAIL_API_KEY || 'no-key', }), }; @@ -82,7 +82,7 @@ export default class InitialStats extends Command { await createInitialTootStats(db, account); await axios.post( - `https://${flags.host}/api/mail/firststats`, + `${flags.host}/api/mail/firststats`, {userID: `${user._id}`, accounts: [account._id]}, { headers: { diff --git a/src/commands/fetch/tootstats.ts b/src/commands/fetch/tootstats.ts index 99387b7..9575453 100644 --- a/src/commands/fetch/tootstats.ts +++ b/src/commands/fetch/tootstats.ts @@ -14,7 +14,7 @@ export default class TootStats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/commands/hello/index.ts b/src/commands/hello/index.ts index 38494a6..5e2982f 100644 --- a/src/commands/hello/index.ts +++ b/src/commands/hello/index.ts @@ -1,11 +1,13 @@ import {Command, Flags} from '@oclif/core'; +import {logger} from '../../helpers/logger'; +import {completeLogging} from '../../helpers/completeLogging'; export default class Hello extends Command { static description = 'Say hello'; static examples = [ `$ oex hello friend --from oclif -hello friend from oclif! (./src/commands/hello/index.ts) +hello friend from oclif! `, ]; @@ -18,6 +20,8 @@ hello friend from oclif! (./src/commands/hello/index.ts) async run(): Promise { const {args, flags} = await this.parse(Hello); - this.log(`hello ${args.person} from ${flags.from}! (./src/commands/hello/index.ts)`); + logger.info(`hello ${args.person} from ${flags.from}!`); + + await completeLogging(); } } diff --git a/src/commands/mail/weeklystats.ts b/src/commands/mail/weeklystats.ts index 71b0dba..e3e9fe2 100644 --- a/src/commands/mail/weeklystats.ts +++ b/src/commands/mail/weeklystats.ts @@ -14,7 +14,7 @@ export default class WeeklyStats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', @@ -24,12 +24,12 @@ export default class WeeklyStats extends Command { host: Flags.string({ char: 'h', description: 'App host URL', - default: 'app.analytodon.com', + default: process.env.APP_URL || 'https://app.analytodon.com', }), authorization: Flags.string({ char: 't', description: 'Authorization header', - required: true, + default: process.env.EMAIL_API_KEY || 'no-key', }), user: Flags.string({ char: 'u', @@ -97,7 +97,7 @@ export default class WeeklyStats extends Command { logger.info(`Send weekly stats: Trigger mail for user ${user._id} with accounts ${accountIds.join(',')}`); await axios.post( - `https://${flags.host}/api/mail/weeklystats`, + `${flags.host}/api/mail/weeklystats`, {userID: `${user._id}`, accounts: accountIds}, { headers: { diff --git a/src/commands/tools/rebuilddailytootstats.ts b/src/commands/tools/rebuilddailytootstats.ts index d54f828..d0c2cf8 100644 --- a/src/commands/tools/rebuilddailytootstats.ts +++ b/src/commands/tools/rebuilddailytootstats.ts @@ -13,7 +13,7 @@ export default class RebuildDailyTootStats extends Command { connectionString: Flags.string({ char: 'c', description: 'MongoDB connection string', - default: 'mongodb://localhost:27017', + default: process.env.MONGODB_URI || 'mongodb://localhost:27017', }), database: Flags.string({ char: 'd', diff --git a/src/helpers/completeLogging.ts b/src/helpers/completeLogging.ts index 7b5eeae..8959c95 100644 --- a/src/helpers/completeLogging.ts +++ b/src/helpers/completeLogging.ts @@ -3,5 +3,5 @@ const newrelic = require('newrelic'); export const completeLogging = async () => new Promise((resolve) => { - newrelic.shutdown({collectPendingData: true, timeout: 10000}, () => resolve()); + newrelic.shutdown({collectPendingData: true, timeout: 10000, waitForIdle: true}, () => resolve()); }); diff --git a/src/helpers/logger.ts b/src/helpers/logger.ts index 629c4d8..591cc40 100644 --- a/src/helpers/logger.ts +++ b/src/helpers/logger.ts @@ -1,5 +1,8 @@ /* eslint-disable */ -const logger = require('pino')(); +const nrPino = require('@newrelic/pino-enricher'); +const pino = require('pino'); + +const logger = pino(nrPino()); export {logger}; /* eslint-enable */