Adapt for digital ocean hosting

commit e380094
Author: Raphael Stäbler <raphael.staebler@productionbuild.de>
Date:   Sat Aug 26 09:19:22 2023 +0200

    Enable all cronjobs

commit ff6957e
Author: Raphael Stäbler <raphael.staebler@productionbuild.de>
Date:   Fri Aug 25 20:08:04 2023 +0200

    Fix crontab

commit 6e705d3
Author: Raphael Stäbler <raphael.staebler@productionbuild.de>
Date:   Fri Aug 25 18:01:15 2023 +0200

    Add support for env variables

commit 779f300
Author: Raphael Stäbler <raphael.staebler@productionbuild.de>
Date:   Fri Aug 25 17:39:29 2023 +0200

    Update crontab

commit 86480d2
Author: Raphael Stäbler <raphael.staebler@productionbuild.de>
Date:   Wed Aug 23 18:21:43 2023 +0200

    Fix new relic and update docker image

commit 1cf8e4c
Author: Raphael Stäbler <raphael.staebler@productionbuild.de>
Date:   Wed Aug 23 10:29:50 2023 +0200

    Update for digitalocean deployment

commit 4bcadaf
Author: Raphael Stäbler <raphael.staebler@productionbuild.de>
Date:   Fri Aug 25 14:01:34 2023 +0200

    Add cleanup tootstats command

Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG NEWRELIC_LICENSE_KEY
+
+WORKDIR /analytodon-cli
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install -y cron curl ca-certificates
+
+RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && apt-get install -y nodejs
+
+# Remove package lists for smaller image sizes
+RUN rm -rf /var/lib/apt/lists/* \
+    && which cron \
+    && rm -rf /etc/cron.*/*
+
+COPY crontab hello-cron
+COPY entrypoint.sh entrypoint.sh
+COPY package.json package.json
+COPY newrelic.js newrelic.js
+COPY tsconfig.json tsconfig.json
+COPY README.md README.md
+COPY bin bin
+COPY src src
+
+RUN npm i typescript
+RUN npm run build
+
+RUN ./bin/run hello analytodon -f "the build process"
+
+RUN crontab hello-cron
+RUN chmod +x entrypoint.sh
+
+ENTRYPOINT ["/analytodon-cli/entrypoint.sh"]
+
+# https://manpages.ubuntu.com/manpages/trusty/man8/cron.8.html
+# -f | Stay in foreground mode, don't daemonize.
+# -L loglevel | Tell  cron  what to log about jobs (errors are logged regardless of this value) as the sum of the following values:
+CMD ["cron","-f", "-L", "2"]

README.md

@@ -3,52 +3,25 @@
 CLI for [Analytodon](https://www.analytodon.com).
 
 <!-- toc -->
-
--   [Usage](#usage)
--   [Commands](#commands)
+* [Analytodon CLI](#analytodon-cli)
+* [Usage](#usage)
+* [Commands](#commands)
 <!-- tocstop -->
 
 # Usage
 
 <!-- usage -->
-
 ```sh-session
 $ npm install -g analytodon-cli
 $ analytodon-cli COMMAND
 running command...
 $ analytodon-cli (--version)
-analytodon-cli/0.0.0 darwin-arm64 node-v16.19.0
+analytodon-cli/0.0.0 darwin-arm64 node-v18.17.1
 $ analytodon-cli --help [COMMAND]
 USAGE
   $ analytodon-cli COMMAND
 ...
 ```
-
-Recommended cron configuration:
-
-```crontab
-# Analytodon initial stats
-* * * * * /path/to/analytodon-cli fetch initialstats
-
-# Analytodon fetch
-35 * * * * /path/to/analytodon-cli fetch accountstats
-35 * * * * /path/to/analytodon-cli fetch tootstats
-
-# Analytodon aggregate
-5 * * * * /path/to/analytodon-cli aggregate dailyaccountstats
-5 * * * * /path/to/analytodon-cli aggregate dailytootstats
-
-# Analytodon weekly mail
-0 19,20,21,22,23 * * 0 /path/to/analytodon-cli mail weeklystats
-0 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17 * * 1 /path/to/analytodon-cli mail weeklystats
-
-# Analytodon cleanup
-45 11 * * * /path/to/analytodon-cli cleanup users
-46 11 * * * /path/to/analytodon-cli cleanup usercredentials
-47 11 * * * /path/to/analytodon-cli cleanup accounts
-48 11 * * * /path/to/analytodon-cli cleanup accountdata
-```
-
 <!-- usagestop -->
 
 # Commands
@@ -325,7 +298,7 @@ DESCRIPTION
 
 EXAMPLES
   $ oex hello friend --from oclif
-  hello friend from oclif! (./src/commands/hello/index.ts)
+  hello friend from oclif!
 ```
 
 _See code: [dist/commands/hello/index.ts](https://github.com/blazer82/analytodon-cli/blob/v0.0.0/dist/commands/hello/index.ts)_
@@ -440,7 +413,7 @@ ALIASES
   $ analytodon-cli plugins add
 
 EXAMPLES
-  $ analytodon-cli plugins:install myplugin
+  $ analytodon-cli plugins:install myplugin 
 
   $ analytodon-cli plugins:install https://github.com/someuser/someplugin
 
@@ -500,7 +473,7 @@ ALIASES
   $ analytodon-cli plugins add
 
 EXAMPLES
-  $ analytodon-cli plugins:install myplugin
+  $ analytodon-cli plugins:install myplugin 
 
   $ analytodon-cli plugins:install https://github.com/someuser/someplugin
 

crontab

@@ -0,0 +1,27 @@
+# Analytodon cron jobs
+
+#*/15 * * * * /analytodon-cli/bin/run hello analytodon -f "the cron tab"
+
+# analytodon initial stats
+* * * * * /analytodon-cli/bin/run fetch initialstats
+
+# analytodon fetch
+35 * * * * /analytodon-cli/bin/run fetch accountstats
+35 * * * * /analytodon-cli/bin/run fetch tootstats -a
+
+# analytodon aggregate
+5 * * * * /analytodon-cli/bin/run aggregate dailyaccountstats
+5 * * * * /analytodon-cli/bin/run aggregate dailytootstats
+
+# analytodon weekly mail
+0 19,20,21,22,23 * * 0 /analytodon-cli/bin/run mail weeklystats
+0 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17 * * 1 /analytodon-cli/bin/run mail weeklystats
+
+# analytodon cleanup
+45 11 * * * /analytodon-cli/bin/run cleanup users
+46 11 * * * /analytodon-cli/bin/run cleanup usercredentials
+47 11 * * * /analytodon-cli/bin/run cleanup accounts
+48 11 * * * /analytodon-cli/bin/run cleanup accountdata
+40 2 * * * /analytodon-cli/bin/run cleanup tootstats
+
+# An empty line is required at the end of this file for a valid cron file.

entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+env >> /etc/environment
+
+# execute CMD
+echo "$@"
+exec "$@"

package.json

@@ -17,14 +17,15 @@
         "/oclif.manifest.json"
     ],
     "dependencies": {
+        "@newrelic/pino-enricher": "^1.1.1",
         "@oclif/core": "^1.19.1",
         "@oclif/plugin-help": "^5",
         "@oclif/plugin-plugins": "^2.1.7",
         "axios": "^0.27.2",
         "megalodon": "^4.1.3",
         "mongodb": "^4.12.1",
-        "newrelic": "^9.7.0",
-        "pino": "^8.7.0",
+        "newrelic": "^10.6.2",
+        "pino": "^8.15.0",
         "timezone-support": "^3.1.0"
     },
     "devDependencies": {
@@ -63,11 +64,11 @@
         }
     },
     "scripts": {
-        "build": "shx rm -rf dist && tsc -b && oclif manifest && oclif readme",
+        "build": "rm -rf dist && tsc -b && oclif manifest && oclif readme",
         "prettier:check": "npx prettier --check '{src,test}/**/*.ts'",
         "prettier:write": "npx prettier --write '{src,test}/**/*.ts'",
         "lint": "eslint . --ext .ts --config .eslintrc && npm run prettier:check",
-        "postpack": "shx rm -f oclif.manifest.json",
+        "postpack": "rm -f oclif.manifest.json",
         "posttest": "npm run lint",
         "pack": "npm run build && oclif pack tarballs --no-xz -t linux-x64 && npm run postpack",
         "test": "mocha --forbid-only \"test/**/*.test.ts\"",

src/commands/aggregate/dailyaccountstats.ts

@@ -14,7 +14,7 @@ export default class DailyAccountStats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/aggregate/dailytootstats.ts

@@ -14,7 +14,7 @@ export default class DailyTootStats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/cleanup/accountdata.ts

@@ -12,7 +12,7 @@ export default class Accountdata extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/cleanup/accounts.ts

@@ -15,7 +15,7 @@ export default class Accounts extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/cleanup/tootstats.ts

@@ -16,7 +16,7 @@ export default class Tootstats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/cleanup/usercredentials.ts

@@ -15,7 +15,7 @@ export default class Usercredentials extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/cleanup/users.ts

@@ -15,7 +15,7 @@ export default class Users extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/fetch/accountstats.ts

@@ -14,7 +14,7 @@ export default class AccountStats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/fetch/initialstats.ts

@@ -16,7 +16,7 @@ export default class InitialStats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',
@@ -30,12 +30,12 @@ export default class InitialStats extends Command {
         host: Flags.string({
             char: 'h',
             description: 'App host URL',
-            default: 'app.analytodon.com',
+            default: process.env.APP_URL || 'https://app.analytodon.com',
         }),
         authorization: Flags.string({
             char: 't',
             description: 'Authorization header',
-            required: true,
+            default: process.env.EMAIL_API_KEY || 'no-key',
         }),
     };
 
@@ -82,7 +82,7 @@ export default class InitialStats extends Command {
                 await createInitialTootStats(db, account);
 
                 await axios.post(
-                    `https://${flags.host}/api/mail/firststats`,
+                    `${flags.host}/api/mail/firststats`,
                     {userID: `${user._id}`, accounts: [account._id]},
                     {
                         headers: {

src/commands/fetch/tootstats.ts

@@ -14,7 +14,7 @@ export default class TootStats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/commands/hello/index.ts

@@ -1,11 +1,13 @@
 import {Command, Flags} from '@oclif/core';
+import {logger} from '../../helpers/logger';
+import {completeLogging} from '../../helpers/completeLogging';
 
 export default class Hello extends Command {
     static description = 'Say hello';
 
     static examples = [
         `$ oex hello friend --from oclif
-hello friend from oclif! (./src/commands/hello/index.ts)
+hello friend from oclif!
 `,
     ];
 
@@ -18,6 +20,8 @@ hello friend from oclif! (./src/commands/hello/index.ts)
     async run(): Promise<void> {
         const {args, flags} = await this.parse(Hello);
 
-        this.log(`hello ${args.person} from ${flags.from}! (./src/commands/hello/index.ts)`);
+        logger.info(`hello ${args.person} from ${flags.from}!`);
+
+        await completeLogging();
     }
 }

src/commands/mail/weeklystats.ts

@@ -14,7 +14,7 @@ export default class WeeklyStats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',
@@ -24,12 +24,12 @@ export default class WeeklyStats extends Command {
         host: Flags.string({
             char: 'h',
             description: 'App host URL',
-            default: 'app.analytodon.com',
+            default: process.env.APP_URL || 'https://app.analytodon.com',
         }),
         authorization: Flags.string({
             char: 't',
             description: 'Authorization header',
-            required: true,
+            default: process.env.EMAIL_API_KEY || 'no-key',
         }),
         user: Flags.string({
             char: 'u',
@@ -97,7 +97,7 @@ export default class WeeklyStats extends Command {
                             logger.info(`Send weekly stats: Trigger mail for user ${user._id} with accounts ${accountIds.join(',')}`);
 
                             await axios.post(
-                                `https://${flags.host}/api/mail/weeklystats`,
+                                `${flags.host}/api/mail/weeklystats`,
                                 {userID: `${user._id}`, accounts: accountIds},
                                 {
                                     headers: {

src/commands/tools/rebuilddailytootstats.ts

@@ -13,7 +13,7 @@ export default class RebuildDailyTootStats extends Command {
         connectionString: Flags.string({
             char: 'c',
             description: 'MongoDB connection string',
-            default: 'mongodb://localhost:27017',
+            default: process.env.MONGODB_URI || 'mongodb://localhost:27017',
         }),
         database: Flags.string({
             char: 'd',

src/helpers/completeLogging.ts

@@ -3,5 +3,5 @@ const newrelic = require('newrelic');
 
 export const completeLogging = async () =>
     new Promise<void>((resolve) => {
-        newrelic.shutdown({collectPendingData: true, timeout: 10000}, () => resolve());
+        newrelic.shutdown({collectPendingData: true, timeout: 10000, waitForIdle: true}, () => resolve());
     });

src/helpers/logger.ts

@@ -1,5 +1,8 @@
 /* eslint-disable */
-const logger = require('pino')();
+const nrPino = require('@newrelic/pino-enricher');
+const pino = require('pino');
+
+const logger = pino(nrPino());
 
 export {logger};
 /* eslint-enable */