Merge pull request 'Import upstream patches from The Lounge (Feb 2024…

…), bump version to v4.4.1-2' (#2) from upstream-patches-202402 into master

Reviewed-on: https://git.supernets.org/supernets/hardlounge/pulls/2

.github/workflows/build.yml

@@ -12,9 +12,7 @@ jobs:
     strategy:
       matrix:
         include:
-          # EOL: April 2024
-          - os: ubuntu-latest
-            node_version: 16.x
+          # EOL: April 2025
           - os: macOS-latest
             node_version: 18.x
           - os: windows-latest
@@ -25,6 +23,9 @@ jobs:
           # EOL: April 2026
           - os: ubuntu-latest
             node_version: 20.x
+          # EOL: April/June 2024
+          - os: ubuntu-latest
+            node_version: 21.x
 
     runs-on: ${{ matrix.os }}
 

client/js/router.ts

@@ -110,26 +110,23 @@ router.beforeEach((to, from, next) => {
 	next();
 });
 
-router.beforeEach((to, from, next) => {
+router.beforeEach((to, from) => {
 	// Disallow navigating to non-existing routes
 	if (!to.matched.length) {
-		next(false);
-		return;
+		return false;
 	}
 
 	// Disallow navigating to invalid channels
 	if (to.name === "RoutedChat" && !store.getters.findChannel(Number(to.params.id))) {
-		next(false);
-		return;
+		return false;
 	}
 
 	// Disallow navigating to invalid networks
 	if (to.name === "NetworkEdit" && !store.getters.findNetwork(String(to.params.uuid))) {
-		next(false);
-		return;
+		return false;
 	}
 
-	next();
+	return true;
 });
 
 router.afterEach((to) => {

client/tsconfig.json

@@ -11,6 +11,7 @@
     "../server/log.ts",
     "../server/config.ts",
     "../server/client.ts",
+    "../server/storageCleaner.ts",
     "../server/clientManager.ts",
     "../server/identification.ts",
     "../server/plugins/changelog.ts",
@@ -44,15 +45,17 @@
   "compilerOptions": {
     "sourceMap": false /*Create source map files for emitted JavaScript files. See more: https://www.typescriptlang.org/tsconfig#sourceMap */,
     "jsx": "preserve" /* Specify what JSX code is generated. */,
-    "lib": ["DOM", "DOM.Iterable", "ESNext"],
-
+    "lib": [
+      "DOM",
+      "DOM.Iterable",
+      "ESNext"
+    ],
     // this enables stricter inference for data properties on `this`
     "strict": true,
     // if using webpack 2+ or rollup, to leverage tree shaking:
     "module": "es2020",
     "moduleResolution": "node",
-
     // TODO: Remove eventually
     "noImplicitAny": false /*Enable error reporting for expressions and declarations with an implied any type. See more: https://www.typescriptlang.org/tsconfig#noImplicitAny */
   } /* Instructs the TypeScript compiler how to compile .ts files. */
-}
+}

defaults/config.js

@@ -308,6 +308,26 @@ module.exports = {
 	// This value is set to `["sqlite", "text"]` by default.
 	messageStorage: ["sqlite", "text"],
 
+	// ### `storagePolicy`
+
+	// When the sqlite storage is in use, control the maximum storage duration.
+	// A background task will periodically clean up messages older than the limit.
+
+	// The available keys for the `storagePolicy` object are:
+	//
+	// - `enabled`: If this is false, the cleaning task is not running.
+	// - `maxAgeDays`: Maximum age of an entry in days.
+	// - `deletionPolicy`: Controls what types of messages are being deleted.
+	//   Valid options are:
+	//   - `statusOnly`: Only delete message types which are status related (e.g. away, back, join, parts, mode, ctcp...)
+	//     but keep actual messages from nicks. This keeps the DB size down while retaining "precious" messages.
+	//   - `everything`: Delete everything, including messages from irc nicks
+	storagePolicy: {
+		enabled: false,
+		maxAgeDays: 7,
+		deletionPolicy: "statusOnly",
+	},
+
 	// ### `useHexIp`
 	//
 	// When set to `true`, users' IP addresses will be encoded as hex.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@supernets/hardlounge",
   "description": "The self-hosted Web IRC client",
-  "version": "4.4.1-1",
+  "version": "4.4.1-2",
   "preferGlobal": true,
   "bin": {
     "hardlounge": "index.js"
@@ -24,8 +24,9 @@
     "lint:stylelint": "stylelint --color \"client/**/*.css\"",
     "lint": "run-p --aggregate-output --continue-on-error lint:*",
     "start": "node index start",
-    "test": "run-p --aggregate-output --continue-on-error lint:* test:*",
-    "test:mocha": "cross-env NODE_ENV=test webpack --mode=development && cross-env NODE_ENV=test TS_NODE_PROJECT='./test/tsconfig.json' nyc --nycrc-path=test/.nycrc-mocha.json mocha --require ts-node/register --colors --config=test/.mocharc.yml",
+    "test": "run-p --aggregate-output --continue-on-error lint:* test:mocha",
+    "test:mocha": "webpack --mode=development && cross-env NODE_ENV=test TS_NODE_PROJECT='./test/tsconfig.json' mocha --config=test/.mocharc.yml 'test/**/*.ts'",
+    "test:nospec": "webpack --mode=development && cross-env NODE_ENV=test TS_NODE_PROJECT='./test/tsconfig.json' mocha --config=test/.mocharc.yml",
     "watch": "webpack --watch"
   },
   "keywords": [
@@ -40,7 +41,7 @@
   ],
   "license": "MIT",
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   },
   "files": [
     "./.thelounge_home",
@@ -52,6 +53,7 @@
     "./public/**"
   ],
   "dependencies": {
+    "@babel/traverse": "7.23.6",
     "@fastify/busboy": "1.0.0",
     "bcryptjs": "2.4.3",
     "caniuse-lite": "^1.0.30001561",
@@ -83,7 +85,7 @@
     "yarn": "1.22.17"
   },
   "optionalDependencies": {
-    "sqlite3": "5.1.6"
+    "sqlite3": "5.1.7"
   },
   "devDependencies": {
     "@babel/core": "7.17.10",
@@ -92,15 +94,15 @@
     "@fortawesome/fontawesome-free": "5.15.4",
     "@istanbuljs/nyc-config-typescript": "1.0.2",
     "@textcomplete/core": "0.1.10",
-    "@textcomplete/textarea": "0.1.12",
-    "@types/bcryptjs": "2.4.5",
+    "@textcomplete/textarea": "0.1.13",
+    "@types/bcryptjs": "2.4.6",
     "@types/chai": "4.3.5",
-    "@types/cheerio": "0.22.33",
-    "@types/content-disposition": "0.5.7",
+    "@types/cheerio": "0.22.35",
+    "@types/content-disposition": "0.5.8",
     "@types/express": "4.17.13",
-    "@types/is-utf8": "0.2.2",
+    "@types/is-utf8": "0.2.3",
     "@types/ldapjs": "2.2.2",
-    "@types/linkify-it": "3.0.3",
+    "@types/linkify-it": "3.0.5",
     "@types/lodash": "4.14.200",
     "@types/mime-types": "2.1.1",
     "@types/mocha": "9.1.1",

server/client.ts

@@ -18,6 +18,7 @@ import TextFileMessageStorage from "./plugins/messageStorage/text";
 import Network, {IgnoreListItem, NetworkConfig, NetworkWithIrcFramework} from "./models/network";
 import ClientManager from "./clientManager";
 import {MessageStorage, SearchQuery, SearchResponse} from "./plugins/messageStorage/types";
+import { StorageCleaner } from "./storageCleaner";
 
 type OrderItem = Chan["id"] | Network["uuid"];
 type Order = OrderItem[];
@@ -138,6 +139,13 @@ class Client {
 		if (!Config.values.public && client.config.log) {
 			if (Config.values.messageStorage.includes("sqlite")) {
 				client.messageProvider = new SqliteMessageStorage(client.name);
+				if (Config.values.storagePolicy.enabled) {
+					log.info(
+						`Activating storage cleaner. Policy: ${Config.values.storagePolicy.deletionPolicy}. MaxAge: ${Config.values.storagePolicy.maxAgeDays} days`
+					);
+					const cleaner = new StorageCleaner(client.messageProvider);
+					cleaner.start();
+				}
 				client.messageStorage.push(client.messageProvider);
 			}
 

server/command-line/storage.ts

@@ -3,14 +3,15 @@ import { Command } from "commander";
 import ClientManager from "../clientManager";
 import Utils from "./utils";
 import SqliteMessageStorage from "../plugins/messageStorage/sqlite";
+import {StorageCleaner} from "../storageCleaner";
 
 const program = new Command("storage").description(
 	"various utilities related to the message storage"
 );
 
 program
 	.command("migrate")
-	.argument("[user]", "migrate a specific user only, all if not provided")
+	.argument("[username]", "migrate a specific user only, all if not provided")
 	.description("Migrate message storage where needed")
 	.on("--help", Utils.extraHelp)
 	.action(function (user) {
@@ -20,7 +21,19 @@ program
 		});
 	});
 
-async function runMigrations(user: string) {
+program
+	.command("clean")
+	.argument("[user]", "clean messages for a specific user only, all if not provided")
+	.description("Delete messages from the DB based on the storage policy")
+	.on("--help", Utils.extraHelp)
+	.action(function (user) {
+		runCleaning(user).catch((err) => {
+			log.error(err.toString());
+			process.exit(1);
+		});
+	});
+
+async function runMigrations(user?: string) {
 	const manager = new ClientManager();
 	const users = manager.getUsers();
 
@@ -65,4 +78,46 @@ function isUserLogEnabled(manager: ClientManager, user: string): boolean {
 	return conf.log;
 }
 
+async function runCleaning(user: string) {
+	const manager = new ClientManager();
+	const users = manager.getUsers();
+
+	if (user) {
+		if (!users.includes(user)) {
+			throw new Error(`invalid user ${user}`);
+		}
+
+		return cleanUser(manager, user);
+	}
+
+	for (const name of users) {
+		await cleanUser(manager, name);
+		// if any migration fails we blow up,
+		// chances are the rest won't complete either
+	}
+}
+
+async function cleanUser(manager: ClientManager, user: string) {
+	log.info("handling user", user);
+
+	if (!isUserLogEnabled(manager, user)) {
+		log.info("logging disabled for user", user, ". Skipping");
+		return;
+	}
+
+	const sqlite = new SqliteMessageStorage(user);
+	await sqlite.enable();
+	const cleaner = new StorageCleaner(sqlite);
+	const num_deleted = await cleaner.runDeletesNoLimit();
+	log.info(`deleted ${num_deleted} messages`);
+	log.info("running a vacuum now, this might take a while");
+
+	if (num_deleted > 0) {
+		await sqlite.vacuum();
+	}
+
+	await sqlite.close();
+	log.info(`cleaning messages for ${user} has been successful`);
+}
+
 export default program;

server/command-line/utils.ts

@@ -143,11 +143,11 @@ class Utils {
 				data.toString()
 					.trim()
 					.split("\n")
-					.forEach((line) => {
+					.forEach((line: string) => {
 						try {
-							line = JSON.parse(line);
+							const json = JSON.parse(line);
 
-							if (line.type === "success") {
+							if (json.type === "success") {
 								success = true;
 							}
 						} catch (e: any) {
@@ -163,11 +163,24 @@ class Utils {
 					.trim()
 					.split("\n")
 					.forEach((line: string) => {
-						const json = JSON.parse(line);
-
-						if (json.type === "error") {
-							log.error(json.data);
+						try {
+							const json = JSON.parse(line);
+
+							switch (json.type) {
+								case "error":
+									log.error(json.data);
+									break;
+								case "warning":
+									// this includes pointless things like "ignored scripts due to flag"
+									// so let's hide it
+									break;
+							}
+						return;
+						} catch (e: any) {
+							// we simply fall through and log at debug... chances are there's nothing the user can do about it
+							// as it includes things like deprecation warnings, but we might want to know as developers						
 						}
+					log.debug(line);
 					});
 			});
 

server/config.ts

@@ -76,6 +76,12 @@ type Debug = {
 	raw: boolean;
 };
 
+type StoragePolicy = {
+	enabled: boolean;
+	maxAgeDays: number;
+	deletionPolicy: "statusOnly" | "everything";
+};
+
 export type ConfigType = {
 	public: boolean;
 	host: string | undefined;
@@ -97,6 +103,7 @@ export type ConfigType = {
 	defaults: Defaults;
 	lockNetwork: boolean;
 	messageStorage: string[];
+	storagePolicy: StoragePolicy;
 	useHexIp: boolean;
 	webirc?: WebIRC;
 	identd: Identd;