Merge branch 'master' into patch-1

.github/workflows/build-and-test.yml

@@ -25,7 +25,7 @@ jobs:
     needs: build
     steps:
     - name: Add hosts for integration tests
-      run: sudo echo "127.0.0.1 localhost auth.example.com matrix.example.com matrix1.example.com matrix2.example.com matrix3.example.com federation.example.com opensearch.example.com" | sudo tee -a /etc/hosts
+      run: sudo echo "127.0.0.1 localhost auth.example.com matrix.example.com matrix1.example.com matrix2.example.com matrix3.example.com federated-identity.example.com opensearch.example.com" | sudo tee -a /etc/hosts
     - uses: actions/checkout@v3
     - name: Set up Node LTS
       uses: actions/setup-node@v3

.github/workflows/publish-tom-federation-server.yml → .github/workflows/publish-tom-federated-identity-service.yml

@@ -1,7 +1,7 @@
 # This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
 
-name: publish-tom-federation-server
+name: publish-tom-federated-identity-service
 
 on:
   workflow_call:
@@ -19,15 +19,15 @@ jobs:
         uses: tj-actions/changed-files@v41
         with:
           files: |
-            packages/federation-server/**/*
+            packages/federated-identity-service/**/*
             .github/workflows/**
       - name: Publish to dockerhub
         uses: elgohr/Publish-Docker-Github-Action@v5
         with:
-          name: linagora/tom-federation-server
+          name: linagora/tom-federated-identity-service
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
           workdir: "${{ github.workspace }}"
           context: .
-          buildoptions: "-t linagora/tom-federation-server -f packages/federation-server/Dockerfile"
+          buildoptions: "-t linagora/tom-federated-identity-service -f packages/federated-identity-service/Dockerfile"
           tags: "latest,${{ steps.date.outputs.date }}"

.github/workflows/publish-tom-server.yml

@@ -19,7 +19,7 @@ jobs:
         uses: tj-actions/changed-files@v41
         with:
           files: |
-            packages/federation-server/**/*
+            packages/federated-identity-service/**/*
             .github/workflows/**
       - name: Publish to dockerhub
         uses: elgohr/Publish-Docker-Github-Action@v5

.github/workflows/push.yml

@@ -9,9 +9,9 @@ jobs:
     name: Build And Test
     uses: ./.github/workflows/build-and-test.yml
     secrets: inherit
-  publish-federation-server:
-    name: Publish Federation Server
-    uses: ./.github/workflows/publish-tom-federation-server.yml
+  publish-federated-identity-service:
+    name: Publish Federated Identity Service
+    uses: ./.github/workflows/publish-tom-federated-identity-service.yml
     needs: [test]
     secrets: inherit
   publish-tom-server:
@@ -24,8 +24,3 @@ jobs:
     uses: ./.github/workflows/publish-matrix-invite.yml
     needs: [test]
     secrets: inherit
-  upadte-docs:
-    name: Update Docs
-    uses: ./.github/workflows/update-docs.yml
-    needs: [test]
-    secrets: inherit

.github/workflows/update-docs.yml

@@ -11,6 +11,8 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.ref }}
+      - name: Setup Graphviz
+        uses: tlylt/install-graphviz@v1
       - name: Set up Node LTS
         uses: actions/setup-node@v3
         with:
@@ -40,4 +42,4 @@ jobs:
         uses: EndBug/add-and-commit@v9
         with:
           add: docs
-          message: 'chore: update documentation'
+          message: 'chore: update documentation'

.gitlab-ci.yml

@@ -43,7 +43,7 @@ test:
     - npm
   script:
     - cp /etc/hosts ~/hosts.new
-    - sed -i "s#\tdocker#\tdocker auth.example.com matrix.example.com matrix1.example.com matrix2.example.com matrix3.example.com federation.example.com#g" ~/hosts.new
+    - sed -i "s#\tdocker#\tdocker auth.example.com matrix.example.com matrix1.example.com matrix2.example.com matrix3.example.com federated-identity.example.com#g" ~/hosts.new
     - cp -f ~/hosts.new /etc/hosts
     - apt-get update && apt-get --assume-yes install ca-certificates curl gnupg coreutils
     - install -m 0755 -d /etc/apt/keyrings

Dockerfile

@@ -10,7 +10,7 @@ env BASE_URL= \
     DATABASE_USER= \
     DATABASE_PASSWORD= \
     DATABASE_SSL= \
-    FEDERATION_SERVERS= \
+    FEDERATED_IDENTITY_SERVICES= \
     JITSI_BASE_URL= \
     JITSI_JWT_ALGORITHM= \
     JITSI_JWT_ISSUER= \
@@ -22,7 +22,7 @@ env BASE_URL= \
     LDAP_USER= \
     LDAP_PASSWORD= \
     LDAP_URI= \
-    LOGGER=error \
+    LOG_LEVEL=error \
     LOG_TRANSPORTS=Console \
     MATRIX_SERVER= \
     MATRIX_DATABASE_ENGINE= \
@@ -45,7 +45,7 @@ env BASE_URL= \
     OPENSEARCH_WAIT_FOR_ACTIVE_SHARDS= \
     SERVER_NAME= \
     TEMPLATE_DIR=/usr/src/app/packages/tom-server/templates \
-    UDPATE_FEDERATION_HASHES_CRON="3 3 * * *" \
+    UPDATE_FEDERATED_IDENTITY_HASHES_CRON="3 3 * * *" \
     UPDATE_USERS_CRON="*/15 * * * *" \
     SMS_API_LOGIN= \
     SMS_API_URL= \

README.md

@@ -1,4 +1,4 @@
-# Twake Matrix extension server
+# Twake-Chat Matrix extension server
 
 <br />
 <div align="center">
@@ -22,11 +22,23 @@
 
 ---
 
-This is a multi-packages repository. See [packages](./packages) directory.
+This repository is a multi-packages repository. See [Modules](#modules) for details.
+
+**ToM server** enhances a [Matrix Synapse server](https://github.com/element-hq/synapse) with several features:
+ * first, **Tom** is a [Matrix Identity Server](https://spec.matrix.org/latest/identity-service-api/) but with additional features:
+   * inside an organization, it adds some search APIs that allows to find internal users like do the mail clients, for autocompletion for example
+   * it extends also [Matrix Identity Service](https://spec.matrix.org/latest/identity-service-api/) search responses by adding inactive users
+ * it provide also an "application service" that allows administrator to create channels with automatic join
+ * it implements also the [federated identity mechanism](https://github.com/matrix-org/matrix-spec-proposals/pull/4004) that extend the
+   [Matrix Identity Service](https://spec.matrix.org/latest/identity-service-api/) to join Matrix identity services to provide a better search
+
+Here is the architecture principle:
+
+![architecture principle](./docs/arch.png)
 
 REST API Endpoints documentation is available on https://linagora.github.io/ToM-server/
 
-[Try it with docker](./docker.md)
+[Try it with docker](#twake-chat-docker)
 
 ## Scripts
 
@@ -39,7 +51,7 @@ REST API Endpoints documentation is available on https://linagora.github.io/ToM-
   [Matrix Identity Service](https://spec.matrix.org/v1.6/identity-service-api/) implementation for Node.js
 * [@twake/matrix-invite](./packages/matrix-invite): matrix invitation web application
 * [@twake/server](./packages/tom-server): the main Twake Chat Server, extends [@twake/matrix-identity-server](./packages/matrix-identity-server)
-* [@twakeg/federation-server](./packages/federation-server): Twake Federation Server
+* [@twake/federated-identity-service](./packages/federated-identity-service): Twake Federated Identity Service
 * [@twake/config-parser](./packages/config-parser): simple file parser that uses also environment variables
 * [@twake/crypto](./packages/crypto): cryptographic methods for Twake Chat
 * [@twake/logger](./packages/logger): logger for Twake
@@ -49,6 +61,75 @@ REST API Endpoints documentation is available on https://linagora.github.io/ToM-
   [Matrix specification](https://spec.matrix.org/latest/server-server-api/#server-discovery)
 * [@twake/retry-promise](packages/retry-promise): simple module extending javascript Promise with retry strategy
 
+## Twake-Chat docker
+
+This repository provides different docker images. The main is [Tom Server](./Dockerfile) itself. Here are its environment variables:
+
+* Required:
+  * `BASE_URL`: Public URL
+  * Database:
+    * `DATABASE_ENGINE` _(`pg` or `sqlite`)_
+    * `DATABASE_HOST` _(path for `sqlite`)_
+    * `DATABASE_NAME`
+    * `DATABASE_USER`
+    * `DATABASE_PASSWORD`
+    * `DATABASE_SSL`
+  * `OIDC_ISSUER`: URL of SSO server
+  * LDAP service:
+    * `LDAP_BASE`
+    * `LDAP_FILTER`
+    * `LDAP_USER`
+    * `LDAP_PASSWORD`
+    * `LDAP_URI`
+  * Matrix server:
+    * `SERVER_NAME` _(same value than in Matrix's homeserver.yaml)_
+    * `MATRIX_SERVER` _(real Matrix server)_
+  * `TEMPLATE_DIR` _(default: `node_modules/@twake/server/templates`)_
+* Recommended:
+  * `ADDITIONAL_FEATURES`: set true to have all search features; false for a public instance
+  * Cron service:
+    * `CRON_SERVICE` _(default: true)_: enable cron tasks
+    * `PEPPER_CRON` _(default: `9 1 * * *`)_
+    * `UPDATE_USERS_CRON` _(default: `*/15 * * * *`)_
+    * `UPDATE_FEDERATED_IDENTITY_HASHES_CRON` _(default: `3 3 * * *`)_
+  * Logs:
+    * `LOG_TRANSPORTS`: set to `Console`
+    * `LOG_LEVEL`: default to "error", possible values: "error", "warn", "info", "http", "verbose", "debug", "silly"
+  * `TRUSTED_PROXIES`: IP list of server allowed to set `X-Frowarded-For` header
+  * Rate limits _(see [express-rate-limit](https://www.npmjs.com/package/express-rate-limit))_:
+    * `RATE_LIMITING_WINDOW`
+    * `RATE_LIMITING_NB_REQUESTS`
+* Optional:
+  * `FEDERATED_IDENTITY_SERVICES`: list of federated identity services
+  * Use a CrowdSec service:
+    * `CROWDSEC_URI`
+    * `CROWDSEC_KEY`
+  * Add Jitsi into metadata:
+    * `JITSI_BASE_URL`
+    * `JITSI_JWT_ALGORITHM`
+    * `JITSI_JWT_ISSUER`
+    * `JITSI_SECRET`
+    * `JITSI_PREFERRED_DOMAIN`
+    * `JITSI_USE_JWT`
+  * Matrix database _(for automatic channels)_:
+    * `MATRIX_DATABASE_ENGINE`
+    * `MATRIX_DATABASE_HOST`
+    * `MATRIX_DATABASE_NAME`
+    * `MATRIX_DATABASE_PASSWORD`
+    * `MATRIX_DATABASE_SSL`
+    * `MATRIX_DATABASE_USER`
+  * Opensearch features:
+    * `OPENSEARCH_CA_CERT_PATH`
+    * `OPENSEARCH_HOST`
+    * `OPENSEARCH_IS_ACTIVATED`
+    * `OPENSEARCH_MAX_RETRIES`
+    * `OPENSEARCH_NUMBER_OF_SHARDS`
+    * `OPENSEARCH_NUMBER_OF_REPLICAS`
+    * `OPENSEARCH_PASSWORD`
+    * `OPENSEARCH_SSL`
+    * `OPENSEARCH_USER`
+    * `OPENSEARCH_WAIT_FOR_ACTIVE_SHARDS`
+
 ## Copyright and license
 
 Copyright (c) 2023-present Linagora <https://linagora.com>

docker.md

@@ -2,7 +2,7 @@
 
 Image are published in docker hub:
  * [The ToM Server itself](https://hub.docker.com/r/linagora/tom-server)
- * [The Federation Identity Server](https://hub.docker.com/r/linagora/tom-federation-server)
+ * [The Federated Identity Service](https://hub.docker.com/r/linagora/tom-federated-identity-service)
 
 ## The ToM server image
 

docs/arch.dot

@@ -0,0 +1,54 @@
+digraph { 
+  nodesep=1
+  subgraph cluster_external {
+    style=dotted
+    label=External
+    fontcolor=olivedrab3
+    "External Twake-Chat" [color=olivedrab3]
+    "External Matrix" [color=olivedrab3]
+    "Client" [color=green]
+  }
+  subgraph cluster_frontends {
+    style=invisible
+    shape=doubleoctagon
+    margin=20
+    label=frontends
+    color=gray50
+    fontcolor=gray50
+    "SSO" [color=blue,shape=box,group=frontend]
+    "Synapse" [color=blue,fontcolor=black,shape=box]
+    "Tom Server" [color=blue,fontcolor=black,shape=box]
+  }
+  subgraph cluster_db {
+    bgcolor="#eeeeff"
+    style=dashed
+    label=DB
+    fontcolor=blue
+    "Filesystem" [color=blue,fontcolor=blue,shape=cylinder,group=db]
+    "Postgres" [color=blue,fontcolor=blue,shape=cylinder,group=db]
+    "Redis" [color=blue,fontcolor=blue,shape=cylinder,group=db]
+    "LDAP" [color=blue,fontcolor=blue,shape=cylinder,group=db]
+  }
+  subgraph cluster_fede {
+    style=dotted
+    color=gray50
+    fontcolor=gray50
+    label=Common
+    "Federation Server" [color=red,shape=box,group=frontend]
+  }
+  "Client" -> "SSO" [color=red,fontcolor=red]
+  "Client" -> "Synapse" [color=red,fontcolor=red,label=Chat,penwidth=3]
+  "Client" -> "Tom Server" [color=red,fontcolor=red,label=Search,penwidth=3]
+  "Client" -> "Federation Server" [color=red,fontcolor=red,label=Search]
+  "Tom Server" -> "Postgres" [color=blue,label="Synapse + own DB",fontcolor=blue,penwidth=2]
+  "Tom Server" -> "LDAP" [color=blue,fontcolor=blue,label="Get user data"]
+  "Tom Server" -> "Redis" [color=blue,fontcolor=blue,label="Cache user data"]
+  "Tom Server" -> "Federation Server" [constraint=false,color=coral3,fontcolor=coral3,label="Push hashes",style=dashed]
+  "Tom Server" -> "Synapse" [constraint=false,color=coral3,fontcolor=red,style=dashed]
+  "External Matrix" -> "Synapse" [color=olivedrab3,fontcolor=olivedrab3,label=Federation]
+  "External Twake-Chat" -> "Synapse" [color=olivedrab3,fontcolor=olivedrab3,label=Federation]
+  "External Twake-Chat" -> "Federation Server" [constraint=false,color=olivedrab3,fontcolor=olivedrab3,label="Push hashes",style=dashed]
+  "Synapse" -> "Postgres" [color=blue,fontcolor=blue,penwidth=2]
+  "Synapse" -> "Filesystem" [color=blue,label="medias",fontcolor=blue,penwidth=2]
+  "Synapse" -> "SSO" [constraint=false,color=coral3,fontcolor=coral3,label="Validate SSO token",style=dashed]
+}