AB#24741: Publish timetable rendering API along with other fixes (#392)

* fix combining rows

* MM-484: Allow launching separate services as different instances (#363)

* MM-484: Allow launching separate services as different instances

* Add default value to PUBLISHER_RENDER_URL

* MM-497: Endpoint for render URL generation, Update README about service splitting (#364)

* MM-497: Endpoint for render URL generation

* Update README about the sectioning of the service, new start commands and explanations.

* Remove extra slash from fallback url

* Allow route title to enter the padding area

This allow rendering to continue and not to fail if the route title is too long.

* MM-495: Rename ticket-zones.json, update README (#365)

* MM-502: Add more functionality to generateRenderUrl endpoint and rendering (#366)

* MM-502: Add current date by default unless date is included in generateRenderUrl params

* Tweak order of local setup commands, add moment to dependencies

* MM-502: Add parameter to hide address info from timetable render

* MM-502: Add print button to rendered document, and a parameter to hide it from document

* Update yarn.locl

---------

Co-authored-by: Juho Hänninen <juho.hanninen@cgi.com>

* Fix url env variables for server (#368)

* Fix url env variables for server

* Fix webpack errors due to fs reference in browser app

* Fix build and type errors

* Fix server host checking

* Add link to publisher-server for worker

* MM-504: Add redirect parameter to generateRenderUrl endpoint

* Remove forever from deployment

* Add publisher server url to env to be build in the image

* Fix and enhance puppeteer console logging

* Remove trailing slash from server url env

* Upgrade hsl-map-style

* Fix single departure grouping with same minutes

* Add filtering for duplicate departures

* Drop console statement

* day specific a4 timetable departures

* bug fixes

* l_rail color added

* legend updated

* translations

* style reference update

* yarn.lock

* revert package.json and yarn.lock

* new color and background color for stopPoster

* yarn.lock update

* L-rail icon and color update

* MM-507: Add Docker Compose setup for development (#383)

* MM-507: Add docker compose configuration for development

* MM-507: Add setup instructions in README

* Add font instructions to README, tidy up

* Persist postgresql volume, add db dependency for startup sequence

* Use tram diagram also on light rail stops

(cherry picked from commit d0932d1)

* Resize Pikaratikka's icon to render it correctly

* fixed StopPoster timetable columns

* removed redundant prop

* #26114: language change for print button (#388)

* #26114: Add language choices into the print button on timetable rendering

* #26236: Fix StopPosterA3 generating error (#391)

* Fix StopPosterA3 generating error

* Refactor stopId property from A3Timetable to prevent same error

* Merge conflict fix

---------

Co-authored-by: Anton <anton.jyrkiainen@helsinki.fi>
Co-authored-by: Juho Hänninen <juho.hanninen@cgi.com>

.env.dev

@@ -1,5 +1,7 @@
 PG_CONNECTION_STRING=
 REDIS_CONNECTION_STRING=
+PUBLISHER_RENDER_URL=
+REACT_APP_PUBLISHER_SERVER_URL=http://julistegeneraattori-server:4000
 
 DIGITRANSIT_URL=https://dev-api.digitransit.fi/
 DIGITRANSIT_APIKEY=

.env.local

@@ -1,5 +1,7 @@
-PG_CONNECTION_STRING=
-REDIS_CONNECTION_STRING=
+PG_CONNECTION_STRING=postgres://postgres:postgres@publisher-postgres:5432/postgres
+REDIS_CONNECTION_STRING=redis://redis:6379
+PUBLISHER_RENDER_URL=http://publisher-render:5000
+REACT_APP_PUBLISHER_SERVER_URL=http://publisher-server:4000
 
 DIGITRANSIT_URL=https://dev-api.digitransit.fi/
 DIGITRANSIT_APIKEY=

.env.prod

@@ -1,5 +1,7 @@
 PG_CONNECTION_STRING=
 REDIS_CONNECTION_STRING=
+PUBLISHER_RENDER_URL=
+REACT_APP_PUBLISHER_SERVER_URL=http://julistegeneraattori-server:4000
 
 DIGITRANSIT_URL=https://cdn.digitransit.fi/
 DIGITRANSIT_APIKEY=

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16-buster-slim
+FROM node:16.20-buster-slim
 
 RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -yq wget curl gnupg fontconfig fonts-liberation ca-certificates --no-install-recommends \
@@ -30,11 +30,7 @@ ARG DIGITRANSIT_APIKEY
 ENV DIGITRANSIT_APIKEY=${DIGITRANSIT_APIKEY}
 RUN yarn run build
 
-CMD \
-  ./fonts.sh && \
-  fc-cache -f -v && \
-  yarn run start:production && \
-  yarn run server:production && \
-  yarn run worker:production && \
-  sleep 3 && \
-  node_modules/.bin/forever -f logs 1
+ARG SERVICE='start:production'
+ENV SERVICE=${SERVICE}
+
+CMD yarn run ${SERVICE}

README.md

@@ -4,6 +4,9 @@ This project is the server-side component of the poster publisher (the UI can be
 
 In production everything runs from a docker container that takes care of all dependencies and environment setup.
 
+## Development
+If ticket zone regions are changed, remember to update [ticket-zones-polygons.json](./src/components/map/ticket-zones-polygons.json) to reflect the new zones !
+
 ### Dependencies
 
 Install dependencies:
@@ -16,7 +19,10 @@ Also install Chrome. Refer the installation instructions of Chrome for your oper
 
 ### About the app
 
-This project is split in two parts. The first is a server that receives the poster requests and fires up a Puppeteer instance which runs a React app that renders the posters. This component lives in the `scripts` directory. The other part is the React app itself which lives in the `src` directory. It is a more complicated app, as it needs to load data from both the publisher database as well as from the JORE database.
+This app is split into three parts:
+- Server (receives poster requests and stores them into the Redis database)
+- Worker (Fetches poster jobs from Redis, fires up Puppeteer to render and save the posters)
+- Render (Receives worker's requests for certain poster parameters, fetches relevant data and renders the poster in the browser for Worker's Puppeteer instance to save into PDFs)
 
 ### Writing components
 
@@ -77,24 +83,17 @@ REDIS_CONNECTION_STRING=redis://localhost:6379
 
 In development, start the Publisher backend server like this, (make sure you have connection strings in `.env`)
 ```bash
-yarn run server:hot
-```
-
-That command will run a Forever instance that watches for changes and restarts the server when they happen.
-
-Alternatively, to run the server with plain Node, leave off `hot`:
-```bash
 yarn run server
 ```
 
 Then, start generator worker. (You can start multiple workers.)
-```
-yarn worker
+```bash
+yarn run worker
 ```
 
-Finally, start the React app
+Finally, start the React app/Rendering
 ```bash
-yarn start
+yarn run start
 ```
 
 Now you can use the UI with the server, or open a poster separately in your browser. The poster app needs `component` and `props` query parameters, and the server will echo the currently rendering URL in its console. But if you just need to open the poster app, you can use this link that will show H0454, Snellmaninkatu:
@@ -109,7 +108,35 @@ If Azure credentials are not set in the .env file the posters will be stored loc
 
 See [hsl-map-publisher-ui](https://github.com/HSLdevcom/hsl-map-publisher-ui) for UI.
 
-### Running in local Docker
+
+## Two different ways to setup local environment:
+
+### A) Setup local environment using Docker Compose
+
+For fonts you have three options:
+- Create `fonts/` -directory inside project folder. Place `Gotham Rounded` and `Gotham XNarrow` OpenType fonts there from Azure.
+- Place `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_KEY` either via `.env.local` or Docker secrets. Fonts will be downloaded from Azure on startup.
+- If no fonts or credentials are provided, the app use just the default fonts found inside Debian image.
+
+Due to licensing, we cannot include the fonts in the public repository.
+
+Ensure you have the correct environment variables defined in your .env file (by default `.env.local`):
+
+Remember to also include `DIGITRANSIT_APIKEY` !
+
+Build the local version of `hsl-map-publisher`:
+
+``` 
+docker build --build-arg BUILD_ENV=local -t hsl-map-publisher .
+```
+
+Setup the development environment:
+
+```
+docker compose up
+```
+
+### B) Manually setup local Docker environment
 
 As before, make sure you are running a database and broker for the publisher:
 
@@ -133,8 +160,19 @@ Build the Docker image with the following command:
 docker build --build-arg BUILD_ENV=local -t hsl-map-publisher .
 ```
 
-And run the Docker container with this command (you can leave font-directory mounting away if you don't have them locally):
+And run the Docker container with these commands (you can leave font-directory mounting away if you don't have them locally):
+
+Server:
+```bash
+docker run -d -p 4000:4000 --name publisher-server -v $(pwd)/output:/output -v $(pwd)/fonts:/fonts --link publisher-postgres --link redis -e SERVICE=server:production hsl-map-publisher
+```
 
+Rendering:
 ```bash
-docker run -d -p 4000:4000 --name publisher -v $(pwd)/output:/output -v $(pwd)/fonts:/fonts --link publisher-postgres --link redis hsl-map-publisher
+docker run -d -p 5000:5000 --name publisher-render -v $(pwd)/output:/output -v $(pwd)/fonts:/fonts --link publisher-postgres --link redis -e SERVICE=start:production hsl-map-publisher
 ```
+
+And finally a Worker, which is linked to the rendering instance:
+```bash
+docker run -d --name publisher-worker -v $(pwd)/output:/output -v $(pwd)/fonts:/fonts --link publisher-postgres --link redis --link publisher-render --link publisher-server -e SERVICE=worker:production hsl-map-publisher
+```

compose.yaml

@@ -0,0 +1,61 @@
+services:
+
+  redis:
+    image: redis
+    ports:
+      - 6379:6379
+
+  publisher-postgres:
+    image: postgres
+    ports:
+      - 5433:5432
+    environment:
+      - POSTGRES_PASSWORD=postgres
+    volumes:
+      - dev_postgres:/var/lib/postgresql/data
+
+  publisher-server:
+    image: hsl-map-publisher
+    ports:
+      - 4000:4000
+    environment:
+      - SERVICE=server:production
+    volumes:
+      - ./output:/output
+      - ./fonts:/fonts
+    depends_on:
+      publisher-postgres:
+        condition: service_started
+      redis:
+        condition: service_started
+
+  publisher-render:
+    image: hsl-map-publisher
+    ports:
+      - 5000:5000
+    environment:
+      - SERVICE=start:production
+    volumes:
+      - ./output:/output
+      - ./fonts:/fonts
+    depends_on:
+      publisher-postgres:
+        condition: service_started
+      redis:
+        condition: service_started
+
+  publisher-worker:
+    image: hsl-map-publisher
+    environment:
+      - SERVICE=worker:production
+    volumes:
+      - ./output:/output
+      - ./fonts:/fonts
+    depends_on:
+      publisher-postgres:
+        condition: service_started
+      redis:
+        condition: service_started
+
+volumes:
+  dev_postgres:

constants.js

@@ -33,6 +33,7 @@ const secretsEnv = mapValues(process.env, (value, key) => {
 module.exports = {
   PG_CONNECTION_STRING: secretsEnv.PG_CONNECTION_STRING || '',
   REDIS_CONNECTION_STRING: secretsEnv.REDIS_CONNECTION_STRING || '',
+  PUBLISHER_RENDER_URL: secretsEnv.PUBLISHER_RENDER_URL || 'http://localhost:5000',
   DIGITRANSIT_URL: secretsEnv.DIGITRANSIT_URL || '',
   DIGITRANSIT_APIKEY: secretsEnv.DIGITRANSIT_APIKEY || '',
   JORE_GRAPHQL_URL: secretsEnv.JORE_GRAPHQL_URL || '',

package.json

@@ -10,12 +10,11 @@
     "build": "webpack --config webpack.prod.js",
     "serve": "serve -l 5000 dist",
     "start": "webpack-serve ./webpack.dev.js",
-    "start:production": "forever start -c \"serve -l 5000\" dist",
+    "start:production": "serve -l 5000 dist",
     "server": "node -r dotenv/config scripts/server",
-    "server:production": "forever start -c \"node --max_old_space_size=8192 -r dotenv/config\" scripts/server.js",
-    "server:hot": "forever -w -t --killSignal=SIGTERM --watchDirectory scripts -c \"node --max_old_space_size=8192 -r dotenv/config\" scripts/server.js",
+    "server:production": "node -r dotenv/config scripts/server.js",
     "worker": "node -r dotenv/config scripts/worker",
-    "worker:production": "forever start -c \"node -r dotenv/config\" scripts/worker.js",
+    "worker:production": "./fonts.sh && fc-cache -f -v && node -r dotenv/config scripts/worker.js",
     "knex": "PG_CONNECTION_STRING=postgres://postgres:postgres@localhost:5432/postgres knex",
     "docker:build": "docker build -t hsl-map-publisher .",
     "docker:run": "docker run -d -p 4000:4000 --name publisher -v $(pwd)/output:/output -v $(pwd)/fonts:/fonts --link publisher-postgres -e \"PG_CONNECTION_STRING=postgres://postgres:postgres@publisher-postgres:5432/postgres\" --shm-size=1G hsl-map-publisher",
@@ -84,13 +83,12 @@
     "babel-polyfill": "^6.26.0",
     "babel-preset-stage-0": "^6.24.1",
     "bullmq": "^1.86.2",
-    "cheerio": "^1.0.0-rc.2",
+    "cheerio": "=1.0.0-rc.3",
     "classnames": "^2.2.5",
     "clean-webpack-plugin": "^0.1.19",
     "dotenv": "^8.1.0",
     "dotenv-webpack": "^1.7.0",
-    "forever": "^4.0.3",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "9.0.0",
     "graphql": "^0.11.7",
     "graphql-tag": "^2.5.0",
     "haversine": "^1.1.1",
@@ -99,10 +97,12 @@
     "ioredis": "^5.0.6",
     "knex": "^2.0.0",
     "koa": "^2.5.3",
+    "koa-body": "^6.0.1",
     "koa-json-body": "^5.3.0",
     "koa-router": "^7.4.0",
     "koa-session": "^5.10.1",
     "lodash": "^4.17.21",
+    "moment": "^2.29.4",
     "node-fetch": "^1.7.3",
     "p-map": "^1.2.0",
     "pdf-merger-js": "^3.4.0",

scripts/generator.js

@@ -4,10 +4,11 @@ const puppeteer = require('puppeteer');
 const qs = require('qs');
 const log = require('./util/log');
 const { uploadPosterToCloud } = require('./cloudService');
+const moment = require('moment');
 
-const { AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY } = require('../constants');
+const { AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, PUBLISHER_RENDER_URL } = require('../constants');
 
-const CLIENT_URL = 'http://localhost:5000';
+const CLIENT_URL = PUBLISHER_RENDER_URL;
 const RENDER_TIMEOUT = 10 * 60 * 1000;
 const MAX_RENDER_ATTEMPTS = 3;
 const SCALE = 96 / 72;
@@ -28,6 +29,15 @@ async function initialize() {
   });
 }
 
+function generateRenderUrl(component, template, props) {
+  const generationProps = props.date
+    ? props
+    : Object.assign(props, { date: moment(Date()).format('YYYY-MM-DD') }); // Add current date by default if request props do not contain it
+  const encodedProps = qs.stringify({ component, props: generationProps, template });
+  const pageUrl = `${PUBLISHER_RENDER_URL}/?${encodedProps}`;
+  return pageUrl;
+}
+
 /**
  * Renders component to PDF file
  * @returns {Promise}
@@ -45,14 +55,16 @@ async function renderComponent(options) {
     onError(error);
   });
 
-  page.on('console', ({ type, text }) => {
-    if (['error', 'warning', 'log'].includes(type)) {
-      onInfo(`Console(${type}): ${text}`);
+  page.on('console', message => {
+    const { url, lineNumber, columnNumber } = message.location();
+    if (['error', 'warning', 'log'].includes(message.type())) {
+      onInfo(
+        `Console(${message.type()}) on (${url}:${lineNumber}:${columnNumber}):\n${message.text()}`,
+      );
     }
   });
 
-  const encodedProps = qs.stringify({ component, props, template });
-  const pageUrl = `${CLIENT_URL}/?${encodedProps}`;
+  const pageUrl = generateRenderUrl(component, template, props);
 
   console.log(`Opening ${pageUrl} in Puppeteer.`);
 
@@ -137,4 +149,5 @@ async function generate(options) {
 
 module.exports = {
   generate,
+  generateRenderUrl,
 };