General settings (#28)

adding ui for general settings | adding 'working hours' as new feature

CHANGELOG.md

@@ -1,3 +1,9 @@
+###### [V5.1.0]
+- Upgrading dependencies
+- NodeJS 12.13 is now the minimum supported version
+- Adding general settings as new configuration page to ui
+- Adding new feature working hours 
+
 ###### [V5.0.0]
 - Upgrading dependencies
 - NodeJS 12 is now the minimum supported version

README.md

@@ -18,7 +18,7 @@ yarn run start
 _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening a browser `http://localhost:9998`. The default login is `admin` for username and password. (You should change the password asap when you plan to run Fredy on your server.)
 
 <p align="center">
-  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_1.png" width="30%">
+  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot__1.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;
   <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;  
@@ -31,7 +31,7 @@ _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_
 ## Immoscout
 I have added **EXPERIMENTAL** support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successful validated) re-send the cookies each time.
 
-To be able to use Immoscout, you need to create an account and copy the apiKey into the config file under /conf/config.json.
+To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the ApiKey in the "General Settings" tab (visible when logged in as administrator).
 The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time it works pretty good :)
 
 If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefor they've decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (No I don't get any money for recommending good services...)

conf/config.json

@@ -1,7 +1 @@
-{
-  "interval": 30,
-  "port": 9998,
-  "scrapingAnt": {
-    "apiKey": ""
-  }
-}
+{"interval":"30","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}

index.js

@@ -13,6 +13,8 @@ const jobStorage = require('./lib/services/storage/jobStorage');
 const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
 const FredyRuntime = require('./lib/FredyRuntime');
 
+const { duringWorkingHoursOrNotSet } = require('./lib/utils');
+
 //starting the api service
 require('./lib/api/api');
 
@@ -24,31 +26,39 @@ console.log(`Started Fredy successfully. Ui can be accessed via http://localhost
 /* eslint-enable no-console */
 setInterval(
   (function exec() {
-    config.lastRun = Date.now();
-    jobStorage
-      .getJobs()
-      .filter((job) => job.enabled)
-      .forEach((job) => {
-        const providerIds = job.provider.map((provider) => provider.id);
-
-        provider
-          .filter((provider) => provider.endsWith('.js'))
-          .map((pro) => require(`${path}/${pro}`))
-          .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
-          .forEach(async (pro) => {
-            const providerId = pro.metaInformation.id;
-            if (providerId == null || providerId.length === 0) {
-              throw new Error('Provider id must not be empty. => ' + pro);
-            }
-            const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
-            if (providerConfig == null) {
-              throw new Error(`Provider Config for provider with id ${providerId} not found.`);
-            }
-            pro.init(providerConfig, job.blacklist);
-            await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute();
-            setLastJobExecution(job.id);
-          });
-      });
+    const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+
+    if (isDuringWorkingHoursOrNotSet) {
+      config.lastRun = Date.now();
+      jobStorage
+        .getJobs()
+        .filter((job) => job.enabled)
+        .forEach((job) => {
+          const providerIds = job.provider.map((provider) => provider.id);
+
+          provider
+            .filter((provider) => provider.endsWith('.js'))
+            .map((pro) => require(`${path}/${pro}`))
+            .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
+            .forEach(async (pro) => {
+              const providerId = pro.metaInformation.id;
+              if (providerId == null || providerId.length === 0) {
+                throw new Error('Provider id must not be empty. => ' + pro);
+              }
+              const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
+              if (providerConfig == null) {
+                throw new Error(`Provider Config for provider with id ${providerId} not found.`);
+              }
+              pro.init(providerConfig, job.blacklist);
+              await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute();
+              setLastJobExecution(job.id);
+            });
+        });
+    } else {
+      /* eslint-disable no-console */
+      console.debug('Working hours set. Skipping as outside of working hours.');
+      /* eslint-enable no-console */
+    }
     return exec;
   })(),
   INTERVAL

lib/api/api.js

@@ -1,5 +1,6 @@
 const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
 const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
+const { generalSettingsRouter } = require('./routes/generalSettingsRoute');
 const { analyticsRouter } = require('./routes/analyticsRouter');
 const { providerRouter } = require('./routes/providerRouter');
 const { loginRouter } = require('./routes/loginRoute');
@@ -28,6 +29,7 @@ service.use('/api/jobs', authInterceptor());
 service.use('/api/admin', adminInterceptor());
 
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
+service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);

lib/api/routes/generalSettingsRoute.js

@@ -0,0 +1,24 @@
+const service = require('restana')();
+const generalSettingsRouter = service.newRouter();
+const config = require('../../../conf/config.json');
+const fs = require('fs');
+
+generalSettingsRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, config);
+  res.send();
+});
+
+generalSettingsRouter.post('/', async (req, res) => {
+  const settings = req.body;
+
+  try {
+    fs.writeFileSync(`${__dirname}/../../../conf/config.json`, JSON.stringify(settings));
+  } catch (err) {
+    console.error(err);
+    res.send(new Error('Error while trying to write settings.'));
+    return;
+  }
+  res.send();
+});
+
+exports.generalSettingsRouter = generalSettingsRouter;

lib/utils.js

@@ -8,4 +8,29 @@ function isOneOf(word, arr) {
   return blacklist.test(word);
 }
 
-module.exports = { isOneOf };
+function nullOrEmpty(val) {
+  return val == null || val.length === 0;
+}
+
+function timeStringToMs(timeString, now) {
+  const d = new Date(now);
+  const parts = timeString.split(':');
+  d.setHours(parts[0]);
+  d.setMinutes(parts[1]);
+  d.setSeconds(0);
+  return d.getTime();
+}
+
+function duringWorkingHoursOrNotSet(config, now) {
+  const { workingHours } = config;
+  if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
+    return true;
+  }
+
+  const toDate = timeStringToMs(workingHours.to, now);
+  const fromDate = timeStringToMs(workingHours.from, now);
+
+  return fromDate <= now && toDate >= now;
+}
+
+module.exports = { isOneOf, nullOrEmpty, duringWorkingHoursOrNotSet };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -41,7 +41,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=12.0.0",
+    "node": ">=12.13.0",
     "npm": ">=6.0.0"
   },
   "browserslist": [
@@ -63,7 +63,7 @@
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
     "nanoid": "3.1.23",
-    "node-mailjet": "3.3.1",
+    "node-mailjet": "3.3.4",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-redux": "7.2.4",
@@ -86,9 +86,9 @@
     "babel-loader": "8.2.2",
     "chai": "4.3.4",
     "clean-webpack-plugin": "3.0.0",
-    "copy-webpack-plugin": "8.1.1",
-    "css-loader": "5.2.4",
-    "eslint": "7.26.0",
+    "copy-webpack-plugin": "9.0.0",
+    "css-loader": "5.2.6",
+    "eslint": "7.27.0",
     "eslint-config-prettier": "8.3.0",
     "eslint-plugin-react": "7.23.2",
     "file-loader": "6.2.0",

test/provider/utils.test.js

@@ -1,5 +1,13 @@
 const utils = require('../../lib/utils');
 const assert = require('assert');
+const expect = require('chai').expect;
+
+const fakeWorkingHoursConfig = (from, to) => ({
+  workingHours: {
+    to,
+    from,
+  },
+});
 
 describe('utils', () => {
   describe('#isOneOf()', () => {
@@ -10,4 +18,22 @@ describe('utils', () => {
       assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
     });
   });
+
+  describe('#duringWorkingHoursOrNotSet()', () => {
+    it('should be false', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;
+    });
+    it('should be true', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true;
+    });
+    it('should be true if nothing set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true;
+    });
+    it('should be true if only to is set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true;
+    });
+    it('should be true if only from is set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true;
+    });
+  });
 });

ui/src/App.js

@@ -2,6 +2,7 @@ import React, { useEffect } from 'react';
 
 import InsufficientPermission from './components/permission/InsufficientPermission';
 import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
+import GeneralSettings from './views/generalSettings/GeneralSettings';
 import ToastsContainer from './components/toasts/ToastContainer';
 import JobMutation from './views/jobs/mutation/JobMutation';
 import UserMutator from './views/user/mutation/UserMutator';
@@ -78,6 +79,12 @@ export default function FredyApp() {
               currentUser={currentUser}
             />
             <PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
+            <PermissionAwareRoute
+              name="General Settings"
+              path="/generalSettings"
+              component={<GeneralSettings />}
+              currentUser={currentUser}
+            />
 
             <Redirect from="/" to={'/jobs'} />
           </Switch>

ui/src/components/menu/Menu.js

@@ -1,6 +1,6 @@
 import React from 'react';
 import { useHistory } from 'react-router-dom';
-import { Menu } from 'semantic-ui-react';
+import { Icon, Menu } from 'semantic-ui-react';
 
 import './Menu.less';
 import { useLocation } from 'react-router';
@@ -19,7 +19,7 @@ const TopMenu = function TopMenu({ isAdmin }) {
         className={isActiveRoute('jobs') ? 'topMenu__active' : 'topMenu__item'}
         onClick={() => history.push('/jobs')}
       >
-        Job Configuration
+        <Icon name="search" /> Job Configuration
       </Menu.Item>
 
       {isAdmin && (
@@ -29,7 +29,18 @@ const TopMenu = function TopMenu({ isAdmin }) {
           className={isActiveRoute('users') ? 'topMenu__active' : 'topMenu__item'}
           onClick={() => history.push('/users')}
         >
-          User configuration
+          <Icon name="user" /> User configuration
+        </Menu.Item>
+      )}
+
+      {isAdmin && (
+        <Menu.Item
+          name="general"
+          active={isActiveRoute('general')}
+          className={isActiveRoute('general') ? 'topMenu__active' : 'topMenu__item'}
+          onClick={() => history.push('/generalSettings')}
+        >
+          <Icon name="cog" /> General Settings
         </Menu.Item>
       )}
     </Menu>

ui/src/services/rematch/models/generalSettings.js

@@ -0,0 +1,26 @@
+import { xhrGet } from '../../xhr';
+
+export const generalSettings = {
+  state: {
+    settings: {},
+  },
+  reducers: {
+    //only admins
+    setGeneralSettings: (state, payload) => {
+      return {
+        ...state,
+        settings: payload,
+      };
+    },
+  },
+  effects: {
+    async getGeneralSettings() {
+      try {
+        const response = await xhrGet('/api/admin/generalSettings');
+        this.setGeneralSettings(response.json);
+      } catch (Exception) {
+        console.error('Error while trying to get resource for api/admin/generalSettings. Error:', Exception);
+      }
+    },
+  },
+};

ui/src/services/rematch/store.js

@@ -1,4 +1,5 @@
 import { notificationAdapter } from './models/notificationAdapter';
+import { generalSettings } from './models/generalSettings';
 import createLoadingPlugin from '@rematch/loading';
 import { provider } from './models/provider';
 import { createLogger } from 'redux-logger';
@@ -17,6 +18,7 @@ const store = init({
   name: 'fredy',
   models: {
     notificationAdapter,
+    generalSettings,
     provider,
     jobs,
     user,