sending tracking information

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}

index.js

@@ -6,6 +6,7 @@ import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyRuntime from './lib/FredyRuntime.js';
 import { duringWorkingHoursOrNotSet } from './lib/utils.js';
 import './lib/api/api.js';
+import {track} from './lib/services/tracking/Tracker.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
@@ -25,6 +26,7 @@ setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
     if (isDuringWorkingHoursOrNotSet) {
+      track();
       config.lastRun = Date.now();
       jobStorage
         .getJobs()

lib/api/routes/generalSettingsRoute.js

@@ -1,5 +1,5 @@
 import restana from 'restana';
-import { config, getDirName } from '../../utils.js';
+import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
 import fs from 'fs';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
@@ -10,7 +10,9 @@ generalSettingsRouter.get('/', async (req, res) => {
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
   try {
-    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify(settings));
+    const currentConfig = await readConfigFromStorage();
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
+    await refreshConfig();
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));

lib/api/routes/jobRouter.js

@@ -5,6 +5,7 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import * as immoscoutProvider from '../../provider/immoscout.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
+import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
 const service = restana();
 const jobRouter = service.newRouter();
 function doesJobBelongsToUser(job, req) {
@@ -25,8 +26,8 @@ jobRouter.get('/', async (req, res) => {
   res.send();
 });
 jobRouter.get('/processingTimes', async (req, res) => {
-  let scrapingAntData = null;
-  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+  let scrapingAntData = {};
+  if (isScrapingAntApiKeySet()) {
     try {
       const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
       scrapingAntData = await response.json();
@@ -38,6 +39,7 @@ jobRouter.get('/processingTimes', async (req, res) => {
     interval: config.interval,
     lastRun: config.lastRun || null,
     scrapingAntData,
+    error: scrapingAntData?.detail == null ? null : scrapingAntData?.detail
   };
   res.send();
 });

lib/defaultConfig.js

@@ -0,0 +1,8 @@
+export const DEFAULT_CONFIG = {
+            'interval': '60',
+            'port': 9998,
+            'scrapingAnt': {'apiKey': '', 'proxy': 'datacenter'},
+            'workingHours': {'from': '', 'to': ''},
+            'demoMode': false,
+            'analyticsEnabled': null
+    };

lib/services/scrapingAnt.js

@@ -22,7 +22,7 @@ export const transformUrlForScrapingAnt = (url, id) => {
   return url;
 };
 export const isScrapingAntApiKeySet = () => {
-  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 8;
 };
 export const makeUrlResidential = (url) => {
   return url.replace('datacenter', 'residential');

lib/services/tracking/Tracker.js

@@ -0,0 +1,42 @@
+import Mixpanel from 'mixpanel';
+import {getJobs} from '../storage/jobStorage.js';
+
+import {config} from '../../utils.js';
+
+export const track = function () {
+    //only send tracking information if the user allowed to do so.
+    if (config.analyticsEnabled) {
+
+        const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+
+        const activeProvider = new Set();
+        const activeAdapter = new Set();
+        const platform = process.platform;
+        const arch = process.arch;
+        const language = process.env.LANG || 'en';
+        const nodeVersion = process.version || 'N/A';
+
+        const jobs = getJobs();
+
+        if (jobs != null && jobs.length > 0) {
+            jobs.forEach(job => {
+                job.provider.forEach(provider => {
+                    activeProvider.add(provider.id);
+                });
+                job.notificationAdapter.forEach(adapter => {
+                    activeAdapter.add(adapter.id);
+                });
+            });
+
+            mixpanelTracker.track('fredy_tracking', {
+                adapter: Array.from(activeAdapter),
+                provider: Array.from(activeProvider),
+                isDemo: config.demoMode,
+                platform,
+                arch,
+                nodeVersion,
+                language
+            });
+        }
+    }
+};

lib/utils.js

@@ -2,6 +2,7 @@ import {dirname} from 'node:path';
 import {fileURLToPath} from 'node:url';
 import {readFile} from 'fs/promises';
 import {createHash} from 'crypto';
+import {DEFAULT_CONFIG} from './defaultConfig.js';
 
 function isOneOf(word, arr) {
     if (arr == null || arr.length === 0) {
@@ -52,7 +53,23 @@ function buildHash(...inputs) {
         .digest('hex');
 }
 
-const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+let config = {};
+export async function readConfigFromStorage(){
+    return JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+}
+
+export async function refreshConfig(){
+    try {
+        config = await readConfigFromStorage();
+        //backwards compatability...
+        config.analyticsEnabled ??= null;
+        config.demoMode ??= false;
+    } catch (error) {
+        config = {...DEFAULT_CONFIG};
+        console.error('Error reading config file', error);
+    }
+}
+await refreshConfig();
 
 export {isOneOf};
 export {nullOrEmpty};

package.json

@@ -64,6 +64,7 @@
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
     "markdown": "^0.5.0",
+    "mixpanel": "^0.18.0",
     "nanoid": "5.0.8",
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.6",

ui/src/App.jsx

@@ -17,11 +17,13 @@ import Jobs from './views/jobs/Jobs';
 import { Route } from 'react-router';
 
 import './App.less';
+import TrackingModal from './components/tracking/TrackingModal.jsx';
 
 export default function FredyApp() {
   const dispatch = useDispatch();
   const [loading, setLoading] = React.useState(true);
   const currentUser = useSelector((state) => state.user.currentUser);
+  const settings = useSelector((state) => state.generalSettings.settings);
 
   useEffect(() => {
     async function init() {
@@ -31,6 +33,7 @@ export default function FredyApp() {
         await dispatch.jobs.getJobs();
         await dispatch.jobs.getProcessingTimes();
         await dispatch.notificationAdapter.getAdapter();
+        await dispatch.generalSettings.getGeneralSettings();
       }
       setLoading(false);
     }
@@ -59,6 +62,7 @@ export default function FredyApp() {
         <Logout />
         <Logo width={190} white />
         <Menu isAdmin={isAdmin()} />
+        {settings.analyticsEnabled === null && <TrackingModal/>}
         <Switch>
           <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
           <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />

ui/src/components/tracking/TrackingModal.jsx

@@ -0,0 +1,48 @@
+import React from 'react';
+import {Modal} from '@douyinfe/semi-ui';
+import Logo from '../logo/Logo.jsx';
+import {xhrPost} from '../../services/xhr.js';
+
+import './TrackingModal.less';
+
+const saveResponse = async (analyticsEnabled) => {
+    await xhrPost('/api/admin/generalSettings', {
+        analyticsEnabled
+    });
+};
+
+export default function TrackingModal() {
+
+    return <Modal
+        visible={true}
+        onOk={async () => {
+            await saveResponse(true);
+            location.reload();
+        }}
+        onCancel={async () => {
+            await saveResponse(false);
+            location.reload();
+        }}
+        maskClosable={false}
+        closable={false}
+        okText="Yes! I want to help"
+        cancelText="No, thanks"
+    >
+        <Logo white/>
+        <div className="trackingModal__description">
+            <p>Hey 👋</p>
+            <p>Fed up with popups? Yeah, me too. But this one’s important, and I promise it will only appear once ;)</p>
+            <p>Fredy is completely free (and will always remain free). If you’d like, you can support me by donating
+                through my GitHub, but there’s absolutely no obligation to do so.</p>
+            <p>However, it would be a huge
+                help if you’d allow me to collect some analytical data. Wait, before you click “no,” let me explain. If
+                you
+                agree, Fredy will send a ping to my Mixpanel project each time it runs.</p>
+            <p>The data includes: names of
+                active adapters/providers, OS, architecture, Node version, and language. The information is entirely
+                anonymous and helps me understand which adapters/providers are most frequently used.</p>
+            <p>Thanks🤘</p>
+        </div>
+    </Modal>;
+
+}

ui/src/components/tracking/TrackingModal.less

@@ -0,0 +1,5 @@
+.trackingModal {
+  &__description {
+    margin-top:10rem;
+  }
+}