Merge branch '3.x'

.env.example

@@ -0,0 +1,7 @@
+APP_ENV=production
+APP_PORT=8080
+HEADLESS=true
+HTTP=true
+WEBSOCKET=false
+WITH_DATABASE=false
+DB_URI=mongodb://localhost/shallty_crawler

.gitignore

@@ -1,6 +1,6 @@
 /node_modules
 /logs
 /Server/public/images/screenshot/*
-config.json
 deploy-key
-ecosystem.config.js
+ecosystem.config.js
+.env

.travis.yml

@@ -2,4 +2,4 @@ language: node_js
 node_js:
 - node
 before_install:
-- mv config.example.json config.json
+- mv .env.example .env

Browser/index.js

@@ -1,5 +1,4 @@
 const puppeteer = require('puppeteer')
-const { app_env, headless } = require('../config.json')
 const Util = require('../utils/utils')
 
 class Browser {
@@ -11,7 +10,7 @@ class Browser {
     async init() {
         if (!this.browser) {
             this.browser = await puppeteer.launch({
-                headless: headless,
+                headless: process.env.HEADLESS === 'true',
                 args: ['--disable-gpu', '--no-sandbox', '--disable-setuid-sandbox']
             })
         }
@@ -24,7 +23,7 @@ class Browser {
     async newBrowser() {
         const browsers = this.browsers
         const browser = await puppeteer.launch({
-            headless: app_env == 'local' ? false : true,
+            headless: process.env.HEADLESS === 'true',
             args: ['--no-sandbox']
         })
         const newBrowser = {

Models/Shortlink.js

@@ -0,0 +1,25 @@
+const mongoose = require('mongoose')
+const Schema = mongoose.Schema
+
+const ShortlinkSchema = new Schema({
+    original: {
+        type: String,
+        required: true,
+        unique: true,
+    },
+    parsed: {
+        type: String,
+    },
+    createdAt: {
+        type: Date,
+        default: new Date()
+    },
+    updatedAt: {
+        type: Date,
+        default: new Date()
+    },
+})
+
+const Shortlink = mongoose.model('Shortlink', ShortlinkSchema)
+
+module.exports = Shortlink

Queue/index.js

@@ -0,0 +1,51 @@
+const crypto = require('crypto')
+const { sleep } = require('../utils/utils')
+
+class Queue {
+    constructor() {
+        this.runningJobs = 0
+    }
+
+    register(task) {
+        return {
+            id: crypto.randomBytes(8).toString('hex'),
+            task: task,
+            result: undefined,
+        }
+    }
+
+    async run(job) {
+        if (this.runningJobs > 4) {
+            return {
+                finished: false
+            }
+        }
+
+        this.runningJobs = this.runningJobs + 1
+
+        const result = await job.task()
+
+        this.runningJobs = this.runningJobs - 1
+
+        return {
+            finished: true,
+            result: result,
+        }
+    }
+
+    async dispatch(job) {
+        let run = {
+            finished: false
+        }
+
+        while (!run.finished) {
+            await sleep(2000)
+
+            run = await this.run(job)
+        }
+
+        return run.result
+    }
+}
+
+module.exports = new Queue

Shortlinks/index.js

@@ -1,9 +1,11 @@
 const fs = require('fs')
 const path = require('path')
 const Handler = require('../exceptions/Handler')
+const ShortlinkModel = require('../Models/Shortlink')
+const Queue = require('../Queue')
 
+const WITH_DATABASE = process.env.WITH_DATABASE === 'true'
 const shortlinkFiles = fs.readdirSync(path.join(__dirname, './')).filter(file => file !== 'index.js' && file.endsWith('.js'))
-
 const shortlinks = []
 
 for (const file of shortlinkFiles) {
@@ -15,7 +17,55 @@ class Shortlink {
         this.shorterners = shortlinks
     }
 
-    async parse(link) {
+    /**
+     * Save new parsed shortlink to the database.
+     * 
+     * @param {String} originalLink 
+     * @param {String} parsedLink 
+     */
+    async cacheShortlink(originalLink, parsedLink) {
+        const newParsed = new ShortlinkModel({
+            original: originalLink,
+            parsed: parsedLink
+        })
+
+        await newParsed.save()
+    }
+
+    /**
+     * Get parsed shortlink by the original link from the database.
+     * 
+     * @param {String} link 
+     */
+    async getCachedShortlink(link) {
+        return await ShortlinkModel.findOne({
+            original: link
+        })
+    }
+
+    /**
+     * Response with the cached shortlink.
+     * 
+     * @param cachedShortlink 
+     */
+    async responseWithCached(cachedShortlink) {
+        return {
+            success: true,
+            cached: true,
+            id: cachedShortlink._id,
+            original: cachedShortlink.original,
+            url: cachedShortlink.parsed,
+            createdAt: cachedShortlink.createdAt,
+            updatedAt: cachedShortlink.updatedAt,
+        }
+    }
+
+    /**
+     * Select the correct parser for the link.
+     * 
+     * @param {String} link 
+     */
+    selectParser(link) {
         let shorterner = null
 
         for (const i of this.shorterners) {
@@ -38,11 +88,39 @@ class Shortlink {
             }
         }
 
-        if (!shorterner) {
-            return Handler.error('Error: Unknown shortlink.')
+        return shorterner
+    }
+
+    async parse(link, options = {}) {
+        if (WITH_DATABASE) {
+            const cachedShortlink = await this.getCachedShortlink(link)
+
+            if (cachedShortlink) return this.responseWithCached(cachedShortlink)
+        }
+
+        let parsed
+        let shorterner = this.selectParser(link)
+
+        if (!shorterner) return Handler.error('Error: Unknown shortlink.')
+
+        if (options.queue) {
+            const job = Queue.register(async () => {
+                return await shorterner.parse(link)
+            })
+
+            parsed = await Queue.dispatch(job)
+        } else {
+            parsed = await shorterner.parse(link)
+        }
+
+        if (!parsed.error) {
+            parsed.success = true
+            parsed.cached = false
+
+            if (WITH_DATABASE) await this.cacheShortlink(link, parsed.url)
         }
 
-        return await shorterner.parse(link)
+        return parsed
     }
 }
 

Socket/listeners/index.js

@@ -63,9 +63,8 @@ module.exports = {
 
     shortlinkListener: (io, socket) => {
         socket.on('parse', async function (params) {
-            const data = await Shortlink.parse(params.link)
-
-            io.emit('parse', data)
+            const data = await Shortlink.parse(params.link, { queue: true })
+            io.to(socket.id).emit('parse', data)
         })
     }
 }

config.example.json → config.json

@@ -1,8 +1,4 @@
 {
-    "app_env": "production",
-    "app_port": 8080,
-    "headless": true,
-    "websocket": false,
     "meownime_url": "https://meownime.com",
     "samehadaku_url": "https://samehadaku.vip",
     "neonime_url": "https://neonime.org",
@@ -11,4 +7,4 @@
     "mangaku_url": "https://mangaku.in",
     "kiryuu_url": "https://kiryuu.co",
     "moenime_url": "https://moenime.id"
-}
+}

exceptions/Handler.js

@@ -1,6 +1,5 @@
 const { createLogger, format, transports } = require('winston')
 require('winston-daily-rotate-file')
-const { app_env } = require('../config.json')
 
 const transport = new(transports.DailyRotateFile)({
     filename: 'shallty-%DATE%.log',
@@ -30,7 +29,7 @@ const logger = createLogger({
     ]
 })
 
-if (app_env !== 'production') {
+if (process.env.APP_ENV !== 'production') {
     logger.add(new transports.Console({
         format: format.combine(
             format.colorize()
@@ -44,14 +43,13 @@ class Handler {
      * 
      * @param {Error} err Error instance.
      */
-    error(err, isReturn = true) {
+    error(err) {
         logger.error(err)
 
-        if (isReturn)
-            return {
-                error: true,
-                message: 'Something went wrong. ' + err
-            }
+        return {
+            error: true,
+            message: 'Something went wrong. ' + err
+        }
     }
 }
 

index.js

@@ -1,22 +1,32 @@
+require('dotenv').config()
+const mongoose = require('mongoose')
 const Server = require('./Server')
 const Socket = require('./Socket')
 const Browser = require('./Browser')
-const { app_port, websocket } = require('./config.json')
-const runningPort = process.env.PORT || app_port
 
 let appplication = null
 
 ;(function () {
-    const [http, app] = Server.init(runningPort)
+    if (process.env.WITH_DATABASE === 'true') {
+        mongoose.connect(process.env.DB_URI, {
+            useNewUrlParser: true,
+            useUnifiedTopology: true,
+        })
+    }
+
+    const [http, app] = Server.init(process.env.APP_PORT)
 
     appplication = app
 
-    if (websocket) {
-        Socket.init(http, runningPort)
+    console.log(process.env.WEBSOCKET)
+    if (process.env.WEBSOCKET === 'true') {
+        Socket.init(http, process.env.APP_PORT)
     }
-
-    Server.initRoute()
 
+    if (process.env.HTTP === 'true') {
+        Server.initRoute()
+    }
+
     Browser.init()
 })()