Separate json and csv export

Add buttons to export json and csv in the ui

Update readme

Update readme

Clean up readme

README.md

@@ -4,15 +4,20 @@
 
 immo-feed scrapes french real estate websites like leboncoin, ouestfrance, bienici etc. for listings and makes the aggregated results available with a simple api and frontend. it's super easy to create new scraper sources and customise the existing ones to suit your search.
 
-![immo-feed](https://user-images.githubusercontent.com/1336344/42247317-a0cec83e-7f20-11e8-9051-6070580f1fa0.png)
+![immo-feed-1.2.0](https://user-images.githubusercontent.com/1336344/42539414-619a6b70-849b-11e8-8b51-5db87fe398b5.png)
 
 # Changelog
 
+## 1.2.0
+- [#25](https://github.com/tamarasaurus/immo-feed/issues/25) Add email notifications with Mailgun (see [Setting up notifications](https://github.com/tamarasaurus/immo-feed#setting-up-notifications))
+- [#26](https://github.com/tamarasaurus/immo-feed/issues/26) Add CSV and JSON export endpoint with a download option
+- [#30](https://github.com/tamarasaurus/immo-feed/issues/30) Fix pagination when results are hidden in the UI
+
 ## 1.1.0
-- Add tests for scraper sources #18 #17
+- Add tests for scraper sources [#18](https://github.com/tamarasaurus/immo-feed/issues/18) [#17](https://github.com/tamarasaurus/immo-feed/issues/17)
     - These are run nightly on Travis CI for every source
     - Can also be run locally (see [Testing](https://github.com/tamarasaurus/immo-feed#testing))
-- Add handling of multiple pages in the runner #21
+- Add handling of multiple pages in the runner [#21](https://github.com/tamarasaurus/immo-feed/issues/21)
     - Now you can specify these new properties in your sources (see [Adding a new source](https://github.com/tamarasaurus/immo-feed#adding-a-new-scraper-source)):
     ```
         // The selector for the next page button
@@ -21,9 +26,9 @@ immo-feed scrapes french real estate websites like leboncoin, ouestfrance, bieni
         // How many pages do you want to scrape
         public pagesToScrape: number = 5
     ```
-- Add pagination in the database, api and frontend #23
+- Add pagination in the database, api and frontend [#23](https://github.com/tamarasaurus/immo-feed/issues/23)
 - Run the scrapers sequentially to improve stability
-- Add some clearer and prettier logging to the runner #1
+- Add some clearer and prettier logging to the runner [#1](https://github.com/tamarasaurus/immo-feed/issues/30)
 
 ## 1.0.0
 - First release of the app !
@@ -37,7 +42,7 @@ immo-feed is made of:
 - a mongo database to store the results
 - a tiny api to access the results
 - a frontend app to view the results
-- a way to send notifications for new results (only Slack is available for now)
+- a way to send notifications for new results (only email is available for now)
 
 ## Requirements
 
@@ -117,7 +122,7 @@ immo-feed supports email notifications for newly scraped results. Emails will be
 
 1. Set the `NOTIFY` environment variable to true
 2. Sign up for a [Mailgun](https://www.mailgun.com/) account
-3. Set the following environment variables for immo-feed defined in your Mailgun domain page: 
+3. Set the following environment variables for immo-feed defined in your Mailgun domain page:
     - `MAILGUN_API_KEY` - API key
     - `MAILGUN_API_BASE_URL` - API Base URL
     - `MAILGUN_NOTIFY_EMAIL` - Your email address

frontend/public/index.html

@@ -12,6 +12,8 @@
     <div id="app">
         <nav>
             <h1>🏠 immo-feed</h1>
+            <a href="http://localhost:3000/export/csv?download=true" class="button">export csv</a>
+            <a href="http://localhost:3000/export/json?download=true" class="button">export json</a>
         </nav>
 
         <div class="filters">

frontend/public/main.css

@@ -125,6 +125,24 @@ nav label {
     cursor: pointer;
 }
 
+nav .button {
+    background: black;
+    border: 0;
+    color: white;
+    height: 30px;
+    width: 100px;
+    margin-left: 10px;
+    cursor: pointer;
+    text-decoration: none;
+    text-align: center;
+    line-height: 30px;
+    font-size: 14px;
+}
+
+nav .button:hover {
+    opacity: 0.85
+}
+
 footer {
     margin-top: 40px;
     text-align: center;
@@ -153,7 +171,7 @@ footer {
 }
 
 .filters {
-  display: flex; 
+  display: flex;
   justify-content: flex-end;
 }
 

frontend/public/main.js

@@ -71,4 +71,3 @@ document.getElementById('sort').addEventListener('change', function () {
         return parseInt(b[this.value]) - parseInt(a[this.value])
     })
 })
-

scraper/src/api/server.ts

@@ -36,24 +36,44 @@ app.post('/results/:id/hide', cors(), async (req: any, res: any) => {
     }
 })
 
-app.get('/export', cors(), async (req: any, res: any) => {
-    const { since, type, download } = req.query
+app.get('/export/csv', cors(), async (req: any, res: any) => {
+    const { since, download } = req.query
     let records = await storage.findUpdatedSince(since)
 
-    if (type === 'csv') {
-        records = parse(records, {
-            fields: [ '_id', 'date', 'name', 'description', 'price', 'size', 'link', 'photo' ]
-        })
+    records = parse(records, {
+        fields: [
+            '_id',
+            'date',
+            'name',
+            'description',
+            'price',
+            'size',
+            'link',
+            'photo'
+        ]
+    })
 
-        if (download === 'true') {
-            res.setHeader('Content-disposition', `attachment; filename=immo-feed-${new Date().getTime()}.csv`);
-            res.set('Content-Type', 'text/csv');
-        }
+    if (!!download) {
+        const fileName = `attachment; filename=immo-feed-${new Date().getTime()}.csv`
+        res.setHeader('Content-disposition', fileName);
+        res.set('Content-Type', 'text/csv');
+    }
+
+    return res.status(200).send(records);
+})
 
-        return res.status(200).send(records);
+app.get('/export/json', cors(), async (req: any, res: any) => {
+    const { since, download } = req.query
+    let records = await storage.findUpdatedSince(since)
+
+    if (!!download) {
+        const fileName = `attachment; filename=immo-feed-${new Date().getTime()}.json`
+        res.setHeader('Content-disposition', fileName);
+        res.set('Content-Type', 'application/json');
     }
 
     res.json(records)
 })
 
 app.listen(process.env.PORT || 3000)
+

scraper/src/notification/email.ts

@@ -19,7 +19,7 @@ export default function notify(updatedResults: any[]) {
         form: {
             from: email,
             to: email,
-            subject: `🏠 ${updatedResults.length} new result${updatedResults.length > 1 ? 's' : ''} found`,
+            subject: `🏠 immo-feed - ${updatedResults.length} new result${updatedResults.length > 1 ? 's' : ''} found`,
             html: emailTemplate(updatedResults)
         }
     })

scraper/src/source/leboncoin.ts

@@ -4,7 +4,7 @@ export default class LebonCoin extends HTMLSource {
     public url = 'https://www.leboncoin.fr/ventes_immobilieres/offres/'
     public resultSelector = 'li[itemtype="http://schema.org/Offer"]'
     public nextPageSelector = '.bgMain nav ul li:last-child > a'
-    public pagesToScrape = 10
+    public pagesToScrape = 5
 
     public resultAttributes = [
         { type: 'name', selector: '[itemprop="name"]' },