diff --git a/README.md b/README.md new file mode 100644 index 0000000..31c9a18 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +## Chingu Solo Project Feedback Scraper + +This firebase function scrapes feedback from [soloproject-evaluation](https://github.com/chingu-voyages/soloproject-evaluation) using Puppeteer. + +For Puppeteer to work with firebase functions, +1. Make sure enough memory is allocated (512Mb in this case) +2. Use Puppeteer `16.2.0`. Version 18 might work according to some reports. However, the latest version (21), does not work. +3. `.puppeteerrc.cjs` to configure cache +4. `"gcp-build": "node node_modules/puppeteer/install.js"` to add a custom build step + +The function does 4 things +1. Scrapes the Github repo for feedback +2. Stores the json file in firebase store +3. Revalidates feedback tags in the next.js Evaluation app + https://chingu-soloproject-evaluation-app.vercel.app/ + https://github.com/cherylli/chingu-soloproject-evaluation-app +4. Returns the scrape results (in JSON) diff --git a/functions/src/index.ts b/functions/src/index.ts index 0a2135e..73631c3 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -17,13 +17,23 @@ exports.scrape = onRequest({ memory: "512MiB", region: "australia-southeast1", }, async (req, res) => { + // 1. Scrape const feedback = await runScraper(); + // 2. update firebase store const jsonFile = JSON.stringify(feedback); const feedbackRef = ref(storage, "feedback.json"); uploadString(feedbackRef, jsonFile).then((snapshot)=>{ console.log(snapshot); }); + + // 3. revalidate next.js app + await fetch(`${process.env.BASE_URL}/api/revalidate?tag=feedback&secret= + ${process.env.NEXT_REVALIDATION_SECRET}`, { + method: "POST", + }); + + // 4. return scrape result res.send(feedback); }); diff --git a/functions/src/scraper.ts b/functions/src/scraper.ts index 0f28e6c..2a58975 100644 --- a/functions/src/scraper.ts +++ b/functions/src/scraper.ts @@ -32,7 +32,7 @@ export const runScraper = async () => { for (let i = 0; i < data.length; i = i + col) { const row: Row = {}; for (let j = 0; j < col; j++) { - row[headers[j]] = data[i + j]; + row[headers[j].toLowerCase()] = data[i + j]; } rows.push(row); }