add readme

README.md

@@ -0,0 +1,17 @@
+## Chingu Solo Project Feedback Scraper
+
+This firebase function scrapes feedback from [soloproject-evaluation](https://github.com/chingu-voyages/soloproject-evaluation) using Puppeteer.
+
+For Puppeteer to work with firebase functions, 
+1. Make sure enough memory is allocated (512Mb in this case)
+2. Use Puppeteer `16.2.0`. Version 18 might work according to some reports. However, the latest version (21), does not work.
+3. `.puppeteerrc.cjs` to configure cache
+4. `"gcp-build": "node node_modules/puppeteer/install.js"` to add a custom build step
+
+The function does 4 things
+1. Scrapes the Github repo for feedback
+2. Stores the json file in firebase store
+3. Revalidates feedback tags in the next.js Evaluation app
+   https://chingu-soloproject-evaluation-app.vercel.app/
+   https://github.com/cherylli/chingu-soloproject-evaluation-app
+4. Returns the scrape results (in JSON)

functions/src/index.ts

@@ -17,13 +17,23 @@ exports.scrape = onRequest({
   memory: "512MiB",
   region: "australia-southeast1",
 }, async (req, res) => {
+  // 1. Scrape
   const feedback = await runScraper();
 
+  // 2. update firebase store
   const jsonFile = JSON.stringify(feedback);
   const feedbackRef = ref(storage, "feedback.json");
   uploadString(feedbackRef, jsonFile).then((snapshot)=>{
     console.log(snapshot);
   });
+
+  // 3. revalidate next.js app
+  await fetch(`${process.env.BASE_URL}/api/revalidate?tag=feedback&secret=
+  ${process.env.NEXT_REVALIDATION_SECRET}`, {
+    method: "POST",
+  });
+
+  // 4. return scrape result
   res.send(feedback);
 });
 

functions/src/scraper.ts

@@ -32,7 +32,7 @@ export const runScraper = async () => {
       for (let i = 0; i < data.length; i = i + col) {
         const row: Row = {};
         for (let j = 0; j < col; j++) {
-          row[headers[j]] = data[i + j];
+          row[headers[j].toLowerCase()] = data[i + j];
         }
         rows.push(row);
       }