write to json file

atlantabitdevs · Jul 26, 2023 · c8a7faa · c8a7faa
1 parent a666969
commit c8a7faa
Show file tree

Hide file tree

Showing 4 changed files with 350 additions and 160 deletions.
diff --git a/ai/src/summerize.ts → ai/src/get_scraped_data.ts b/ai/src/summerize.ts → ai/src/get_scraped_data.ts
@@ -10,7 +10,7 @@ type Link = {
   link: string;
 };
 
-export const summarize = async ({}) => {
+const getScrapedData = async ({}) => {
   const eventsDir = path.join(__dirname, '../', '../', 'posts');
 
   const files = readdirSync(eventsDir);
@@ -35,18 +35,15 @@ export const summarize = async ({}) => {
 
   const eventPath = path.join(eventsDir, mostRecentEvent);
 
-  console.log(eventPath);
-  console.log(mostRecentEvent);
-
-  const summaryPath = path.join(__dirname, '../', '../' , 'summaries', mostRecentEvent.replace('new-event', 'summary')).replace('.md', '.json');
-
-  console.log(summaryPath);
+  const summaryPath = path
+    .join(__dirname, '../', '../', 'summaries', mostRecentEvent.replace('new-event', 'summary'))
+    .replace('.md', '.json');
 
   const links = await getLinks({
     path: eventPath,
   });
 
-  const results = (await map(links, scrapeFiles)).filter(n => n.text !== '' || n.title !== '');
+  const results = (await map(links, scrapeFiles)).filter(n => !!n.text && !!n.title);
 
   return { results, summaryPath };
 };
@@ -56,3 +53,5 @@ const scrapeFiles = async (link: Link) => {
 
   return { text, title: link.title };
 };
+
+export default getScrapedData;
diff --git a/ai/src/index.ts b/ai/src/index.ts
@@ -4,16 +4,18 @@ import { Configuration, OpenAIApi } from 'openai';
 import { ScrapedDataResults, SummaryResults } from './types';
 import { auto, mapLimit } from 'async';
 
-import { summarize } from './summerize';
+import getScrapedData from './get_scraped_data';
 import writeSummary from './write_summary';
 
 dotenv.config();
 
+const model = 'gpt-3.5-turbo-16k';
+
 type Tasks = {
   initAi: {
     openai: OpenAIApi;
   };
-  getSummary: {
+  getScrapedData: {
     results: ScrapedDataResults[];
     summaryPath: string;
   };
@@ -33,10 +35,10 @@ const main = async () => {
         return { openai };
       },
 
-      getSummary: [
+      getScrapedData: [
         'initAi',
         async ({ initAi }) => {
-          const { results, summaryPath } = await summarize({});
+          const { results, summaryPath } = await getScrapedData({});
 
           // try {
           //   const aiResults = await Promise.all([
@@ -64,32 +66,42 @@ const main = async () => {
           //   console.log(err.response.data);
           // }
 
-          // await writeSummary({ path: summaryPath, data: results });
+          // const newResults = results.map(n => {
+          //   return {
+          //     title: n.title,
+          //     summary: n.text,
+          //     summaryeli15: '',
+          //   };
+          // });
+
+          // await writeSummary({ path: summaryPath, data: newResults });
 
           return { results, summaryPath };
         },
       ],
 
       summarize: [
-        'getSummary',
+        'getScrapedData',
         'initAi',
-        async ({ getSummary, initAi }) => {
+        async ({ getScrapedData, initAi }) => {
           const aiResults = async (summary: ScrapedDataResults) => {
             try {
               const aiCompletion = await Promise.all([
                 initAi.openai.createChatCompletion({
-                  model: 'gpt-3.5-turbo',
-                  messages: [{ role: 'user', content: `Can you explaint his for me? \n ${summary.text}` }],
+                  model,
+                  messages: [
+                    { role: 'user', content: `Can you explaint this in great detail for me? \n ${summary.text}` },
+                  ],
                 }),
                 initAi.openai.createChatCompletion({
-                  model: 'gpt-3.5-turbo',
-                  messages: [{ role: 'user', content: `Can you explain this like I am 15? ${summary.text} ` }],
+                  model,
+                  messages: [
+                    { role: 'user', content: `Can you explain this in great detail like I am 15? ${summary.text} ` },
+                  ],
                 }),
               ]);
-
               const chatCompletion = aiCompletion[0].data.choices[0].message?.content || 'No summary generated';
               const chatCompletionEli15 = aiCompletion[1].data.choices[0].message?.content || 'No summary generated';
-
               return {
                 summary: chatCompletion,
                 summaryeli15: chatCompletionEli15,
@@ -100,12 +112,10 @@ const main = async () => {
               console.log(err.response.data);
             }
           };
-
-          const results = (await mapLimit(getSummary.results, 10, aiResults)).filter(
+          const results = (await mapLimit(getScrapedData.results, 3, aiResults)).filter(
             (result): result is SummaryResults => result !== undefined
           );
-
-          await writeSummary({ path: getSummary.summaryPath, data: results });
+          await writeSummary({ path: getScrapedData.summaryPath, data: results });
         },
       ],
     });

diff --git a/ai/src/scrape.ts b/ai/src/scrape.ts
@@ -31,6 +31,7 @@ const scrape = async ({ url }: Args): Promise<Return> => {
         return text;
       })
       .get()
+      .filter(n => !!n)
       .filter(text => text.length > 50 && text.length < 500) // Filters out too short or too long paragraphs
       .join('\n');
 
@@ -43,8 +44,13 @@ const scrape = async ({ url }: Args): Promise<Return> => {
         return text;
       })
       .get()
+      .filter(n => !!n)
       .join('\n');
 
+    if (!paragraphs && !preTags) {
+      return { title, text: '' };
+    }
+
     const scrapedText = [paragraphs, preTags].join('\n\n');
 
     return { title, text: scrapedText };