update

.github/workflows/mkdocs-deploy.yml

@@ -35,4 +35,17 @@ jobs:
           restore-keys: |
             mkdocs-material-
       - run: pip install mkdocs-material 
-      - run: mkdocs gh-deploy --force
+      - run: mkdocs gh-deploy --force
+      # commit
+      - name: Commit files
+        run: |
+         git config --local user.email "action@github.com"
+         git config --local user.name "GitHub Action"
+         git add -A
+         git commit -m "update data" -a
+      # push
+      - name: Push changes
+        uses: ad-m/github-push-action@v0.6.0
+        with:
+         github_token: ${{ secrets.GITHUB_TOKEN }}
+         branch: main

app/code/literatureFetch.py

@@ -19,13 +19,11 @@
     # 'limit': 5,
     'publicationTypes': 'JournalArticle',
     # 'year': '2020-',
-    'fields': 'paperId,url,journal,\
-               title,publicationTypes,publicationDate,\
-               citationCount,publicationVenue',
+    'fields': 'paperId,url,journal,title,publicationTypes,publicationDate,citationCount,publicationVenue',
     # 'sort': 'citationCount:desc',
     'token': None
 }
-N = 10
+N = 50
 DIC = {}
 
 def fetch_articles(search_query,
@@ -48,7 +46,6 @@ def fetch_articles(search_query,
         while True:
             # Make the GET request to the paper search endpoint with the URL and query parameters
             search_response = requests.get(URL, params=query_params, timeout=None)
-            print ('status code', search_response.status_code)
             # WHen the status code is 429, sleep for 5 minutes
             # if search_response.status_code == 429:
             #     status_code_429 += 1
@@ -58,8 +55,13 @@ def fetch_articles(search_query,
             #         time.sleep(310)
             #         continue
             # When the status code is 200, break the loop
-            if search_response.status_code == 200:
+            if search_response.status_code != 200:
+                print ('status code', search_response.status_code)
+                print ('Sleeping for 3 seconds....')
+                time.sleep(3)
+            else:
                 break
+
         search_response_json = search_response.json()
         fetched_data += search_response_json['data']
         # End the loop if we have fetched enough data
@@ -70,9 +72,25 @@ def fetch_articles(search_query,
         # if the token is not None
         if search_response_json['token'] is not None:
             query_params['token'] = search_response_json['token']
+
+    # fix the publicationVenue and journal
+    for paper in fetched_data:
+        # print (paper)
+        publication_venue = paper['publicationVenue']
+        if publication_venue is None:
+            paper['publicationVenue'] = 'NA'
+        elif 'name' in publication_venue:
+            paper['publicationVenue'] = publication_venue['name']
+        else:
+            paper['publicationVenue'] = ''
+        journal = paper['journal']
+        if journal is None:
+            paper['journal'] = 'NA'
+        elif 'name' in journal:
+            paper['journal'] = journal['name']
     return fetched_data
 
-def create_template(template_file, category_name) -> str:
+def create_template(template_file, category_name, df) -> str:
     """
     Return the markdown content for a given template
 
@@ -94,7 +112,9 @@ def create_template(template_file, category_name) -> str:
         category_name=category_name,
         title=DIC[category_name]['title'],
         query=DIC[category_name]['query'],
-        hide_nav="---\nhide:\n\t- navigation---\n",
+        x_year=df['Year'].tolist(),
+        y_num_articles=df['num_articles'].tolist(),
+        y_num_citations=df['num_citations'].tolist()
     )
     # return markdownify.markdownify(content)
     return content
@@ -109,6 +129,7 @@ def main():
     Returns:
         None
     """
+
     # Work with all the categories in the file
     with open('../data/query.tsv', 'r', encoding='utf-8') as f:
         for line in f:
@@ -122,14 +143,15 @@ def main():
             ## Fetch the most cited articles
             data = fetch_articles(query)
             DIC[category_name] = {'title': title, 'query': query, 'most_cited_articles': data}
-            plot = utils.metrics_over_time(data, category_name, title)
-            plot.savefig(f'../../docs/assets/{category_name}.png')
+            # plot = utils.metrics_over_time_js(data, category_name, title)
+            # plot.savefig(f'../../docs/assets/{category_name}.png')
+            df = utils.metrics_over_time_js(data, category_name, title)
             ################################
             ## Fetch the most recent articles
             data = fetch_articles(query, sort = 'publicationDate:desc')
             DIC[category_name]['most_recent_articles'] = data
             # print (data[0])
-            markdown_text = create_template("category.txt", category_name)
+            markdown_text = create_template("category.txt", category_name, df)
                                             # DIC[category_name]['most_cited_articles'][0:N],
                                             # DIC[category_name]['most_recent_articles'][0:N])
             # Add the hide navigation
@@ -147,14 +169,15 @@ def main():
     ## Fetch the most cited articles
     data = fetch_articles(query)
     DIC[category_name] = {'title': title, 'query': query, 'most_cited_articles': data}
-    plot = utils.metrics_over_time(data, category_name, title)
-    plot.savefig(f'../../docs/assets/{category_name}.png')
+    # plot = utils.metrics_over_time_js(data, category_name, title)
+    # plot.savefig(f'../../docs/assets/{category_name}.png')
+    df = utils.metrics_over_time_js(data, category_name, title)
     ################################
     ## Fetch the most recent articles
     data = fetch_articles(query, sort = 'publicationDate:desc')
     DIC[category_name]['most_recent_articles'] = data
     # print (data[0])
-    markdown_text = create_template("category.txt", category_name)
+    markdown_text = create_template("category.txt", category_name, df)
                                     # DIC[category_name]['most_cited_articles'][0:N],
                                     # DIC[category_name]['most_recent_articles'][0:N])
     # Add the hide navigation
@@ -163,6 +186,20 @@ def main():
     with open(f'../../docs/{category_name}.md', 'w', encoding='utf-8') as file:
         file.write(markdown_text)
     ################################
+
+    # Read YAML file
+    file_path = '../../base.yml'
+    data = utils.read_yaml(file_path)
+    print (data['nav'])
+
+    # Add more stuff to the YAML data
+    data['nav'] = []
+    for category_name, category_items in DIC.items():
+        data['nav'].append({category_items['title']: category_name + '.md'})
+
+    print (data['nav'])
+    # Write modified YAML data back to file
+    # write_yaml(data, file_path)
 
 if __name__ == '__main__':
     # Run the main function

app/code/utils.py

@@ -6,6 +6,7 @@
 
 import matplotlib.pyplot as plt
 import pandas as pd
+import yaml
 
 def metrics_over_time(data, category_name, title) -> plt:
     """
@@ -75,4 +76,63 @@ def metrics_over_time(data, category_name, title) -> plt:
     return plt
 
 
-    #A26, B1
+    #A26, B1
+
+def metrics_over_time_js(data, category_name, title) -> plt:
+    """
+    Return the metrics over time
+
+    Args:
+        data (list): list of dictionaries
+        category_name (str): category name
+        title (str): title of the graph
+    
+    Returns:
+        None
+
+    Example:
+    data = [
+        {
+            'title': 'title1',
+            'publicationDate': '2020-01-01',
+            'citationCount': 10
+        },
+        {
+            'title': 'title2',
+            'publicationDate': '2020-01-01',
+            'citationCount': 10
+        }
+    ]
+    """
+    dic = {}
+    for paper in data:
+        publication_date = paper['publicationDate']
+        if publication_date is None or publication_date == '':
+            continue
+        year = publication_date.split('-')[0]
+        if year not in dic:
+            dic[year] = {'num_articles': 0, 'num_citations': 0}
+        dic[year]['num_articles'] += 1
+        citation_count = paper['citationCount']
+        if citation_count is None or citation_count == '':
+            continue
+        dic[year]['num_citations'] += citation_count
+    # Using noc and yop, plot the line graph with years on x-axis and number of citations on y-axis
+    df = pd.DataFrame(dic).T
+    # Make another colum for the year
+    df['Year'] = df.index
+    # Sort by year
+    df = df.sort_values(by='Year', ascending=True)
+    # print (df)
+    return df
+
+# Function to read YAML file
+def read_yaml(file_path):
+    with open(file_path, 'r') as file:
+        data = yaml.safe_load(file)
+    return data
+
+# Function to write YAML file
+def write_yaml(data, file_path):
+    with open(file_path, 'w') as file:
+        yaml.dump(data, file, default_flow_style=False)

app/data/query2.tsv

@@ -0,0 +1,2 @@
+Title	Query
+Neural ODEs	Neural ODEs