Fix nb_exec.py

.github/workflows/execute-notebooks.yml

@@ -1,8 +1,11 @@
 name: Execute notebooks
 
-# Run on demand
 on:
+  # Run on demand
   workflow_dispatch:
+  # 6.49 am (GMT) every Monday; time chosen at random
+  schedule:
+  - cron: "49 6 * * MON"
 
 # Limit workflow permissions
 permissions:
@@ -18,10 +21,12 @@ env:
 
 jobs:
   execute:
-    name: Execute
+    name: Update notebooks
+    if: github.repository_owner == 'Econ-ARK'
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      pull-requests: write
 
     steps:
     - uses: actions/checkout@v3
@@ -32,28 +37,39 @@ jobs:
         cache: 'pip'
         cache-dependency-path: |
           requirements/base.txt
+          .github/workflows/execute-notebooks.yml
 
-    - name: Install HARK
+    - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         python -m pip install .
-        python -m pip install nbformat nbclient nbstripout 
+        # For LabeledModels.ipynb
+        python -m pip install estimagic
+        # For nbstripout
+        python -m pip install nbstripout
+        # For nb_exec.py
+        python -m pip install ipykernel nbclient nbformat
 
     - name: Strip output
       run: nbstripout examples/**/*.ipynb
 
+    # This step takes c. 20 minutes
     - name: Execute notebooks
       run: python tools/nb_exec.py examples/**/*.ipynb
+      env:
+        PYTHONUNBUFFERED: "1"
 
     - name: Open PR
       uses: peter-evans/create-pull-request@v5
       with:
         author: "Econ-ARK Bot <noreply@econ-ark.org>"
         branch: "bot/update-notebooks"
-        token: ${{ secrets.GITHUB_TOKEN }}
         commit-message: "[bot] updated notebooks"
         delete-branch: true
         title: "[bot] Execute example notebooks"
-        body: |
-          This PR was automatically generated to re-execute
+        # language=Markdown
+        body: >
+          This PR was [automatically generated] to re-execute
           the example notebooks for use in the documentation.
+          
+          [automatically generated]: https://github.com/Econ-ARK/HARK/actions/workflows/execute-notebooks.yml

tools/nb_exec.py

@@ -10,34 +10,43 @@
 per processor in the host machine.
 """
 
-from pathlib import Path
+import multiprocessing
+import os
 import sys
-from concurrent.futures import ProcessPoolExecutor
+import time
+from pathlib import Path
 
 import nbformat
 from nbclient import NotebookClient
 
-ROOT_DIR = Path(__file__).parent.parent
+ROOT_DIR = Path(__file__).resolve().parent.parent
 
 
-def run_notebook(notebook_file):
+def run_notebook(notebook_file: Path):
     rel_file_name = notebook_file.relative_to(ROOT_DIR).as_posix()
     print(f'{rel_file_name}: Loading notebook')
-    nb = nbformat.read(notebook_file, as_version=4)
-    client = NotebookClient(nb, timeout=600, kernel_name='python3', record_timing=False)
-    print(f'{rel_file_name}: Executing')
-    client.execute()
-    print(f'{rel_file_name}: Writing')
-    nbformat.write(nb, notebook_file)
-    print(f'{rel_file_name}: Finished')
-    del nb, client
+    try:
+        # Journey-PhD and LifecycleModel expect execution from their own directory
+        os.chdir(notebook_file.parent)
+        nb = nbformat.read(notebook_file, as_version=4)
+        client = NotebookClient(nb, timeout=600, kernel_name='python3', record_timing=False)
+        print(f'{rel_file_name}: Executing')
+        start = time.perf_counter()
+        client.execute()
+        elapsed = time.perf_counter() - start
+        print(f'{rel_file_name}: Writing')
+        nbformat.write(nb, notebook_file)
+        print(f'{rel_file_name}: Finished (executed in {elapsed:.2f}s)')
+        del nb, client, start, elapsed
+    except Exception as err:
+        print(f'{rel_file_name}: Failed to execute\n   {err}', file=sys.stderr)
 
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:
-        notebooks = sys.argv[1:]
+        notebooks = (Path(p).resolve() for p in sys.argv[1:])
     else:
         notebooks = ROOT_DIR.joinpath('examples').rglob('*.ipynb')
 
-    with ProcessPoolExecutor() as pool:
+    with multiprocessing.Pool() as pool:
         pool.map(run_notebook, notebooks)