API key for semantic scholar (#7)

* added api key for semantic scholar

* added docs preview and test with poetry

.github/workflows/docs-preview.yaml

@@ -0,0 +1,41 @@
+name: Deploy PR Docs previews
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - closed
+
+concurrency: preview-${{ github.ref }}
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.6
+    - uses: actions/setup-python@v5.1.0
+      with:
+        python-version: "3.12"
+    - uses: pre-commit/action@v3.0.1
+  deploy-preview:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.1.6
+      - uses: actions/setup-python@v5.1.0
+        with:
+          python-version: "3.12"
+      - name: Setup Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: 1.8.3
+      - name: Install Packages
+        run: poetry install --with docs,dev
+      - run: git config user.name 'github-actions[bot]' && git config user.email 'github-actions[bot]@users.noreply.github.com'
+      - name: Build Docs
+        run: poetry run mkdocs build
+      - name: Deploy preview
+        uses: rossjrw/pr-preview-action@v1.4.7
+        with:
+          source-dir: ./site/

.github/workflows/docs-publish.yaml

@@ -5,23 +5,40 @@ on:
       - main
   workflow_dispatch:
     inputs:
-      name:
-        description: 'Publish docs'
+      directory:
+        description: 'Build docs'
+        default: 'build it now'
         required: false
-        default: 'I Just wanna publish the docs'
+
 
 jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.6
+    - uses: actions/setup-python@v5.1.0
+      with:
+        python-version: "3.12"
+    - uses: pre-commit/action@v3.0.1
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4.1.6
+      - uses: actions/setup-python@v5.1.0
+        with:
+          python-version: "3.12"
+      - name: Setup Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: 1.8.3
+      - name: Install Packages
+        run: poetry install --with docs,dev
+      - run: git config user.name 'github-actions[bot]' && git config user.email 'github-actions[bot]@users.noreply.github.com'
+      - name: Build Docs
+        run: poetry run mkdocs build
+      - name: Deploy 🚀
+        uses: JamesIves/github-pages-deploy-action@v4.6.1
         with:
-          python-version: 3.8
-      - run: pip install mkdocs
-      - run: pip install mkdocs-material
-      - run: pip install mkdocs-material-extensions
-      - run: pip install mkdocs-autorefs
-      - run: pip install mkdocstrings
-      - run: pip install -e ".[all]"
-      - run: mkdocs gh-deploy --force
+          folder: site
+          clean-exclude: pr-preview/
+          force: false

.github/workflows/publish-with-poetry.yaml

@@ -0,0 +1,55 @@
+name: Build and Publish Package to Pypi
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  tests:
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.6
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5.1.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: 1.8.3
+      - name: Install Packages
+        run: poetry install --with dev,docs
+      - name: Build coverage file
+        run: |
+          poetry run pytest
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.1.6
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5.1.0
+        with:
+          python-version: "3.12"
+      - name: Setup Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: 1.8.3
+      - name: Install dependencies
+        run: |
+          poetry --version
+          poetry install
+      - name: Build and publish
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        env:
+          PYPI_USERNAME: __token__
+          PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          poetry publish --build -u $PYPI_USERNAME -p $PYPI_PASSWORD

.github/workflows/tests.yaml

@@ -1,10 +1,6 @@
-name: Test Code with Pip
+name: Test Code
 
 on:
-  push:
-    branches:
-      - "**"
-      - "!gh-pages"
   pull_request:
     branches:
       - main
@@ -19,20 +15,28 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ["3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4.1.6
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install from source (required for the pre-commit tests)
-        run: pip install ".[all]"
-      - name: Test with pytest
+      - name: Setup Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: 1.8.3
+      - name: Install Packages
+        run: poetry install --with docs,dev
+      - name: Build coverage file
         run: |
-          pytest --cache-clear --cov=app tests/ > pytest-coverage.txt
-      - name: Comment coverage
-        uses: coroo/pytest-coverage-commentator@v1.0.2
+          poetry run pytest --cache-clear --junitxml=pytest.xml --cov=kirsche tests/ > pytest-coverage.txt
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@v1.1.51
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml
+          title: Coverage Report (${{ github.event.pull_request.head.sha || github.sha }}, ${{ matrix.python-version }}, ${{ matrix.os }})

.pre-commit-config.yaml

@@ -1,29 +1,47 @@
+exclude: "^notebooks/|^notes/"
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files
       - id: debug-statements
       - id: detect-private-key
       - id: end-of-file-fixer
       - id: requirements-txt-fixer
       - id: trailing-whitespace
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.960
-    hooks:
-      - id: mypy
-        args: [--no-strict-optional, --ignore-missing-imports]
-        files: ^(eerily/|tests/)
   - repo: https://github.com/ambv/black
-    rev: 22.6.0
+    rev: 24.4.2
     hooks:
     - id: black
       language: python
-      args:
-        - "--line-length=120"
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.13.2
     hooks:
       - id: isort
         name: isort (python)
-        args: ["--profile", "black"]
+        args: ["--multi-line", "3", "--profile", "black", "--treat-comment-as-code", "# %%", "--float-to-top"]
+  - repo: local
+    # We do not use pre-commit/mirrors-mypy,
+    # as it comes with opinionated defaults
+    # (like --ignore-missing-imports)
+    # and is difficult to configure to run
+    # with the dependencies correctly installed.
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        language: python
+        # language_version: python3.12
+        additional_dependencies:
+          - mypy
+          - pandas-stubs
+          - pydantic
+          - pytest
+        types:
+          - python
+        # use require_serial so that script
+        # is only called once per commit
+        require_serial: true
+        # Print the number of files as a sanity-check
+        verbose: true
+        exclude: ^docs

kirsche/utils/web.py

@@ -1,10 +1,11 @@
+import os
 import random
+from typing import Optional, Union
 
 import requests
 from loguru import logger
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
-from typing import Union, Optional
 
 
 def get_random_user_agent(browsers: Optional[Union[str, list]] = None) -> dict:
@@ -129,6 +130,9 @@ def get_session_query_configs(
     if headers is None:
         headers = get_random_user_agent()
 
+    if os.getenv("SC_API_KEY") and "x-api-key" not in headers:
+        headers["x-api-key"] = os.getenv("SC_API_KEY")
+
     if timeout is None:
         timeout = (5, 14)
 

kirsche/version.py

@@ -1 +1 @@
-__version__ = "0.2.8"
+__version__ = "0.2.9"