Merge branch 'feat/moderation-log' into feat/moderation-log

.github/workflows/docker-publish.yml

@@ -0,0 +1,92 @@
+name: Publish Docker Image
+
+on:
+  push:
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+    name: Build and push Docker image 
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        uses: sigstore/cosign-installer@v3.6.0
+        with:
+          cosign-release: 'v2.4.0'
+
+      # Set up BuildKit Docker container builder to be able to build
+      # multi-platform images and export cache
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.6.1
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3.3.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5.5.1
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v6.7.0
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        env:
+          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
+
+      # Delete old package versions
+      # https://github.com/actions/delete-package-versions
+      - name: Delete old package versions
+        uses: actions/delete-package-versions@v5
+        with:
+          package-name: 'hikka'
+          package-type: 'container'
+          min-versions-to-keep: 8

.github/workflows/run-test.yml

@@ -0,0 +1,51 @@
+name: Run Test
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  tests:
+    name: Test
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_DB: postgres        
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: user
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          
+    steps:
+      - name: Checkout git repository
+        uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-root
+
+      - name: Copy settings.toml for configuration test database
+        run: cp docs/settings.example.toml settings.toml        
+
+      - name: Run tests
+        run: poetry run pytest
+

Dockerfile

@@ -0,0 +1,43 @@
+FROM python:3.12.5-alpine3.20 as base
+
+ENV VIRTUAL_ENV=/project/.venv \
+    PATH="/project/.venv/bin:$PATH"
+
+
+FROM base as builder
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache \
+    PIP_ROOT_USER_ACTION=ignore
+
+RUN apk add gcc python3-dev musl-dev linux-headers
+
+RUN pip install poetry==1.8.3
+
+WORKDIR /project
+
+COPY pyproject.toml poetry.lock ./
+RUN touch README.md
+
+RUN poetry install --no-root && rm -rf $POETRY_CACHE_DIR
+
+
+
+FROM base as runtime
+
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+WORKDIR /project
+
+# app source files
+COPY sync.py .
+COPY aggregator.py .
+COPY app ./app
+
+# db migrations files
+COPY alembic ./alembic
+COPY docs/alembic.example.ini ./alembic.ini
+
+CMD uvicorn --factory app:create_app --host 0.0.0.0 --port 8000

aggregator.py

@@ -28,24 +28,41 @@ async def import_aggregator():
 
     sessionmanager.init(settings.database.endpoint)
 
+    print("Genres")
     await aggregator_genres()
+    print("Roles")
     await aggregator_roles()
+    print("Characters")
     await aggregator_characters()
+    print("Companies")
     await aggregator_companies()
+    print("Magazines")
     await aggregator_magazines()
+    print("People")
     await aggregator_people()
+    print("Anime")
     await aggregator_anime()
+    print("Manga")
     await aggregator_manga()
+    print("Novel")
     await aggregator_novel()
+    print("Anime info")
     await aggregator_anime_info()
+    print("Manga info")
     await aggregator_manga_info()
+    print("Novel info")
     await aggregator_novel_info()
+    print("Franchises")
     await aggregator_franchises()
+    print("Schedule")
     await update_schedule_build()
+    print("Search")
     await update_search()
+    print("Content")
     await update_content()
 
     # TODO: improve performance
+    print("Weights")
     await update_weights()
 
     await sessionmanager.close()

alembic/env.py

@@ -8,6 +8,8 @@
 from alembic import context
 
 from app.models import Base
+from app.utils import get_settings
+
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -28,7 +30,8 @@
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
-
+settings = get_settings()
+config.set_main_option('sqlalchemy.url', settings.database.endpoint)
 
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.

alembic/versions/2024_07_26_1356-21009df4f5f5_introduce_third_party_clients.py

@@ -0,0 +1,90 @@
+"""Introduce third-party clients
+
+Revision ID: 21009df4f5f5
+Revises: 4c13fdf8868d
+Create Date: 2024-07-26 13:56:21.385422
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "21009df4f5f5"
+down_revision = "4c13fdf8868d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "service_clients",
+        sa.Column("secret", sa.String(length=128), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("description", sa.String(), nullable=False),
+        sa.Column("endpoint", sa.String(), nullable=False),
+        sa.Column("user_id", sa.Uuid(), nullable=True),
+        sa.Column("created", sa.DateTime(), nullable=False),
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["service_users.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "service_auth_token_requests",
+        sa.Column("user_id", sa.Uuid(), nullable=True),
+        sa.Column("expiration", sa.DateTime(), nullable=False),
+        sa.Column("created", sa.DateTime(), nullable=False),
+        sa.Column("client_id", sa.Uuid(), nullable=True),
+        sa.Column(
+            "scope", postgresql.JSONB(astext_type=sa.Text()), nullable=False
+        ),
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["client_id"], ["service_clients.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["service_users.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.add_column(
+        "service_auth_tokens", sa.Column("client_id", sa.Uuid(), nullable=True)
+    )
+    op.add_column(
+        "service_auth_tokens",
+        sa.Column(
+            "scope",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="[]",
+            nullable=False,
+        ),
+    )
+    op.create_foreign_key(
+        "service_auth_tokens_client_id_fkey",
+        "service_auth_tokens",
+        "service_clients",
+        ["client_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "service_auth_tokens_client_id_fkey",
+        "service_auth_tokens",
+        type_="foreignkey",
+    )
+    op.drop_column("service_auth_tokens", "scope")
+    op.drop_column("service_auth_tokens", "client_id")
+    op.drop_table("service_auth_token_requests")
+    op.drop_table("service_clients")
+    # ### end Alembic commands ###

alembic/versions/2024_07_26_2028-96c0d6b7aeba_add_client_updated_column.py

@@ -0,0 +1,28 @@
+"""Add client updated column
+
+Revision ID: 96c0d6b7aeba
+Revises: 21009df4f5f5
+Create Date: 2024-07-26 20:28:17.602711
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '96c0d6b7aeba'
+down_revision = '21009df4f5f5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('service_clients', sa.Column('updated', sa.DateTime(), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('service_clients', 'updated')
+    # ### end Alembic commands ###