diff --git a/.github/workflows/bump_version.yaml b/.github/workflows/bump_version.yaml new file mode 100755 index 0000000..78b31c6 --- /dev/null +++ b/.github/workflows/bump_version.yaml @@ -0,0 +1,64 @@ +# Bump version with Commitizen + +name: ๐Ÿš€ Bump version + +on: + push: + branches: + - main + +jobs: + bump_version: + name: ๐Ÿš€ Bump and changelog + runs-on: ubuntu-latest + if: contains(github.event.head_commit.message, 'Merge') + environment: + name: release + url: https://github.com/Michele-Alberti/data-lunch/tags + outputs: + version: ${{ steps.cz.outputs.version }} + steps: + - name: Set pre-release + id: vars + env: + TITLE: ${{ github.event.pull_request.title }} + run: | + pre_release=$(awk -F'[][]' '{print $2}' <<< "$TITLE") + echo "pre_release=$pre_release" >> $GITHUB_OUTPUT + echo "pre-release: $pre_release" + - name: Check out + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.CUSTOM_GITHUB_TOKEN }} + - name: Bump and changelog + id: cz + uses: commitizen-tools/commitizen-action@0.16.1 + with: + github_token: ${{ secrets.CUSTOM_GITHUB_TOKEN }} + prerelease: "${{ steps.vars.outputs.pre_release }}" + - name: Print version + run: echo "Bumped to version ${{ steps.cz.outputs.version }}" + + merge_back_to_dev: + name: ๐Ÿ”— Merge back + needs: bump_version + runs-on: ubuntu-latest + environment: + name: release + url: https://github.com/Michele-Alberti/data-lunch/commits/development + steps: + - name: Check out + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: main + - name: Set Git config + run: | + git config --local user.name "github-actions[bot]" + git config --local user.email "github-actions[bot]@users.noreply.github.com" + - name: Merge master back to dev + run: | + git checkout development + git merge --no-ff main -m "Merge branch 'main' into development" + git push diff --git a/.github/workflows/ci_cd.yaml b/.github/workflows/ci_cd.yaml new file mode 100644 index 0000000..610edc9 --- /dev/null +++ b/.github/workflows/ci_cd.yaml @@ -0,0 +1,196 @@ +# Continuous improvement and delivery + +name: ๐Ÿ Publish and release ๐Ÿ“ฆ + +on: + push: + branches: + - main + - development + +jobs: + # Build the package for PyPI + build: + name: ๐Ÿ› ๏ธ Build ๐Ÿ“ฆ + # Only publish to PyPI on tag pushes and on TestPyPI if branch is development + if: startsWith(github.ref_name, 'refs/tags/v') + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + # Publish to PyPI only for tags that start with 'v' + publish-to-pypi: + name: >- + ๐Ÿ Publish ๐Ÿ“ฆ to PyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: release + url: https://pypi.org/p/dlunch + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + # Then release to github + github-release: + name: ๐Ÿ›ซ Release ๐Ÿ“ฆ on GitHub + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v1.2.3 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' + + # And build container + build_container: + name: ๐Ÿ‹ Build Docker image + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + environment: + name: release + url: https://hub.docker.com/r/michelealberti/data-lunch-app + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: ./ + file: ./docker/web/Dockerfile.web + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ secrets.DOCKER_HUB_USERNAME }}/data-lunch-app:stable + cache-from: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/data-lunch-app:buildcache + cache-to: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/data-lunch-app:buildcache,mode=max + + # If on development just publish to TestPyPI + # Build the package for TestPyPI + test-build: + name: ๐Ÿšง Build test ๐Ÿ“ฆ + # Only publish to PyPI on tag pushes and on TestPyPI if branch is development + if: github.ref_name == 'development' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Change package version to unique value + run: | + test_ver=$(git describe | sed 's/\(.*-.*\)-.*/\1/') + echo test version: $test_ver + sed -i "s/version = \".*\"/version = \"$test_ver\"/g" pyproject.toml + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + publish-to-testpypi: + name: ๐Ÿงช Publish ๐Ÿ“ฆ to TestPyPI + needs: + - test-build + runs-on: ubuntu-latest + + environment: + name: test-release + url: https://test.pypi.org/p/dlunch + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution package to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/manual_build.yaml b/.github/workflows/manual_build.yaml index e73f3a9..e66e2dc 100755 --- a/.github/workflows/manual_build.yaml +++ b/.github/workflows/manual_build.yaml @@ -1,6 +1,6 @@ -# Bump version with Commitizen +# Manual Docker build -name: Manual build +name: ๐Ÿช› Manual build on: workflow_dispatch: @@ -14,15 +14,18 @@ on: jobs: build_container: - name: Build container image + name: ๐Ÿ‹ Build Docker image runs-on: ubuntu-latest + environment: + name: test-release + url: https://hub.docker.com/r/michelealberti/data-lunch-app steps: - name: Print build info run: | echo "Image tag: ${{ github.event.inputs.tag }}" echo "Description: ${{ github.event.inputs.build_description }}" - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Login to DockerHub uses: docker/login-action@v2 with: diff --git a/.github/workflows/release_new_version.yaml b/.github/workflows/release_new_version.yaml deleted file mode 100755 index 6c29fd3..0000000 --- a/.github/workflows/release_new_version.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Bump version with Commitizen - -name: Release new version - -on: - push: - branches: - - main - -jobs: - bump_version: - runs-on: ubuntu-latest - if: contains(github.event.head_commit.message, 'Merge') - name: Bump version and log - outputs: - version: ${{ steps.cz.outputs.version }} - steps: - - name: Check out - uses: actions/checkout@v3 - with: - fetch-depth: 0 - token: ${{ secrets.CUSTOM_GITHUB_TOKEN }} - - name: Bump and changelog - id: cz - uses: commitizen-tools/commitizen-action@0.16.1 - with: - github_token: ${{ secrets.CUSTOM_GITHUB_TOKEN }} - - name: Print version - run: echo "Bumped to version ${{ steps.cz.outputs.version }}" - merge_back_to_dev: - needs: bump_version - runs-on: ubuntu-latest - if: contains(github.event.head_commit.message, 'Merge') - name: Merge back to dev - steps: - - name: Check out - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: main - token: ${{ secrets.CUSTOM_GITHUB_TOKEN }} - - name: Set Git config - run: | - git config --local user.name "github-actions[bot]" - git config --local user.email "github-actions[bot]@users.noreply.github.com" - - name: Merge master back to dev - run: | - git checkout development - git merge --no-ff main -m "Merge branch 'main' into development" - git push - build_container: - needs: merge_back_to_dev - name: Build container image - runs-on: ubuntu-latest - if: contains(github.event.head_commit.message, 'Merge') - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: main - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: ./ - file: ./docker/web/Dockerfile.web - builder: ${{ steps.buildx.outputs.name }} - push: true - tags: ${{ secrets.DOCKER_HUB_USERNAME }}/data-lunch-app:stable - cache-from: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/data-lunch-app:buildcache - cache-to: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/data-lunch-app:buildcache,mode=max diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bde2658..aa4e644 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: black language_version: python3 - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 - repo: https://github.com/commitizen-tools/commitizen diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..5580e3c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include dlunch/conf * +recursive-include dlunch/static * +recursive-include dlunch/templates * +include dlunch/quotes.xlsx \ No newline at end of file diff --git a/README.md b/README.md index 90390e5..097cd0a 100755 --- a/README.md +++ b/README.md @@ -8,6 +8,10 @@ The ultimate web app for a well organized lunch. - [2. Development environment setup](#2-development-environment-setup) - [2.1. Miniconda](#21-miniconda) - [2.2. Environment variables](#22-environment-variables) + - [2.2.1. General](#221-general) + - [2.2.2. Docker and Google Cloud Platform](#222-docker-and-google-cloud-platform) + - [2.2.3. TLS/SSL Certificate](#223-tlsssl-certificate) + - [2.2.4. Encryption and Authorization](#224-encryption-and-authorization) - [2.3. Setup the development environment](#23-setup-the-development-environment) - [2.4. Manually install data-lunch CLI](#24-manually-install-data-lunch-cli) - [2.5. Running the docker-compose system](#25-running-the-docker-compose-system) @@ -42,27 +46,48 @@ conda activate data-lunch ### 2.2. Environment variables The following environment variables are required for running the _web app_, the _makefile_ or _utility scripts_. -| Variable | Type | Example Value | + +#### 2.2.1. General +| Variable | Type | Description | |----------|:------:|-------| -`PANEL_APP` | _str_ | data-lunch-app (used by `makefile`) -`PANEL_ENV` | _str_ | development +`PANEL_APP` | _str_ | app name, _data-lunch-app_ by default (used by `makefile`) +`PANEL_ENV` | _str_ | environment, e.g. _development_, _quality_, _production_ `PANEL_ARGS` | _str_ | additional arguments passed to _Hydra_ (e.g. `panel/gui=major_release`) -`PORT` | _int_ | 5000 +`PORT` | _int_ | port used bu the web app (or the container), default to _5000_ + +#### 2.2.2. Docker and Google Cloud Platform +| Variable | Type | Description | +|----------|:------:|-------| `DOCKER_USERNAME` | _str_ | your _Docker Hub_ username, used by `makefile` and stats panel to extract container name (optional) +`IMAGE_VERSION` | _str_ | _Docker_ image version, typically `stable` or `latest` `GCLOUD_PROJECT` | _str_ | _Google Cloud Platform_ `project_id`, used by `makefile` for _GCP's CLI_ authentication and for uploading the database to _gcp_ storage, if active in web app configuration files (see panel.scheduled_tasks) `GCLOUD_BUCKET` | _str_ | _Google Cloud Platform_ `bucket`, used for uploading database to _gcp_ storage, if active in web app configuration files (see panel.scheduled_tasks) -`CERT_EMAIL` | _str_ | email for _SSL certificates_ -`DOMAIN` | _str_ | mywebapp.com (domain name) -`MAIL_USER` | _str_ | mywebappemail@email.com (email client user, used for sending emails with the current instance IP) -`MAIL_APP_PASSWORD` | _str_ | email client password (used for sending emails with the current instance IP) -`MAIL_RECIPIENTS` | _str_ | email recipients as string, separated by `,` (used for sending emails with the current instance IP) -`DUCKDNS_URL` | _str_ | _URL_ used in `compose_init.sh` to update dynamic address (see _Duck DNS's_ instructions for details) -`IMAGE_VERSION` | _str_ | _stable_ (_Docker_ image version, typically `stable` or `latest`) +`MAIL_USER` | _str_ | email client user, used for sending emails containing the instance IP, e.g._mywebappemail@email.com_ (used only for _Google Cloud Platform_) +`MAIL_APP_PASSWORD` | _str_ | email client password used for sending emails containing the instance IP (used only for _Google Cloud Platform_) +`MAIL_RECIPIENTS` | _str_ | email recipients as string, separated by `,` (used for sending emails containing the instance IP when hosted on _Google Cloud Platform_) +`DUCKDNS_URL` | _str_ | _URL_ used in `compose_init.sh` to update dynamic address (see _Duck DNS's_ instructions for details, used when hosted on _Google Cloud Platform_) + +#### 2.2.3. TLS/SSL Certificate +| Variable | Type | Description | +|----------|:------:|-------| +`CERT_EMAIL` | _str_ | email for registering _SSL certificates_, shared with the authority _Let's Encrypt_ (via `certbot`) +`DOMAIN` | _str_ | domain name, e.g. _mywebapp.com_ + +#### 2.2.4. Encryption and Authorization +| Variable | Type | Description | +|----------|:------:|-------| `DATA_LUNCH_COOKIE_SECRET` | _str_ | _Secret_ used for securing the authentication cookie (use `make generate-secrets` to generate a valid secret) `DATA_LUNCH_OAUTH_ENC_KEY` | _str_ | _Encription key_ used by the OAuth algorithm for encryption (use `make generate-secrets` to generate a valid secret) -`DATA_LUNCH_OAUTH_KEY` | _str_ | _OAuth key_ used for configuring the OAuth provider (_GitHub_) -`DATA_LUNCH_OAUTH_SECRET` | _str_ | _OAuth secret_ used for configuring the OAuth provider (_GitHub_) -`DATA_LUNCH_OAUTH_REDIRECT_URI` | _str_ | _OAuth redirect uri_ used for configuring the OAuth provider (_GitHub_), leave empty to panel use default value +`DATA_LUNCH_OAUTH_KEY` | _str_ | _OAuth key_ used for configuring the OAuth provider (_GitHub_, _Azure_) +`DATA_LUNCH_OAUTH_SECRET` | _str_ | _OAuth secret_ used for configuring the OAuth provider (_GitHub_, _Azure_) +`DATA_LUNCH_OAUTH_REDIRECT_URI` | _str_ | _OAuth redirect uri_ used for configuring the OAuth provider (_GitHub_, _Azure_), do not set to use default value +`DATA_LUNCH_OAUTH_TENANT_ID` | _str_ | _OAuth tenant id_ used for configuring the OAuth provider (_Azure_), do not set to use default value +`DATA_LUNCH_DB_USER` | _str_ | _Postgresql_ user, do not set to use default value +`DATA_LUNCH_DB_PASSWORD` | _str_ | _Postgresql_ password +`DATA_LUNCH_DB_HOST` | _str_ | _Postgresql_ host, do not set to use default value +`DATA_LUNCH_DB_PORT` | _str_ | _Postgresql_ port, do not set to use default value +`DATA_LUNCH_DB_DATABASE` | _str_ | _Postgresql_ database, do not set to use default value +`DATA_LUNCH_DB_SCHEMA` | _str_ | _Postgresql_ schema, do not set to use default value ### 2.3. Setup the development environment diff --git a/data_lunch_app/conf/auth/default.yaml b/data_lunch_app/conf/auth/default.yaml deleted file mode 100644 index 6726a38..0000000 --- a/data_lunch_app/conf/auth/default.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# oauth config (not used by panel.serve) -oauth_encryption_key: ${oc.env:DATA_LUNCH_OAUTH_ENC_KEY} -oauth_expiry: 15 -# Template not set by panel.serve -auth_error_template: data_lunch_app/templates/error.html -# Autorization callback (_partial_ is required for usage with lambda functions) -authorization_callback: - _target_: data_lunch_app.auth.authorize - _partial_: True - # Flag to authorize access of guest users to the home page - # They can only place orders for guests - authorize_guest_users: true -# Set cookie function arguments (set to empty dict to turn off) -cookie_kwargs: - httponly: true - secure: true - samesite: strict - -# BASIC AUTHENTICATION DATA -basic_auth: - # Those values are used by basic authentication specific features - # PASSWORD (used only if basic auth is active) - # Special charachters and reg expression used to check (or create) a password - psw_special_chars: ';:,.+-=_#!$%&?^*@' - psw_regex: ^(?=.*[0-9])(?=.*[a-z])(?=.*[A-Z])(?=.*[${auth.basic_auth.psw_special_chars}]).{8,}$ - # Set generated password length (call to generate_password) - generated_psw_length: 12 - # GUEST USER (used only if basic auth is active) - # If true create default guest user credentials (i.e. a user named "guest") - # It is an unprivileged user (is_guest = true) named guest - guest_user: true - # Guest user password flag default value - # If true a new password for guest user is set everytime the main function is called - default_reset_guest_user_password_flag: false \ No newline at end of file diff --git a/data_lunch_app/conf/server/basic_auth.yaml b/data_lunch_app/conf/server/basic_auth.yaml deleted file mode 100644 index 77b01cc..0000000 --- a/data_lunch_app/conf/server/basic_auth.yaml +++ /dev/null @@ -1,7 +0,0 @@ -defaults: - - /${oc.env:PANEL_ENV} - -auth_provider: - _target_: data_lunch_app.auth.DataLunchProvider - login_template: data_lunch_app/templates/login_basic.html - logout_template: data_lunch_app/templates/logout.html diff --git a/data_lunch_app/__init__.py b/dlunch/__init__.py similarity index 90% rename from data_lunch_app/__init__.py rename to dlunch/__init__.py index d084892..b160f7e 100755 --- a/data_lunch_app/__init__.py +++ b/dlunch/__init__.py @@ -1,6 +1,7 @@ # App metadata __version__ = "2.7.0" +import importlib.resources import pathlib import hydra import logging @@ -14,9 +15,15 @@ from . import core from . import gui from . import auth +from .auth import pn_user log = logging.getLogger(__name__) +# OMEGACONF RESOLVER ---------------------------------------------------------- +OmegaConf.register_new_resolver( + "pkg_path", lambda pkg: str(importlib.resources.files(pkg)) +) + # APP FACTORY FUNCTION -------------------------------------------------------- @@ -45,13 +52,13 @@ def create_app(config: DictConfig) -> pn.Template: # Set guest override flag if it is None (not found in flags table) # Guest override flag is per-user and is not set for guests if ( - models.get_flag(config=config, id=f"{pn.state.user}_guest_override") + models.get_flag(config=config, id=f"{pn_user(config)}_guest_override") is None ) and not auth.is_guest( - user=pn.state.user, config=config, allow_override=False + user=pn_user(config), config=config, allow_override=False ): models.set_flag( - config=config, id=f"{pn.state.user}_guest_override", value=False + config=config, id=f"{pn_user(config)}_guest_override", value=False ) # DASHBOARD BASE TEMPLATE @@ -104,7 +111,7 @@ def create_app(config: DictConfig) -> pn.Template: gi.reload_on_guest_override( toggle=models.get_flag( config=config, - id=f"{pn.state.user}_guest_override", + id=f"{pn_user(config)}_guest_override", value_if_missing=False, ), reload=False, diff --git a/data_lunch_app/__main__.py b/dlunch/__main__.py similarity index 85% rename from data_lunch_app/__main__.py rename to dlunch/__main__.py index ebcb6f7..3095dfc 100755 --- a/data_lunch_app/__main__.py +++ b/dlunch/__main__.py @@ -63,37 +63,38 @@ def run_app(config: DictConfig): # ensure that each invocation has a dedicated state variable (users' # selections are not shared between instances) # Backend exist only if auth is active + # Health is an endpoint for app health assessments # Pass a dictionary for a multipage app - pages = {"": lambda: create_app(config=config)} + pages = { + "": lambda: create_app(config=config), + "health": pn.Column("Data-Lunch is alive!"), + } if auth.is_auth_active(config=config): pages["backend"] = lambda: create_backend(config=config) - # If config.server.auth_provider exists, update - # config.server.auth_provider key with the instantiated object - try: + # If basic authentication is active, instantiate ta special auth object + # otherwise leave an empty dict + # This step is done before panel.serve because auth_provider requires that + # the whole config is passed as an input + if auth.is_basic_auth_active(config=config): auth_object = { "auth_provider": hydra.utils.instantiate( - config.server.auth_provider, config + config.basic_auth.auth_provider, config ) } log.debug( "auth_object dict set to instantiated object from config.server.auth_provider" ) - except ConfigAttributeError: + else: auth_object = {} log.debug( "missing config.server.auth_provider, auth_object dict left empty" ) - # Mask the auth_provider key from config.server to avoid a TypeError - # (multiple values for keyword argument 'auth_provider') - masked_server_config = OmegaConf.masked_copy( - config.server, - [k for k in config.server.keys() if k != "auth_provider"], + pn.serve( + panels=pages, **hydra.utils.instantiate(config.server), **auth_object ) - pn.serve(panels=pages, **masked_server_config, **auth_object) - def schedule_task( name: str, diff --git a/data_lunch_app/auth.py b/dlunch/auth.py similarity index 94% rename from data_lunch_app/auth.py rename to dlunch/auth.py index d92942f..33b2e77 100644 --- a/data_lunch_app/auth.py +++ b/dlunch/auth.py @@ -8,6 +8,7 @@ import panel as pn from panel.auth import OAuthProvider from panel.util import base64url_encode +import re import secrets import string from sqlalchemy.sql import true as sql_true @@ -235,14 +236,30 @@ def from_str(cls, password: str): # FUNCTIONS ------------------------------------------------------------------- +def pn_user(config: DictConfig) -> str: + """Return the user from Panel state object. + If remove_email_domain is True, remove the domain from the user.""" + # Store user + user = pn.state.user + + if user: + # Check if username is an email + if re.fullmatch(r"[^@]+@[^@]+\.[^@]+", user): + # Remove domain from username + if config.auth.remove_email_domain: + user = user.split("@")[0] + + return user + + def is_basic_auth_active(config: DictConfig) -> bool: """Check config object and return true if basic authentication is active. Return false otherwise.""" # Check if a valid auth key exists - auth_provider = config.server.get("auth_provider", None) + auth_provider = config.get("basic_auth", None) - return auth_provider + return auth_provider is not None def is_auth_active(config: DictConfig) -> bool: @@ -265,14 +282,13 @@ def authorize( """Authorization callback, read config, user info and target path. Return True (authorized) or False (not authorized) by checking current user and target path""" - # Set current user and existing users info - if is_basic_auth_active(config=config): - # For basic authentication username is under the 'user' key - current_user_key = "user" - else: - # For github is under the 'login' key - current_user_key = "login" - current_user = user_info[current_user_key] + + # If authorization is not active authorize every user + if not is_auth_active(config=config): + return True + + # Set current user from panel state + current_user = pn_user(config) privileged_users = list_users(config=config) log.debug(f"target path: {target_path}") # If user is not authenticated block it @@ -462,11 +478,15 @@ def is_guest( The guest override chached value (per-user) can force the function to always return True. If allow_override is set to False the guest override value is ignored.""" + # If authorization is not active always return false (user is not guest) + if not is_auth_active(config=config): + return False + # Load guest override from flag table (if the button is pressed its value # is True). If not available use False. guest_override = models.get_flag( config=config, - id=f"{pn.state.user}_guest_override", + id=f"{pn_user(config)}_guest_override", value_if_missing=False, ) @@ -484,6 +504,11 @@ def is_guest( def is_admin(user: str, config: DictConfig) -> bool: """Check if a user is an dmin by checking the 'privileged_users' table""" + + # If authorization is not active always return false (ther is no admin) + if not is_auth_active(config=config): + return False + # Create session session = models.create_session(config) diff --git a/data_lunch_app/cli.py b/dlunch/cli.py similarity index 99% rename from data_lunch_app/cli.py rename to dlunch/cli.py index 569001c..beb09b5 100755 --- a/data_lunch_app/cli.py +++ b/dlunch/cli.py @@ -13,7 +13,7 @@ from . import auth # Version -__version__ = pkg_resources.require("data_lunch")[0].version +__version__ = pkg_resources.require("dlunch")[0].version # CLI COMMANDS ---------------------------------------------------------------- diff --git a/data_lunch_app/cloud.py b/dlunch/cloud.py similarity index 100% rename from data_lunch_app/cloud.py rename to dlunch/cloud.py diff --git a/data_lunch_app/conf/__init__.py b/dlunch/conf/__init__.py similarity index 100% rename from data_lunch_app/conf/__init__.py rename to dlunch/conf/__init__.py diff --git a/dlunch/conf/auth/default.yaml b/dlunch/conf/auth/default.yaml new file mode 100644 index 0000000..37023e7 --- /dev/null +++ b/dlunch/conf/auth/default.yaml @@ -0,0 +1,19 @@ +# Remove email domain from usernames (e.g. "user@example.com" -> "user") +remove_email_domain: true +# oauth config (not used by panel.serve) +oauth_encryption_key: ${oc.env:DATA_LUNCH_OAUTH_ENC_KEY} +oauth_expiry: 15 +# Template not set by panel.serve +auth_error_template: ${package_path}/templates/error.html +# Autorization callback (_partial_ is required for usage with lambda functions) +authorization_callback: + _target_: dlunch.auth.authorize + _partial_: true + # Flag to authorize access of guest users to the home page + # They can only place orders for guests + authorize_guest_users: true +# Set cookie function arguments (set to empty dict to turn off) +cookie_kwargs: + httponly: true + secure: true + samesite: strict diff --git a/data_lunch_app/conf/auth/development.yaml b/dlunch/conf/auth/development.yaml similarity index 100% rename from data_lunch_app/conf/auth/development.yaml rename to dlunch/conf/auth/development.yaml diff --git a/data_lunch_app/conf/auth/production.yaml b/dlunch/conf/auth/production.yaml similarity index 100% rename from data_lunch_app/conf/auth/production.yaml rename to dlunch/conf/auth/production.yaml diff --git a/dlunch/conf/basic_auth/basic_auth.yaml b/dlunch/conf/basic_auth/basic_auth.yaml new file mode 100644 index 0000000..6eaa31a --- /dev/null +++ b/dlunch/conf/basic_auth/basic_auth.yaml @@ -0,0 +1,24 @@ +# BASIC AUTHENTICATION DATA +# These values are added to server config only if server is "basic_auth" + +# AUTH PROVIDER OBJECT +auth_provider: + _target_: dlunch.auth.DataLunchProvider + login_template: ${package_path}/templates/login_basic.html + logout_template: ${package_path}/templates/logout.html +# Those values are used by basic authentication specific features + +# PASSWORD +# Special charachters and reg expression used to check (or create) a password +psw_special_chars: ';:,.+-=_#!$%&?^*@' +psw_regex: ^(?=.*[0-9])(?=.*[a-z])(?=.*[A-Z])(?=.*[${basic_auth.psw_special_chars}]).{8,}$ +# Set generated password length (call to generate_password) +generated_psw_length: 12 + +# GUEST USER +# If true create default guest user credentials (i.e. a user named "guest") +# It is an unprivileged user (is_guest = true) named guest +guest_user: true +# Guest user password flag default value +# If true a new password for guest user is set everytime the main function is called +default_reset_guest_user_password_flag: false \ No newline at end of file diff --git a/data_lunch_app/conf/config.yaml b/dlunch/conf/config.yaml similarity index 86% rename from data_lunch_app/conf/config.yaml rename to dlunch/conf/config.yaml index 0ae2b42..81d8f90 100755 --- a/data_lunch_app/conf/config.yaml +++ b/dlunch/conf/config.yaml @@ -4,9 +4,13 @@ defaults: - db: sqlite - server: no_oauth - auth: ${oc.env:PANEL_ENV} + - optional basic_auth: ${server} - override hydra/job_logging: custom # - _self_ #config.yaml is overriding configs from the Defaults List +# PACKAGE PATH +package_path: ${pkg_path:dlunch} + # LOCAL TIMEZONE local_timezone: Europe/Rome diff --git a/data_lunch_app/conf/db/common.yaml b/dlunch/conf/db/common.yaml similarity index 100% rename from data_lunch_app/conf/db/common.yaml rename to dlunch/conf/db/common.yaml diff --git a/data_lunch_app/conf/db/development.yaml b/dlunch/conf/db/development.yaml similarity index 100% rename from data_lunch_app/conf/db/development.yaml rename to dlunch/conf/db/development.yaml diff --git a/data_lunch_app/conf/db/postgresql.yaml b/dlunch/conf/db/postgresql.yaml similarity index 100% rename from data_lunch_app/conf/db/postgresql.yaml rename to dlunch/conf/db/postgresql.yaml diff --git a/data_lunch_app/conf/db/production.yaml b/dlunch/conf/db/production.yaml similarity index 100% rename from data_lunch_app/conf/db/production.yaml rename to dlunch/conf/db/production.yaml diff --git a/data_lunch_app/conf/db/sqlite.yaml b/dlunch/conf/db/sqlite.yaml similarity index 100% rename from data_lunch_app/conf/db/sqlite.yaml rename to dlunch/conf/db/sqlite.yaml diff --git a/data_lunch_app/conf/hydra/job_logging/custom.yaml b/dlunch/conf/hydra/job_logging/custom.yaml similarity index 100% rename from data_lunch_app/conf/hydra/job_logging/custom.yaml rename to dlunch/conf/hydra/job_logging/custom.yaml diff --git a/data_lunch_app/conf/panel/default.yaml b/dlunch/conf/panel/default.yaml similarity index 92% rename from data_lunch_app/conf/panel/default.yaml rename to dlunch/conf/panel/default.yaml index dad3623..ff8e028 100755 --- a/data_lunch_app/conf/panel/default.yaml +++ b/dlunch/conf/panel/default.yaml @@ -82,7 +82,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: null period: 1d callable: - _target_: data_lunch_app.scheduled_tasks.reset_guest_user_password + _target_: dlunch.scheduled_tasks.reset_guest_user_password - kwargs: name: scheduled cleaning enabled: true @@ -90,7 +90,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: 00 period: 1d callable: - _target_: data_lunch_app.scheduled_tasks.clean_files_db + _target_: dlunch.scheduled_tasks.clean_files_db - kwargs: name: database upload enabled: ${db.ext_storage_upload.enabled} @@ -98,7 +98,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: 25 period: 60min callable: - _target_: data_lunch_app.scheduled_tasks.upload_db_to_gcp_storage + _target_: dlunch.scheduled_tasks.upload_db_to_gcp_storage source_file_name: ${db.ext_storage_upload.source_file_name} destination_blob_name: ${db.ext_storage_upload.destination_blob_name} bucket_name: ${db.ext_storage_upload.bucket_name} diff --git a/data_lunch_app/conf/panel/gui/1st_of_april.yaml b/dlunch/conf/panel/gui/1st_of_april.yaml similarity index 100% rename from data_lunch_app/conf/panel/gui/1st_of_april.yaml rename to dlunch/conf/panel/gui/1st_of_april.yaml diff --git a/data_lunch_app/conf/panel/gui/4th_of_may.yaml b/dlunch/conf/panel/gui/4th_of_may.yaml similarity index 100% rename from data_lunch_app/conf/panel/gui/4th_of_may.yaml rename to dlunch/conf/panel/gui/4th_of_may.yaml diff --git a/data_lunch_app/conf/panel/gui/default.yaml b/dlunch/conf/panel/gui/default.yaml similarity index 100% rename from data_lunch_app/conf/panel/gui/default.yaml rename to dlunch/conf/panel/gui/default.yaml diff --git a/data_lunch_app/conf/panel/gui/easter.yaml b/dlunch/conf/panel/gui/easter.yaml similarity index 100% rename from data_lunch_app/conf/panel/gui/easter.yaml rename to dlunch/conf/panel/gui/easter.yaml diff --git a/data_lunch_app/conf/panel/gui/major_release.yaml b/dlunch/conf/panel/gui/major_release.yaml similarity index 100% rename from data_lunch_app/conf/panel/gui/major_release.yaml rename to dlunch/conf/panel/gui/major_release.yaml diff --git a/data_lunch_app/conf/panel/gui/women_day.yaml b/dlunch/conf/panel/gui/women_day.yaml similarity index 100% rename from data_lunch_app/conf/panel/gui/women_day.yaml rename to dlunch/conf/panel/gui/women_day.yaml diff --git a/data_lunch_app/conf/panel/no_sched_clean.yaml b/dlunch/conf/panel/no_sched_clean.yaml similarity index 78% rename from data_lunch_app/conf/panel/no_sched_clean.yaml rename to dlunch/conf/panel/no_sched_clean.yaml index a5f04b5..dc509a8 100644 --- a/data_lunch_app/conf/panel/no_sched_clean.yaml +++ b/dlunch/conf/panel/no_sched_clean.yaml @@ -10,7 +10,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: null period: 1d callable: - _target_: data_lunch_app.scheduled_tasks.reset_guest_user_password + _target_: dlunch.scheduled_tasks.reset_guest_user_password - kwargs: name: scheduled cleaning enabled: false @@ -18,7 +18,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: null period: 30min callable: - _target_: data_lunch_app.scheduled_tasks.clean_files_db + _target_: dlunch.scheduled_tasks.clean_files_db - kwargs: name: database upload enabled: ${db.ext_storage_upload.enabled} @@ -26,7 +26,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: null period: 30min callable: - _target_: data_lunch_app.scheduled_tasks.upload_db_to_gcp_storage + _target_: dlunch.scheduled_tasks.upload_db_to_gcp_storage source_file_name: ${db.ext_storage_upload.source_file_name} destination_blob_name: ${db.ext_storage_upload.destination_blob_name} bucket_name: ${db.ext_storage_upload.bucket_name} diff --git a/data_lunch_app/conf/panel/quality.yaml b/dlunch/conf/panel/quality.yaml similarity index 78% rename from data_lunch_app/conf/panel/quality.yaml rename to dlunch/conf/panel/quality.yaml index deb67e0..f2ca294 100644 --- a/data_lunch_app/conf/panel/quality.yaml +++ b/dlunch/conf/panel/quality.yaml @@ -10,7 +10,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: null period: 1d callable: - _target_: data_lunch_app.scheduled_tasks.reset_guest_user_password + _target_: dlunch.scheduled_tasks.reset_guest_user_password - kwargs: name: scheduled cleaning enabled: true @@ -18,7 +18,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: null period: 30min callable: - _target_: data_lunch_app.scheduled_tasks.clean_files_db + _target_: dlunch.scheduled_tasks.clean_files_db - kwargs: name: database upload enabled: ${db.ext_storage_upload.enabled} @@ -26,7 +26,7 @@ scheduled_tasks: # Set to [] empty list to turn it off minute: null period: 30min callable: - _target_: data_lunch_app.scheduled_tasks.upload_db_to_gcp_storage + _target_: dlunch.scheduled_tasks.upload_db_to_gcp_storage source_file_name: ${db.ext_storage_upload.source_file_name} destination_blob_name: ${db.ext_storage_upload.destination_blob_name} bucket_name: ${db.ext_storage_upload.bucket_name} diff --git a/dlunch/conf/server/azure_oauth.yaml b/dlunch/conf/server/azure_oauth.yaml new file mode 100644 index 0000000..6f7dacd --- /dev/null +++ b/dlunch/conf/server/azure_oauth.yaml @@ -0,0 +1,13 @@ +defaults: + - /${oc.env:PANEL_ENV} + +oauth_provider: azure +oauth_key: ${oc.env:DATA_LUNCH_OAUTH_KEY} +oauth_secret: ${oc.env:DATA_LUNCH_OAUTH_SECRET} +oauth_redirect_uri: ${oc.env:DATA_LUNCH_OAUTH_REDIRECT_URI} +oauth_extra_params: + scope: ${server.oauth_key}/.default + tenant: ${oc.env:DATA_LUNCH_OAUTH_TENANT_ID} +login_template: ${package_path}/templates/login_oauth.html +logout_template: ${package_path}/templates/logout.html +oauth_error_template: ${package_path}/templates/error.html diff --git a/data_lunch_app/conf/server/no_oauth.yaml b/dlunch/conf/server/basic_auth.yaml similarity index 100% rename from data_lunch_app/conf/server/no_oauth.yaml rename to dlunch/conf/server/basic_auth.yaml diff --git a/data_lunch_app/conf/server/common.yaml b/dlunch/conf/server/common.yaml similarity index 65% rename from data_lunch_app/conf/server/common.yaml rename to dlunch/conf/server/common.yaml index 20f867e..e3b71c0 100644 --- a/data_lunch_app/conf/server/common.yaml +++ b/dlunch/conf/server/common.yaml @@ -1,3 +1,6 @@ +# Server config are used with instantiate +_target_: builtins.dict +_convert_: all address: 0.0.0.0 port: ${oc.decode:${oc.env:PORT, 5000}} websocket_origin: "*" @@ -5,8 +8,8 @@ xheaders: true verbose: false open: false static_dirs: - images: ./data_lunch_app/static/images - css: ./data_lunch_app/static/css + images: ${package_path}/static/images + css: ${package_path}/static/css compress_response: true cookie_secret: ${oc.env:DATA_LUNCH_COOKIE_SECRET} # Secure cookies are required (this vakue can't be null) enable_xsrf_cookies: true \ No newline at end of file diff --git a/data_lunch_app/conf/server/development.yaml b/dlunch/conf/server/development.yaml similarity index 100% rename from data_lunch_app/conf/server/development.yaml rename to dlunch/conf/server/development.yaml diff --git a/data_lunch_app/conf/server/github_oauth.yaml b/dlunch/conf/server/github_oauth.yaml similarity index 54% rename from data_lunch_app/conf/server/github_oauth.yaml rename to dlunch/conf/server/github_oauth.yaml index 5214040..41c1b2d 100644 --- a/data_lunch_app/conf/server/github_oauth.yaml +++ b/dlunch/conf/server/github_oauth.yaml @@ -5,6 +5,6 @@ oauth_provider: github oauth_key: ${oc.env:DATA_LUNCH_OAUTH_KEY} oauth_secret: ${oc.env:DATA_LUNCH_OAUTH_SECRET} oauth_redirect_uri: ${oc.env:DATA_LUNCH_OAUTH_REDIRECT_URI} -login_template: data_lunch_app/templates/login_oauth.html -logout_template: data_lunch_app/templates/logout.html -oauth_error_template: data_lunch_app/templates/error.html +login_template: ${package_path}/templates/login_oauth.html +logout_template: ${package_path}/templates/logout.html +oauth_error_template: ${package_path}/templates/error.html diff --git a/dlunch/conf/server/no_oauth.yaml b/dlunch/conf/server/no_oauth.yaml new file mode 100644 index 0000000..d814523 --- /dev/null +++ b/dlunch/conf/server/no_oauth.yaml @@ -0,0 +1,2 @@ +defaults: + - /${oc.env:PANEL_ENV} diff --git a/data_lunch_app/conf/server/production.yaml b/dlunch/conf/server/production.yaml similarity index 100% rename from data_lunch_app/conf/server/production.yaml rename to dlunch/conf/server/production.yaml diff --git a/data_lunch_app/core.py b/dlunch/core.py similarity index 96% rename from data_lunch_app/core.py rename to dlunch/core.py index 19cbb0c..2b67f7f 100644 --- a/data_lunch_app/core.py +++ b/dlunch/core.py @@ -22,6 +22,7 @@ # Authentication from . import auth +from .auth import pn_user import cryptography.fernet # LOGGER ---------------------------------------------------------------------- @@ -88,10 +89,18 @@ def set_guest_user_password(config: DictConfig) -> str: """If guest user is requested return a password, otherwise return "" This function always returns "" if basic auth is not used """ - if config.auth.basic_auth.guest_user and auth.is_basic_auth_active( - config=config - ): - # If flag does not exist use the default value + # Check if basic auth is active + if auth.is_basic_auth_active(config=config): + # If active basic_auth.guest_user is true if guest user is active + is_guest_user_active = config.basic_auth.guest_user + else: + # Otherwise the guest user feature is not applicable + is_guest_user_active = False + + # Set the guest password variable + if is_guest_user_active: + # If flag for resetting the password does not exist use the default + # value if ( models.get_flag(config=config, id="reset_guest_user_password") is None @@ -99,7 +108,7 @@ def set_guest_user_password(config: DictConfig) -> str: models.set_flag( config=config, id="reset_guest_user_password", - value=config.auth.basic_auth.default_reset_guest_user_password_flag, + value=config.basic_auth.default_reset_guest_user_password_flag, ) # Generate a random password only if requested (check on flag) # otherwise load from pickle @@ -114,8 +123,8 @@ def set_guest_user_password(config: DictConfig) -> str: ) # Create password guest_password = auth.generate_password( - special_chars=config.auth.basic_auth.psw_special_chars, - length=config.auth.basic_auth.generated_psw_length, + special_chars=config.basic_auth.psw_special_chars, + length=config.basic_auth.generated_psw_length, ) # Add hashed password to database auth.add_user_hashed_password( @@ -299,13 +308,13 @@ def reload_menu( # Check guest override button status (if not in table use False) gi.toggle_guest_override_button.value = models.get_flag( config=config, - id=f"{pn.state.user}_guest_override", + id=f"{pn_user(config)}_guest_override", value_if_missing=False, ) # Set no more orders toggle button visibility and activation if auth.is_guest( - user=pn.state.user, config=config, allow_override=False + user=pn_user(config), config=config, allow_override=False ): # Deactivate the no_more_orders button for guest users gi.toggle_no_more_order_button.disabled = True @@ -316,7 +325,7 @@ def reload_menu( gi.toggle_no_more_order_button.visible = True # Guest graphic configuration - if auth.is_guest(user=pn.state.user, config=config): + if auth.is_guest(user=pn_user(config), config=config): # If guest show guest type selection group gi.person_widget.widgets["guest"].disabled = False gi.person_widget.widgets["guest"].visible = True @@ -500,7 +509,7 @@ def reload_menu( stats_and_info_text = gi.build_stats_and_info_text( config=config, df_stats=df_stats, - user=pn.state.user, + user=pn_user(config), version=__version__, host_name=get_host_name(config), stylesheets=[config.panel.gui.css_files.stats_info_path], @@ -563,7 +572,7 @@ def send_order( # If auth is active, check if a guests is using a name reserved to a # privileged user if ( - auth.is_guest(user=pn.state.user, config=config) + auth.is_guest(user=pn_user(config), config=config) and (person.username in auth.list_users(config=config)) and (auth.is_auth_active(config=config)) ): @@ -583,7 +592,7 @@ def send_order( # Check if a privileged user is ordering for an invalid name if ( - not auth.is_guest(user=pn.state.user, config=config) + not auth.is_guest(user=pn_user(config), config=config) and ( person.username not in ( @@ -626,7 +635,7 @@ def send_order( try: # Add User (note is empty by default) # Do not pass guest for privileged users (default to NotAGuest) - if auth.is_guest(user=pn.state.user, config=config): + if auth.is_guest(user=pn_user(config), config=config): new_user = models.Users( id=person.username, guest=person.guest, @@ -726,7 +735,7 @@ def delete_order( # If auth is active, check if a guests is deleting an order of a # privileged user if ( - auth.is_guest(user=pn.state.user, config=config) + auth.is_guest(user=pn_user(config), config=config) and (person.username in auth.list_users(config=config)) and (auth.is_auth_active(config=config)) ): @@ -929,12 +938,12 @@ def submit_password(gi: gui.GraphicInterface, config: DictConfig) -> bool: """Same as backend_submit_password with an additional check on old password""" # Get user's password hash - password_hash = auth.get_hash_from_user(pn.state.user, config=config) + password_hash = auth.get_hash_from_user(pn_user(config), config=config) # Check if old password is correct if password_hash == gi.password_widget.object.old_password: # Check if new password match repeat password return backend_submit_password( - gi=gi, config=config, user=pn.state.user, logout_on_success=True + gi=gi, config=config, user=pn_user(config), logout_on_success=True ) else: pn.state.notifications.error( @@ -973,7 +982,7 @@ def backend_submit_password( ): # Check if new password is valid with regex if re.fullmatch( - config.auth.basic_auth.psw_regex, + config.basic_auth.psw_regex, gi.password_widget.object.new_password, ): # If is_guest and is_admin are None (not passed) use the ones diff --git a/data_lunch_app/gui.py b/dlunch/gui.py similarity index 98% rename from data_lunch_app/gui.py rename to dlunch/gui.py index d9d2fd2..5ec7123 100644 --- a/data_lunch_app/gui.py +++ b/dlunch/gui.py @@ -17,6 +17,7 @@ # Auth from . import auth +from .auth import pn_user log = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def __init__(self, config, **params): self.param.guest.default = config.panel.guest_types[0] self.guest = config.panel.guest_types[0] # Check user (a username is already set for privileged users) - username = pn.state.user + username = pn_user(config) if not auth.is_guest(user=username, config=config) and ( username is not None ): @@ -203,11 +204,11 @@ def __init__( if auth.is_auth_active(config=config): self.header_row.append(pn.HSpacer()) # Backend only for admin - if auth.is_admin(user=pn.state.user, config=config): + if auth.is_admin(user=pn_user(config), config=config): self.header_row.append(self.backend_button) # Guest override only for non guests if not auth.is_guest( - user=pn.state.user, config=config, allow_override=False + user=pn_user(config), config=config, allow_override=False ): self.header_row.append(self.toggle_guest_override_button) self.header_row.append(self.logout_button) @@ -230,11 +231,11 @@ def reload_on_guest_override_callback( # Only non guest can store this value in 'flags' table (guest users # are always guests, there is no use in sotring a flag for them) if not auth.is_guest( - user=pn.state.user, config=config, allow_override=False + user=pn_user(config), config=config, allow_override=False ): models.set_flag( config=config, - id=f"{pn.state.user}_guest_override", + id=f"{pn_user(config)}_guest_override", value=toggle, ) # Show banner if override is active @@ -719,7 +720,7 @@ def load_sidebar_tabs( # Append upload, download and stats only for non-guest # Append password only for non-guest users if auth is active if not auth.is_guest( - user=pn.state.user, config=config, allow_override=False + user=pn_user(config), config=config, allow_override=False ): self.sidebar_tabs.append(self.sidebar_menu_upload_col) self.sidebar_tabs.append(self.sidebar_download_orders_col) @@ -1009,7 +1010,7 @@ def __init__( min_height=450, ) # Add controls only for admin users - if not auth.is_admin(user=pn.state.user, config=config): + if not auth.is_admin(user=pn_user(config), config=config): self.backend_controls.append(self.access_denied_text) self.backend_controls.append(pn.Spacer(height=15)) else: diff --git a/data_lunch_app/models.py b/dlunch/models.py similarity index 99% rename from data_lunch_app/models.py rename to dlunch/models.py index c7ca070..044799e 100755 --- a/data_lunch_app/models.py +++ b/dlunch/models.py @@ -667,7 +667,7 @@ def create_database_with_retries(config: DictConfig) -> None: # Check if guest exists if ( session.get(Credentials, "guest") is None - ) and config.auth.basic_auth.guest_user: + ) and config.basic_auth.guest_user: # Add only credentials for guest (guest users are not included # in privileged_users table) auth.add_user_hashed_password( diff --git a/data_lunch_app/quotes.xlsx b/dlunch/quotes.xlsx similarity index 100% rename from data_lunch_app/quotes.xlsx rename to dlunch/quotes.xlsx diff --git a/data_lunch_app/scheduled_tasks.py b/dlunch/scheduled_tasks.py similarity index 100% rename from data_lunch_app/scheduled_tasks.py rename to dlunch/scheduled_tasks.py diff --git a/data_lunch_app/static/css/app_header.css b/dlunch/static/css/app_header.css similarity index 100% rename from data_lunch_app/static/css/app_header.css rename to dlunch/static/css/app_header.css diff --git a/data_lunch_app/static/css/custom_tabulator.css b/dlunch/static/css/custom_tabulator.css similarity index 100% rename from data_lunch_app/static/css/custom_tabulator.css rename to dlunch/static/css/custom_tabulator.css diff --git a/data_lunch_app/static/css/guest_override.css b/dlunch/static/css/guest_override.css similarity index 100% rename from data_lunch_app/static/css/guest_override.css rename to dlunch/static/css/guest_override.css diff --git a/data_lunch_app/static/css/labels.css b/dlunch/static/css/labels.css similarity index 100% rename from data_lunch_app/static/css/labels.css rename to dlunch/static/css/labels.css diff --git a/data_lunch_app/static/css/no_more_orders.css b/dlunch/static/css/no_more_orders.css similarity index 100% rename from data_lunch_app/static/css/no_more_orders.css rename to dlunch/static/css/no_more_orders.css diff --git a/data_lunch_app/static/css/stats_and_info.css b/dlunch/static/css/stats_and_info.css similarity index 100% rename from data_lunch_app/static/css/stats_and_info.css rename to dlunch/static/css/stats_and_info.css diff --git a/data_lunch_app/static/css/stats_tabulator.css b/dlunch/static/css/stats_tabulator.css similarity index 100% rename from data_lunch_app/static/css/stats_tabulator.css rename to dlunch/static/css/stats_tabulator.css diff --git a/data_lunch_app/static/images/favicon.ico b/dlunch/static/images/favicon.ico similarity index 100% rename from data_lunch_app/static/images/favicon.ico rename to dlunch/static/images/favicon.ico diff --git a/data_lunch_app/static/images/logo.png b/dlunch/static/images/logo.png similarity index 100% rename from data_lunch_app/static/images/logo.png rename to dlunch/static/images/logo.png diff --git a/data_lunch_app/templates/error.html b/dlunch/templates/error.html similarity index 100% rename from data_lunch_app/templates/error.html rename to dlunch/templates/error.html diff --git a/data_lunch_app/templates/login_basic.html b/dlunch/templates/login_basic.html similarity index 100% rename from data_lunch_app/templates/login_basic.html rename to dlunch/templates/login_basic.html diff --git a/data_lunch_app/templates/login_oauth.html b/dlunch/templates/login_oauth.html similarity index 100% rename from data_lunch_app/templates/login_oauth.html rename to dlunch/templates/login_oauth.html diff --git a/data_lunch_app/templates/logout.html b/dlunch/templates/logout.html similarity index 100% rename from data_lunch_app/templates/logout.html rename to dlunch/templates/logout.html diff --git a/docker/compose_init.sh b/docker/compose_init.sh index dbd4d99..184ed5d 100644 --- a/docker/compose_init.sh +++ b/docker/compose_init.sh @@ -40,5 +40,5 @@ echo "dhparam already exists" fi else - make ssl-cert + make ssl-gen-certificate fi \ No newline at end of file diff --git a/docker/web/Dockerfile.web b/docker/web/Dockerfile.web index 4dc9024..059eab0 100755 --- a/docker/web/Dockerfile.web +++ b/docker/web/Dockerfile.web @@ -45,4 +45,4 @@ ENV PANEL_APP=data-lunch-app EXPOSE 5000 # Set command -ENTRYPOINT ["python", "-m", "data_lunch_app"] \ No newline at end of file +ENTRYPOINT ["python", "-m", "dlunch"] \ No newline at end of file diff --git a/makefile b/makefile index 2d99cf6..f3321a9 100755 --- a/makefile +++ b/makefile @@ -1,3 +1,13 @@ +# Colors +YELLOW := \033[1;33m +GREEN := \033[1;32m +RED := \033[1;31m +CYAN := \033[1;36m +ORANGE := \033[0;33m +PURPLE := \033[1;35m +WHITE := \033[1;37m +NC := \033[0m + # Data-Lunch variables APP=${PANEL_APP} IMAGENAME=${DOCKER_USERNAME}/${APP} @@ -5,40 +15,152 @@ RUNNAME=${DOCKER_USERNAME}_${APP} VERSION=${IMAGE_VERSION} IMAGEFULLNAME=${IMAGENAME}:${VERSION} PROJECTNAME=${DOCKER_USERNAME}_${APP} + # Database variables (used if not sqlite) DBNAME:=postgres DBSERVICE:=db DBCONTAINERNAME:=${DBNAME}_${DBSERVICE} DBIMAGEFULLNAME:=${DBNAME}:${VERSION} DBPORT:=5432 + # Docker compose up UP_SERVICES:=web nginx -.PHONY: help build push all clean +# Directories +CERT_DIR := ssl + +# Conda commands +CONDA_ACTIVATE_BASE:=source ${CONDA_ROOT}/etc/profile.d/conda.sh; conda activate; help: - @echo "Makefile commands:" - @echo "build" - @echo "push" - @echo "all" + @echo -e " ${PURPLE} LIST OF AVAILABLE COMMANDS ${NC}" + @echo -e " ${RED}======================|=========================================================================${NC}" + @echo -e " ${RED}Command | Description ${NC}" + @echo -e " ${RED}======================|=========================================================================${NC}" + @echo -e " ${YELLOW}SETUP ------------------------------------------------------------------------------------------${NC}" + @echo -e " ${WHITE} help :${NC} prints this help message" + @echo -e " ${WHITE} gcp-config :${NC} configures and authenticates GCP" + @echo -e " ${WHITE} gcp-revoke :${NC} removes GCP configurations and authentication" + @echo -e " ${WHITE} ssl-gen-certificate :${NC} creates SSL certificate" + @echo -e " ${WHITE} ssl-rm-certificate :${NC} removes SSL certificate" + @echo -e " ${WHITE} generate-secrets :${NC} print random cookies and encryption secrets" + @echo -e " ${YELLOW}DOCKER -----------------------------------------------------------------------------------------${NC}" + @echo -e " ${WHITE} docker-build :${NC} builds docker image from Dockerfile" + @echo -e " ${WHITE} docker-push :${NC} pushes docker image to DockerHub repository" + @echo -e " ${WHITE} docker-pull :${NC} pulls docker image from DockerHub repository" + @echo -e " ${WHITE} docker-build-push :${NC} builds and pushes docker image to DockerHub repository" + @echo -e " ${WHITE} docker-run :${NC} runs docker image" + @echo -e " ${WHITE} docker-run-it :${NC} runs docker image in interactive mode" + @echo -e " ${WHITE} docker-stop :${NC} stops docker image" + @echo -e " ${WHITE} docker-db-run :${NC} runs database image" + @echo -e " ${WHITE} docker-db-stop :${NC} stops database image" + @echo -e " ${WHITE} docker-up :${NC} starts docker compose" + @echo -e " ${WHITE} docker-up-build :${NC} builds and start docker compose" + @echo -e " ${WHITE} docker-down :${NC} stops docker compose" + @echo -e " ${WHITE} docker-up-init :${NC} initializes docker compose" + @echo -e " ${YELLOW}CLEAN ------------------------------------------------------------------------------------------${NC}" + @echo -e " ${WHITE} clean-folders :${NC} cleans all folders nb checkpoints, pycache & pytest folders" + @echo -e " ${WHITE} clean-docker :${NC} cleans docker containers and images" + @echo -e " ${WHITE} clean :${NC} runs clean-notebooks, clean-docker, clean-folders, clean-k8s" + @echo -e " ${YELLOW}MISC -------------------------------------------------------------------------------------------${NC}" + @echo -e " ${WHITE} interrogate :${NC} runs interrogate to check code quality" + @echo -e " ${WHITE} package-build :${NC} build python package" + @echo -e " ${WHITE} package-publish :${NC} publish python package to PyPI" + @echo -e " ${WHITE} package-test-publish:${NC} publish python package to TestPyPI" + @echo -e " ${WHITE} package-test-install:${NC} install package with pip from TestPyPI (use only in a test env)" + @echo -e " ${WHITE} pre-commit-run :${NC} runs pre-commit hooks" + @echo -e " ${WHITE} commitizen-bump :${NC} runs commitizen for releasing a new version on master branch" + @echo -e " ${WHITE} commitizen-push :${NC} use git to push commits on 'development' and 'master' branches" + @echo -e "${RED}=======================|=========================================================================${NC}" + @echo "" + +default: help -.DEFAULT_GOAL := help +# Force execution every time +.PHONY: clean ${K8S_MANIFEST_FILES} + +# Pre-check ------------------------------------------------------------------- +check-version: +ifndef IMAGE_VERSION + $(error "IMAGE_VERSION is not set, use 'make [COMMAND] IMAGE_VERSION=latest' to build with 'latest' as version") +endif check-dialect: ifndef DB_DIALECT $(error DB_DIALECT is not set, add DB_DIALECT=postgresql or DB_DIALECT=sqlite after the make command) endif -build: +# Setup rules ----------------------------------------------------------------- +gcp-config: + @echo -e "${YELLOW}start GCP config and auth${NC}" + gcloud config configurations create ${APP} + gcloud config set project ${GCLOUD_PROJECT} + gcloud auth login + gcloud auth application-default login + gcloud auth application-default set-quota-project ${GCLOUD_PROJECT} + @echo -e "${GREEN}GCP config and auth done${NC}" + +gcp-revoke: + @echo -e "${YELLOW}remove GCP config and auth${NC}" + gcloud config configurations activate default + gcloud config configurations delete ${APP} + gcloud auth application-default revoke + @echo -e "${GREEN}GCP config and auth removed${NC}" + +ssl-gen-certificate: ssl-rm-certificate + @echo -e "${YELLOW}create test SSL certificate${NC}" + mkdir -p ./${CERT_DIR}/conf/live/${DOMAIN}/ + curl https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > \ + ./ssl/conf/options-ssl-nginx.conf + find ./ssl/conf/ -name "ssl-dhparams.pem" -type f -mtime +1 -delete + if [[ ! -f ./ssl/conf/ssl-dhparams.pem ]] ; then \ + echo "building dhparam"; \ + openssl dhparam -out ./ssl/conf/ssl-dhparams.pem 2048; \ + else \ + echo "dhparam already exists" ;\ + fi; + openssl req -nodes -x509 \ + -newkey rsa:2048 \ + -keyout ./ssl/conf/live/${DOMAIN}/privkey.pem \ + -out ./ssl/conf/live/${DOMAIN}/fullchain.pem \ + -subj "/C=IT/ST=Lombardia/L=Milan/O=MIC/OU=IT/CN=${DOMAIN}/" + @echo -e "${GREEN}certificate created${NC}" + +ssl-rm-certificate: + @echo -e "${YELLOW}remove test SSL certificate${NC}" + @-rm -R ssl + @echo -e "${GREEN}certificate folder removed${NC}" + +generate-secrets: + @echo -e "${YELLOW}print secrets${NC}" + @echo "\nCOOKIE SECRET:" + @panel secret + @echo "\nENCRIPTION KEY:" + @panel oauth-secret + @echo -e "\n${GREEN}done${NC}" + +# Docker rules ---------------------------------------------------------------- + +docker-build: check-version + @echo -e "${YELLOW}build dl_optima docker image from ./docker/Dockerfile${NC}" docker build -t ${IMAGEFULLNAME} -f docker/web/Dockerfile.web . + docker system prune -f + @echo -e "${GREEN}build done${NC}" -push: +docker-push: check-version + @echo -e "${YELLOW}push image to ECR (with version '${IMAGE_VERSION}')${NC}" docker push ${IMAGEFULLNAME} + @echo -e "${GREEN}push done${NC}" -pull: +docker-pull: check-version + @echo -e "${YELLOW}pull image from ECR (selected version: '${IMAGE_VERSION}')${NC}" docker pull ${IMAGEFULLNAME} + @echo -e "${GREEN}pull done${NC}" + +docker-build-push: docker-build docker-push -run: +docker-run: check-version + @echo -e "${YELLOW}run container locally (selected version: '${IMAGE_VERSION}')${NC}" docker run -d --name ${RUNNAME} \ -v ${PWD}/shared_data:/app/shared_data \ -p 127.0.0.1:${PORT}:${PORT} \ @@ -53,134 +175,213 @@ run: -e DATA_LUNCH_OAUTH_SECRET=${DATA_LUNCH_OAUTH_SECRET} \ -e DATA_LUNCH_OAUTH_SECRET=${DATA_LUNCH_OAUTH_REDIRECT_URI} \ ${IMAGEFULLNAME} ${PANEL_ARGS} + @echo -e "${GREEN}done${NC}" -run-it: +docker-run-it: check-version + @echo -e "${YELLOW}run interactive session locally (selected version: '${IMAGE_VERSION}')${NC}" docker run --rm --entrypoint "" -e PANEL_ENV=development -it ${IMAGEFULLNAME} /bin/sh + @echo -e "${GREEN}done${NC}" -run-development: - docker run -d --name ${RUNNAME} \ - -v ${PWD}/shared_data:/app/shared_data \ - -p 127.0.0.1:${PORT}:${PORT} \ - -e PANEL_ENV=development \ - -e PORT=${PORT} \ - -e GCLOUD_PROJECT=${GCLOUD_PROJECT} \ - -e GCLOUD_BUCKET=${GCLOUD_BUCKET} \ - -e DOCKER_USERNAME=${DOCKER_USERNAME} \ - -e DATA_LUNCH_COOKIE_SECRET=${DATA_LUNCH_COOKIE_SECRET} \ - -e DATA_LUNCH_OAUTH_ENC_KEY=${DATA_LUNCH_OAUTH_ENC_KEY} \ - -e DATA_LUNCH_OAUTH_KEY=${DATA_LUNCH_OAUTH_KEY} \ - -e DATA_LUNCH_OAUTH_SECRET=${DATA_LUNCH_OAUTH_SECRET} \ - -e DATA_LUNCH_OAUTH_SECRET=${DATA_LUNCH_OAUTH_REDIRECT_URI} \ - ${IMAGEFULLNAME} ${PANEL_ARGS} - - -stop: +docker-stop: + @echo -e "${YELLOW}stop running container${NC}" docker stop ${RUNNAME} + @echo -e "${GREEN}done${NC}" -run-db: +docker-db-run: + @echo -e "${YELLOW}run database locally${NC}" docker run -d --name ${DBCONTAINERNAME} \ - -v ${PWD}/shared_data/db_pg:/var/lib/postgresql/data \ - -p 127.0.0.1:${DBPORT}:${DBPORT} \ - -e POSTGRES_USER=data_lunch_rw \ - -e POSTGRES_PASSWORD=${DATA_LUNCH_DB_PASSWORD} \ - -e POSTGRES_DB=data_lunch_database \ - ${DBIMAGEFULLNAME} - -stop-db: - docker stop ${DBCONTAINERNAME} + -v ${PWD}/shared_data/db_pg:/var/lib/postgresql/data \ + -p 127.0.0.1:${DBPORT}:${DBPORT} \ + -e POSTGRES_USER=data_lunch_rw \ + -e POSTGRES_PASSWORD=${DATA_LUNCH_DB_PASSWORD} \ + -e POSTGRES_DB=data_lunch_database \ + ${DBIMAGEFULLNAME} + @echo -e "${GREEN}done${NC}" -send-ip-email: - docker run --rm --name send_email \ - --entrypoint "" \ - -v ${PWD}/scripts:/app/scripts \ - -v ${PWD}/shared_data:/app/shared_data \ - -e MAIL_USER=${MAIL_USER} \ - -e MAIL_APP_PASSWORD=${MAIL_APP_PASSWORD} \ - -e MAIL_RECIPIENTS=${MAIL_RECIPIENTS} \ - -e DOMAIN=${DOMAIN} \ - ${IMAGEFULLNAME} /bin/sh -c "python /app/scripts/send_email_with_ip.py ${PANEL_ARGS}" +docker-db-stop: + @echo -e "${YELLOW}stop running database${NC}" + docker stop ${DBCONTAINERNAME} + @echo -e "${GREEN}done${NC}" -create-users-credentials: - docker run --rm --name create_users \ - --entrypoint "" \ - -v ${PWD}/scripts:/app/scripts \ - -v ${PWD}/shared_data:/app/shared_data \ - -e PANEL_ENV=${PANEL_ENV} \ - -e GCLOUD_BUCKET=${GCLOUD_BUCKET} \ - -e GCLOUD_PROJECT=${GCLOUD_PROJECT} \ - -e MAIL_USER=${MAIL_USER} \ - -e MAIL_APP_PASSWORD=${MAIL_APP_PASSWORD} \ - -e MAIL_RECIPIENTS=${MAIL_RECIPIENTS} \ - -e DOMAIN=${DOMAIN} \ - ${IMAGEFULLNAME} /bin/sh -c "python /app/scripts/create_users_from_list.py ${PANEL_ARGS}" - -up: check-dialect +docker-up: check-dialect + @echo -e "${YELLOW}start docker compose system${NC}" if [[ ${PANEL_ENV} == "production" ]] ; then \ - docker compose -p ${PROJECTNAME} -f docker/docker-compose.yaml --project-directory . up -d --scale web=3; \ + docker compose -p ${PROJECTNAME} \ + -f docker/docker-compose.yaml \ + --project-directory . \ + up -d --scale web=3; \ else \ if [[ ${DB_DIALECT} == "postgresql" ]] ; then \ - docker compose -p ${PROJECTNAME} -f docker/docker-compose.yaml --project-directory . up -d ${UP_SERVICES} ${DBSERVICE} --scale web=3; \ + docker compose -p ${PROJECTNAME} \ + -f docker/docker-compose.yaml \ + --project-directory . \ + up -d ${UP_SERVICES} ${DBSERVICE} --scale web=3; \ else \ - docker compose -p ${PROJECTNAME} -f docker/docker-compose.yaml --project-directory . up -d ${UP_SERVICES} --scale web=3; \ + docker compose -p ${PROJECTNAME} \ + -f docker/docker-compose.yaml \ + --project-directory . \ + up -d ${UP_SERVICES} --scale web=3; \ fi; \ fi; + @echo -e "${GREEN}done${NC}" -up-build: check-dialect build +docker-up-build: check-dialect build + @echo -e "${YELLOW}start docker compose system${NC}" if [[ ${PANEL_ENV} == "production" ]] ; then \ - docker compose -p ${PROJECTNAME} -f docker/docker-compose.yaml --project-directory . up -d --build --scale web=3; \ + docker compose -p ${PROJECTNAME} \ + -f docker/docker-compose.yaml \ + --project-directory . \ + up -d --build --scale web=3; \ else \ if [[ ${DB_DIALECT} == "postgresql" ]] ; then \ - docker compose -p ${PROJECTNAME} -f docker/docker-compose.yaml --project-directory . up -d --build ${UP_SERVICES} ${DBSERVICE} --scale web=3; \ + docker compose -p ${PROJECTNAME} \ + -f docker/docker-compose.yaml \ + --project-directory . \ + up -d --build ${UP_SERVICES} ${DBSERVICE} --scale web=3; \ else \ - docker compose -p ${PROJECTNAME} -f docker/docker-compose.yaml --project-directory . up -d --build ${UP_SERVICES} --scale web=3; \ + docker compose -p ${PROJECTNAME} \ + -f docker/docker-compose.yaml \ + --project-directory . \ + up -d --build ${UP_SERVICES} --scale web=3; \ fi; \ fi; + @echo -e "${GREEN}done${NC}" -down: +docker-down: + @echo -e "${YELLOW}stop docker compose system${NC}" docker compose -p ${PROJECTNAME} -f docker/docker-compose.yaml --project-directory . down + @echo -e "${GREEN}done${NC}" -up-init: +docker-up-init: + @echo -e "${YELLOW}init docker compose system${NC}" bash docker/compose_init.sh + @echo -e "${GREEN}done${NC}" -db-clean: - docker run --rm --name clean_database --entrypoint "" -v ${PWD}/shared_data:/app/shared_data -e PANEL_ENV=production ${IMAGEFULLNAME} /bin/sh -c "data-lunch db clean --yes" +docker-db-clean: + @echo -e "${YELLOW}clean database${NC}" + docker run --rm --name clean_database --entrypoint "" -v ${PWD}/shared_data:/app/shared_data -e PANEL_ENV=production ${IMAGEFULLNAME} /bin/sh -c "data-lunch -o ${PANEL_ARGS} db clean --yes" + @echo -e "${GREEN}done${NC}" -gcp-config: - gcloud config configurations create ${APP} - gcloud config set project ${GCLOUD_PROJECT} - gcloud auth login - gcloud auth application-default login - gcloud auth application-default set-quota-project ${GCLOUD_PROJECT} +# Clean rules ----------------------------------------------------------------- +clean-folders: + @echo -e "${YELLOW}clean folders${NC}" + rm -rf .ipynb_checkpoints __pycache__ .pytest_cache */.ipynb_checkpoints */__pycache__ */.pytest_cache dist + @echo -e "${GREEN}done${NC}" -gcp-revoke: - gcloud config configurations activate default - gcloud config configurations delete ${APP} - gcloud auth application-default revoke +clean-docker: + @echo -e "${YELLOW}clean docker${NC}" + docker system prune -f + @echo -e "${GREEN}done${NC}" -ssl-cert: - mkdir -p ./ssl/conf/live/${DOMAIN}/ - curl https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > ./ssl/conf/options-ssl-nginx.conf - find ./ssl/conf/ -name "ssl-dhparams.pem" -type f -mtime +1 -delete - if [[ ! -f ./ssl/conf/ssl-dhparams.pem ]] ; then \ - echo "building dhparam"; \ - openssl dhparam -out ./ssl/conf/ssl-dhparams.pem 2048; \ - else \ - echo "dhparam already exists" ;\ - fi; - openssl req -nodes -x509 -newkey rsa:2048 -keyout ./ssl/conf/live/${DOMAIN}/privkey.pem -out ./ssl/conf/live/${DOMAIN}/fullchain.pem -subj "/C=IT/ST=Lombardia/L=Milan/O=MIC/OU=IT/CN=${DOMAIN}/" +# Other rules ----------------------------------------------------------------- -rm-ssl-cert: - rm -R ./ssl +interrogate: + @echo -e "${YELLOW}check docstrings${NC}" + @${CONDA_ACTIVATE_BASE} \ + conda activate ci-cd;\ + interrogate -vv \ + --ignore-module \ + --ignore-init-method \ + --ignore-private \ + --ignore-magic \ + --ignore-property-decorators \ + --fail-under=80 \ + dlunch + @echo -e "${GREEN}done${NC}" -generate-secrets: - @echo "\n*** START ***" - @echo "\nCOOKIE SECRET:" - @panel secret - @echo "\nENCRIPTION KEY:" - @panel oauth-secret - @echo "\n*** END ***\n" +package-build: + @echo -e "${YELLOW}build python package${NC}" + @${CONDA_ACTIVATE_BASE} \ + conda activate ci-cd;\ + python -m build + @echo -e "${GREEN}done${NC}" + +package-publish: + @echo -e "${YELLOW}publish python package to PyPI${NC}" + @${CONDA_ACTIVATE_BASE} \ + conda activate ci-cd;\ + twine upload --repository dlunch dist/* + @echo -e "${GREEN}done${NC}" + +package-test-install: + @echo -e "${YELLOW}install package from PyPI${NC}" + pip install dlunch + @echo -e "${GREEN}done${NC}" + +package-test-publish: + @echo -e "${YELLOW}publish python package to TestPyPI${NC}" + @${CONDA_ACTIVATE_BASE} \ + conda activate ci-cd;\ + twine upload --repository testpypi dist/* + @echo -e "${GREEN}done${NC}" + +package-test-install: + @echo -e "${YELLOW}install package from TestPyPI${NC}" + pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple dlunch + @echo -e "${GREEN}done${NC}" + +pre-commit-run: + @echo -e "${YELLOW}run pre-commit hooks${NC}" + @${CONDA_ACTIVATE_BASE} \ + conda activate ci-cd;\ + pre-commit run + @echo -e "${GREEN}done${NC}" + +commitizen-bump: + @echo -e "${YELLOW}run commitizen bump${NC}" + @${CONDA_ACTIVATE_BASE} \ + conda activate ci-cd && \ + git checkout development && \ + git pull --ff-only && \ + git checkout master && \ + git pull --ff-only && \ + git merge development --no-ff && \ + cz bump --no-verify && \ + git checkout development && \ + git merge master --no-ff + @echo -e "${GREEN}done${NC}" -all: up-init up-build +commitizen-push: + @echo -e "${YELLOW}run commitizen push${NC}" + @${CONDA_ACTIVATE_BASE} \ + conda activate ci-cd && \ + git checkout development && \ + git pull --ff-only && \ + git checkout master && \ + git pull --ff-only && \ + git push &&\ + git push --tags &&\ + git checkout development && \ + git push + @echo -e "${GREEN}done${NC}" + +send-ip-email: + @echo -e "${YELLOW}send email with GCP instance IP${NC}" + docker run --rm --name send_email \ + --entrypoint "" \ + -v ${PWD}/scripts:/app/scripts \ + -v ${PWD}/shared_data:/app/shared_data \ + -e MAIL_USER=${MAIL_USER} \ + -e MAIL_APP_PASSWORD=${MAIL_APP_PASSWORD} \ + -e MAIL_RECIPIENTS=${MAIL_RECIPIENTS} \ + -e DOMAIN=${DOMAIN} \ + ${IMAGEFULLNAME} /bin/sh -c "python /app/scripts/send_email_with_ip.py ${PANEL_ARGS}" + @echo -e "${GREEN}done${NC}" + +create-users-credentials: + @echo -e "${YELLOW}create user credentials from list in GCP storage${NC}" + docker run --rm --name create_users \ + --entrypoint "" \ + -v ${PWD}/scripts:/app/scripts \ + -v ${PWD}/shared_data:/app/shared_data \ + -e PANEL_ENV=${PANEL_ENV} \ + -e GCLOUD_BUCKET=${GCLOUD_BUCKET} \ + -e GCLOUD_PROJECT=${GCLOUD_PROJECT} \ + -e MAIL_USER=${MAIL_USER} \ + -e MAIL_APP_PASSWORD=${MAIL_APP_PASSWORD} \ + -e MAIL_RECIPIENTS=${MAIL_RECIPIENTS} \ + -e DOMAIN=${DOMAIN} \ + ${IMAGEFULLNAME} /bin/sh -c "python /app/scripts/create_users_from_list.py ${PANEL_ARGS}" + @echo -e "${GREEN}done${NC}" -clean: - docker system prune -f \ No newline at end of file +clean: clean-docker clean-folders diff --git a/pyproject.toml b/pyproject.toml index f972c74..f14aed4 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,23 +3,32 @@ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "data_lunch" +name = "dlunch" version = "2.7.0" +authors = [ + { name="Michele Alberti", email="michele.alberti90@gmail.com" }, +] description = "The ultimate web app for a well organized lunch." readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = ["python", "webapp", "lunch"] license = {text = "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International"} classifiers = [ "Programming Language :: Python :: 3", + "License :: Free for non-commercial use", + "Operating System :: OS Independent", ] dynamic = ["dependencies"] +[project.urls] +Homepage = "https://github.com/Michele-Alberti/data-lunch" +Issues = "https://github.com/Michele-Alberti/data-lunch/issues" + [project.scripts] -data-lunch = "data_lunch_app.cli:main" +data-lunch = "dlunch.cli:main" [tool.setuptools] -packages = ["data_lunch_app"] +packages = ["dlunch"] [tool.setuptools.dynamic] dependencies = {file = ["requirements/requirements.txt"]} @@ -45,7 +54,7 @@ exclude = ''' name = "cz_conventional_commits" version = "2.7.0" version_files = [ - "data_lunch_app/__init__.py", + "dlunch/__init__.py", "pyproject.toml:version", ] tag_format = "v$version" diff --git a/requirements/ci-cd.yml b/requirements/ci-cd.yml new file mode 100644 index 0000000..ce8c111 --- /dev/null +++ b/requirements/ci-cd.yml @@ -0,0 +1,13 @@ +name: ci-cd +channels: + - defaults + - conda-forge +dependencies: + - python=3.11.5 + - pre_commit=3.4.0 + - interrogate=1.5.0 + - twine=4.0.2 + - pip=23.3.1 + - pip: + - commitizen==3.13.0 + - build==1.0.3 diff --git a/requirements/commitizen.yml b/requirements/commitizen.yml deleted file mode 100755 index 84005ae..0000000 --- a/requirements/commitizen.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: commitizen -channels: - - defaults -dependencies: - - python=3.11.5 - - pip=23.3.1 - - pip: - - commitizen==3.13.0 diff --git a/requirements/pre-commit.yml b/requirements/pre-commit.yml deleted file mode 100755 index b31781c..0000000 --- a/requirements/pre-commit.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: pre-commit -channels: - - defaults - - conda-forge -dependencies: - - python=3.11.5 - - pre_commit=3.4.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 5fa55c7..3218ff3 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -10,7 +10,7 @@ tenacity==8.2.2 tqdm==4.65.0 panel==1.3.6 sqlalchemy==2.0.23 -psycopg==3.1.16 +psycopg[binary]==3.1.16 hydra-core==1.3.2 google-cloud-storage==2.6.0 pytesseract==0.3.10 diff --git a/scripts/create_users_from_list.py b/scripts/create_users_from_list.py index d377008..3935a76 100644 --- a/scripts/create_users_from_list.py +++ b/scripts/create_users_from_list.py @@ -18,8 +18,8 @@ import pathlib # Password -import data_lunch_app.auth as auth -from data_lunch_app.cloud import download_from_gcloud_as_bytes +import dlunch.auth as auth +from dlunch.cloud import download_from_gcloud_as_bytes import pandas as pd from hydra import compose, initialize import sys @@ -30,7 +30,7 @@ # global initialization initialize( - config_path="../data_lunch_app/conf", + config_path="../dlunch/conf", job_name="script_create_users_from_list", version_base="1.3", ) @@ -76,7 +76,7 @@ for user in new_users_names_df.itertuples(): # Generate a random password password = auth.generate_password( - special_chars=config.auth.basic_auth.psw_special_chars + special_chars=config.basic_auth.psw_special_chars ) # Add hashed password to credentials file auth.add_user_hashed_password(user.name, password, config=config) diff --git a/setup_dev_env.sh b/setup_dev_env.sh index 64e76d9..f4a7f67 100755 --- a/setup_dev_env.sh +++ b/setup_dev_env.sh @@ -9,13 +9,9 @@ echo -e "${YELLOW}installing development environment${NC}" conda env create -f requirements/environment.yml echo -e "${YELLOW}activate with ${CODE} conda activate data-lunch ${NC}\n" -echo -e "${YELLOW}installing pre-commit environment${NC}" -conda env create -f requirements/pre-commit.yml -echo -e "${YELLOW}activate with ${CODE} conda activate pre-commit ${NC}\n" - -echo -e "${YELLOW}installing commitizen environment${NC}" -conda env create -f requirements/commitizen.yml -echo -e "${YELLOW}activate with ${CODE} conda activate commitizen ${NC}\n" +echo -e "${YELLOW}installing ci-cd environment${NC}" +conda env create -f requirements/ci-cd.yml +echo -e "${YELLOW}activate with ${CODE} conda activate ci-cd ${NC}\n" echo -e "${YELLOW}installing google cloud environment${NC}" conda env create -f requirements/gc-sdk.yml @@ -35,7 +31,7 @@ done # Install pre-commit hooks echo -e "${YELLOW}installing pre-commit environment${NC}" -conda activate pre-commit +conda activate ci-cd pre-commit install echo -e "\n${GREEN}pre-commit hooks installed${NC}\n" while true; do