From a0f177b48bcc39e8b00cb67bd267ea71298793cd Mon Sep 17 00:00:00 2001 From: Dyakov Roman Date: Thu, 16 Mar 2023 22:17:32 +0300 Subject: [PATCH] Marketing auth (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Удалил дэш Сделал возможность писать любой мусор Добавил мэппинг на пользователей Твой ФФ --- Makefile | 19 ++++ logging_dev.conf | 21 +++++ marketing_api/__main__.py | 3 +- marketing_api/dashboard/__init__.py | 0 marketing_api/dashboard/base.py | 27 ------ marketing_api/methods/__init__.py | 3 - marketing_api/methods/dash_db.py | 91 ------------------- marketing_api/models/__init__.py | 1 + marketing_api/models/base.py | 3 +- marketing_api/models/db.py | 56 ++++++++---- marketing_api/routes/__init__.py | 1 + marketing_api/routes/base.py | 77 ++++++++++------ marketing_api/routes/models.py | 7 +- marketing_api/settings.py | 4 +- ...f17f_no_foreign_key_constraint_for_user.py | 24 +++++ .../d1136ec942ac_no_nesessary_fields.py | 24 +++++ .../versions/e2c2d4fe34f1_auth_user_id.py | 28 ++++++ requirements.dev.txt | 3 + requirements.txt | 5 - tests/base/__init__.py | 0 tests/conftest.py | 6 +- tests/{base/base.py => routes.py} | 26 ++++-- 22 files changed, 237 insertions(+), 192 deletions(-) create mode 100644 Makefile create mode 100644 logging_dev.conf delete mode 100644 marketing_api/dashboard/__init__.py delete mode 100644 marketing_api/dashboard/base.py delete mode 100644 marketing_api/methods/__init__.py delete mode 100644 marketing_api/methods/dash_db.py create mode 100644 migrations/versions/cdb1cd1ef17f_no_foreign_key_constraint_for_user.py create mode 100644 migrations/versions/d1136ec942ac_no_nesessary_fields.py create mode 100644 migrations/versions/e2c2d4fe34f1_auth_user_id.py delete mode 100644 tests/base/__init__.py rename tests/{base/base.py => routes.py} (60%) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4cde912 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +run: + source ./venv/bin/activate && uvicorn --reload --log-config logging_dev.conf marketing_api.routes.base:app + +configure: venv + source ./venv/bin/activate && pip install -r requirements.dev.txt -r requirements.txt + +venv: + python3.11 -m venv venv + +format: + source ./venv/bin/activate && autoflake -r --in-place --remove-all-unused-imports ./marketing_api + source ./venv/bin/activate && isort ./marketing_api + source ./venv/bin/activate && black ./marketing_api + +db: + docker run -d -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust --name db-marketing-backend postgres:15 + +migrate: + source ./venv/bin/activate && alembic upgrade head diff --git a/logging_dev.conf b/logging_dev.conf new file mode 100644 index 0000000..7837272 --- /dev/null +++ b/logging_dev.conf @@ -0,0 +1,21 @@ +[loggers] +keys=root + +[handlers] +keys=all + +[formatters] +keys=main + +[logger_root] +level=DEBUG +handlers=all + +[handler_all] +class=StreamHandler +formatter=main +level=DEBUG +args=(sys.stdout,) + +[formatter_main] +format=%(asctime)s %(levelname)-8s %(name)-15s %(message)s diff --git a/marketing_api/__main__.py b/marketing_api/__main__.py index 17acb27..25fca11 100644 --- a/marketing_api/__main__.py +++ b/marketing_api/__main__.py @@ -1,6 +1,7 @@ -from marketing_api.routes.base import app import uvicorn +from marketing_api.routes.base import app + if __name__ == '__main__': uvicorn.run(app) diff --git a/marketing_api/dashboard/__init__.py b/marketing_api/dashboard/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/marketing_api/dashboard/base.py b/marketing_api/dashboard/base.py deleted file mode 100644 index 1c0da3d..0000000 --- a/marketing_api/dashboard/base.py +++ /dev/null @@ -1,27 +0,0 @@ -from dash import Dash, dcc, html -import plotly.express as px -from marketing_api.methods.dash_db import graph_dau, graph_wau, graph_mau -from sqlalchemy.engine import create_engine -from sqlalchemy.orm import sessionmaker -from marketing_api.settings import get_settings - - -engine = create_engine(get_settings().DB_DSN) -session = sessionmaker(engine) - -dash_app = Dash(__name__, requests_pathname_prefix="/dashboard/") - -with session() as db_session: - dau = graph_dau(db_session) - wau = graph_wau(db_session) - mau = graph_mau(db_session) - - -dash_app.layout = html.Div(children=[ - html.H1( - children='Profcomff app dashboard', - ), - dau, - wau, - mau, -]) diff --git a/marketing_api/methods/__init__.py b/marketing_api/methods/__init__.py deleted file mode 100644 index c773676..0000000 --- a/marketing_api/methods/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .dash_db import graph_mau, graph_wau, graph_dau - -__all__ = ["graph_wau", "graph_mau", "graph_dau"] diff --git a/marketing_api/methods/dash_db.py b/marketing_api/methods/dash_db.py deleted file mode 100644 index d234970..0000000 --- a/marketing_api/methods/dash_db.py +++ /dev/null @@ -1,91 +0,0 @@ -from sqlalchemy.orm import Session -from marketing_api.models.db import ActionsInfo -from datetime import datetime, timedelta -from dash.dcc import Graph - - -def count_users_in_daterange(session: Session, start_ts: datetime, end_ts: datetime) -> int: - """ - Counts ActionsInfo rows with distinct user_id in daterange [from,to) - :param start_ts: start date - :param end_ts: end date - :param session: sqlalchemy.orm.Session - :return: int number of distinct users - """ - res = session.query(ActionsInfo). \ - filter( - start_ts <= ActionsInfo.create_ts, - ActionsInfo.create_ts < end_ts, - ). \ - distinct(ActionsInfo.user_id). \ - count() - return res - - -def graph_dau(session: Session, start_ts: datetime = datetime(2022, 9, 1), end_ts=datetime.utcnow()) -> Graph: - res = dict() - curr = start_ts - while end_ts >= curr: - res[ - (curr + timedelta(days=1)).date().isoformat() - ] = count_users_in_daterange(session, start_ts=curr, end_ts=(curr + timedelta(days=1))) - curr += timedelta(days=1) - return Graph( - id='dau-graph', - figure={ - 'data': [ - {'x': list(res.keys()), 'y': list(res.values()), 'type': 'line', 'name': 'DAU'}, - ], - 'layout': { - 'title': 'Daily active users', - 'xlabel': 'date', - 'ylabel': 'users', - } - } - ) - - -def graph_wau(session: Session, start_ts: datetime = datetime(2022, 9, 1), end_ts=datetime.utcnow()) -> Graph: - res = dict() - curr = start_ts - timedelta(days=abs(start_ts.weekday() - (end_ts.weekday() - 6))) - while end_ts >= curr + timedelta(days=6): - res[ - f"{curr.date().isoformat()} - {(curr + timedelta(days=7)).date().isoformat()}" - ] = count_users_in_daterange(session, start_ts=curr, end_ts=(curr + timedelta(days=7))) - curr += timedelta(days=7) - return Graph( - id='wau-graph', - figure={ - 'data': [ - {'x': list(res.keys()), 'y': list(res.values()), 'type': 'line', 'name': 'WAU'}, - ], - 'layout': { - 'title': 'Weekly active users', - 'xaxis_title': 'date', - 'yaxis_title': 'users', - } - } - ) - - -def graph_mau(session: Session, start_ts: datetime = datetime(2022, 9, 1), end_ts=datetime.utcnow()) -> Graph: - res = dict() - curr = start_ts - timedelta(days=29 - (end_ts.day - start_ts.day) % 30) - while end_ts >= curr + timedelta(days=29): - res[ - f"{curr.date().isoformat()} - {(curr + timedelta(days=30)).date().isoformat()}" - ] = count_users_in_daterange(session, start_ts=curr, end_ts=(curr + timedelta(days=30))) - curr += timedelta(days=30) - return Graph( - id='mau-graph', - figure={ - 'data': [ - {'x': list(res.keys()), 'y': list(res.values()), 'type': 'line', 'name': 'MAU'}, - ], - 'layout': { - 'title': 'Monthly active users', - 'xaxis_title': 'date', - 'yaxis_title': 'users', - } - } - ) \ No newline at end of file diff --git a/marketing_api/models/__init__.py b/marketing_api/models/__init__.py index 41beaaa..f06858d 100644 --- a/marketing_api/models/__init__.py +++ b/marketing_api/models/__init__.py @@ -1,3 +1,4 @@ from .db import ActionsInfo + __all__ = ["ActionsInfo"] diff --git a/marketing_api/models/base.py b/marketing_api/models/base.py index aeff06e..86c2930 100644 --- a/marketing_api/models/base.py +++ b/marketing_api/models/base.py @@ -1,5 +1,6 @@ import re -from sqlalchemy.ext.declarative import as_declarative, declared_attr + +from sqlalchemy.orm import as_declarative, declared_attr @as_declarative() diff --git a/marketing_api/models/db.py b/marketing_api/models/db.py index 943bfd2..f7fb4b2 100644 --- a/marketing_api/models/db.py +++ b/marketing_api/models/db.py @@ -1,8 +1,11 @@ +import enum from datetime import datetime -import sqlalchemy.orm + +import sqlalchemy as sa +from sqlalchemy.orm import relationship from sqlalchemy import Column + from .base import Base -import enum class Actions(str, enum.Enum): @@ -12,26 +15,41 @@ class Actions(str, enum.Enum): INSTALLED: str = "installed" -class ActionsInfo(Base): - """Actions from user""" - - id = Column(sqlalchemy.Integer, primary_key=True) - user_id = Column(sqlalchemy.Integer, sqlalchemy.ForeignKey("user.id")) - action = Column(sqlalchemy.String, nullable=False) - path_from = Column(sqlalchemy.String, nullable=False) - path_to = Column(sqlalchemy.String, nullable=True) - additional_data = Column(sqlalchemy.String, nullable=True) - create_ts = Column(sqlalchemy.DateTime, nullable=False, default=datetime.utcnow) +class User(Base): + id = Column(sa.Integer, primary_key=True) + union_number = Column(sa.String, nullable=True) + auth_user_id = Column(sa.Integer, nullable=True) + modify_ts = Column(sa.DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) + create_ts = Column(sa.DateTime, nullable=False, default=datetime.utcnow) + + actions = relationship( + "ActionsInfo", + primaryjoin="foreign(ActionsInfo.user_id)==User.id", + uselist=True, + back_populates="user", + ) def __repr__(self): - return f"ActionInfo(user_id: {self.user_id}, action: {self.action}" + return f"User(id={self.id}, union_number={self.union_number}, auth_user_id={self.auth_user_id})" -class User(Base): - id = Column(sqlalchemy.Integer, primary_key=True) - union_number = Column(sqlalchemy.String, nullable=True) - modify_ts = Column(sqlalchemy.DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) - create_ts = Column(sqlalchemy.DateTime, nullable=False, default=datetime.utcnow) +class ActionsInfo(Base): + """Actions from user""" + + id = Column(sa.Integer, primary_key=True) + user_id = Column(sa.Integer, nullable=False) + action = Column(sa.String, nullable=False) + path_from = Column(sa.String, nullable=True) + path_to = Column(sa.String, nullable=True) + additional_data = Column(sa.String, nullable=True) + create_ts = Column(sa.DateTime, nullable=False, default=datetime.utcnow) + + user = relationship( + User, + primaryjoin="foreign(ActionsInfo.user_id)==User.id", + uselist=False, + back_populates="actions" + ) def __repr__(self): - return f"User(id: {self.id}, union_number: {self.union_number}" + return f"ActionInfo(user_id={self.user_id}, action={self.action})" diff --git a/marketing_api/routes/__init__.py b/marketing_api/routes/__init__.py index 7c8d47d..5086daf 100644 --- a/marketing_api/routes/__init__.py +++ b/marketing_api/routes/__init__.py @@ -1,3 +1,4 @@ from .base import app + __all__ = ["app"] diff --git a/marketing_api/routes/base.py b/marketing_api/routes/base.py index fecd641..f739c18 100644 --- a/marketing_api/routes/base.py +++ b/marketing_api/routes/base.py @@ -1,30 +1,27 @@ -from pydantic import ValidationError -from fastapi import FastAPI -from fastapi import Path -from fastapi.middleware.cors import CORSMiddleware -from fastapi_sqlalchemy import DBSessionMiddleware +import logging + +from auth_lib.fastapi import UnionAuth +from fastapi import FastAPI, Depends from fastapi.exceptions import HTTPException -from fastapi_sqlalchemy import db -from starlette.responses import PlainTextResponse, RedirectResponse -from starlette.middleware.wsgi import WSGIMiddleware -from sqlalchemy.exc import IntegrityError -import starlette +from fastapi.middleware.cors import CORSMiddleware +from fastapi_sqlalchemy import DBSessionMiddleware, db +from pydantic import ValidationError +from starlette.responses import PlainTextResponse from marketing_api import __version__ -from marketing_api.settings import get_settings from marketing_api.models import ActionsInfo -from marketing_api.dashboard.base import dash_app from marketing_api.models.db import User as DbUser +from marketing_api.settings import get_settings from .models import ActionInfo, User, UserPatch settings = get_settings() +logger = logging.getLogger(__name__) app = FastAPI( title='Сервис мониторинга активности пользователей', description='API для проведения маркетинговых исследований', version=__version__, - # Настраиваем интернет документацию root_path=settings.ROOT_PATH if __version__ != 'dev' else '/', docs_url=None if __version__ != 'dev' else '/docs', @@ -33,35 +30,60 @@ @app.post('/v1/action') -async def write_action(user_action_info: ActionInfo): - db.session.add(ActionsInfo(**user_action_info.dict())) +async def write_action( + user_action_info: ActionInfo, + user=Depends(UnionAuth(auto_error=False, allow_none=True)), +): + """Создать действие""" + user_id = user.get("id") if user else None + logger.debug(f"write_action by {user_id=}") + ai = ActionsInfo(**user_action_info.dict()) + db.session.add(ai) db.session.flush() + if ai.user: + logger.debug(ai.user) + ai.user.auth_user_id = user_id + db.session.flush() + else: + logger.warning(f"write_action with user {user_action_info.user_id} not exists!") return PlainTextResponse(status_code=200) @app.post('/v1/user', response_model=User) -async def create_user(): - user = DbUser() - db.session.add(user) +async def create_user(user=Depends(UnionAuth(auto_error=False, allow_none=True))): + """Создать уникальный идентификатор установки""" + user_id = user.get("id") if user else None + logger.debug(f"create_user by {user_id=}") + dbuser = DbUser() + dbuser.auth_user_id = user.get("id") if user else None + db.session.add(dbuser) db.session.flush() - return user + return dbuser @app.patch('/v1/user/{id}', response_model=User) -async def patch_user(id: int, patched_user: UserPatch): +async def patch_user( + id: int, + patched_user: UserPatch, + user=Depends(UnionAuth(["marketing.user.patch"])) +): + """Изменить пользователя в маркетинге + + Необходимые scopes: `marketing.user.patch` + """ + user_id = user.get("id") if user else None + logger.debug(f"patch_user by {user_id=}") result: DbUser = db.session.query(DbUser).filter(DbUser.id == id).one_or_none() if not result: raise HTTPException(404, "No user found") - result.union_number = patched_user.union_number + if patched_user.union_number: + result.union_number = patched_user.union_number + if patched_user.auth_user_id: + result.union_number = patched_user.auth_user_id db.session.flush() return result -@app.get('/') -async def to_dashboard(): - return RedirectResponse("/dashboard") - - @app.exception_handler(ValidationError) async def http_validation_error_handler(req, exc): return PlainTextResponse("Invalid data", status_code=422) @@ -85,6 +107,3 @@ async def http_error_handler(req, exc): allow_methods=settings.CORS_ALLOW_METHODS, allow_headers=settings.CORS_ALLOW_HEADERS, ) - -app.mount("/dashboard", WSGIMiddleware(dash_app.server)) - diff --git a/marketing_api/routes/models.py b/marketing_api/routes/models.py index 9290250..f895ef5 100644 --- a/marketing_api/routes/models.py +++ b/marketing_api/routes/models.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, AnyHttpUrl +from pydantic import BaseModel class Base(BaseModel): @@ -10,7 +10,7 @@ class ActionInfo(Base): user_id: int | None action: str additional_data: str | None - path_from: str + path_from: str | None path_to: str | None @@ -20,4 +20,5 @@ class User(Base): class UserPatch(Base): - union_number: str + union_number: str | None + auth_user_id: int | None diff --git a/marketing_api/settings.py b/marketing_api/settings.py index 4920fe5..a602307 100644 --- a/marketing_api/settings.py +++ b/marketing_api/settings.py @@ -1,13 +1,13 @@ import os from functools import lru_cache -from pydantic import BaseSettings, PostgresDsn, AnyHttpUrl +from pydantic import BaseSettings, PostgresDsn class Settings(BaseSettings): """Application settings""" - DB_DSN: PostgresDsn + DB_DSN: PostgresDsn = 'postgresql://postgres@localhost:5432/postgres' ROOT_PATH: str = '/' + os.getenv('APP_NAME', '') CORS_ALLOW_ORIGINS: list[str] = ['*'] diff --git a/migrations/versions/cdb1cd1ef17f_no_foreign_key_constraint_for_user.py b/migrations/versions/cdb1cd1ef17f_no_foreign_key_constraint_for_user.py new file mode 100644 index 0000000..6033c1b --- /dev/null +++ b/migrations/versions/cdb1cd1ef17f_no_foreign_key_constraint_for_user.py @@ -0,0 +1,24 @@ +"""No foreign key constraint for user + +Revision ID: cdb1cd1ef17f +Revises: 6b75dd50398f +Create Date: 2023-03-16 21:02:42.333305 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'cdb1cd1ef17f' +down_revision = '6b75dd50398f' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint('actions_info_user_id_fkey', 'actions_info', type_='foreignkey') + + +def downgrade(): + op.create_foreign_key('actions_info_user_id_fkey', 'actions_info', 'user', ['user_id'], ['id']) diff --git a/migrations/versions/d1136ec942ac_no_nesessary_fields.py b/migrations/versions/d1136ec942ac_no_nesessary_fields.py new file mode 100644 index 0000000..fa562a6 --- /dev/null +++ b/migrations/versions/d1136ec942ac_no_nesessary_fields.py @@ -0,0 +1,24 @@ +"""No nesessary fields + +Revision ID: d1136ec942ac +Revises: e2c2d4fe34f1 +Create Date: 2023-03-16 21:31:00.557581 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd1136ec942ac' +down_revision = 'e2c2d4fe34f1' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column('actions_info', 'path_from', existing_type=sa.VARCHAR(), nullable=True) + + +def downgrade(): + op.alter_column('actions_info', 'user_id', existing_type=sa.INTEGER(), nullable=True) diff --git a/migrations/versions/e2c2d4fe34f1_auth_user_id.py b/migrations/versions/e2c2d4fe34f1_auth_user_id.py new file mode 100644 index 0000000..6efd209 --- /dev/null +++ b/migrations/versions/e2c2d4fe34f1_auth_user_id.py @@ -0,0 +1,28 @@ +"""Auth user id + +Revision ID: e2c2d4fe34f1 +Revises: cdb1cd1ef17f +Create Date: 2023-03-16 21:13:20.798843 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e2c2d4fe34f1' +down_revision = 'cdb1cd1ef17f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('user', sa.Column('auth_user_id', sa.Integer(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'auth_user_id') + # ### end Alembic commands ### diff --git a/requirements.dev.txt b/requirements.dev.txt index fe2d178..4b4faa3 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -3,3 +3,6 @@ pytest pytest-cov black httpx +autoflake +isort +pytest_mock diff --git a/requirements.txt b/requirements.txt index 554dfda..888de4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,10 +6,5 @@ uvicorn alembic SQLAlchemy gunicorn -dash -dash-html-components -dash-core-components -dash-table -pandas auth-lib-profcomff[fastapi] logging-profcomff diff --git a/tests/base/__init__.py b/tests/base/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py index 978d2d9..20f9ca2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,10 +8,10 @@ from sqlalchemy.orm import Session, sessionmaker -@pytest.fixture(scope='session') +@pytest.fixture() def client(): client = TestClient(app) - return client + yield client @pytest.fixture(scope='session') @@ -20,7 +20,7 @@ def dbsession(): engine = create_engine(settings.DB_DSN) TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) Base.metadata.create_all(bind=engine) - return TestingSessionLocal() + yield TestingSessionLocal() @pytest.fixture diff --git a/tests/base/base.py b/tests/routes.py similarity index 60% rename from tests/base/base.py rename to tests/routes.py index b2d93fc..87df1c4 100644 --- a/tests/base/base.py +++ b/tests/routes.py @@ -1,17 +1,18 @@ +import json + from fastapi.testclient import TestClient +from pytest_mock import MockerFixture from sqlalchemy.orm import Session -from marketing_api.routes.models import ActionInfo, User -import json -def test_can_post_without_user_id(client: TestClient, dbsession: Session): +def test_can_post_without_user_id(client: TestClient): action = dict( action="INSTALL", path_from="http://127.0.0.1:8000/", path_to="http://127.0.0.1:8000/me" ) response = client.post(f"/v1/action", json=action) - assert response.status_code == 200 + assert response.status_code == 200, response.json() def test_can_post_with_user_id(client: TestClient, user_id: int): @@ -31,10 +32,19 @@ def test_cannot_post_invalid_info(client: TestClient): "path_to": "http://127.0.0.1:8000/me", } response = client.post(f"/v1/action", json=action) - assert response.status_code == 422 - - -def test_can_patch_user(client: TestClient): + assert response.status_code == 200 # Маркетинг принимает вообще все, что угодно + + +def test_can_patch_user(client: TestClient, mocker: MockerFixture): + user_mock = mocker.patch('auth_lib.fastapi.UnionAuth.__call__') + user_mock.return_value = { + "session_scopes": [{"id": 0, "name": "string", "comment": "string"}], + "user_scopes": [{"id": 0, "name": "string", "comment": "string"}], + "indirect_groups": [{"id": 0, "name": "string", "parent_id": 0}], + "groups": [{"id": 0, "name": "string", "parent_id": 0}], + "id": 0, + "email": "string", + } db_user = client.post("/v1/user") user_id = db_user.json()['id'] patch = json.dumps({'union_number': '666'})