From efedf16c640780ec76066cf277bf313f55023d9d Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Sun, 8 Dec 2024 22:27:19 +0100 Subject: [PATCH 1/3] refactor: update API and improves data retrieval. --- README.md | 21 ++++++++++++- api_data_schemas.py | 75 +++++++++++++++++++++++++++++++++++++++++++++ api_v1.py | 67 ++++++++++------------------------------ data_retriever.py | 46 ++++----------------------- response_models.py | 35 --------------------- 5 files changed, 118 insertions(+), 126 deletions(-) create mode 100644 api_data_schemas.py delete mode 100644 response_models.py diff --git a/README.md b/README.md index 098de5f..e2019c1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ -# RelaySMS-Telemetry-Aggregator +# RelaySMS Telemetry Aggregator + Collects, analyzes, and exposes RelaySMS usage data via a unified API for transparent telemetry insights. + +## References + +4. [REST API V1 Resources](https://api.telemetry.smswithoutborders.com/docs) + +## Contributing + +To contribute: + +1. Fork the repository. +2. Create a feature branch: `git checkout -b feature-branch`. +3. Commit your changes: `git commit -m 'Add a new feature'`. +4. Push to the branch: `git push origin feature-branch`. +5. Open a Pull Request. + +## License + +This project is licensed under the GNU General Public License (GPL). See the [LICENSE](LICENSE) file for details. diff --git a/api_data_schemas.py b/api_data_schemas.py new file mode 100644 index 0000000..a455098 --- /dev/null +++ b/api_data_schemas.py @@ -0,0 +1,75 @@ +""" +This program is free software: you can redistribute it under the terms +of the GNU General Public License, v. 3.0. If a copy of the GNU General +Public License was not distributed with this file, see . +""" + +from typing import Literal +from pydantic import BaseModel, Field +from fastapi import Query + + +class SummaryDetails(BaseModel): + """Details of the summary metrics.""" + + total_signup_users: int + total_retained_users: int + + +class SummaryParams(BaseModel): + """Parameters for filtering and grouping summary metrics.""" + + start_date: str = Field(description="Start date in 'YYYY-MM-DD' format.") + end_date: str = Field(description="End date in 'YYYY-MM-DD' format.") + country_code: str = Field( + default=None, description="2-character ISO region code.", max_length=2 + ) + + +class SummaryResponse(BaseModel): + """Response model containing summary metrics.""" + + summary: SummaryDetails + + +class SignupDetails(BaseModel): + """Details of the signup metrics.""" + + total_signup_users: int + total_retained_users: int + + +class SignupParams(BaseModel): + """Parameters for filtering and grouping signup metrics.""" + + start_date: str = Field(description="Start date in 'YYYY-MM-DD' format.") + end_date: str = Field(description="End date in 'YYYY-MM-DD' format.") + country_code: str = Field(description="2-character ISO region code.", max_length=2) + granularity: Literal["day", "month"] = Field( + default="day", description="Granularity of data (day or month)." + ) + group_by: Literal["country", "date"] = Field( + default="date", + description="Criteria to group results (e.g., 'country', 'date').", + ) + top: int = Field( + default=None, + description="Maximum number of results to return. " + "(cannot be used with 'page' or 'page_size')", + ) + page: int = Query(default=1, ge=1, description="Page number for paginated results.") + page_size: int = Query( + default=10, ge=10, le=100, description="Number of records per page." + ) + + +class SignupResponse(BaseModel): + """Response model containing signup metrics.""" + + signup: SignupDetails + + +class ErrorResponse(BaseModel): + """Response model for errors.""" + + error: str diff --git a/api_v1.py b/api_v1.py index d64723d..0677f47 100644 --- a/api_v1.py +++ b/api_v1.py @@ -4,35 +4,16 @@ Public License was not distributed with this file, see . """ -import datetime +from typing import Annotated import requests from fastapi import APIRouter, HTTPException, Query from fastapi.responses import JSONResponse from data_retriever import get_summary -from response_models import SummaryResponse, ErrorResponse +from api_data_schemas import SummaryResponse, ErrorResponse, SummaryParams router = APIRouter(prefix="/v1", tags=["API V1"]) -def parse_date(date_str: str = None, default_days: int = 0) -> datetime.datetime: - """ - Parse and validate the date string. - """ - if not date_str: - default_date = datetime.datetime.now() + datetime.timedelta(days=default_days) - return default_date.strftime("%Y-%m-%d") - - try: - return datetime.datetime.strptime(date_str, "%Y-%m-%d") - except ValueError as e: - raise HTTPException( - status_code=400, - detail={ - "error": f"Date must be in the format 'YYYY-MM-DD', but got {date_str}" - }, - ) from e - - def get_security_headers() -> dict: """ Return a dictionary of security headers. @@ -50,47 +31,33 @@ def get_security_headers() -> dict: @router.get( "/summary", - response_model=SummaryResponse, responses={ 400: {"model": ErrorResponse}, 422: {"model": ErrorResponse}, + 500: {"model": ErrorResponse}, }, + response_model=SummaryResponse, ) -def summary( - start_date: str, - end_date: str, - page: int = Query(default=1, ge=1), - page_size: int = Query(default=10, ge=10), -): - """ - Fetch metrics summary. - - Args: - page (int): Pagination page number. - page_size (int): Number of items per page. - start_date (datetime): Start date for filtering. - end_date (datetime): End date for filtering. - - Returns: - dict: Summary of metrics. - """ - start_date = parse_date(start_date, default_days=0) - end_date = parse_date(end_date, default_days=30) +def summary(query: Annotated[SummaryParams, Query()]) -> SummaryResponse: + """Fetch metrics summary.""" try: params = { - "page": page, - "page_size": page_size, - "start": start_date.strftime("%Y-%m-%d"), - "end": end_date.strftime("%Y-%m-%d"), + "start_date": query.start_date, + "end_date": query.end_date, + "country_code": query.country_code, } summary_data = get_summary(params) - return JSONResponse( - content=summary_data, - headers=get_security_headers(), - ) + response_data = { + "summary": { + "total_signup_users": summary_data["total_signup_users"], + "total_retained_users": summary_data["total_retained_users"], + } + } + + return JSONResponse(content=response_data, headers=get_security_headers()) except requests.HTTPError as e: raise HTTPException( status_code=e.response.status_code, detail=e.response.json() diff --git a/data_retriever.py b/data_retriever.py index e7620db..3c8d84d 100644 --- a/data_retriever.py +++ b/data_retriever.py @@ -28,8 +28,8 @@ def get_summary(params: dict): Raises: HTTPError: If any of the external API calls fail. """ - retained_metrics_url = f"{VAULT_URL}/v3/retained-user-metrics" - signup_metrics_url = f"{VAULT_URL}/v3/signup-metrics" + retained_metrics_url = f"{VAULT_URL}/v3/metrics/retained" + signup_metrics_url = f"{VAULT_URL}/v3/metrics/signup" try: retained_response = requests.get( @@ -43,46 +43,12 @@ def get_summary(params: dict): retained_metrics = retained_response.json() signup_metrics = signup_response.json() - combined_metrics = { - "summary": { - "total_signup_users": signup_metrics["total_signup_count"], - "total_active_users": retained_metrics["total_retained_user_count"], - "total_signup_countries": signup_metrics["total_country_count"], - "total_active_countries": retained_metrics["total_country_count"], - "data": [], - } + metrics_summary = { + "total_signup_users": signup_metrics["total_signup_users"], + "total_retained_users": retained_metrics["total_retained_users"], } - all_dates = set(signup_metrics["data"].keys()).union( - retained_metrics["data"].keys() - ) - - sorted_dates = sorted(all_dates, reverse=True) - - for date in sorted_dates: - combined_date_data = {"date": date, "stats": []} - - signup_countries = signup_metrics["data"].get(date, {}) - retained_countries = retained_metrics["data"].get(date, {}) - all_countries = set(signup_countries.keys()).union( - retained_countries.keys() - ) - - for country in all_countries: - stats = { - "country": country, - "signup_users": signup_countries.get(country, {}).get( - "signup_count", 0 - ), - "active_users": retained_countries.get(country, {}).get( - "retained_user_count", 0 - ), - } - combined_date_data["stats"].append(stats) - - combined_metrics["summary"]["data"].append(combined_date_data) - - return combined_metrics + return metrics_summary except requests.RequestException as e: raise e diff --git a/response_models.py b/response_models.py deleted file mode 100644 index 4435a16..0000000 --- a/response_models.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -This program is free software: you can redistribute it under the terms -of the GNU General Public License, v. 3.0. If a copy of the GNU General -Public License was not distributed with this file, see . -""" - -from pydantic import BaseModel -from typing import List, Dict - - -class Stats(BaseModel): - country: str - signup_users: int - active_users: int - - -class SummaryData(BaseModel): - date: str - stats: List[Stats] - - -class Overview(BaseModel): - total_signup_users: int - total_active_users: int - total_signup_countries: int - total_active_countries: int - - -class SummaryResponse(BaseModel): - summary: Overview - data: List[SummaryData] - - -class ErrorResponse(BaseModel): - error: str From e3ecc967d6502103f62525c735ae88b01264be16 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Sun, 8 Dec 2024 22:42:58 +0100 Subject: [PATCH 2/3] build(ci): add staging deployment pipeline. --- .github/dependabot.yml | 15 ++++++++ .github/workflows/staging-deploy.yml | 55 ++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/staging-deploy.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a37eb23 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 99 + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 99 + allow: + - dependency-type: "direct" + - dependency-type: "indirect" diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml new file mode 100644 index 0000000..a2f4974 --- /dev/null +++ b/.github/workflows/staging-deploy.yml @@ -0,0 +1,55 @@ +name: Staging Server Build Pipeline + +on: + push: + branches: + - staging + +jobs: + deploy: + name: Deploy to Staging Server + runs-on: ubuntu-latest + environment: + name: staging + steps: + - name: Execute Remote SSH Commands + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.HOST }} + username: ${{ secrets.USERNAME }} + key: ${{ secrets.KEY }} + script: | + set -e + + echo "============================" + echo "Updating repository ..." + echo "============================" + if ! assembler clone --branch staging --project telemetry_aggregator; then + echo "❌ Error updating repository!" + exit 1 + fi + echo "===============================" + echo "✅ Repository update complete" + echo "===============================" + + echo "=========================" + echo "Building project ..." + echo "=========================" + if ! assembler deploy --project telemetry_aggregator; then + echo "❌ Error building project!" + exit 1 + fi + echo "===========================" + echo "✅ Project build complete" + echo "===========================" + + echo "=============================" + echo "Cleaning up staging builds ..." + echo "=============================" + if ! ${{ secrets.CLEANUP_CMD }}; then + echo "❌ Error cleaning up builds!" + exit 1 + fi + echo "=============================" + echo "✅ Cleanup complete" + echo "=============================" From fd934c6a6aee001f60372646b05e0f9772e98746 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Mon, 9 Dec 2024 14:18:03 +0100 Subject: [PATCH 3/3] feat: Add signup and retained user metrics APIs. --- api_data_schemas.py | 81 +++++++++++++++++++++++++++++++++++++-------- api_v1.py | 81 +++++++++++++++++++++++++++++++++++++++++++-- data_retriever.py | 62 ++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 16 deletions(-) diff --git a/api_data_schemas.py b/api_data_schemas.py index a455098..26965cd 100644 --- a/api_data_schemas.py +++ b/api_data_schemas.py @@ -4,7 +4,7 @@ Public License was not distributed with this file, see . """ -from typing import Literal +from typing import Literal, List, Union from pydantic import BaseModel, Field from fastapi import Query @@ -32,25 +32,19 @@ class SummaryResponse(BaseModel): summary: SummaryDetails -class SignupDetails(BaseModel): - """Details of the signup metrics.""" - - total_signup_users: int - total_retained_users: int - - -class SignupParams(BaseModel): - """Parameters for filtering and grouping signup metrics.""" +class MetricsParams(BaseModel): + """Parameters for filtering and grouping metrics.""" start_date: str = Field(description="Start date in 'YYYY-MM-DD' format.") end_date: str = Field(description="End date in 'YYYY-MM-DD' format.") - country_code: str = Field(description="2-character ISO region code.", max_length=2) + country_code: str = Field( + default=None, description="2-character ISO region code.", max_length=2 + ) granularity: Literal["day", "month"] = Field( - default="day", description="Granularity of data (day or month)." + default="day", description="Granularity of data." ) group_by: Literal["country", "date"] = Field( - default="date", - description="Criteria to group results (e.g., 'country', 'date').", + default="date", description="Criteria to group results." ) top: int = Field( default=None, @@ -63,12 +57,71 @@ class SignupParams(BaseModel): ) +class PaginationDetails(BaseModel): + """Pagination details for paginated responses.""" + + page: int + page_size: int + total_pages: int + total_records: int + + +class CountrySignupData(BaseModel): + """Signup data grouped by country.""" + + country_code: str + signup_users: int + + +class TimeframeSignupData(BaseModel): + """Signup data grouped by timeframe.""" + + timeframe: str + signup_users: int + + +class SignupDetails(BaseModel): + """Details of the signup metrics.""" + + total_signup_users: int + pagination: PaginationDetails + data: List[Union[CountrySignupData, TimeframeSignupData]] + + class SignupResponse(BaseModel): """Response model containing signup metrics.""" signup: SignupDetails +class CountryRetainedData(BaseModel): + """Retained data grouped by country.""" + + country_code: str + retained_users: int + + +class TimeframeRetainedData(BaseModel): + """Retained data grouped by timeframe.""" + + timeframe: str + retained_users: int + + +class RetainedDetails(BaseModel): + """Details of the retained metrics.""" + + total_retained_users: int + pagination: PaginationDetails + data: List[Union[CountryRetainedData, TimeframeRetainedData]] + + +class RetainedResponse(BaseModel): + """Response model containing retained metrics.""" + + retained: RetainedDetails + + class ErrorResponse(BaseModel): """Response model for errors.""" diff --git a/api_v1.py b/api_v1.py index 0677f47..9ceedfc 100644 --- a/api_v1.py +++ b/api_v1.py @@ -8,8 +8,15 @@ import requests from fastapi import APIRouter, HTTPException, Query from fastapi.responses import JSONResponse -from data_retriever import get_summary -from api_data_schemas import SummaryResponse, ErrorResponse, SummaryParams +from data_retriever import get_summary, get_signup, get_retained +from api_data_schemas import ( + MetricsParams, + ErrorResponse, + SummaryResponse, + SummaryParams, + SignupResponse, + RetainedResponse, +) router = APIRouter(prefix="/v1", tags=["API V1"]) @@ -62,3 +69,73 @@ def summary(query: Annotated[SummaryParams, Query()]) -> SummaryResponse: raise HTTPException( status_code=e.response.status_code, detail=e.response.json() ) from e + + +@router.get( + "/signup", + responses={ + 400: {"model": ErrorResponse}, + 422: {"model": ErrorResponse}, + 500: {"model": ErrorResponse}, + }, + response_model=SignupResponse, +) +def signup(query: Annotated[MetricsParams, Query()]) -> SignupResponse: + """Fetch signup users metrics.""" + + try: + params = { + "start_date": query.start_date, + "end_date": query.end_date, + "country_code": query.country_code, + "granularity": query.granularity, + "group_by": query.group_by, + "top": query.top, + "page": query.page, + "page_size": query.page_size, + } + + signup_data = get_signup(params) + + response_data = {"signup": signup_data} + + return JSONResponse(content=response_data, headers=get_security_headers()) + except requests.HTTPError as e: + raise HTTPException( + status_code=e.response.status_code, detail=e.response.json() + ) from e + + +@router.get( + "/retained", + responses={ + 400: {"model": ErrorResponse}, + 422: {"model": ErrorResponse}, + 500: {"model": ErrorResponse}, + }, + response_model=RetainedResponse, +) +def retained(query: Annotated[MetricsParams, Query()]) -> RetainedResponse: + """Fetch retained users metrics.""" + + try: + params = { + "start_date": query.start_date, + "end_date": query.end_date, + "country_code": query.country_code, + "granularity": query.granularity, + "group_by": query.group_by, + "top": query.top, + "page": query.page, + "page_size": query.page_size, + } + + retained_data = get_retained(params) + + response_data = {"retained": retained_data} + + return JSONResponse(content=response_data, headers=get_security_headers()) + except requests.HTTPError as e: + raise HTTPException( + status_code=e.response.status_code, detail=e.response.json() + ) from e diff --git a/data_retriever.py b/data_retriever.py index 3c8d84d..162a5f6 100644 --- a/data_retriever.py +++ b/data_retriever.py @@ -52,3 +52,65 @@ def get_summary(params: dict): except requests.RequestException as e: raise e + + +def get_signup(params: dict): + """ + Fetches signup metrics data from the metrics API. + + Args: + params (dict): Query parameters to include in the API request. Expected keys may include: + - start_date (str): Start date for the metrics in 'YYYY-MM-DD' format. + - end_date (str): End date for the metrics in 'YYYY-MM-DD' format. + - granularity (str, optional): Level of detail, e.g., 'day' or 'month'. + - group_by (str, optional): Dimension to group the metrics by, e.g., 'country'. + - top (int, optional): Maximum number of results to return. + - page (int, optional): Pagination page number. + - page_size (int, optional): Number of records per page. + + Returns: + dict: The JSON response from the metrics API containing signup data. + """ + signup_metrics_url = f"{VAULT_URL}/v3/metrics/signup" + + try: + signup_response = requests.get(signup_metrics_url, params=params, timeout=30) + + signup_response.raise_for_status() + + return signup_response.json() + + except requests.RequestException as e: + raise e + + +def get_retained(params: dict): + """ + Fetches retained metrics data from the metrics API. + + Args: + params (dict): Query parameters to include in the API request. Expected keys may include: + - start_date (str): Start date for the metrics in 'YYYY-MM-DD' format. + - end_date (str): End date for the metrics in 'YYYY-MM-DD' format. + - granularity (str, optional): Level of detail, e.g., 'day' or 'month'. + - group_by (str, optional): Dimension to group the metrics by, e.g., 'country'. + - top (int, optional): Maximum number of results to return. + - page (int, optional): Pagination page number. + - page_size (int, optional): Number of records per page. + + Returns: + dict: The JSON response from the metrics API containing retained data. + """ + retained_metrics_url = f"{VAULT_URL}/v3/metrics/retained" + + try: + retained_response = requests.get( + retained_metrics_url, params=params, timeout=30 + ) + + retained_response.raise_for_status() + + return retained_response.json() + + except requests.RequestException as e: + raise e