From f88c3cb46680c48a7d8f1c87911dbd252d7ce3ab Mon Sep 17 00:00:00 2001 From: Robin Holzinger Date: Thu, 14 Dec 2023 03:35:13 +0100 Subject: [PATCH] chore: Fix ruff and add stricter linting (#16) Co-authored-by: Ege Kocabas --- .pre-commit-config.yaml | 4 +++ parma_mining/github/analytics_client.py | 11 ++++--- parma_mining/github/api/main.py | 15 ++++----- parma_mining/github/client.py | 16 ++++----- parma_mining/github/model.py | 44 ++++++++++++------------- parma_mining/mining_common/README.md | 3 ++ parma_mining/mining_common/const.py | 6 ++++ pyproject.toml | 5 +++ tests/test_organization_details.py | 9 +++-- 9 files changed, 68 insertions(+), 45 deletions(-) create mode 100644 parma_mining/mining_common/README.md create mode 100644 parma_mining/mining_common/const.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69d4e3b..dd86aff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,11 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: "v0.1.4" hooks: + - id: ruff + types_or: [python, pyi, jupyter] + args: [--fix] - id: ruff-format + types_or: [python, pyi, jupyter] - repo: https://github.com/psf/black-pre-commit-mirror rev: 23.10.1 hooks: diff --git a/parma_mining/github/analytics_client.py b/parma_mining/github/analytics_client.py index ca675d2..7014788 100644 --- a/parma_mining/github/analytics_client.py +++ b/parma_mining/github/analytics_client.py @@ -1,9 +1,11 @@ +import json +import os +import urllib.parse + import httpx from dotenv import load_dotenv + from parma_mining.github.model import ResponseModel -import os -import json -import urllib.parse class AnalyticsClient: @@ -24,7 +26,8 @@ def send_post_request(self, api_endpoint, data): return response.json() else: raise Exception( - f"API request failed with status code {response.status_code}, response: {response.text}" + f"API request failed with status code {response.status_code}," + f"response: {response.text}" ) def register_measurements(self, mapping, parent_id=None, source_module_id=None): diff --git a/parma_mining/github/api/main.py b/parma_mining/github/api/main.py index 2120b4e..a82b779 100644 --- a/parma_mining/github/api/main.py +++ b/parma_mining/github/api/main.py @@ -1,19 +1,18 @@ """Main entrypoint for the API routes in of parma-analytics.""" import json -from typing import List +import os + +from dotenv import load_dotenv from fastapi import FastAPI from starlette import status + from parma_mining.github.analytics_client import AnalyticsClient from parma_mining.github.client import GitHubClient from parma_mining.github.model import ( - OrganizationModel, - DiscoveryModel, CompaniesRequest, + DiscoveryModel, ResponseModel, ) -from dotenv import load_dotenv -import os - from parma_mining.github.normalization_map import GithubNormalizationMap load_dotenv() @@ -72,7 +71,7 @@ def get_organization_details(companies: CompaniesRequest): # Write data to db via endpoint in analytics backend try: analytics_client.feed_raw_data(data) - except: + except Exception: print("Error writing to db") else: # To be included in logging @@ -82,7 +81,7 @@ def get_organization_details(companies: CompaniesRequest): @app.get( "/search/companies", - response_model=List[DiscoveryModel], + response_model=list[DiscoveryModel], status_code=status.HTTP_200_OK, ) def search_organizations(query: str): diff --git a/parma_mining/github/client.py b/parma_mining/github/client.py index 6060cad..bdf25a8 100644 --- a/parma_mining/github/client.py +++ b/parma_mining/github/client.py @@ -1,16 +1,16 @@ -from typing import List from fastapi import HTTPException -from github import Github, GithubException, Auth +from github import Auth, Github, GithubException from starlette import status -from parma_mining.github.model import OrganizationModel, RepositoryModel, DiscoveryModel -from parma_mining.github.normalization_map import GithubNormalizationMap + +from parma_mining.github.model import DiscoveryModel, OrganizationModel, RepositoryModel class GitHubClient: def __init__(self, token: str): self.client = Github(auth=Auth.Token(token)) - # Retrieve organization details and statistics on all repositories of the organization + # Get organization details and statistics + # on all repositories of the organization def get_organization_details(self, org_name: str) -> OrganizationModel: try: organization = self.client.get_organization(org_name) @@ -52,17 +52,17 @@ def get_organization_details(self, org_name: str) -> OrganizationModel: org_info["repos"].append(parsed_repo) return OrganizationModel.model_validate(org_info) - except GithubException as e: + except GithubException: raise GithubException - def search_organizations(self, query: str) -> List[DiscoveryModel]: + def search_organizations(self, query: str) -> list[DiscoveryModel]: try: organizations = self.client.search_users(query + " type:org") return [ DiscoveryModel.model_validate({"name": org.login, "url": org.html_url}) for org in organizations ] - except GithubException as e: + except GithubException: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error searching organizations", diff --git a/parma_mining/github/model.py b/parma_mining/github/model.py index 375351f..d1b5a92 100644 --- a/parma_mining/github/model.py +++ b/parma_mining/github/model.py @@ -1,36 +1,36 @@ import json from datetime import datetime + from pydantic import BaseModel -from typing import Optional class RepositoryModel(BaseModel): """Model to structure the JSON Data.""" name: str - description: Optional[str] - stars: Optional[int] - forks: Optional[int] - language: Optional[str] - created_at: Optional[datetime] - updated_at: Optional[datetime] - pushed_at: Optional[datetime] - html_url: Optional[str] - clone_url: Optional[str] - svn_url: Optional[str] - homepage: Optional[str] - size: Optional[int] - stargazers_count: Optional[int] - watchers_count: Optional[int] - forks_count: Optional[int] - open_issues_count: Optional[int] + description: str | None + stars: int | None + forks: int | None + language: str | None + created_at: datetime | None + updated_at: datetime | None + pushed_at: datetime | None + html_url: str | None + clone_url: str | None + svn_url: str | None + homepage: str | None + size: int | None + stargazers_count: int | None + watchers_count: int | None + forks_count: int | None + open_issues_count: int | None class OrganizationModel(BaseModel): - name: Optional[str] - description: Optional[str] + name: str | None + description: str | None url: str - repos: Optional[list[RepositoryModel]] + repos: list[RepositoryModel] | None def updated_model_dump(self) -> str: """Dump the CompanyModel instance to a JSON string.""" @@ -47,8 +47,8 @@ def updated_model_dump(self) -> str: class DiscoveryModel(BaseModel): - name: Optional[str] - url: Optional[str] + name: str | None + url: str | None class CompaniesRequest(BaseModel): diff --git a/parma_mining/mining_common/README.md b/parma_mining/mining_common/README.md new file mode 100644 index 0000000..2ccb4bd --- /dev/null +++ b/parma_mining/mining_common/README.md @@ -0,0 +1,3 @@ +# Mining Common + +Generic mining logic that we might abstract into a shared subrepository in the future. diff --git a/parma_mining/mining_common/const.py b/parma_mining/mining_common/const.py new file mode 100644 index 0000000..e271c19 --- /dev/null +++ b/parma_mining/mining_common/const.py @@ -0,0 +1,6 @@ +HTTP_200 = 200 +HTTP_201 = 201 +HTTP_400 = 400 +HTTP_401 = 401 +HTTP_403 = 403 +HTTP_404 = 404 diff --git a/pyproject.toml b/pyproject.toml index 85d31ff..a0818fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,11 @@ exclude = ''' line-length = 88 select = ["F", "E", "W", "I", "N", "UP", "D", "PL"] target-version = "py311" +ignore = ["D100", "D101", "D102", "D103", "D104", "D107"] # TODO: remove after midterm + +[tool.ruff.lint.isort] +case-sensitive = true +detect-same-package = true [tool.mypy] python_version = '3.11' diff --git a/tests/test_organization_details.py b/tests/test_organization_details.py index 5652a59..fedfb7d 100644 --- a/tests/test_organization_details.py +++ b/tests/test_organization_details.py @@ -1,9 +1,12 @@ +from unittest.mock import MagicMock + import pytest from fastapi import HTTPException from fastapi.testclient import TestClient from starlette import status -from unittest.mock import MagicMock + from parma_mining.github.api.main import app +from parma_mining.mining_common.const import HTTP_200, HTTP_404 client = TestClient(app) @@ -65,7 +68,7 @@ def test_get_organization_details( mock_analytics_client.assert_called() - assert response.status_code == 200 + assert response.status_code == HTTP_200 def test_get_organization_details_bad_request(mocker): @@ -84,4 +87,4 @@ def test_get_organization_details_bad_request(mocker): } response = client.post("/companies", json=payload) - assert response.status_code == 404 + assert response.status_code == HTTP_404