diff --git a/parma_mining/reddit/api/main.py b/parma_mining/reddit/api/main.py index 7e5f1cf..d798471 100644 --- a/parma_mining/reddit/api/main.py +++ b/parma_mining/reddit/api/main.py @@ -101,7 +101,7 @@ def get_company_details(body: CompaniesRequest, token: str = Depends(authenticat ) except CrawlingError as e: logger.error( - f"Can't fetch company details from GitHub. Error: {e}" + f"Can't fetch company details from Reddit. Error: {e}" ) collect_errors(company_id, errors, e) continue diff --git a/parma_mining/reddit/client.py b/parma_mining/reddit/client.py index 657bb88..de72983 100644 --- a/parma_mining/reddit/client.py +++ b/parma_mining/reddit/client.py @@ -90,9 +90,6 @@ def get_company_details( "is_video": submission.is_video, "over18": submission.over_18, "permalink": submission.permalink, - "scraped_at": datetime.utcnow().strftime( - "%Y-%m-%dT%H:%M:%S.%fZ" - ), "score": submission.score, "subreddit_name": submission.subreddit.display_name, "subreddit_description": ( diff --git a/parma_mining/reddit/model.py b/parma_mining/reddit/model.py index 1f8fc29..3f181b6 100644 --- a/parma_mining/reddit/model.py +++ b/parma_mining/reddit/model.py @@ -27,7 +27,6 @@ class SubmissionModel(BaseModel): is_video: bool | None over18: bool | None permalink: str | None - scraped_at: datetime | None score: int | None subreddit_name: str | None subreddit_description: str | None diff --git a/parma_mining/reddit/normalization_map.py b/parma_mining/reddit/normalization_map.py index b303078..5fe7753 100644 --- a/parma_mining/reddit/normalization_map.py +++ b/parma_mining/reddit/normalization_map.py @@ -8,7 +8,7 @@ class RedditNormalizationMap: """Normalization map for Reddit.""" map_json = { - "SourceName": "reddit", + "Source": "reddit", "Mappings": [ { "SourceField": "name", @@ -80,11 +80,6 @@ class RedditNormalizationMap: "DataType": "link", "MeasurementName": "repository homepage url", }, - { - "SourceField": "scraped_at", - "DataType": "date", - "MeasurementName": "timestamp of scraping", - }, { "SourceField": "score", "DataType": "int", @@ -148,7 +143,7 @@ class RedditNormalizationMap: { "SourceField": "downvotes", "DataType": "int", - "MeasurementName": "downvote coun of comment", + "MeasurementName": "downvote count of comment", }, ], },