From 148d4dd88226428733e9ef0721501f26b3795af0 Mon Sep 17 00:00:00 2001 From: Rarescode Date: Fri, 12 Apr 2024 22:51:09 +0300 Subject: [PATCH 1/3] Switched methods to push data to the validator instead of prod --- sites/setup_api.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/sites/setup_api.py b/sites/setup_api.py index bac8db95..7ae2c97a 100644 --- a/sites/setup_api.py +++ b/sites/setup_api.py @@ -22,37 +22,37 @@ def __call__(self): """ Perform the data update process. """ - self._send_clean_request() + self.get_token() time.sleep(0.2) - self._send_post_request() + self.add_jobs() + + def get_token(self): - def _send_clean_request(self): - """ - Send the clean request to the Peviitor API. - """ + payload = json.dumps({ + "email": "irimusrares7@gmail.com" + }) - clean_url = 'https://api.peviitor.ro/v4/clean/' - clean_header = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'apikey': self.api_key + post_header = { + 'Content-Type': 'application/json' } - requests.post(clean_url, headers=clean_header, data={'company': self.company_name}) - def _send_post_request(self): - """ - Send the post request to update the data - """ + self.access_token = requests.request("POST", "https://api.laurentiumarian.ro/get_token", headers=post_header, data=payload).json()['access'] + + + def add_jobs(self): + post_header = { - 'Content-Type': 'application/json', - 'apikey': self.api_key + 'Authorization': f'Bearer {self.access_token}', + 'Content-Type': 'application/json' } - # print(json.dumps(self.data_list)) - requests.post('https://api.peviitor.ro/v4/update/', headers=post_header, data=json.dumps(self.data_list)) + requests.request("POST", "https://api.laurentiumarian.ro/jobs/add/", headers=post_header, data=json.dumps(self.data_list)) + # don't delete this lines if you want to see the graph on scraper's page file = self.company_name.lower() + '.py' data = {'data': len(self.data_list)} dataset_url = f'https://dev.laurentiumarian.ro/dataset/JobsScrapers/{file}/' requests.post(dataset_url, json=data) ######################################################## + From c16b27a10374443b492482e0ede4a3b124177bc5 Mon Sep 17 00:00:00 2001 From: Rarescode Date: Sat, 13 Apr 2024 19:17:25 +0300 Subject: [PATCH 2/3] Modified to use secrets --- sites/setup_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sites/setup_api.py b/sites/setup_api.py index 7ae2c97a..09d8e522 100644 --- a/sites/setup_api.py +++ b/sites/setup_api.py @@ -4,7 +4,6 @@ import time -# Inspired from the Andrei Cojocaru update_peviitor_api Decorator class UpdatePeviitorAPI: """ Class for updating data on pe viitor API @@ -16,7 +15,7 @@ def __init__(self, company_name, data_list): """ self.company_name = company_name self.data_list = data_list - self.api_key = os.environ.get('API_KEY') + self.email = os.environ.get('EMAIL') def __call__(self): """ @@ -29,7 +28,7 @@ def __call__(self): def get_token(self): payload = json.dumps({ - "email": "irimusrares7@gmail.com" + "email": self.email }) post_header = { From ef48e9d3b1244714b59c305b75479dad8cfe112a Mon Sep 17 00:00:00 2001 From: Rarescode Date: Sat, 20 Apr 2024 10:12:42 +0300 Subject: [PATCH 3/3] Modified validator urls --- sites/setup_api.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/sites/setup_api.py b/sites/setup_api.py index 09d8e522..52349111 100644 --- a/sites/setup_api.py +++ b/sites/setup_api.py @@ -35,7 +35,7 @@ def get_token(self): 'Content-Type': 'application/json' } - self.access_token = requests.request("POST", "https://api.laurentiumarian.ro/get_token", headers=post_header, data=payload).json()['access'] + self.access_token = requests.request("POST", "https://api.peviitor.ro/v5/get_token/", headers=post_header, data=payload).json()['access'] def add_jobs(self): @@ -45,13 +45,5 @@ def add_jobs(self): 'Content-Type': 'application/json' } - requests.request("POST", "https://api.laurentiumarian.ro/jobs/add/", headers=post_header, data=json.dumps(self.data_list)) - - # don't delete this lines if you want to see the graph on scraper's page - file = self.company_name.lower() + '.py' - data = {'data': len(self.data_list)} - dataset_url = f'https://dev.laurentiumarian.ro/dataset/JobsScrapers/{file}/' - requests.post(dataset_url, json=data) - ######################################################## - + requests.request("POST", "https://api.peviitor.ro/v5/add/", headers=post_header, data=json.dumps(self.data_list))