From f5c28aba90922d3cfa0fdf7c58c07a53b945413e Mon Sep 17 00:00:00 2001 From: Jongmin Kim Date: Tue, 7 Nov 2023 05:00:50 +0900 Subject: [PATCH] refactor: refactor plugin server --- Dockerfile | 10 +- pkg/pip_requirements.txt | 8 +- .../cost_analysis/conf => plugin}/__init__.py | 0 .../connector}/__init__.py | 0 .../connector/aws_s3_connector.py | 2 +- .../connector/spaceone_connector.py | 5 +- src/plugin/main.py | 126 ++++++++++++++++++ src/plugin/manager/__init__.py | 1 + .../manager/cost_manager.py | 29 ++-- src/plugin/manager/data_source_manager.py | 43 ++++++ .../manager/job_manager.py | 13 +- src/setup.py | 6 +- src/spaceone/__init__.py | 1 - src/spaceone/cost_analysis/__init__.py | 1 - .../cost_analysis/conf/global_conf.py | 22 --- src/spaceone/cost_analysis/conf/proto_conf.py | 5 - .../cost_analysis/connector/__init__.py | 2 - .../connector/aws_hyperbilling_connector.py | 111 --------------- src/spaceone/cost_analysis/error/__init__.py | 1 - src/spaceone/cost_analysis/error/cost.py | 5 - src/spaceone/cost_analysis/info/__init__.py | 4 - .../cost_analysis/info/common_info.py | 7 - src/spaceone/cost_analysis/info/cost_info.py | 37 ----- .../cost_analysis/info/data_source_info.py | 13 -- src/spaceone/cost_analysis/info/job_info.py | 36 ----- .../cost_analysis/interface/grpc/__init__.py | 0 .../interface/grpc/plugin/__init__.py | 0 .../interface/grpc/plugin/cost.py | 16 --- .../interface/grpc/plugin/data_source.py | 21 --- .../interface/grpc/plugin/job.py | 14 -- .../cost_analysis/manager/__init__.py | 3 - .../manager/data_source_manager.py | 28 ---- src/spaceone/cost_analysis/model/__init__.py | 2 - .../cost_analysis/model/cost_model.py | 17 --- .../cost_analysis/model/data_source_model.py | 56 -------- src/spaceone/cost_analysis/model/job_model.py | 29 ---- .../cost_analysis/service/__init__.py | 3 - .../cost_analysis/service/cost_service.py | 43 ------ .../service/data_source_service.py | 60 --------- .../cost_analysis/service/job_service.py | 47 ------- 40 files changed, 197 insertions(+), 630 deletions(-) rename src/{spaceone/cost_analysis/conf => plugin}/__init__.py (100%) rename src/{spaceone/cost_analysis/interface => plugin/connector}/__init__.py (100%) rename src/{spaceone/cost_analysis => plugin}/connector/aws_s3_connector.py (98%) rename src/{spaceone/cost_analysis => plugin}/connector/spaceone_connector.py (96%) create mode 100644 src/plugin/main.py create mode 100644 src/plugin/manager/__init__.py rename src/{spaceone/cost_analysis => plugin}/manager/cost_manager.py (90%) create mode 100644 src/plugin/manager/data_source_manager.py rename src/{spaceone/cost_analysis => plugin}/manager/job_manager.py (89%) delete mode 100644 src/spaceone/__init__.py delete mode 100644 src/spaceone/cost_analysis/__init__.py delete mode 100644 src/spaceone/cost_analysis/conf/global_conf.py delete mode 100644 src/spaceone/cost_analysis/conf/proto_conf.py delete mode 100644 src/spaceone/cost_analysis/connector/__init__.py delete mode 100644 src/spaceone/cost_analysis/connector/aws_hyperbilling_connector.py delete mode 100644 src/spaceone/cost_analysis/error/__init__.py delete mode 100644 src/spaceone/cost_analysis/error/cost.py delete mode 100644 src/spaceone/cost_analysis/info/__init__.py delete mode 100644 src/spaceone/cost_analysis/info/common_info.py delete mode 100644 src/spaceone/cost_analysis/info/cost_info.py delete mode 100644 src/spaceone/cost_analysis/info/data_source_info.py delete mode 100644 src/spaceone/cost_analysis/info/job_info.py delete mode 100644 src/spaceone/cost_analysis/interface/grpc/__init__.py delete mode 100644 src/spaceone/cost_analysis/interface/grpc/plugin/__init__.py delete mode 100644 src/spaceone/cost_analysis/interface/grpc/plugin/cost.py delete mode 100644 src/spaceone/cost_analysis/interface/grpc/plugin/data_source.py delete mode 100644 src/spaceone/cost_analysis/interface/grpc/plugin/job.py delete mode 100644 src/spaceone/cost_analysis/manager/__init__.py delete mode 100644 src/spaceone/cost_analysis/manager/data_source_manager.py delete mode 100644 src/spaceone/cost_analysis/model/__init__.py delete mode 100644 src/spaceone/cost_analysis/model/cost_model.py delete mode 100644 src/spaceone/cost_analysis/model/data_source_model.py delete mode 100644 src/spaceone/cost_analysis/model/job_model.py delete mode 100644 src/spaceone/cost_analysis/service/__init__.py delete mode 100644 src/spaceone/cost_analysis/service/cost_service.py delete mode 100644 src/spaceone/cost_analysis/service/data_source_service.py delete mode 100644 src/spaceone/cost_analysis/service/job_service.py diff --git a/Dockerfile b/Dockerfile index b15aee0..a34fde7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim +FROM cloudforet/python-core:2.0 ENV PYTHONUNBUFFERED 1 ENV SPACEONE_PORT 50051 @@ -13,15 +13,11 @@ COPY pkg/*.txt ${PKG_DIR}/ RUN pip install --upgrade pip && \ pip install --upgrade -r ${PKG_DIR}/pip_requirements.txt -ARG CACHEBUST=1 -RUN pip install --upgrade --pre spaceone-core spaceone-api - COPY src ${SRC_DIR} WORKDIR ${SRC_DIR} -RUN python3 setup.py install && \ - rm -rf /tmp/* +RUN python3 setup.py install && rm -rf /tmp/* EXPOSE ${SPACEONE_PORT} ENTRYPOINT ["spaceone"] -CMD ["grpc", "spaceone.cost_analysis"] +CMD ["run", "plugin-server", "plugin"] diff --git a/pkg/pip_requirements.txt b/pkg/pip_requirements.txt index d6adef9..c7559fe 100644 --- a/pkg/pip_requirements.txt +++ b/pkg/pip_requirements.txt @@ -1,8 +1,4 @@ + spaceone-core spaceone-api -schematics -requests -boto3 -pyarrow -pandas - +spaceone-cost-analysis \ No newline at end of file diff --git a/src/spaceone/cost_analysis/conf/__init__.py b/src/plugin/__init__.py similarity index 100% rename from src/spaceone/cost_analysis/conf/__init__.py rename to src/plugin/__init__.py diff --git a/src/spaceone/cost_analysis/interface/__init__.py b/src/plugin/connector/__init__.py similarity index 100% rename from src/spaceone/cost_analysis/interface/__init__.py rename to src/plugin/connector/__init__.py diff --git a/src/spaceone/cost_analysis/connector/aws_s3_connector.py b/src/plugin/connector/aws_s3_connector.py similarity index 98% rename from src/spaceone/cost_analysis/connector/aws_s3_connector.py rename to src/plugin/connector/aws_s3_connector.py index d7ee700..25cca59 100644 --- a/src/spaceone/cost_analysis/connector/aws_s3_connector.py +++ b/src/plugin/connector/aws_s3_connector.py @@ -6,7 +6,7 @@ from spaceone.core import utils from spaceone.core.connector import BaseConnector -from spaceone.cost_analysis.error import * +from spaceone.core.error import * __all__ = ['AWSS3Connector'] diff --git a/src/spaceone/cost_analysis/connector/spaceone_connector.py b/src/plugin/connector/spaceone_connector.py similarity index 96% rename from src/spaceone/cost_analysis/connector/spaceone_connector.py rename to src/plugin/connector/spaceone_connector.py index 1a04b37..5120d87 100644 --- a/src/spaceone/cost_analysis/connector/spaceone_connector.py +++ b/src/plugin/connector/spaceone_connector.py @@ -1,12 +1,11 @@ import logging import re import requests -from requests import HTTPError from google.protobuf.json_format import MessageToDict from spaceone.core.connector.space_connector import SpaceConnector from spaceone.core.connector import BaseConnector -from spaceone.cost_analysis.error import * +from spaceone.core.error import * __all__ = ['SpaceONEConnector'] @@ -96,7 +95,7 @@ def request(self, method, params, **kwargs): response = requests.post(url, json=params, headers=headers) if response.status_code >= 400: - raise HTTPError(f'HTTP {response.status_code} Error: {response.json()["detail"]}') + raise requests.HTTPError(f'HTTP {response.status_code} Error: {response.json()["detail"]}') response = response.json() return response diff --git a/src/plugin/main.py b/src/plugin/main.py new file mode 100644 index 0000000..e31fa81 --- /dev/null +++ b/src/plugin/main.py @@ -0,0 +1,126 @@ +from typing import Generator +from spaceone.cost_analysis.plugin.data_source.lib.server import DataSourcePluginServer +from .manager.data_source_manager import DataSourceManager +from .manager.job_manager import JobManager +from .manager.cost_manager import CostManager + +app = DataSourcePluginServer() + + +@app.route('DataSource.init') +def data_source_init(params: dict) -> dict: + """ init plugin by options + + Args: + params (DataSourceInitRequest): { + 'options': 'dict', # Required + 'domain_id': 'str' # Required + } + + Returns: + PluginResponse: { + 'metadata': 'dict' + } + """ + options = params['options'] + + data_source_mgr = DataSourceManager() + return data_source_mgr.init_response(options) + + +@app.route('DataSource.verify') +def data_source_verify(params: dict) -> None: + """ Verifying data source plugin + + Args: + params (CollectorVerifyRequest): { + 'options': 'dict', # Required + 'secret_data': 'dict', # Required + 'schema': 'str', + 'domain_id': 'str' + } + + Returns: + None + """ + + options = params['options'] + secret_data = params['secret_data'] + schema = params.get('schema') + + data_source_mgr = DataSourceManager() + data_source_mgr.verify_plugin(options, secret_data, schema) + + +@app.route('Job.get_tasks') +def job_get_tasks(params: dict) -> dict: + """ Get job tasks + + Args: + params (JobGetTaskRequest): { + 'options': 'dict', # Required + 'secret_data': 'dict', # Required + 'schema': 'str', + 'start': 'str', + 'last_synchronized_at': 'datetime', + 'domain_id': 'str' + } + + Returns: + TasksResponse: { + 'tasks': 'list', + 'changed': 'list' + } + + """ + + domain_id = params['domain_id'] + options = params['options'] + secret_data = params['secret_data'] + schema = params.get('schema') + start = params.get('start') + last_synchronized_at = params.get('last_synchronized_at') + + job_mgr = JobManager() + return job_mgr.get_tasks(domain_id, options, secret_data, schema, start, last_synchronized_at) + + +@app.route('Cost.get_data') +def cost_get_data(params: dict) -> Generator[dict, None, None]: + """ Get external cost data + + Args: + params (CostGetDataRequest): { + 'options': 'dict', # Required + 'secret_data': 'dict', # Required + 'schema': 'str', + 'task_options': 'dict', + 'domain_id': 'str' + } + + Returns: + Generator[ResourceResponse, None, None] + { + 'cost': 'float', + 'usage_quantity': 'float', + 'usage_unit': 'str', + 'provider': 'str', + 'region_code': 'str', + 'product': 'str', + 'usage_type': 'str', + 'resource': 'str', + 'tags': 'dict' + 'additional_info': 'dict' + 'data': 'dict' + 'billed_date': 'str' + } + """ + + options = params['options'] + secret_data = params['secret_data'] + + task_options = params.get('task_options', {}) + schema = params.get('schema') + + cost_mgr = CostManager() + return cost_mgr.get_data(options, secret_data, task_options, schema) diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/plugin/manager/__init__.py @@ -0,0 +1 @@ + diff --git a/src/spaceone/cost_analysis/manager/cost_manager.py b/src/plugin/manager/cost_manager.py similarity index 90% rename from src/spaceone/cost_analysis/manager/cost_manager.py rename to src/plugin/manager/cost_manager.py index c28f57d..5f195c9 100644 --- a/src/spaceone/cost_analysis/manager/cost_manager.py +++ b/src/plugin/manager/cost_manager.py @@ -1,13 +1,13 @@ import logging +from typing import Generator from datetime import datetime from dateutil import rrule from spaceone.core import utils from spaceone.core.manager import BaseManager -from spaceone.cost_analysis.error import * -from spaceone.cost_analysis.connector.aws_s3_connector import AWSS3Connector -from spaceone.cost_analysis.connector.spaceone_connector import SpaceONEConnector -from spaceone.cost_analysis.model.cost_model import Cost +from spaceone.core.error import * +from ..connector.aws_s3_connector import AWSS3Connector +from ..connector.spaceone_connector import SpaceONEConnector _LOGGER = logging.getLogger(__name__) @@ -51,10 +51,11 @@ class CostManager(BaseManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.aws_s3_connector: AWSS3Connector = self.locator.get_connector('AWSS3Connector') - self.space_connector: SpaceONEConnector = self.locator.get_connector('SpaceONEConnector') + self.aws_s3_connector = AWSS3Connector() + self.space_connector = SpaceONEConnector() - def get_data(self, options, secret_data, schema, task_options): + def get_data(self, options: dict, secret_data: dict, task_options: dict, schema: str = None) \ + -> Generator[dict, None, None]: self.aws_s3_connector.create_session(options, secret_data, schema) self._check_task_options(task_options) @@ -79,7 +80,9 @@ def get_data(self, options, secret_data, schema, task_options): for results in response_stream: yield self._make_cost_data(results, account_id) - yield [] + yield { + 'results': [] + } def _update_sync_state(self, options, secret_data, schema, service_account_id): self.space_connector.init_client(options, secret_data, schema) @@ -156,13 +159,9 @@ class CostSummaryItem(BaseModel): costs_data.append(data) - # Excluded because schema validation is too slow - # cost_data = Cost(data) - # cost_data.validate() - # - # costs_data.append(cost_data.to_primitive()) - - return costs_data + return { + 'results': costs_data + } @staticmethod def _get_tags_from_cost_data(cost_data: dict) -> dict: diff --git a/src/plugin/manager/data_source_manager.py b/src/plugin/manager/data_source_manager.py new file mode 100644 index 0000000..43bd5cf --- /dev/null +++ b/src/plugin/manager/data_source_manager.py @@ -0,0 +1,43 @@ +import logging + +from spaceone.core.manager import BaseManager +from ..connector.aws_s3_connector import AWSS3Connector +from ..connector.spaceone_connector import SpaceONEConnector + +_LOGGER = logging.getLogger(__name__) + + +class DataSourceManager(BaseManager): + + @staticmethod + def init_response(options: dict) -> dict: + return { + 'metadata': { + 'currency': 'USD', + 'supported_secret_types': ['MANUAL'], + 'data_source_rules': [ + { + 'name': 'match_service_account', + 'conditions_policy': 'ALWAYS', + 'actions': { + 'match_service_account': { + 'source': 'additional_info.Account ID', + 'target': 'data.account_id' + } + }, + 'options': { + 'stop_processing': True + } + } + ] + } + } + + @staticmethod + def verify_plugin(options: dict, secret_data: dict, schema: str = None) -> None: + space_connector = SpaceONEConnector() + space_connector.init_client(options, secret_data, schema) + space_connector.verify_plugin() + + aws_s3_connector = AWSS3Connector() + aws_s3_connector.create_session(options, secret_data, schema) diff --git a/src/spaceone/cost_analysis/manager/job_manager.py b/src/plugin/manager/job_manager.py similarity index 89% rename from src/spaceone/cost_analysis/manager/job_manager.py rename to src/plugin/manager/job_manager.py index 6786f05..925df72 100644 --- a/src/spaceone/cost_analysis/manager/job_manager.py +++ b/src/plugin/manager/job_manager.py @@ -3,8 +3,7 @@ from spaceone.core.error import * from spaceone.core.manager import BaseManager -from spaceone.cost_analysis.connector.spaceone_connector import SpaceONEConnector -from spaceone.cost_analysis.model.job_model import Tasks +from ..connector.spaceone_connector import SpaceONEConnector _LOGGER = logging.getLogger(__name__) _DEFAULT_DATABASE = 'MZC' @@ -14,9 +13,10 @@ class JobManager(BaseManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.space_connector: SpaceONEConnector = self.locator.get_connector('SpaceONEConnector') + self.space_connector = SpaceONEConnector() - def get_tasks(self, options, secret_data, schema, start, last_synchronized_at, domain_id): + def get_tasks(self, domain_id: str, options: dict, secret_data: dict, schema: str = None, start: str = None, + last_synchronized_at: datetime = None) -> dict: tasks = [] changed = [] @@ -75,11 +75,6 @@ def get_tasks(self, options, secret_data, schema, start, last_synchronized_at, d _LOGGER.debug(f'[get_tasks] tasks: {tasks}') _LOGGER.debug(f'[get_tasks] changed: {changed}') - tasks = Tasks({'tasks': tasks, 'changed': changed}) - - tasks.validate() - return tasks.to_primitive() - else: _LOGGER.debug(f'[get_tasks] no project: tags.domain_id = {domain_id}') diff --git a/src/setup.py b/src/setup.py index 090baf5..37cdc4f 100644 --- a/src/setup.py +++ b/src/setup.py @@ -33,11 +33,7 @@ install_requires=[ 'spaceone-core', 'spaceone-api', - 'schematics', - 'requests', - 'boto3', - 'pyarrow', - 'pandas' + 'spaceone-cost-analysis' ], zip_safe=False, ) diff --git a/src/spaceone/__init__.py b/src/spaceone/__init__.py deleted file mode 100644 index 69e3be5..0000000 --- a/src/spaceone/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/src/spaceone/cost_analysis/__init__.py b/src/spaceone/cost_analysis/__init__.py deleted file mode 100644 index 9baac51..0000000 --- a/src/spaceone/cost_analysis/__init__.py +++ /dev/null @@ -1 +0,0 @@ -name = 'cost_analysis' diff --git a/src/spaceone/cost_analysis/conf/global_conf.py b/src/spaceone/cost_analysis/conf/global_conf.py deleted file mode 100644 index 018ab7c..0000000 --- a/src/spaceone/cost_analysis/conf/global_conf.py +++ /dev/null @@ -1,22 +0,0 @@ -CONNECTORS = { - 'SpaceONEConnector': {}, - 'AWSS3Connector': {} -} - -LOG = { - 'filters': { - 'masking': { - 'rules': { - 'DataSource.verify': [ - 'secret_data' - ], - 'Job.get_tasks': [ - 'secret_data' - ], - 'Cost.get_data': [ - 'secret_data' - ] - } - } - } -} diff --git a/src/spaceone/cost_analysis/conf/proto_conf.py b/src/spaceone/cost_analysis/conf/proto_conf.py deleted file mode 100644 index 29c2f5a..0000000 --- a/src/spaceone/cost_analysis/conf/proto_conf.py +++ /dev/null @@ -1,5 +0,0 @@ -PROTO = { - 'spaceone.cost_analysis.interface.grpc.plugin.data_source': ['DataSource'], - 'spaceone.cost_analysis.interface.grpc.plugin.job': ['Job'], - 'spaceone.cost_analysis.interface.grpc.plugin.cost': ['Cost'], -} diff --git a/src/spaceone/cost_analysis/connector/__init__.py b/src/spaceone/cost_analysis/connector/__init__.py deleted file mode 100644 index 32d9dfe..0000000 --- a/src/spaceone/cost_analysis/connector/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from spaceone.cost_analysis.connector.spaceone_connector import SpaceONEConnector -from spaceone.cost_analysis.connector.aws_s3_connector import AWSS3Connector diff --git a/src/spaceone/cost_analysis/connector/aws_hyperbilling_connector.py b/src/spaceone/cost_analysis/connector/aws_hyperbilling_connector.py deleted file mode 100644 index 334d0e5..0000000 --- a/src/spaceone/cost_analysis/connector/aws_hyperbilling_connector.py +++ /dev/null @@ -1,111 +0,0 @@ -import logging -import requests -import copy - -from spaceone.core.transaction import Transaction -from spaceone.core.connector import BaseConnector -from typing import List - -from spaceone.cost_analysis.error import * - -__all__ = ['AWSHyperBillingConnector'] - -_LOGGER = logging.getLogger(__name__) - -_DEFAULT_HEADERS = { - 'Content-Type': 'application/json', - 'accept': 'application/json' -} - - -class AWSHyperBillingConnector(BaseConnector): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.endpoint = None - self.headers = copy.deepcopy(_DEFAULT_HEADERS) - - def create_session(self, options: dict, secret_data: dict, schema: str = None) -> None: - self._check_secret_data(secret_data) - - self.headers['X-Client-Id'] = secret_data['client_id'] - self.headers['X-Client-Secret'] = secret_data['client_secret'] - self.endpoint = secret_data['endpoint'] - - @staticmethod - def _check_secret_data(secret_data: dict) -> None: - if 'client_id' not in secret_data: - raise ERROR_REQUIRED_PARAMETER(key='secret_data.client_id') - - if 'client_secret' not in secret_data: - raise ERROR_REQUIRED_PARAMETER(key='secret_data.client_secret') - - if 'endpoint' not in secret_data: - raise ERROR_REQUIRED_PARAMETER(key='secret_data.endpoint') - - def get_linked_accounts(self) -> List[dict]: - url = f'{self.endpoint}/v1/search/linkedaccount' - data = {} - - _LOGGER.debug(f'[list_linked_accounts] ({self.headers["X-Client-Id"]}) {url} => {data}') - - response = requests.get(url, json=data, headers=self.headers) - - if response.status_code == 200: - return response.json().get('Results', []) - else: - _LOGGER.error(f'[get_linked_accounts] error code: {response.status_code}') - try: - error_message = response.json() - except Exception as e: - error_message = str(response) - - _LOGGER.error(f'[get_linked_accounts] error message: {error_message}') - raise ERROR_CONNECTOR_CALL_API(reason=error_message) - - def get_cost_data(self, account, start: str, end: str, next_token: str = None) -> dict: - url = f'{self.endpoint}/v1/search/billing' - - data = { - 'Filter': { - 'LinkedAccount': [account], - 'Granularity': 'DAILY', - 'TimePeriod': { - 'Start': start, - 'End': end - } - }, - 'GroupBy': [ - 'USAGE_DATE', - 'REGION', - 'SERVICE_CODE', - 'USAGE_TYPE', - 'INSTANCE_TYPE' - ], - 'Result': [ - 'USAGE_COST', - 'USAGE_QUANTITY' - ], - } - - if next_token: - data['Filter']['NextDataToken'] = next_token - - _LOGGER.debug(f'[get_cost_data] ({self.headers["X-Client-Id"]}) {url} => {data}') - - response = requests.post(url, json=data, headers=self.headers) - - if response.status_code == 200: - return response.json() - - elif response.status_code == 204: - return {} - else: - _LOGGER.error(f'[get_cost_data] error code: {response.status_code}') - try: - error_message = response.json() - except Exception as e: - error_message = str(response) - - _LOGGER.error(f'[get_cost_data] error message: {error_message}') - raise ERROR_CONNECTOR_CALL_API(reason=error_message) diff --git a/src/spaceone/cost_analysis/error/__init__.py b/src/spaceone/cost_analysis/error/__init__.py deleted file mode 100644 index 2de9618..0000000 --- a/src/spaceone/cost_analysis/error/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from spaceone.cost_analysis.error.cost import * diff --git a/src/spaceone/cost_analysis/error/cost.py b/src/spaceone/cost_analysis/error/cost.py deleted file mode 100644 index 67298ea..0000000 --- a/src/spaceone/cost_analysis/error/cost.py +++ /dev/null @@ -1,5 +0,0 @@ -from spaceone.core.error import * - - -class ERROR_CONNECTOR_CALL_API(ERROR_UNKNOWN): - _message = 'API Call Error: {reason}' diff --git a/src/spaceone/cost_analysis/info/__init__.py b/src/spaceone/cost_analysis/info/__init__.py deleted file mode 100644 index 65f5626..0000000 --- a/src/spaceone/cost_analysis/info/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from spaceone.cost_analysis.info.common_info import * -from spaceone.cost_analysis.info.data_source_info import * -from spaceone.cost_analysis.info.job_info import * -from spaceone.cost_analysis.info.cost_info import * diff --git a/src/spaceone/cost_analysis/info/common_info.py b/src/spaceone/cost_analysis/info/common_info.py deleted file mode 100644 index 10b98f1..0000000 --- a/src/spaceone/cost_analysis/info/common_info.py +++ /dev/null @@ -1,7 +0,0 @@ -from google.protobuf.empty_pb2 import Empty - -__all__ = ['EmptyInfo'] - - -def EmptyInfo(): - return Empty() diff --git a/src/spaceone/cost_analysis/info/cost_info.py b/src/spaceone/cost_analysis/info/cost_info.py deleted file mode 100644 index 8e91da7..0000000 --- a/src/spaceone/cost_analysis/info/cost_info.py +++ /dev/null @@ -1,37 +0,0 @@ -import functools -import logging -from spaceone.api.cost_analysis.plugin import cost_pb2 -from spaceone.core.pygrpc.message_type import * -from spaceone.core import utils - -__all__ = ['CostInfo', 'CostsInfo'] - -_LOGGER = logging.getLogger(__name__) - - -def CostInfo(cost_data): - try: - info = { - 'cost': cost_data['cost'], - 'usage_quantity': cost_data.get('usage_quantity'), - 'usage_unit': cost_data.get('usage_unit'), - 'provider': cost_data.get('provider'), - 'region_code': cost_data.get('region_code'), - 'product': cost_data.get('product'), - 'usage_type': cost_data.get('usage_type'), - 'resource': cost_data.get('resource'), - 'tags': change_struct_type(cost_data['tags']) if 'tags' in cost_data else None, - 'additional_info': change_struct_type(cost_data['additional_info']) if 'additional_info' in cost_data else None, - 'billed_date': cost_data['billed_date'] - } - - return cost_pb2.CostInfo(**info) - - except Exception as e: - _LOGGER.debug(f'[CostInfo] cost data: {cost_data}') - _LOGGER.debug(f'[CostInfo] error reason: {e}', exc_info=True) - raise e - - -def CostsInfo(costs_data, **kwargs): - return cost_pb2.CostsInfo(results=list(map(functools.partial(CostInfo, **kwargs), costs_data))) diff --git a/src/spaceone/cost_analysis/info/data_source_info.py b/src/spaceone/cost_analysis/info/data_source_info.py deleted file mode 100644 index 4b2171b..0000000 --- a/src/spaceone/cost_analysis/info/data_source_info.py +++ /dev/null @@ -1,13 +0,0 @@ -from spaceone.api.cost_analysis.plugin import data_source_pb2 -from spaceone.core.pygrpc.message_type import * - - -__all__ = ['PluginInfo'] - - -def PluginInfo(plugin_data): - info = { - 'metadata': change_struct_type(plugin_data['metadata']), - } - - return data_source_pb2.PluginInfo(**info) diff --git a/src/spaceone/cost_analysis/info/job_info.py b/src/spaceone/cost_analysis/info/job_info.py deleted file mode 100644 index 56ce172..0000000 --- a/src/spaceone/cost_analysis/info/job_info.py +++ /dev/null @@ -1,36 +0,0 @@ -import functools -from spaceone.api.cost_analysis.plugin import job_pb2 -from spaceone.core.pygrpc.message_type import * -from spaceone.core import utils - -__all__ = ['TaskInfo', 'TasksInfo'] - - -def TaskInfo(task_data): - info = { - 'task_options': change_struct_type(task_data['task_options']) - } - - return job_pb2.TaskInfo(**info) - - -def ChangedInfo(changed_data): - info = { - 'start': changed_data['start'] - } - - if 'end' in changed_data: - info['end'] = changed_data['end'] - - if 'filter' in changed_data: - info['filter'] = change_struct_type(changed_data['filter']) - - return job_pb2.ChangedInfo(**info) - - -def TasksInfo(result, **kwargs): - tasks_data = result.get('tasks', []) - changed_data = result.get('changed', []) - - return job_pb2.TasksInfo(tasks=list(map(functools.partial(TaskInfo, **kwargs), tasks_data)), - changed=list(map(functools.partial(ChangedInfo, **kwargs), changed_data))) diff --git a/src/spaceone/cost_analysis/interface/grpc/__init__.py b/src/spaceone/cost_analysis/interface/grpc/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/spaceone/cost_analysis/interface/grpc/plugin/__init__.py b/src/spaceone/cost_analysis/interface/grpc/plugin/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/spaceone/cost_analysis/interface/grpc/plugin/cost.py b/src/spaceone/cost_analysis/interface/grpc/plugin/cost.py deleted file mode 100644 index b10486e..0000000 --- a/src/spaceone/cost_analysis/interface/grpc/plugin/cost.py +++ /dev/null @@ -1,16 +0,0 @@ -from spaceone.api.cost_analysis.plugin import cost_pb2, cost_pb2_grpc -from spaceone.core.pygrpc import BaseAPI - - -class Cost(BaseAPI, cost_pb2_grpc.CostServicer): - - pb2 = cost_pb2 - pb2_grpc = cost_pb2_grpc - - def get_data(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service('CostService', metadata) as cost_service: - response_stream = cost_service.get_data(params) - for costs_data in response_stream: - yield self.locator.get_info('CostsInfo', costs_data) diff --git a/src/spaceone/cost_analysis/interface/grpc/plugin/data_source.py b/src/spaceone/cost_analysis/interface/grpc/plugin/data_source.py deleted file mode 100644 index 2313c61..0000000 --- a/src/spaceone/cost_analysis/interface/grpc/plugin/data_source.py +++ /dev/null @@ -1,21 +0,0 @@ -from spaceone.api.cost_analysis.plugin import data_source_pb2, data_source_pb2_grpc -from spaceone.core.pygrpc import BaseAPI - - -class DataSource(BaseAPI, data_source_pb2_grpc.DataSourceServicer): - - pb2 = data_source_pb2 - pb2_grpc = data_source_pb2_grpc - - def init(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service('DataSourceService', metadata) as data_source_service: - return self.locator.get_info('PluginInfo', data_source_service.init(params)) - - def verify(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service('DataSourceService', metadata) as data_source_service: - data_source_service.verify(params) - return self.locator.get_info('EmptyInfo') diff --git a/src/spaceone/cost_analysis/interface/grpc/plugin/job.py b/src/spaceone/cost_analysis/interface/grpc/plugin/job.py deleted file mode 100644 index 95545a6..0000000 --- a/src/spaceone/cost_analysis/interface/grpc/plugin/job.py +++ /dev/null @@ -1,14 +0,0 @@ -from spaceone.api.cost_analysis.plugin import job_pb2, job_pb2_grpc -from spaceone.core.pygrpc import BaseAPI - - -class Job(BaseAPI, job_pb2_grpc.JobServicer): - - pb2 = job_pb2 - pb2_grpc = job_pb2_grpc - - def get_tasks(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service('JobService', metadata) as job_service: - return self.locator.get_info('TasksInfo', job_service.get_tasks(params)) diff --git a/src/spaceone/cost_analysis/manager/__init__.py b/src/spaceone/cost_analysis/manager/__init__.py deleted file mode 100644 index 78c0e31..0000000 --- a/src/spaceone/cost_analysis/manager/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from spaceone.cost_analysis.manager.data_source_manager import DataSourceManager -from spaceone.cost_analysis.manager.job_manager import JobManager -from spaceone.cost_analysis.manager.cost_manager import CostManager diff --git a/src/spaceone/cost_analysis/manager/data_source_manager.py b/src/spaceone/cost_analysis/manager/data_source_manager.py deleted file mode 100644 index 26c350f..0000000 --- a/src/spaceone/cost_analysis/manager/data_source_manager.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging - -from spaceone.core.manager import BaseManager -from spaceone.cost_analysis.model.data_source_model import PluginMetadata -from spaceone.cost_analysis.connector.spaceone_connector import SpaceONEConnector -from spaceone.cost_analysis.connector.aws_s3_connector import AWSS3Connector - -_LOGGER = logging.getLogger(__name__) - - -class DataSourceManager(BaseManager): - - @staticmethod - def init_response(options): - plugin_metadata = PluginMetadata() - plugin_metadata.validate() - - return { - 'metadata': plugin_metadata.to_primitive() - } - - def verify_plugin(self, options, secret_data, schema): - space_connector: SpaceONEConnector = self.locator.get_connector('SpaceONEConnector') - space_connector.init_client(options, secret_data, schema) - space_connector.verify_plugin() - - aws_s3_connector: AWSS3Connector = self.locator.get_connector('AWSS3Connector') - aws_s3_connector.create_session(options, secret_data, schema) diff --git a/src/spaceone/cost_analysis/model/__init__.py b/src/spaceone/cost_analysis/model/__init__.py deleted file mode 100644 index d6f0f76..0000000 --- a/src/spaceone/cost_analysis/model/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from spaceone.cost_analysis.model.data_source_model import * -from spaceone.cost_analysis.model.job_model import * diff --git a/src/spaceone/cost_analysis/model/cost_model.py b/src/spaceone/cost_analysis/model/cost_model.py deleted file mode 100644 index 0dac6a1..0000000 --- a/src/spaceone/cost_analysis/model/cost_model.py +++ /dev/null @@ -1,17 +0,0 @@ -from schematics.models import Model -from schematics.types import DictType, StringType, FloatType, DateTimeType - -__all__ = ['Cost'] - - -class Cost(Model): - cost = FloatType(required=True) - usage_quantity = FloatType(required=True) - usage_unit = StringType(default=None) - provider = StringType(required=True) - region_code = StringType() - product = StringType() - usage_type = StringType() - billed_date = StringType(required=True) - additional_info = DictType(StringType, default={}) - tags = DictType(StringType, default={}) diff --git a/src/spaceone/cost_analysis/model/data_source_model.py b/src/spaceone/cost_analysis/model/data_source_model.py deleted file mode 100644 index f447272..0000000 --- a/src/spaceone/cost_analysis/model/data_source_model.py +++ /dev/null @@ -1,56 +0,0 @@ -from schematics.models import Model -from schematics.types import ListType, DictType, StringType, BooleanType -from schematics.types.compound import ModelType - -__all__ = ['PluginMetadata'] - - -_DEFAULT_DATA_SOURCE_RULES = [ - { - 'name': 'match_service_account', - 'conditions_policy': 'ALWAYS', - 'actions': { - 'match_service_account': { - 'source': 'additional_info.Account ID', - 'target': 'data.account_id' - } - }, - 'options': { - 'stop_processing': True - } - } -] - - -class MatchServiceAccount(Model): - source = StringType(required=True) - target = StringType(required=True) - - -class Actions(Model): - match_service_account = ModelType(MatchServiceAccount) - - -class Options(Model): - stop_processing = BooleanType(default=False) - - -class Condition(Model): - key = StringType(required=True) - value = StringType(required=True) - operator = StringType(required=True, choices=['eq', 'contain', 'not', 'not_contain']) - - -class DataSourceRule(Model): - name = StringType(required=True) - conditions = ListType(ModelType(Condition), default=[]) - conditions_policy = StringType(required=True, choices=['ALL', 'ANY', 'ALWAYS']) - actions = ModelType(Actions, required=True) - options = ModelType(Options, default={}) - tags = DictType(StringType, default={}) - - -class PluginMetadata(Model): - data_source_rules = ListType(ModelType(DataSourceRule), default=_DEFAULT_DATA_SOURCE_RULES) - supported_secret_types = ListType(StringType(), default=['MANUAL']) - currency = StringType(default='USD') diff --git a/src/spaceone/cost_analysis/model/job_model.py b/src/spaceone/cost_analysis/model/job_model.py deleted file mode 100644 index 437b425..0000000 --- a/src/spaceone/cost_analysis/model/job_model.py +++ /dev/null @@ -1,29 +0,0 @@ -from schematics.models import Model -from schematics.types import ListType, IntType, DateTimeType, StringType, DictType -from schematics.types.compound import ModelType - -__all__ = ['Tasks'] - - -class TaskOptions(Model): - start = StringType(required=True, max_length=7) - service_account_id = StringType(required=True) - service_account_name = StringType(required=True) - account_id = StringType(required=True) - database = StringType(required=True) - is_sync = StringType(required=True, choices=('true', 'false')) - - -class Task(Model): - task_options = ModelType(TaskOptions, required=True) - - -class Changed(Model): - start = StringType(required=True, max_length=7) - end = StringType(default=None, max_length=7) - filter = DictType(StringType, default={}) - - -class Tasks(Model): - tasks = ListType(ModelType(Task), required=True) - changed = ListType(ModelType(Changed), default=[]) diff --git a/src/spaceone/cost_analysis/service/__init__.py b/src/spaceone/cost_analysis/service/__init__.py deleted file mode 100644 index 11523de..0000000 --- a/src/spaceone/cost_analysis/service/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from spaceone.cost_analysis.service.data_source_service import DataSourceService -from spaceone.cost_analysis.service.job_service import JobService -from spaceone.cost_analysis.service.cost_service import CostService diff --git a/src/spaceone/cost_analysis/service/cost_service.py b/src/spaceone/cost_analysis/service/cost_service.py deleted file mode 100644 index 1cdc26f..0000000 --- a/src/spaceone/cost_analysis/service/cost_service.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging - -from spaceone.core.service import * -from spaceone.cost_analysis.error import * -from spaceone.cost_analysis.manager.cost_manager import CostManager - -_LOGGER = logging.getLogger(__name__) - - -@authentication_handler -@authorization_handler -@event_handler -class CostService(BaseService): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.cost_mgr: CostManager = self.locator.get_manager('CostManager') - - @transaction - @check_required(['options', 'secret_data', 'task_options']) - def get_data(self, params): - """Get Cost Data - - Args: - params (dict): { - 'options': 'dict', - 'secret_data': 'dict', - 'schema': 'str', - 'task_options': 'dict', - 'domain_id': 'str' - } - - Returns: - list of cost_data - - """ - - options = params['options'] - secret_data = params['secret_data'] - schema = params.get('schema') - task_options = params['task_options'] - - return self.cost_mgr.get_data(options, secret_data, schema, task_options) diff --git a/src/spaceone/cost_analysis/service/data_source_service.py b/src/spaceone/cost_analysis/service/data_source_service.py deleted file mode 100644 index 8f6f53b..0000000 --- a/src/spaceone/cost_analysis/service/data_source_service.py +++ /dev/null @@ -1,60 +0,0 @@ -import logging - -from spaceone.core.service import * - -from spaceone.cost_analysis.error import * -from spaceone.cost_analysis.manager.data_source_manager import DataSourceManager - -_LOGGER = logging.getLogger(__name__) - - -@authentication_handler -@authorization_handler -@event_handler -class DataSourceService(BaseService): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.data_source_mgr: DataSourceManager = self.locator.get_manager('DataSourceManager') - - @transaction - @check_required(['options']) - def init(self, params): - """ init plugin by options - - Args: - params (dict): { - 'options': 'dict', - 'domain_id': 'str' - } - - Returns: - None - """ - - options = params.get('options', {}) - - return self.data_source_mgr.init_response(options) - - @transaction - @check_required(['options', 'secret_data']) - def verify(self, params): - """ Verifying data source plugin - - Args: - params (dict): { - 'options': 'dict', - 'schema': 'str', - 'secret_data': 'dict', - 'domain_id': 'str' - } - - Returns: - None - """ - - options = params['options'] - secret_data = params['secret_data'] - schema = params.get('schema') - - return self.data_source_mgr.verify_plugin(options, secret_data, schema) diff --git a/src/spaceone/cost_analysis/service/job_service.py b/src/spaceone/cost_analysis/service/job_service.py deleted file mode 100644 index 1a1cd53..0000000 --- a/src/spaceone/cost_analysis/service/job_service.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging - -from spaceone.core.service import * -from spaceone.cost_analysis.error import * -from spaceone.cost_analysis.manager.job_manager import JobManager - -_LOGGER = logging.getLogger(__name__) - - -@authentication_handler -@authorization_handler -@event_handler -class JobService(BaseService): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.job_mgr: JobManager = self.locator.get_manager('JobManager') - - @transaction - @check_required(['options', 'secret_data']) - @change_timestamp_value(['last_synchronized_at'], timestamp_format='iso8601') - def get_tasks(self, params): - """Get Job Tasks - - Args: - params (dict): { - 'options': 'dict', - 'secret_data': 'dict', - 'schema': 'str', - 'start': 'datetime', - 'last_synchronized_at': 'datetime', - 'domain_id': 'str' - } - - Returns: - list of task_data - - """ - - options = params['options'] - secret_data = params['secret_data'] - schema = params.get('schema') - start = params.get('start') - last_synchronized_at = params.get('last_synchronized_at') - domain_id = params['domain_id'] - - return self.job_mgr.get_tasks(options, secret_data, schema, start, last_synchronized_at, domain_id)