-
Notifications
You must be signed in to change notification settings - Fork 36
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[MLOP-636] Create migration classes (#282)
- Loading branch information
1 parent
245eaa5
commit d6ecfa4
Showing
4 changed files
with
115 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
"""Holds available migrations.""" | ||
|
||
from butterfree.migrations.cassandra_migration import CassandraMigration | ||
from butterfree.migrations.metastore_migration import MetastoreMigration | ||
from butterfree.migrations.migration import DatabaseMigration | ||
|
||
__all__ = ["DatabaseMigration", "CassandraMigration", "MetastoreMigration"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
"""Cassandra Migration entity.""" | ||
|
||
from typing import Any, Dict, List | ||
|
||
from butterfree.migrations import DatabaseMigration | ||
|
||
|
||
class CassandraMigration(DatabaseMigration): | ||
"""Cassandra class for Migrations.""" | ||
|
||
def create_query( | ||
self, | ||
fs_schema: List[Dict[str, Any]], | ||
db_schema: List[Dict[str, Any]], | ||
table_name: str, | ||
) -> Any: | ||
"""Create a query regarding Cassandra. | ||
Returns: | ||
Schema object. | ||
""" | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
"""Metastore Migration entity.""" | ||
|
||
from typing import Any, Dict, List | ||
|
||
from butterfree.migrations import DatabaseMigration | ||
|
||
|
||
class MetastoreMigration(DatabaseMigration): | ||
"""Metastore class for Migrations.""" | ||
|
||
def create_query( | ||
self, | ||
fs_schema: List[Dict[str, Any]], | ||
db_schema: List[Dict[str, Any]], | ||
table_name: str, | ||
) -> Any: | ||
"""Create a query regarding Metastore. | ||
Returns: | ||
Schema object. | ||
""" | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
"""Migration entity.""" | ||
|
||
from abc import ABC, abstractmethod | ||
from typing import Any, Callable, Dict, List | ||
|
||
from butterfree.pipelines import FeatureSetPipeline | ||
|
||
|
||
class DatabaseMigration(ABC): | ||
"""Abstract base class for Migrations.""" | ||
|
||
@abstractmethod | ||
def create_query( | ||
self, | ||
fs_schema: List[Dict[str, Any]], | ||
db_schema: List[Dict[str, Any]], | ||
table_name: str, | ||
) -> Any: | ||
"""Create a query regarding a data source. | ||
Returns: | ||
The desired query for the given database. | ||
""" | ||
|
||
def _validate_schema( | ||
self, fs_schema: List[Dict[str, Any]], db_schema: List[Dict[str, Any]] | ||
) -> Any: | ||
"""Provides schema validation for feature sets. | ||
Compares the schema of your local feature set to the | ||
corresponding table in a given database. | ||
Args: | ||
fs_schema: object that contains feature set's schemas. | ||
db_schema: object that contains the table og a given db schema. | ||
""" | ||
|
||
def _get_schema(self, db_client: Callable, table_name: str) -> List[Dict[str, Any]]: | ||
"""Get a table schema in the respective database. | ||
Returns: | ||
Schema object. | ||
""" | ||
pass | ||
|
||
def _apply_migration(self, query: str, db_client: Callable) -> None: | ||
"""Apply the migration in the respective database.""" | ||
|
||
def _send_logs_to_s3(self) -> None: | ||
"""Send all migration logs to S3.""" | ||
pass | ||
|
||
def run(self, pipelines: List[FeatureSetPipeline]) -> None: | ||
"""Runs the migrations. | ||
Args: | ||
pipelines: the feature set pipelines. | ||
""" | ||
pass |