Skip to content

Commit

Permalink
[MLOP-636] Create migration classes (#282)
Browse files Browse the repository at this point in the history
  • Loading branch information
AlvaroMarquesAndrade authored Feb 18, 2021
1 parent 245eaa5 commit d6ecfa4
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 0 deletions.
7 changes: 7 additions & 0 deletions butterfree/migrations/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
"""Holds available migrations."""

from butterfree.migrations.cassandra_migration import CassandraMigration
from butterfree.migrations.metastore_migration import MetastoreMigration
from butterfree.migrations.migration import DatabaseMigration

__all__ = ["DatabaseMigration", "CassandraMigration", "MetastoreMigration"]
23 changes: 23 additions & 0 deletions butterfree/migrations/cassandra_migration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
"""Cassandra Migration entity."""

from typing import Any, Dict, List

from butterfree.migrations import DatabaseMigration


class CassandraMigration(DatabaseMigration):
"""Cassandra class for Migrations."""

def create_query(
self,
fs_schema: List[Dict[str, Any]],
db_schema: List[Dict[str, Any]],
table_name: str,
) -> Any:
"""Create a query regarding Cassandra.
Returns:
Schema object.
"""
pass
23 changes: 23 additions & 0 deletions butterfree/migrations/metastore_migration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
"""Metastore Migration entity."""

from typing import Any, Dict, List

from butterfree.migrations import DatabaseMigration


class MetastoreMigration(DatabaseMigration):
"""Metastore class for Migrations."""

def create_query(
self,
fs_schema: List[Dict[str, Any]],
db_schema: List[Dict[str, Any]],
table_name: str,
) -> Any:
"""Create a query regarding Metastore.
Returns:
Schema object.
"""
pass
62 changes: 62 additions & 0 deletions butterfree/migrations/migration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""Migration entity."""

from abc import ABC, abstractmethod
from typing import Any, Callable, Dict, List

from butterfree.pipelines import FeatureSetPipeline


class DatabaseMigration(ABC):
"""Abstract base class for Migrations."""

@abstractmethod
def create_query(
self,
fs_schema: List[Dict[str, Any]],
db_schema: List[Dict[str, Any]],
table_name: str,
) -> Any:
"""Create a query regarding a data source.
Returns:
The desired query for the given database.
"""

def _validate_schema(
self, fs_schema: List[Dict[str, Any]], db_schema: List[Dict[str, Any]]
) -> Any:
"""Provides schema validation for feature sets.
Compares the schema of your local feature set to the
corresponding table in a given database.
Args:
fs_schema: object that contains feature set's schemas.
db_schema: object that contains the table og a given db schema.
"""

def _get_schema(self, db_client: Callable, table_name: str) -> List[Dict[str, Any]]:
"""Get a table schema in the respective database.
Returns:
Schema object.
"""
pass

def _apply_migration(self, query: str, db_client: Callable) -> None:
"""Apply the migration in the respective database."""

def _send_logs_to_s3(self) -> None:
"""Send all migration logs to S3."""
pass

def run(self, pipelines: List[FeatureSetPipeline]) -> None:
"""Runs the migrations.
Args:
pipelines: the feature set pipelines.
"""
pass

0 comments on commit d6ecfa4

Please sign in to comment.