feat: add support to s3 storage

cli/datalift/datalift/containers.py

@@ -1,6 +1,7 @@
 from dependency_injector import containers, providers
 
 from .storages.infra.ftp import FTPStorage
+from .storages.infra.s3 import S3Storage
 from .storages.infra.dummy import DummyStorage
 
 
@@ -18,8 +19,16 @@ class Container(containers.DeclarativeContainer):
         FTPStorage,
         user=config.storage_backend.conf.user,
         password=config.storage_backend.conf.password,
-        host=config.storage_backend.tools.conf.host,
-        basedir=config.storage_backend.tools.conf.basedir,
+        host=config.storage_backend.conf.host,
+        basedir=config.storage_backend.conf.basedir,
+    )
+
+    s3_storage = providers.Singleton(
+        S3Storage,
+        access_key=config.storage_backend.conf.access_key,
+        secret_key=config.storage_backend.conf.secret_key,
+        endpoint_url=config.storage_backend.conf.endpoint_url,
+        bucket_name=config.storage_backend.conf.bucket,
     )
 
     storage = providers.Selector(

cli/datalift/datalift/requirements.txt

@@ -1,2 +1,3 @@
 dependency-injector>=4.41,<5.0
 PyYAML>=6.0,<7.0
+boto3>1.34,<2.0

cli/datalift/datalift/storages/infra/s3.py

@@ -0,0 +1,71 @@
+import boto3
+from botocore.exceptions import ClientError
+
+from ..base import (
+    StorageDownloader,
+    StorageOutput,
+    StorageUploader,
+)
+
+
+class S3Storage(StorageUploader, StorageDownloader):
+    def __init__(
+        self,
+        access_key: str,
+        secret_key: str,
+        endpoint_url: str,
+        bucket_name: str,
+    ):
+        self.__client = boto3.client(
+            's3',
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            endpoint_url=endpoint_url,
+        )
+        self.__bucket_name = bucket_name
+
+    def upload(self, object_name: str, data: bytes) -> StorageOutput:
+        """Upload file to S3"""
+        try:
+            # Check if bucket exists, create it if it does not
+            if not self.__bucket_exists(self.__bucket_name):
+                self.__create_bucket(self.__bucket_name)
+
+            self.__client.put_object(
+                Bucket=self.__bucket_name,
+                Key=object_name,
+                Body=data
+            )
+            return StorageOutput("s3", f"{self.__bucket_name}/{object_name}")
+        except ClientError as e:
+            raise Exception(f"Error uploading object to S3: {e}")
+
+    def download(self, object_name: str) -> bytes:
+        """Download file from S3"""
+        try:
+            response = self.__client.get_object(
+                Bucket=self.__bucket_name,
+                Key=object_name
+            )
+            return response['Body'].read()
+        except ClientError as e:
+            raise Exception(f"Error downloading object from S3: {e}")
+
+    def __bucket_exists(self, bucket_name: str) -> bool:
+        """Check if bucket exists"""
+        try:
+            self.__client.head_bucket(Bucket=bucket_name)
+            return True
+        except ClientError as e:
+            if e.response['Error']['Code'] == '404':
+                return False
+            else:
+                raise Exception(f"Error checking bucket existence: {e}")
+
+
+    def __create_bucket(self, bucket_name: str):
+        """Create bucket"""
+        try:
+            self.__client.create_bucket(Bucket=bucket_name)
+        except ClientError as e:
+            raise Exception(f"Error creating bucket: {e}")

cli/datalift/setup.py

@@ -3,7 +3,7 @@
 
 setup(
    name='datalift',
-   version='0.1.2',
+   version='0.2.0',
    author='Samuel López Saura',
    author_email='samuellopezsaura@gmail.com',
    packages=find_packages(),
@@ -19,6 +19,7 @@
    long_description_content_type="text/markdown",
    install_requires=[
        "dependency-injector>=4.41,<5.0",
+       "boto3>1.34,<2.0",
    ],
    entry_points = {
        "console_scripts": [