Skip to content

Commit

Permalink
pre-commit run pass
Browse files Browse the repository at this point in the history
  • Loading branch information
Sid Mohan authored and Sid Mohan committed Apr 1, 2024
1 parent 750570e commit ae68c09
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 13 deletions.
5 changes: 2 additions & 3 deletions examples/uploading-file-types.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@
"metadata": {},
"outputs": [],
"source": [
"%pip list\n"
"%pip list"
]
},
{
Expand All @@ -141,8 +141,7 @@
"source": [
"input_file = \"/Users/sidmohan/Desktop/datafog-v2.4.0/datafog-python/tests/files/input_files/agi-builder-meetup.pdf\"\n",
"output = datafog.DataFog.upload_file(uploaded_file_path=input_file)\n",
"print(output)\n",
"\n"
"print(output)"
]
}
],
Expand Down
13 changes: 7 additions & 6 deletions src/datafog/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
import json
import logging
import tempfile
from pathlib import Path

import pandas as pd
import requests
import spacy
from unstructured.partition.auto import partition
from io import BytesIO
from pathlib import Path

from .__about__ import __version__
from .pii_tools import PresidioEngine
Expand Down Expand Up @@ -40,7 +40,6 @@ def __init__(self):
"""
self.nlp = spacy.load("en_spacy_pii_fast")


@staticmethod
def client():
"""
Expand All @@ -50,7 +49,7 @@ def client():
DataFog: A new instance of the DataFog client.
"""
return DataFog()

@staticmethod
def upload_file(uploaded_file_path):
uploaded_file_path = Path(uploaded_file_path)
Expand All @@ -60,7 +59,9 @@ def upload_file(uploaded_file_path):
if not uploaded_file_path.exists():
return "File not found."
else:
temp_file = tempfile.NamedTemporaryFile(delete=True, suffix=uploaded_file_path.suffix)
temp_file = tempfile.NamedTemporaryFile(
delete=True, suffix=uploaded_file_path.suffix
)
temp_file.write(bytes_data)
elements = partition(temp_file.name)
text = ""
Expand All @@ -69,7 +70,7 @@ def upload_file(uploaded_file_path):
texts[uploaded_file_path.name] = text

return texts

def __call__(self, input_source, privacy_operation):
"""
Process the input data and apply the specified privacy operation.
Expand Down
10 changes: 6 additions & 4 deletions tests/test_datafog.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# test_datafog.py
import pytest
import requests

from datafog import DataFog

Expand All @@ -9,11 +8,14 @@
def datafog():
return DataFog()


def test_upload_file(datafog):
uploaded_file="tests/files/input_files/agi-builder-meetup.pdf"
uploaded_file = "tests/files/input_files/agi-builder-meetup.pdf"
result = datafog.upload_file(uploaded_file_path=uploaded_file)
file_text = result[uploaded_file.split('/')[-1]] # Extract the text using the file name as key
assert "Cloudflare" in file_text # confirms that OCR is not on
file_text = result[
uploaded_file.split("/")[-1]
] # Extract the text using the file name as key
assert "Cloudflare" in file_text # confirms that OCR is not on
assert "SF" in file_text
assert "Laurie" in file_text
assert "BentoML" in file_text
Expand Down

0 comments on commit ae68c09

Please sign in to comment.