⚡️ Added read_cache option

.gitignore

@@ -153,6 +153,9 @@ checklink/cookies.txt
 *.dbf
 *.prj
 
+# output folder
+extracao/datasources/arquivos/saida
+
 # VSCODE Files
 .virtual_documents
 

extracao/datasources/base.py

@@ -24,6 +24,7 @@
 @dataclass
 class Base:
 	folder: Union[str, Path] = Path(__file__).parent / 'arquivos' / 'saida'
+	read_cache: bool = False
 
 	def _read(self, stem: str) -> pd.DataFrame:
 		"""Lê o dataframe formado por self.folder / self.stem.parquet.gzip"""

extracao/datasources/sitarweb.py

@@ -50,7 +50,8 @@
 class Sitarweb(Base, GetAttr):
 	"""Common logic from the SITARWEB SQÇ Server Database"""
 
-	def __init__(self, sql_params: dict = SQLSERVER_PARAMS):
+	def __init__(self, sql_params: dict = SQLSERVER_PARAMS, read_cache: bool = False):
+		self.read_cache = read_cache
 		self.default = SQLServer(sql_params)
 
 	@property
@@ -63,15 +64,17 @@ def query(self):
 
 	def extraction(self):
 		"""This method returns a DataFrame with the results of the query"""
-		return pd.read_sql_query(self.query, self.connect(), dtype='category')
+		if self.read_cache:
+			return self._read(f'{self.stem}_raw')
+		return pd.read_sql_query(self.query, self.connect()).astype('string', copy=False)
 
 
 # %% ../../nbs/01c_sitarweb.ipynb 7
 class Radcom(Sitarweb):
 	"""Class for extracting data from SITARWEB refering to the"""
 
-	def __init__(self, sql_params: dict = SQLSERVER_PARAMS):
-		super().__init__(sql_params)
+	def __init__(self, sql_params: dict = SQLSERVER_PARAMS, read_cache: bool = False):
+		super().__init__(sql_params, read_cache)
 
 	@property
 	def query(self):
@@ -115,8 +118,8 @@ def _format(
 
 # %% ../../nbs/01c_sitarweb.ipynb 8
 class Stel(Sitarweb):
-	def __init__(self, sql_params: dict = SQLSERVER_PARAMS):
-		super().__init__(sql_params)
+	def __init__(self, sql_params: dict = SQLSERVER_PARAMS, read_cache: bool = False):
+		super().__init__(sql_params, read_cache)
 
 	@property
 	def query(self):

extracao/datasources/telecom.py

@@ -23,15 +23,17 @@
 load_dotenv(find_dotenv())
 
 # %% ../../nbs/01f_telecom.ipynb 6
-MONGO_URI = os.environ.get('MONGO_URI')
+MONGO_URI = os.environ.get('MONGO_URI', '')
 
 
 # %% ../../nbs/01f_telecom.ipynb 7
 class Telecom(Mosaico):
 	"""This class encapsulates the extraction and processing of Telecommunications Services from the MOSAICO MongoDB"""
 
-	def __init__(self, mongo_uri: str = MONGO_URI, limit: int = 0) -> None:
-		super().__init__(mongo_uri)
+	def __init__(
+		self, mongo_uri: str = MONGO_URI, limit: int = 0, read_cache: bool = False
+	) -> None:
+		super().__init__(mongo_uri, read_cache)
 		self.limit = limit
 
 	@property