Added more filters when querying the MongoDB

InovaFiscaliza · Nov 9, 2023 · fe306b9 · fe306b9
1 parent f3cb8b0
commit fe306b9
Show file tree

Hide file tree

Showing 12 changed files with 516 additions and 469 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,10 +1,2 @@
-*.xlsx filter=lfs diff=lfs merge=lfs -text
-*.parquet.gzip filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.ipynb merge=nbdev-merge
-*.shp filter=lfs diff=lfs merge=lfs -text
-*.shx filter=lfs diff=lfs merge=lfs -text
-*.prj filter=lfs diff=lfs merge=lfs -text
-*.dbf filter=lfs diff=lfs merge=lfs -text
-*.cpg filter=lfs diff=lfs merge=lfs -text
+
diff --git a/extracao/__init__.py b/extracao/__init__.py
@@ -1 +1 @@
-__version__ = "0.8.20"
+__version__ = "0.8.21"
diff --git a/extracao/anatel.py b/extracao/anatel.py
@@ -71,8 +71,8 @@ def extraction(self) -> L:
             Telecom(self.mongo_uri, self.limit),
             SMP(self.mongo_uri, self.limit),
             SRD(self.mongo_uri),
-            # Stel(self.sql_params),
-            # Radcom(self.sql_params),
+            Stel(self.sql_params),
+            Radcom(self.sql_params),
             Aero(),
         ]
 
@@ -152,26 +152,20 @@ def intersect_coordinates_on_poligon(
         gdf_points.crs = regions.crs
 
         # Spatial join points to the regions
-        points_with_regions = gpd.sjoin(
-            gdf_points, regions, how="inner", predicate="within"
-        )
+        gdf = gpd.sjoin(gdf_points, regions, how="inner", predicate="within")
 
         if check_municipio:
             # Check correctness of Coordinates
-            check_coords = (
-                points_with_regions.Código_Município != points_with_regions.CD_MUN
-            )
+            check_coords = gdf.Código_Município != gdf.CD_MUN
 
             log = """[("Colunas", ["Código_Município", "Município", "UF"]),
 					("Processamento", "Informações substituídas  pela localização correta das coordenadas.")		      
 				"""
-            self.register_log(points_with_regions, log, check_coords)
+            self.register_log(gdf, log, check_coords)
 
-            points_with_regions.drop(
-                ["Código_Município", "Município", "UF"], axis=1, inplace=True
-            )
+            gdf.drop(["Código_Município", "Município", "UF"], axis=1, inplace=True)
 
-        points_with_regions.rename(
+        gdf.rename(
             columns={
                 "CD_MUN": "Código_Município",
                 "NM_MUN": "Município",
@@ -180,7 +174,7 @@ def intersect_coordinates_on_poligon(
             inplace=True,
         )
 
-        return points_with_regions
+        return gdf
 
     def validate_coordinates(
         self, df: pd.DataFrame, check_municipio: bool = True
@@ -207,8 +201,9 @@ def _format(
         self,
         dfs: List,  # List with the individual API sources
     ) -> pd.DataFrame:  # Processed DataFrame
-        # aero = self.validate_coordinates(dfs.pop(), False)
-        return self.validate_coordinates(pd.concat(dfs, ignore_index=True))
-        return pd.concat([aero, anatel], ignore_index=True).sort_values(
+        aero = self.validate_coordinates(dfs.pop(), False)
+        anatel = self.validate_coordinates(pd.concat(dfs, ignore_index=True))
+        df = pd.concat([aero, anatel], ignore_index=True).sort_values(
             ["Frequência", "Latitude", "Longitude"], ignore_index=True
         )
+        return df.loc[:, self.columns]
diff --git a/extracao/constants.py b/extracao/constants.py
@@ -82,7 +82,6 @@
 
 AGG_LICENCIAMENTO = [
     "Frequência",
-    "Entidade",
     "Fistel",
     "Código_Município",
     "Longitude",
@@ -389,6 +388,9 @@
         {"NumServico": {"$nin": ["010", "045", "171", "450", "750", "", None]}},
         {"FreqTxMHz": {"$nin": [None, "", 0], "$type": 1.0}},
         {"CodMunicipio": {"$nin": [None, ""]}},
+        {"NumFistel": {"$nin": [None, ""]}},
+        {"CodTipoClasseEstacao": {"$nin": [None, ""]}},
+        {"DesignacaoEmissao": {"$nin": [None, ""]}},
     ]
 }
 
@@ -411,6 +413,10 @@
         {"NumServico": "010"},
         {"FreqTxMHz": {"$nin": [None, "", 0], "$type": 1.0}},
         {"CodMunicipio": {"$nin": [None, ""]}},
+        {"NumFistel": {"$nin": [None, ""]}},
+        {"CodTipoClasseEstacao": {"$nin": [None, ""]}},
+        {"DesignacaoEmissao": {"$nin": [None, ""]}},
+        {"Tecnologia": {"$nin": [None, ""]}},
     ]
 }
 

diff --git a/nbs/00_constants.ipynb b/nbs/00_constants.ipynb
@@ -134,7 +134,6 @@
     "\n",
     "AGG_LICENCIAMENTO = [\n",
     "\t'Frequência',\n",
-    "\t'Entidade',\n",
     "\t'Fistel',\n",
     "\t'Código_Município',\n",
     "\t'Longitude',\n",
@@ -497,6 +496,9 @@
     "\t\t{'NumServico': {'$nin': ['010', '045', '171', '450', '750', '', None]}},\n",
     "\t\t{'FreqTxMHz': {'$nin': [None, '', 0], '$type': 1.0}},\n",
     "\t\t{'CodMunicipio': {'$nin': [None, '']}},\n",
+    "\t\t{'NumFistel': {'$nin': [None, '']}},\n",
+    "\t\t{'CodTipoClasseEstacao': {'$nin': [None, '']}},\n",
+    "\t\t{'DesignacaoEmissao': {'$nin': [None, '']}},\n",
     "\t]\n",
     "}\n",
     "\n",
@@ -519,6 +521,10 @@
     "\t\t{'NumServico': '010'},\n",
     "\t\t{'FreqTxMHz': {'$nin': [None, '', 0], '$type': 1.0}},\n",
     "\t\t{'CodMunicipio': {'$nin': [None, '']}},\n",
+    "\t\t{'NumFistel': {'$nin': [None, '']}},\n",
+    "\t\t{'CodTipoClasseEstacao': {'$nin': [None, '']}},\n",
+    "\t\t{'DesignacaoEmissao': {'$nin': [None, '']}},\n",
+    "\t\t{'Tecnologia': {'$nin': [None, '']}},\n",
     "\t]\n",
     "}"
    ]