Bump SDK to 0.39.0 (#15)

* bump sdk to 0.39.0 and lxml to 5.2.2

* poerty update

* added comment to init

* added main to match template

* moved test and update files to match template

* updated get_records to use Context hint

* refactor of get_streams based on Ruff recommendation

* Line exceptions for Ruff lints

* Changed to double quotes

* exempt lines from linter

* Updated What New 20240801

* bump to version 0.1.3

README.md

@@ -5,6 +5,7 @@
 Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps.
 
 ### Whats New 🛳️🎉
+**2024-08-01 Upgraded to Meltano Singer-SDK 0.39.0**
 
 **2024-04-04 Upgraded to Meltano Singer-SDK 0.36.1:**
 

pyproject.toml

@@ -1,9 +1,9 @@
 [tool.poetry]
 name = "tap-stackoverflow-sampledata"
-version = "0.1.2"
+version = "0.1.3"
 description = "`tap-stackoverflow-sampledata` is a Singer tap for stackoverflow-sampledata, built with the Meltano SDK for Singer Taps."
 readme = "README.md"
-authors = ["Dan Norman"]
+authors = ["Dan Norman <buzzcutnorman@gmail.com>"]
 keywords = [
     "ELT",
     "stackoverflow-sampledata",
@@ -21,14 +21,14 @@ license = "MIT"
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-importlib-resources = { version = "==6.1.*", python = "<3.9" }
-singer-sdk = { version="~=0.36.1" }
+importlib-resources = { version = "==6.4.*", python = "<3.9" }
+singer-sdk = { version="~=0.39.0", extras = [] }
 fs-s3fs = { version = "~=1.1.1", optional = true }
-lxml = "^5.1.0"
+lxml = "^5.2.2"
 
 [tool.poetry.dev-dependencies]
-pytest = ">=7.4.0"
-singer-sdk = { version="~=0.36.1", extras = ["testing"] }
+pytest = ">=8"
+singer-sdk = { version="~=0.39.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

tap_stackoverflow_sampledata/__init__.py

@@ -0,0 +1 @@
+"""Tap for stackoverflow-sampledata."""

tap_stackoverflow_sampledata/__main__.py

@@ -0,0 +1,7 @@
+"""stackoverflow-sampledata entry point."""
+
+from __future__ import annotations
+
+from tap_stackoverflow_sampledata.tap import TapStackOverflowSampleData
+
+TapStackOverflowSampleData.cli()

tap_stackoverflow_sampledata/client.py

@@ -3,11 +3,13 @@
 from __future__ import annotations
 
 import os
-from typing import Iterable, Optional
+from typing import TYPE_CHECKING, Iterable
 
 from lxml import etree
 from singer_sdk.streams import Stream
 
+if TYPE_CHECKING:
+    from singer_sdk.helpers.types import Context
 
 class NonExistentDataDirectoryError(Exception):
     """Exception raised when the give data directory does not exist."""
@@ -34,7 +36,10 @@ def data_file(self) -> str:
             self.get_data_file_path()
         return self._file_path
 
-    def get_records(self, context: Optional[dict]) -> Iterable[dict]:
+    def get_records(
+        self,
+        context: Context | None,  # noqa: ARG002
+    ) -> Iterable[dict]:
         """Return a generator of row-type dictionary objects.
 
         Args:
@@ -45,20 +50,20 @@ def get_records(self, context: Optional[dict]) -> Iterable[dict]:
             One dict per record.
         """
         # Get the Stream Properties Dictornary from the Schema
-        properties: dict = self.schema.get('properties')
+        properties: dict = self.schema.get("properties")
 
         # Blank list to hold all the Primary Key columns
         column_names: list[str] = []
         column_value_types: dict[str, list[str]] = {}
 
         #  We grab the column names from the Stream Schema
         #  Then append them to the columns names list
-        #  We also grab and append the data type to 
+        #  We also grab and append the data type to
         #  Then insert it into column_value_types dict
         for column in properties:
             name = str(column)
             column_names.append(name)
-            column_value_types[name] = properties.get(name).get('type')
+            column_value_types[name] = properties.get(name).get("type")
 
         # Grab the rows from the xml file using by
         # opening the file using using a iteration parser
@@ -84,7 +89,7 @@ def get_records(self, context: Optional[dict]) -> Iterable[dict]:
             for column in column_names:
                 if column in element.attrib:
                     value = element.attrib.get(column)
-                    if 'integer' in column_value_types.get(column):
+                    if "integer" in column_value_types.get(column):
                         row[column] = int(value)
                     else:
                         row[column] = value
@@ -112,7 +117,7 @@ def get_data_file_path(self) -> None:
         """
         # Check that the file_directory from the meltano.yaml exists
         # if it isn't we alert there is an issue
-        if not os.path.exists(self.file_directory):
+        if not os.path.exists(self.file_directory):  # noqa: PTH110
             msg: str = f"File path does not exist {self.file_directory}"
             raise NonExistentDataDirectoryError(msg)
 
@@ -130,7 +135,7 @@ def is_valid_filename(self, file_path: str) -> bool:
         # Check to see if this is a directory turn
         # the is valid flag to false and
         # say you are skipping the file
-        if os.path.isdir(file_path):
+        if os.path.isdir(file_path):  # noqa: PTH112
             is_valid = False
             self.logger.info("Skipping folder %a", file_path)
 

tap_stackoverflow_sampledata/tap.py

@@ -78,7 +78,7 @@ def input_catalog(self) -> Catalog:
                         th.Property(
                             "compression",
                             th.StringType,
-                            description="Currently the only compression options is gzip",
+                            description="Currently the only compression options is gzip",  # noqa: E501
                         )
                     )
                 ),
@@ -88,7 +88,7 @@ def input_catalog(self) -> Catalog:
                         th.Property(
                             "root",
                             th.StringType,
-                            description=("the directory you want batch messages to be placed in\n"
+                            description=("the directory you want batch messages to be placed in\n"  # noqa: E501
                                         "example: file://test/batches"
                             )
                         ),
@@ -130,15 +130,17 @@ def get_streams(self) -> list[Stream]:
             self.logger.error("No stackoverflow_data_directory configured.")
             sys.exit(1)
 
-        if os.path.exists(data_directory):
-            if os.path.isdir(data_directory):
+        if os.path.exists(data_directory):  # noqa: PTH110
+            if os.path.isdir(data_directory):  # noqa: PTH112
                 if len(os.listdir(data_directory)) > 0:
-                    for file in os.listdir(data_directory):
-                        # If the file is in the dictionary
-                        # we add it the Streams Classes that will be
-                        # passed in the dicover_streams process.
-                        if STACKOVERFLOW_FILE_NAMES_TO_STREAMS.get(file.lower()):
-                            stream_types.append(STACKOVERFLOW_FILE_NAMES_TO_STREAMS.get(file.lower()))
+                    # If the file is in the dictionary
+                    # we add it the Streams Classes that will be
+                    # passed in the dicover_streams process.
+                    stream_types.extend(
+                        STACKOVERFLOW_FILE_NAMES_TO_STREAMS.get(file.lower())
+                        for file in os.listdir(data_directory)
+                        if STACKOVERFLOW_FILE_NAMES_TO_STREAMS.get(file.lower())
+                    )
                 else:
                     self.logger.error("There are no files in the directory")
                     sys.exit(1)

tap_stackoverflow_sampledata/tests/test_core.py → tests/test_core.py

@@ -2,9 +2,9 @@
 
 import datetime
 
-from singer_sdk.testing import get_standard_tap_tests
+from singer_sdk.testing import get_tap_test_class
 
-from tap_stackoverflow_sampledata.tap import Tapstackoverflow-sampledata
+from tap_stackoverflow_sampledata.tap import TapStackOverflowSampleData
 
 SAMPLE_CONFIG = {
     "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
@@ -13,14 +13,10 @@
 
 
 # Run standard built-in tap tests from the SDK:
-def test_standard_tap_tests():
-    """Run standard tap tests from the SDK."""
-    tests = get_standard_tap_tests(
-        Tapstackoverflow-sampledata,
-        config=SAMPLE_CONFIG
-    )
-    for test in tests:
-        test()
+TestTapStackOverflowSampleData = get_tap_test_class(
+    tap_class=TapStackOverflowSampleData,
+    config=SAMPLE_CONFIG,
+)
 
 
 # TODO: Create additional tests as appropriate for your tap.