Merge pull request #2 from luabase/jichaos/sorted

Rebase upstream

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      time: "12:00"
+    commit-message:
+      prefix: "feat(deps): "
+      prefix-development: "chore(deps-dev): "
+    versioning-strategy: increase-if-necessary
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: "monthly"
+      time: "12:00"
+    commit-message:
+      prefix: "ci: "

.github/workflows/ci_workflow.yml

@@ -1,31 +1,37 @@
-### A CI workflow template that runs linting and python testing
-### TODO: Modify as needed or as desired.
-
 name: Test tap-klaviyo
 
-# TODO: Re-enable CI/CD testing
-# on: [push]
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    paths:
+    - "tap_klaviyo/**"
+    - "tests/**"
+    - "pyproject.toml"
+    - "poetry.lock"
+    - ".github/workflows/ci_workflow.yml"
 
 jobs:
   pytest:
     runs-on: ubuntu-latest
-    env:
-      GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
     strategy:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Poetry
       run: |
-        pip install poetry
+        pipx install poetry
     - name: Install dependencies
       run: |
         poetry install
     - name: Test with pytest
+      env:
+        TAP_KLAVIYO_AUTH_TOKEN: ${{ secrets.TAP_KLAVIYO_AUTH_TOKEN }}
+        TAP_KLAVIYO_REVISION: ${{ secrets.TAP_KLAVIYO_REVISION }}
       run: |
         poetry run pytest

.gitignore

@@ -134,3 +134,16 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+.vscode/*
+# !.vscode/settings.json
+!.vscode/tasks.json
+# !.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-json
     exclude: "\\.vscode/.*.json"
@@ -16,18 +16,14 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.287
+  rev: v0.1.8
   hooks:
   - id: ruff
-    args: ["--fix"]
-
-- repo: https://github.com/psf/black
-  rev: 23.7.0
-  hooks:
-  - id: black
+    args: [--fix, --exit-non-zero-on-fix, --show-fixes]
+  - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.5.1
+  rev: v1.7.1
   hooks:
   - id: mypy
     pass_filenames: true

pyproject.toml

@@ -8,47 +8,43 @@ keywords = [
     "ELT",
     "Klaviyo",
 ]
-license = "Apache 2.0"
+license = "Apache-2.0"
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.7.1"
+python = ">=3.7.1"
 fs-s3fs = { version = "^1.1.1", optional = true }
-requests = "^2.31.0"
-singer-sdk = { version="^0.26.0" }
+requests = ">=2,<3"
+singer-sdk = { version="~0.34.0" }
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.3.1"
-singer-sdk = { version="^0.26.0", extras = ["testing"] }
+singer-sdk = { version="~0.34.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3 # Vertical Hanging Indent
-src_paths = "tap_klaviyo"
-
 [tool.mypy]
-python_version = "3.9"
+python_version = "3.11"
+warn_redundant_casts = true
+warn_return_any = true
 warn_unused_configs = true
+warn_unused_ignores = true
 
 [build-system]
 requires = ["poetry-core>=1.0.8"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]
-# CLI declaration
 tap-klaviyo = 'tap_klaviyo.tap:TapKlaviyo.cli'
 
-[tool.black]
-max-line-length = 100
-
 [tool.ruff]
-ignore = ["ANN101", "DJ", "PD", "D102"]
 line-length = 100
-select = ["ALL"]
 target-version = "py37"
 
+[tool.ruff.lint]
+ignore = ["ANN101", "DJ", "PD", "D102", "COM812", "ISC001", "FIX002"]
+select = ["ALL"]
+
 [tool.ruff.flake8-import-conventions]
 banned-from = ["typing"]
 

tap_klaviyo/__main__.py

@@ -0,0 +1,5 @@
+"""tap-klaviyo entry point."""
+
+from tap_klaviyo.tap import TapKlaviyo
+
+TapKlaviyo.cli()

tap_klaviyo/client.py

@@ -38,7 +38,7 @@ class KlaviyoPaginator(BaseHATEOASPaginator):
 
     def get_next_url(self, response: requests.Response) -> str:
         data = response.json()
-        return data.get("links").get("next")
+        return data.get("links").get("next")  # type: ignore[no-any-return]
 
 
 class KlaviyoStream(RESTStream):
@@ -99,8 +99,6 @@ def get_url_params(
             if self.is_sorted:
                 params["sort"] = self.replication_key
 
-            params[
-                "filter"
-            ] = f"greater-than({self.replication_key},{filter_timestamp})"
+            params["filter"] = f"greater-than({self.replication_key},{filter_timestamp})"
 
         return params

tap_klaviyo/streams.py

@@ -2,10 +2,14 @@
 
 from __future__ import annotations
 
+import typing as t
 from pathlib import Path
 
 from tap_klaviyo.client import KlaviyoStream
 
+if t.TYPE_CHECKING:
+    from urllib.parse import ParseResult
+
 SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
 
 
@@ -40,6 +44,31 @@ class CampaignsStream(KlaviyoStream):
     replication_key = "updated_at"
     schema_filepath = SCHEMAS_DIR / "campaigns.json"
 
+    @property
+    def partitions(self) -> list[dict] | None:
+        return [
+            {
+                "filter": "equals(messages.channel,'email')",
+            },
+            {
+                "filter": "equals(messages.channel,'sms')",
+            },
+        ]
+
+    def get_url_params(
+        self,
+        context: dict | None,
+        next_page_token: ParseResult | None,
+    ) -> dict[str, t.Any]:
+        url_params = super().get_url_params(context, next_page_token)
+
+        # Apply channel filters
+        if context:
+            parent_filter = url_params["filter"]
+            url_params["filter"] = f"and({parent_filter},{context['filter']})"
+
+        return url_params
+
     def post_process(
         self,
         row: dict,
@@ -106,7 +135,7 @@ def get_child_context(self, record: dict, context: dict | None) -> dict:
         context = context or {}
         context["list_id"] = record["id"]
 
-        return super().get_child_context(record, context)
+        return super().get_child_context(record, context)  # type: ignore[no-any-return]
 
     def post_process(
         self,

tests/test_core.py

@@ -2,7 +2,7 @@
 
 import datetime
 
-from singer_sdk.testing import get_tap_test_class
+from singer_sdk.testing import SuiteConfig, get_tap_test_class
 
 from tap_klaviyo.tap import TapKlaviyo
 
@@ -15,4 +15,14 @@
 TestTapKlaviyo = get_tap_test_class(
     tap_class=TapKlaviyo,
     config=SAMPLE_CONFIG,
+    suite_config=SuiteConfig(
+        # TODO(edgarrmondragon): seed test account with sample data
+        # https://github.com/MeltanoLabs/tap-klaviyo/pull/34
+        ignore_no_records_for_streams=[
+            "campaigns",
+            "events",
+            "flows",
+            "templates",
+        ],
+    ),
 )