[AP-1076] fix pagination for comments (#10)

tap_github/__init__.py

@@ -7,7 +7,7 @@
 
 import requests
 import singer
-from singer import metadata, bookmarks, metrics
+from singer import bookmarks, metadata, metrics
 
 session = requests.Session()
 logger = singer.get_logger()
@@ -643,8 +643,6 @@ def get_all_projects(schemas, repo_path, state, mdata, start_date):
 
                 project_id = r.get('id')
 
-
-
                 # sync project_columns if that schema is present (only there if selected)
                 if schemas.get('project_columns'):
                     for project_column_rec in get_all_project_columns(project_id, schemas['project_columns'], repo_path, state, mdata, start_date):
@@ -825,9 +823,7 @@ def get_reviews_for_pr(pr_number, schema, repo_path, state, mdata):
             with singer.Transformer() as transformer:
                 rec = transformer.transform(review, schema, metadata=metadata.to_map(mdata))
             yield rec
-
-
-        return state
+    return state
 
 def get_review_comments_for_pr(pr_number, schema, repo_path, state, mdata):
     for response in authed_get_all_pages(
@@ -840,9 +836,7 @@ def get_review_comments_for_pr(pr_number, schema, repo_path, state, mdata):
             with singer.Transformer() as transformer:
                 rec = transformer.transform(comment, schema, metadata=metadata.to_map(mdata))
             yield rec
-
-
-        return state
+    return state
 
 def get_commits_for_pr(pr_number, pr_id, schema, repo_path, state, mdata):
     for response in authed_get_all_pages(
@@ -860,7 +854,7 @@ def get_commits_for_pr(pr_number, pr_id, schema, repo_path, state, mdata):
                 rec = transformer.transform(commit, schema, metadata=metadata.to_map(mdata))
             yield rec
 
-        return state
+    return state
 
 
 def get_all_assignees(schema, repo_path, state, mdata, _start_date):

tests/unittests/test_pagination.py

@@ -0,0 +1,47 @@
+import json
+import math
+import unittest
+from unittest import mock
+
+from requests import Response
+from tap_github import get_review_comments_for_pr
+
+response_record = {
+    "type": "RECORD",
+    "stream": "review_comments",
+    "time_extracted": "2000-01-01T00:00:00.000000Z",
+}
+
+
+def generate_paginated_response(total_records, records_per_page):
+    for page in range(math.ceil(total_records / records_per_page)):
+        items_in_page = min(records_per_page, total_records - (records_per_page * page))
+        response_content = [response_record for _ in range(items_in_page)]
+        resp = Response()
+        resp._content = json.dumps(response_content).encode()
+        yield resp
+
+
+class TestReviewComments(unittest.TestCase):
+    def test_review_comments_pagination(self):
+        records = [0, 15, 30, 45]
+        records_per_page = 30
+        for total_records in records:
+            with self.subTest(
+                f"testing review comments pagination with {total_records} records"
+            ):
+                with mock.patch("tap_github.authed_get_all_pages") as mocked_allpage:
+                    mocked_allpage.return_value = generate_paginated_response(
+                        total_records, records_per_page
+                    )
+                    review_comments_generator = get_review_comments_for_pr(
+                        pr_number=452,
+                        schema={},
+                        repo_path="transferwise/pipelinewise",
+                        state=None,
+                        mdata={},
+                    )
+                    counter = 0
+                    for _ in review_comments_generator:
+                        counter += 1
+                self.assertEqual(counter, total_records)