diff --git a/tap_github/__init__.py b/tap_github/__init__.py index 0a8e61b..7105fcb 100644 --- a/tap_github/__init__.py +++ b/tap_github/__init__.py @@ -7,7 +7,7 @@ import requests import singer -from singer import metadata, bookmarks, metrics +from singer import bookmarks, metadata, metrics session = requests.Session() logger = singer.get_logger() @@ -643,8 +643,6 @@ def get_all_projects(schemas, repo_path, state, mdata, start_date): project_id = r.get('id') - - # sync project_columns if that schema is present (only there if selected) if schemas.get('project_columns'): for project_column_rec in get_all_project_columns(project_id, schemas['project_columns'], repo_path, state, mdata, start_date): @@ -825,9 +823,7 @@ def get_reviews_for_pr(pr_number, schema, repo_path, state, mdata): with singer.Transformer() as transformer: rec = transformer.transform(review, schema, metadata=metadata.to_map(mdata)) yield rec - - - return state + return state def get_review_comments_for_pr(pr_number, schema, repo_path, state, mdata): for response in authed_get_all_pages( @@ -840,9 +836,7 @@ def get_review_comments_for_pr(pr_number, schema, repo_path, state, mdata): with singer.Transformer() as transformer: rec = transformer.transform(comment, schema, metadata=metadata.to_map(mdata)) yield rec - - - return state + return state def get_commits_for_pr(pr_number, pr_id, schema, repo_path, state, mdata): for response in authed_get_all_pages( @@ -860,7 +854,7 @@ def get_commits_for_pr(pr_number, pr_id, schema, repo_path, state, mdata): rec = transformer.transform(commit, schema, metadata=metadata.to_map(mdata)) yield rec - return state + return state def get_all_assignees(schema, repo_path, state, mdata, _start_date): diff --git a/tests/unittests/test_pagination.py b/tests/unittests/test_pagination.py new file mode 100644 index 0000000..ae3a7b1 --- /dev/null +++ b/tests/unittests/test_pagination.py @@ -0,0 +1,47 @@ +import json +import math +import unittest +from unittest import mock + +from requests import Response +from tap_github import get_review_comments_for_pr + +response_record = { + "type": "RECORD", + "stream": "review_comments", + "time_extracted": "2000-01-01T00:00:00.000000Z", +} + + +def generate_paginated_response(total_records, records_per_page): + for page in range(math.ceil(total_records / records_per_page)): + items_in_page = min(records_per_page, total_records - (records_per_page * page)) + response_content = [response_record for _ in range(items_in_page)] + resp = Response() + resp._content = json.dumps(response_content).encode() + yield resp + + +class TestReviewComments(unittest.TestCase): + def test_review_comments_pagination(self): + records = [0, 15, 30, 45] + records_per_page = 30 + for total_records in records: + with self.subTest( + f"testing review comments pagination with {total_records} records" + ): + with mock.patch("tap_github.authed_get_all_pages") as mocked_allpage: + mocked_allpage.return_value = generate_paginated_response( + total_records, records_per_page + ) + review_comments_generator = get_review_comments_for_pr( + pr_number=452, + schema={}, + repo_path="transferwise/pipelinewise", + state=None, + mdata={}, + ) + counter = 0 + for _ in review_comments_generator: + counter += 1 + self.assertEqual(counter, total_records)