Return full payload from batch_upsert

docs/source/changelog.rst

@@ -7,6 +7,8 @@ Changelog
 
 See :ref:`Migrating from 1.x to 2.0` for detailed migration notes.
 
+* :meth:`~pyairtable.Table.batch_upsert` now returns the full payload from the Airtable API.
+  - `PR #281 <https://github.com/gtalarico/pyairtable/pull/281>`_.
 * :ref:`ORM` module is no longer experimental and has a stable API.
   - `PR #277 <https://github.com/gtalarico/pyairtable/pull/277>`_.
 * Added :meth:`Model.batch_save <pyairtable.orm.Model.batch_save>`

docs/source/migrations.rst

@@ -72,6 +72,13 @@ Changes to types
 * All functions and methods in this library have full type annotations that will pass ``mypy --strict``.
   See the :ref:`types <Module: pyairtable.api.types>` module for more information on the types this library accepts and returns.
 
+batch_upsert has a different return type
+--------------------------------------------
+
+* :meth:`~pyairtable.Table.batch_upsert` now returns the full payload from the Airtable API,
+  as opposed to just the list of records (with no indication of which were created or updated).
+  See :class:`~pyairtable.api.types.UpsertResultDict` for more details.
+
 
 Migrating from 0.x to 1.0
 ============================

pyairtable/api/table.py

@@ -12,6 +12,7 @@
     RecordDict,
     RecordId,
     UpdateRecordDict,
+    UpsertResultDict,
     assert_typed_dict,
     assert_typed_dicts,
 )
@@ -373,7 +374,7 @@ def batch_upsert(
         replace: bool = False,
         typecast: bool = False,
         return_fields_by_field_id: bool = False,
-    ) -> List[RecordDict]:
+    ) -> UpsertResultDict:
         """
         Updates or creates records in batches, either using ``id`` (if given) or using a set of
         fields (``key_fields``) to look for matches. For more information on how this operation
@@ -390,7 +391,7 @@ def batch_upsert(
             return_fields_by_field_id: |kwarg_return_fields_by_field_id|
 
         Returns:
-            The list of updated records.
+            Lists of created/updated record IDs, along with the list of all records affected.
         """
         # The API will reject a request where a record is missing any of fieldsToMergeOn,
         # but we might not reach that error until we've done several batch operations.
@@ -403,8 +404,13 @@ def batch_upsert(
             if missing:
                 raise ValueError(f"missing {missing!r} in {record['fields'].keys()!r}")
 
-        updated_records = []
         method = "put" if replace else "patch"
+        result: UpsertResultDict = {
+            "updatedRecords": [],
+            "createdRecords": [],
+            "records": [],
+        }
+
         for chunk in self.api.chunked(records):
             formatted_records = [
                 {k: v for (k, v) in record.items() if k in ("id", "fields")}
@@ -420,9 +426,13 @@ def batch_upsert(
                     "performUpsert": {"fieldsToMergeOn": key_fields},
                 },
             )
-            updated_records += assert_typed_dicts(RecordDict, response["records"])
+            result["updatedRecords"].extend(response["updatedRecords"])
+            result["createdRecords"].extend(response["createdRecords"])
+            result["records"].extend(
+                assert_typed_dicts(RecordDict, response["records"])
+            )
 
-        return updated_records
+        return result
 
     def delete(self, record_id: RecordId) -> RecordDeletedDict:
         """

pyairtable/api/types.py

@@ -247,6 +247,27 @@ class RecordDeletedDict(TypedDict):
     deleted: bool
 
 
+class UpsertResultDict(TypedDict):
+    """
+    A ``dict`` representing the payload returned by the Airtable API after an upsert.
+    For more details on this data structure, see the
+    `Update multiple records <https://airtable.com/developers/web/api/update-multiple-records>`__
+    API documentation.
+
+    Usage:
+        >>> table.batch_upsert(records, key_fields=["Name"])
+        {
+            'createdRecords': [...],
+            'updatedRecords': [...],
+            'records': [...]
+        }
+    """
+
+    createdRecords: List[RecordId]
+    updatedRecords: List[RecordId]
+    records: List[RecordDict]
+
+
 class UserAndScopesDict(TypedDict, total=False):
     """
     A ``dict`` representing the `Get user ID & scopes <https://airtable.com/developers/web/api/get-user-id-scopes>`_ endpoint.

tests/conftest.py

@@ -72,11 +72,6 @@ def mock_response_single(mock_records):
     return mock_records[0]
 
 
-@pytest.fixture
-def mock_response_batch(mock_records):
-    return {"records": mock_records * 2}
-
-
 @pytest.fixture
 def mock_response_list(mock_records):
     return [

tests/integration/test_integration_api.py

@@ -176,23 +176,25 @@ def test_batch_upsert(table: Table, cols):
     )
 
     # Test batch_upsert where replace=False
-    results = table.batch_upsert(
+    result = table.batch_upsert(
         [
             {"id": one["id"], "fields": {cols.NUM: 3}},  # use id
             {"fields": {cols.TEXT: "Two", cols.NUM: 4}},  # use key_fields
             {"fields": {cols.TEXT: "Three", cols.NUM: 5}},  # create record
         ],
         key_fields=[cols.TEXT],
     )
-    assert len(results) == 3
-    assert results[0]["id"] == one["id"]
-    assert results[0]["fields"] == {cols.TEXT: "One", cols.NUM: 3}
-    assert results[1]["id"] == two["id"]
-    assert results[1]["fields"] == {cols.TEXT: "Two", cols.NUM: 4}
-    assert results[2]["fields"] == {cols.TEXT: "Three", cols.NUM: 5}
+    assert set(result["updatedRecords"]) == {one["id"], two["id"]}
+    assert len(result["createdRecords"]) == 1
+    assert len(result["records"]) == 3
+    assert result["records"][0]["id"] == one["id"]
+    assert result["records"][0]["fields"] == {cols.TEXT: "One", cols.NUM: 3}
+    assert result["records"][1]["id"] == two["id"]
+    assert result["records"][1]["fields"] == {cols.TEXT: "Two", cols.NUM: 4}
+    assert result["records"][2]["fields"] == {cols.TEXT: "Three", cols.NUM: 5}
 
     # Test batch_upsert where replace=True
-    results = table.batch_upsert(
+    result = table.batch_upsert(
         [
             {"id": one["id"], "fields": {cols.NUM: 3}},  # removes cols.TEXT
             {"fields": {cols.TEXT: "Two"}},  # removes cols.NUM
@@ -202,21 +204,21 @@ def test_batch_upsert(table: Table, cols):
         key_fields=[cols.TEXT],
         replace=True,
     )
-    assert len(results) == 4
-    assert results[0]["id"] == one["id"]
-    assert results[0]["fields"] == {cols.NUM: 3}
-    assert results[1]["id"] == two["id"]
-    assert results[1]["fields"] == {cols.TEXT: "Two"}
-    assert results[2]["fields"] == {cols.TEXT: "Three", cols.NUM: 6}
-    assert results[3]["fields"] == {cols.NUM: 7}
+    assert len(result["records"]) == 4
+    assert result["records"][0]["id"] == one["id"]
+    assert result["records"][0]["fields"] == {cols.NUM: 3}
+    assert result["records"][1]["id"] == two["id"]
+    assert result["records"][1]["fields"] == {cols.TEXT: "Two"}
+    assert result["records"][2]["fields"] == {cols.TEXT: "Three", cols.NUM: 6}
+    assert result["records"][3]["fields"] == {cols.NUM: 7}
 
     # Test that batch_upsert passes along return_fields_by_field_id
-    results = table.batch_upsert(
+    result = table.batch_upsert(
         [{"fields": {cols.TEXT: "Two", cols.NUM: 8}}],
         key_fields=[cols.TEXT],
         return_fields_by_field_id=True,
     )
-    assert results == [
+    assert result["records"] == [
         {
             "id": two["id"],
             "createdTime": two["createdTime"],

tests/test_api_table.py

@@ -5,6 +5,7 @@
 from requests_mock import Mocker
 
 from pyairtable import Api, Base, Table
+from pyairtable.testing import fake_record
 from pyairtable.utils import chunked
 
 
@@ -244,40 +245,51 @@ def test_update(table: Table, mock_response_single, replace, http_method):
 
 
 @pytest.mark.parametrize("replace,http_method", [(False, "PATCH"), (True, "PUT")])
-def test_batch_update(table: Table, mock_response_batch, replace, http_method):
-    records = [
-        {"id": x["id"], "fields": x["fields"]} for x in mock_response_batch["records"]
-    ]
+def test_batch_update(table: Table, replace, http_method):
+    records = [fake_record(fieldvalue=index) for index in range(50)]
     with Mocker() as mock:
-        for chunk in _chunk(mock_response_batch["records"], 10):
-            mock.register_uri(
-                http_method,
-                table.url,
-                status_code=201,
-                json={"records": chunk},
-            )
+        mock.register_uri(
+            http_method,
+            table.url,
+            response_list=[
+                {"json": {"records": chunk}} for chunk in table.api.chunked(records)
+            ],
+        )
         resp = table.batch_update(records, replace=replace)
 
-    assert resp == mock_response_batch["records"]
+    assert resp == records
 
 
 @pytest.mark.parametrize("replace,http_method", [(False, "PATCH"), (True, "PUT")])
-def test_batch_upsert(table: Table, mock_response_batch, replace, http_method):
-    records = [
-        {"id": x["id"], "fields": x["fields"]} for x in mock_response_batch["records"]
+def test_batch_upsert(table: Table, replace, http_method, monkeypatch):
+    field_name = "Name"
+    exists1 = fake_record({field_name: "Exists 1"})
+    exists2 = fake_record({field_name: "Exists 2"})
+    created = fake_record({field_name: "Does not exist"})
+    payload = [
+        {"id": exists1["id"], "fields": {field_name: "Exists 1"}},
+        {"fields": {field_name: "Exists 2"}},
+        {"fields": {field_name: "Does not exist"}},
+    ]
+    responses = [
+        {"createdRecords": [], "updatedRecords": [exists1["id"]], "records": [exists1]},
+        {"createdRecords": [], "updatedRecords": [exists2["id"]], "records": [exists2]},
+        {"createdRecords": [created["id"]], "updatedRecords": [], "records": [created]},
     ]
-    fields = ["Name"]
     with Mocker() as mock:
-        for chunk in _chunk(mock_response_batch["records"], 10):
-            mock.register_uri(
-                http_method,
-                table.url,
-                status_code=201,
-                json={"records": chunk},
-            )
-        resp = table.batch_upsert(records, key_fields=fields, replace=replace)
+        mock.register_uri(
+            http_method,
+            table.url,
+            response_list=[{"json": response} for response in responses],
+        )
+        monkeypatch.setattr(table.api, "MAX_RECORDS_PER_REQUEST", 1)
+        resp = table.batch_upsert(payload, key_fields=[field_name], replace=replace)
 
-    assert resp == mock_response_batch["records"]
+    assert resp == {
+        "createdRecords": [created["id"]],
+        "updatedRecords": [exists1["id"], exists2["id"]],
+        "records": [exists1, exists2, created],
+    }
 
 
 def test_batch_upsert__missing_field(table: Table, requests_mock):

tests/test_typing.py

@@ -42,7 +42,7 @@
     assert_type(table.delete(record_id), T.RecordDeletedDict)
     assert_type(table.batch_create([]), List[T.RecordDict])
     assert_type(table.batch_update([]), List[T.RecordDict])
-    assert_type(table.batch_upsert([], []), List[T.RecordDict])
+    assert_type(table.batch_upsert([], []), T.UpsertResultDict)
     assert_type(table.batch_delete([]), List[T.RecordDeletedDict])
 
     # Ensure we can set all kinds of field values