Salesforce daily test (#3611)

* Add daily salesforce test

* Add more assertions

* Add assertions for data by parsing the key-value strings

* Fix grammar

backend/tests/daily/connectors/salesforce/test_salesforce_connector.py

@@ -1,10 +1,35 @@
+import json
 import os
+import time
+from pathlib import Path
+from typing import Any
 
 import pytest
 
+from onyx.configs.constants import DocumentSource
+from onyx.connectors.models import Document
 from onyx.connectors.salesforce.connector import SalesforceConnector
 
 
+def extract_key_value_pairs_to_set(
+    list_of_unparsed_key_value_strings: list[str],
+) -> set[str]:
+    set_of_key_value_pairs = set()
+    for string_key_value_pairs in list_of_unparsed_key_value_strings:
+        list_of_parsed_key_values = string_key_value_pairs.split("\n")
+        for key_value_pair in list_of_parsed_key_values:
+            set_of_key_value_pairs.add(key_value_pair.strip())
+    return set_of_key_value_pairs
+
+
+def load_test_data(
+    file_name: str = "test_salesforce_data.json",
+) -> dict[str, list[str] | dict[str, Any]]:
+    current_dir = Path(__file__).parent
+    with open(current_dir / file_name, "r") as f:
+        return json.load(f)
+
+
 @pytest.fixture
 def salesforce_connector() -> SalesforceConnector:
     connector = SalesforceConnector(
@@ -20,6 +45,62 @@ def salesforce_connector() -> SalesforceConnector:
     return connector
 
 
+# TODO: make the credentials not expire
+@pytest.mark.xfail(
+    reason=(
+        "Credentials change over time, so this test will fail if run when "
+        "the credentials expire."
+    )
+)
+def test_salesforce_connector_basic(salesforce_connector: SalesforceConnector) -> None:
+    test_data = load_test_data()
+    target_test_doc: Document | None = None
+    all_docs: list[Document] = []
+    for doc_batch in salesforce_connector.poll_source(0, time.time()):
+        for doc in doc_batch:
+            all_docs.append(doc)
+            if doc.id == test_data["id"]:
+                target_test_doc = doc
+
+    assert len(all_docs) == 6
+    assert target_test_doc is not None
+
+    # Set of received links
+    received_links: set[str] = set()
+    # List of received text fields, which contain key-value pairs seperated by newlines
+    recieved_text: list[str] = []
+
+    # Iterate over the sections of the target test doc to extract the links and text
+    for section in target_test_doc.sections:
+        assert section.link
+        assert section.text
+        received_links.add(section.link)
+        recieved_text.append(section.text)
+
+    # Check that the received links match the expected links from the test data json
+    expected_links = set(test_data["expected_links"])
+    assert received_links == expected_links
+
+    # Check that the received key-value pairs from the text fields match the expected key-value pairs from the test data json
+    expected_text = test_data["expected_text"]
+    if not isinstance(expected_text, list):
+        raise ValueError("Expected text is not a list")
+    unparsed_expected_key_value_pairs: list[str] = expected_text
+    received_key_value_pairs = extract_key_value_pairs_to_set(recieved_text)
+    expected_key_value_pairs = extract_key_value_pairs_to_set(
+        unparsed_expected_key_value_pairs
+    )
+    assert received_key_value_pairs == expected_key_value_pairs
+
+    # Check that the rest of the fields match the expected fields from the test data json
+    assert target_test_doc.source == DocumentSource.SALESFORCE
+    assert target_test_doc.semantic_identifier == test_data["semantic_identifier"]
+    assert target_test_doc.metadata == test_data["metadata"]
+    assert target_test_doc.primary_owners == test_data["primary_owners"]
+    assert target_test_doc.secondary_owners == test_data["secondary_owners"]
+    assert target_test_doc.title == test_data["title"]
+
+
 # TODO: make the credentials not expire
 @pytest.mark.xfail(
     reason=(

backend/tests/daily/connectors/salesforce/test_salesforce_data.json

@@ -0,0 +1,20 @@
+{
+  "id": "SALESFORCE_001fI000005drUcQAI",
+  "expected_links": [
+    "https://customization-ruby-2195.my.salesforce.com/001fI000005drUcQAI",
+    "https://customization-ruby-2195.my.salesforce.com/003fI000001jiCPQAY",
+    "https://customization-ruby-2195.my.salesforce.com/017fI00000T7hvsQAB",
+    "https://customization-ruby-2195.my.salesforce.com/006fI000000rDvBQAU"
+  ],
+  "expected_text": [
+    "BillingPostalCode: 60601\nType: Prospect\nWebsite: www.globalistindustries.com\nBillingCity: Chicago\nDescription: Globalist company\nIsDeleted: false\nIsPartner: false\nPhone: (312) 555-0456\nShippingCountry: USA\nShippingState: IL\nIsBuyer: false\nBillingCountry: USA\nBillingState: IL\nShippingPostalCode: 60601\nBillingStreet: 456 Market St\nIsCustomerPortal: false\nPersonActiveTrackerCount: 0\nShippingCity: Chicago\nShippingStreet: 456 Market St",
+    "FirstName: Michael\nMailingCountry: USA\nActiveTrackerCount: 0\nEmail: m.brown@globalindustries.com\nMailingState: IL\nMailingStreet: 456 Market St\nMailingCity: Chicago\nLastName: Brown\nTitle: CTO\nIsDeleted: false\nPhone: (312) 555-0456\nHasOptedOutOfEmail: false\nIsEmailBounced: false\nMailingPostalCode: 60601",
+    "ForecastCategory: Closed\nName: Global Industries Equipment Sale\nIsDeleted: false\nForecastCategoryName: Closed\nFiscalYear: 2024\nFiscalQuarter: 4\nIsClosed: true\nIsWon: true\nAmount: 5000000.0\nProbability: 100.0\nPushCount: 0\nHasOverdueTask: false\nStageName: Closed Won\nHasOpenActivity: false\nHasOpportunityLineItem: false",
+    "Field: created\nDataType: Text\nIsDeleted: false"
+  ],
+  "semantic_identifier": "Unknown Object",
+  "metadata": {},
+  "primary_owners": null,
+  "secondary_owners": null,
+  "title": null
+}