diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index ac09f2fa90b..36b9a8bf3de 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -125,6 +125,7 @@ class DocumentSource(str, Enum): OCI_STORAGE = "oci_storage" XENFORO = "xenforo" NOT_APPLICABLE = "not_applicable" + FRESHDESK = "freshdesk" DocumentSourceRequiringTenantContext: list[DocumentSource] = [DocumentSource.FILE] diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index ce74997333e..bc9196eec59 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -16,6 +16,7 @@ from danswer.connectors.document360.connector import Document360Connector from danswer.connectors.dropbox.connector import DropboxConnector from danswer.connectors.file.connector import LocalFileConnector +from danswer.connectors.freshdesk.connector import FreshdeskConnector from danswer.connectors.github.connector import GithubConnector from danswer.connectors.gitlab.connector import GitlabConnector from danswer.connectors.gmail.connector import GmailConnector @@ -99,6 +100,7 @@ def identify_connector_class( DocumentSource.GOOGLE_CLOUD_STORAGE: BlobStorageConnector, DocumentSource.OCI_STORAGE: BlobStorageConnector, DocumentSource.XENFORO: XenforoConnector, + DocumentSource.FRESHDESK: FreshdeskConnector, } connector_by_source = connector_map.get(source, {}) diff --git a/backend/danswer/connectors/freshdesk/__init__,py b/backend/danswer/connectors/freshdesk/__init__,py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py new file mode 100644 index 00000000000..b1f1dc1d2df --- /dev/null +++ b/backend/danswer/connectors/freshdesk/connector.py @@ -0,0 +1,209 @@ +import json +from collections.abc import Iterator +from datetime import datetime +from datetime import timezone +from typing import List + +import requests + +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.interfaces import GenerateDocumentsOutput +from danswer.connectors.interfaces import LoadConnector +from danswer.connectors.interfaces import PollConnector +from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import ConnectorMissingCredentialError +from danswer.connectors.models import Document +from danswer.connectors.models import Section +from danswer.file_processing.html_utils import parse_html_page_basic +from danswer.utils.logger import setup_logger + +logger = setup_logger() + + +def _create_metadata_from_ticket(ticket: dict) -> dict: + included_fields = { + "fr_escalated", + "spam", + "priority", + "source", + "status", + "type", + "is_escalated", + "tags", + "nr_due_by", + "nr_escalated", + "cc_emails", + "fwd_emails", + "reply_cc_emails", + "ticket_cc_emails", + "support_email", + "to_emails", + } + + metadata = {} + email_data = {} + + for key, value in ticket.items(): + if ( + key in included_fields + and value is not None + and value != [] + and value != {} + and value != "[]" + and value != "" + ): + value_to_str = ( + [str(item) for item in value] if isinstance(value, List) else str(value) + ) + if "email" in key: + email_data[key] = value_to_str + else: + metadata[key] = value_to_str + + if email_data: + metadata["email_data"] = str(email_data) + + # Convert source to human-parsable string + source_types = { + "1": "Email", + "2": "Portal", + "3": "Phone", + "7": "Chat", + "9": "Feedback Widget", + "10": "Outbound Email", + } + if ticket.get("source"): + metadata["source"] = source_types.get( + str(ticket.get("source")), "Unknown Source Type" + ) + + # Convert priority to human-parsable string + priority_types = {"1": "low", "2": "medium", "3": "high", "4": "urgent"} + if ticket.get("priority"): + metadata["priority"] = priority_types.get( + str(ticket.get("priority")), "Unknown Priority" + ) + + # Convert status to human-parsable string + status_types = {"2": "open", "3": "pending", "4": "resolved", "5": "closed"} + if ticket.get("status"): + metadata["status"] = status_types.get( + str(ticket.get("status")), "Unknown Status" + ) + + due_by = datetime.fromisoformat(ticket["due_by"].replace("Z", "+00:00")) + metadata["overdue"] = str(datetime.now(timezone.utc) > due_by) + + return metadata + + +def _create_doc_from_ticket(ticket: dict, domain: str) -> Document: + return Document( + id=str(ticket["id"]), + sections=[ + Section( + link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}", + text=f"description: {parse_html_page_basic(ticket.get('description_text', ''))}", + ) + ], + source=DocumentSource.FRESHDESK, + semantic_identifier=ticket["subject"], + metadata=_create_metadata_from_ticket(ticket), + doc_updated_at=datetime.fromisoformat( + ticket["updated_at"].replace("Z", "+00:00") + ), + ) + + +class FreshdeskConnector(PollConnector, LoadConnector): + def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None: + self.batch_size = batch_size + + def load_credentials(self, credentials: dict[str, str | int]) -> None: + api_key = credentials.get("freshdesk_api_key") + domain = credentials.get("freshdesk_domain") + password = credentials.get("freshdesk_password") + + if not all(isinstance(cred, str) for cred in [domain, api_key, password]): + raise ConnectorMissingCredentialError( + "All Freshdesk credentials must be strings" + ) + + self.api_key = str(api_key) + self.domain = str(domain) + self.password = str(password) + + def _fetch_tickets( + self, start: datetime | None = None, end: datetime | None = None + ) -> Iterator[List[dict]]: + """ + 'end' is not currently used, so we may double fetch tickets created after the indexing + starts but before the actual call is made. + + To use 'end' would require us to use the search endpoint but it has limitations, + namely having to fetch all IDs and then individually fetch each ticket because there is no + 'include' field available for this endpoint: + https://developers.freshdesk.com/api/#filter_tickets + """ + if any(attr is None for attr in [self.api_key, self.domain, self.password]): + raise ConnectorMissingCredentialError("freshdesk") + + base_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets" + params: dict[str, int | str] = { + "include": "description", + "per_page": 50, + "page": 1, + } + + if start: + params["updated_since"] = start.isoformat() + + while True: + response = requests.get( + base_url, auth=(self.api_key, self.password), params=params + ) + response.raise_for_status() + + if response.status_code == 204: + break + + tickets = json.loads(response.content) + logger.info( + f"Fetched {len(tickets)} tickets from Freshdesk API (Page {params['page']})" + ) + + yield tickets + + if len(tickets) < int(params["per_page"]): + break + + params["page"] = int(params["page"]) + 1 + + def _process_tickets( + self, start: datetime | None = None, end: datetime | None = None + ) -> GenerateDocumentsOutput: + doc_batch: List[Document] = [] + + for ticket_batch in self._fetch_tickets(start, end): + for ticket in ticket_batch: + logger.info(_create_doc_from_ticket(ticket, self.domain)) + doc_batch.append(_create_doc_from_ticket(ticket, self.domain)) + + if len(doc_batch) >= self.batch_size: + yield doc_batch + doc_batch = [] + + if doc_batch: + yield doc_batch + + def load_from_state(self) -> GenerateDocumentsOutput: + return self._process_tickets() + + def poll_source( + self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch + ) -> GenerateDocumentsOutput: + start_datetime = datetime.fromtimestamp(start, tz=timezone.utc) + end_datetime = datetime.fromtimestamp(end, tz=timezone.utc) + + yield from self._process_tickets(start_datetime, end_datetime) diff --git a/web/public/Freshdesk.png b/web/public/Freshdesk.png new file mode 100644 index 00000000000..a3343ceb01d Binary files /dev/null and b/web/public/Freshdesk.png differ diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index b74eaffecc5..30f6f0bdccc 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -74,6 +74,7 @@ import slackIcon from "../../../public/Slack.png"; import s3Icon from "../../../public/S3.png"; import r2Icon from "../../../public/r2.png"; import salesforceIcon from "../../../public/Salesforce.png"; +import freshdeskIcon from "../../../public/Freshdesk.png"; import sharepointIcon from "../../../public/Sharepoint.png"; import teamsIcon from "../../../public/Teams.png"; @@ -1293,6 +1294,13 @@ export const AsanaIcon = ({ className = defaultTailwindCSS, }: IconProps) => ; +export const FreshdeskIcon = ({ + size = 16, + className = defaultTailwindCSS, +}: IconProps) => ( + +); + /* EE Icons */ diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx index 8b6e285ec2b..1ec265049d0 100644 --- a/web/src/lib/connectors/connectors.tsx +++ b/web/src/lib/connectors/connectors.tsx @@ -945,6 +945,12 @@ For example, specifying .*-support.* as a "channel" will cause the connector to ], advanced_values: [], }, + freshdesk: { + description: "Configure Freshdesk connector", + values: [], + advanced_values: [], + }, + }; export function createConnectorInitialValues( connector: ConfigurableSources @@ -1202,6 +1208,9 @@ export interface AsanaConfig { asana_team_id?: string; } +export interface FreshdeskConfig {} + + export interface MediaWikiConfig extends MediaWikiBaseConfig { hostname: string; } diff --git a/web/src/lib/connectors/credentials.ts b/web/src/lib/connectors/credentials.ts index 73c788d3a96..c6da2c5f62c 100644 --- a/web/src/lib/connectors/credentials.ts +++ b/web/src/lib/connectors/credentials.ts @@ -181,6 +181,12 @@ export interface AxeroCredentialJson { axero_api_token: string; } +export interface FreshdeskCredentialJson { + freshdesk_domain: string; + freshdesk_password: string; + freshdesk_api_key: string; +} + export interface MediaWikiCredentialJson {} export interface WikipediaCredentialJson extends MediaWikiCredentialJson {} @@ -279,6 +285,11 @@ export const credentialTemplates: Record = { access_key_id: "", secret_access_key: "", } as OCICredentialJson, + freshdesk: { + freshdesk_domain: "", + freshdesk_password: "", + freshdesk_api_key: "", + } as FreshdeskCredentialJson, xenforo: null, google_sites: null, file: null, @@ -419,6 +430,11 @@ export const credentialDisplayNames: Record = { // Axero base_url: "Axero Base URL", axero_api_token: "Axero API Token", + + // Freshdesk + freshdesk_domain: "Freshdesk Domain", + freshdesk_password: "Freshdesk Password", + freshdesk_api_key: "Freshdesk API Key", }; export function getDisplayNameForCredentialKey(key: string): string { return credentialDisplayNames[key] || key; diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index aa053912947..fe71f6b099d 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -36,6 +36,7 @@ import { GoogleStorageIcon, ColorSlackIcon, XenforoIcon, + FreshdeskIcon, } from "@/components/icons/icons"; import { ValidSources } from "./types"; import { @@ -282,6 +283,12 @@ const SOURCE_METADATA_MAP: SourceMap = { displayName: "Ingestion", category: SourceCategory.Other, }, + freshdesk: { + icon: FreshdeskIcon, + displayName: "Freshdesk", + category: SourceCategory.CustomerSupport, + docs: "https://docs.danswer.dev/connectors/freshdesk", + }, // currently used for the Internet Search tool docs, which is why // a globe is used not_applicable: { diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index f8b7fb57536..25d98a1df38 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -263,6 +263,7 @@ const validSources = [ "oci_storage", "not_applicable", "ingestion_api", + "freshdesk", ] as const; export type ValidSources = (typeof validSources)[number];