feat: reusing httpx client to improve performance (#6)

aidial_adapter_dial/app.py

@@ -17,6 +17,7 @@
     HTTPException,
     dial_exception_decorator,
 )
+from aidial_adapter_dial.utils.http_client import get_http_client
 from aidial_adapter_dial.utils.log_config import configure_loggers
 from aidial_adapter_dial.utils.reflection import call_with_extra_body
 from aidial_adapter_dial.utils.sse_stream import to_openai_sse_stream
@@ -103,7 +104,7 @@ async def parse(
             base_url=upstream_endpoint,
             api_key=remote_dial_api_key,
             api_version=query_params.get("api-version"),
-            timeout=DEFAULT_TIMEOUT,
+            http_client=get_http_client(),
         )
 
         attachment_transformer = await AttachmentTransformer.create(

aidial_adapter_dial/utils/http_client.py

@@ -0,0 +1,20 @@
+import functools
+
+import httpx
+
+# connect timeout and total timeout
+DEFAULT_TIMEOUT = httpx.Timeout(600, connect=10)
+
+# Borrowed from openai._constants.DEFAULT_CONNECTION_LIMITS
+DEFAULT_CONNECTION_LIMITS = httpx.Limits(
+    max_connections=1000, max_keepalive_connections=100
+)
+
+
+@functools.cache
+def get_http_client() -> httpx.AsyncClient:
+    return httpx.AsyncClient(
+        timeout=DEFAULT_TIMEOUT,
+        limits=DEFAULT_CONNECTION_LIMITS,
+        follow_redirects=True,
+    )

pyproject.toml

@@ -27,12 +27,12 @@ aiohttp = "3.9.5"
 openai = "1.32.0" # NOTE: used solely for chat completion response types
 pydantic = "^1.10.12"
 aidial-sdk = {version = "^0.8.0", extras = ["telemetry"]}
+respx = "^0.21.1"
 
 [tool.poetry.group.test.dependencies]
 pytest = "7.4.0"
 pytest-asyncio = "0.21.1"
 python-dotenv = "1.0.0"
-respx = "^0.21.1"
 
 [tool.poetry.group.lint.dependencies]
 pyright = "1.1.324"