From db07934d60520e4b6bf35d669d113ece4d961104 Mon Sep 17 00:00:00 2001 From: Alexander Song Date: Fri, 10 Jan 2025 00:54:50 -0800 Subject: [PATCH 1/6] refactor(prompts): validate jsonschema fields using third-party library --- pyproject.toml | 2 + requirements/type-check.txt | 1 + src/phoenix/server/api/helpers/jsonschema.py | 116 ++++++++ .../server/api/helpers/prompts/models.py | 114 +------- tests/unit/server/api/helpers/test_models.py | 274 +----------------- .../api/mutations/test_prompt_mutations.py | 17 +- 6 files changed, 132 insertions(+), 392 deletions(-) create mode 100644 src/phoenix/server/api/helpers/jsonschema.py diff --git a/pyproject.toml b/pyproject.toml index b72933610c..98098ade73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ dependencies = [ "pydantic>=1.0,!=2.0.*,<3", # exclude 2.0.* since it does not support the `json_encoders` configuration setting "authlib", "websockets", + "jsonschema", ] dynamic = ["version"] @@ -102,6 +103,7 @@ dev = [ "portpicker", "uvloop; platform_system != 'Windows'", "grpc-interceptor[testing]", + "types-jsonschema", ] embeddings = [ "fast-hdbscan>=0.2.0", diff --git a/requirements/type-check.txt b/requirements/type-check.txt index 137e38afd5..b4a4d623f5 100644 --- a/requirements/type-check.txt +++ b/requirements/type-check.txt @@ -21,6 +21,7 @@ requests # this is needed to type-check third-party packages strawberry-graphql[opentelemetry]==0.253.1 # need to pin version because we're monkey-patching tenacity types-cachetools +types-jsonschema types-protobuf types-psutil types-requests diff --git a/src/phoenix/server/api/helpers/jsonschema.py b/src/phoenix/server/api/helpers/jsonschema.py new file mode 100644 index 0000000000..4823505b07 --- /dev/null +++ b/src/phoenix/server/api/helpers/jsonschema.py @@ -0,0 +1,116 @@ +from typing import Annotated, Any + +from jsonschema import Draft7Validator, ValidationError +from pydantic import AfterValidator +from typing_extensions import TypeAlias + +# This meta-schema describes valid JSON schemas according to the JSON Schema Draft 7 specification. +# It is copied from https://json-schema.org/draft-07/schema# +JSON_SCHEMA_DRAFT_7_META_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": {"type": "array", "minItems": 1, "items": {"$ref": "#"}}, + "nonNegativeInteger": {"type": "integer", "minimum": 0}, + "nonNegativeIntegerDefault0": { + "allOf": [{"$ref": "#/definitions/nonNegativeInteger"}, {"default": 0}] + }, + "simpleTypes": { + "enum": ["array", "boolean", "integer", "null", "number", "object", "string"] + }, + "stringArray": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + "default": [], + }, + }, + "type": ["object", "boolean"], + "properties": { + "$id": {"type": "string", "format": "uri-reference"}, + "$schema": {"type": "string", "format": "uri"}, + "$ref": {"type": "string", "format": "uri-reference"}, + "$comment": {"type": "string"}, + "title": {"type": "string"}, + "description": {"type": "string"}, + "default": True, + "readOnly": {"type": "boolean", "default": False}, + "writeOnly": {"type": "boolean", "default": False}, + "examples": {"type": "array", "items": True}, + "multipleOf": {"type": "number", "exclusiveMinimum": 0}, + "maximum": {"type": "number"}, + "exclusiveMaximum": {"type": "number"}, + "minimum": {"type": "number"}, + "exclusiveMinimum": {"type": "number"}, + "maxLength": {"$ref": "#/definitions/nonNegativeInteger"}, + "minLength": {"$ref": "#/definitions/nonNegativeIntegerDefault0"}, + "pattern": {"type": "string", "format": "regex"}, + "additionalItems": {"$ref": "#"}, + "items": {"anyOf": [{"$ref": "#"}, {"$ref": "#/definitions/schemaArray"}], "default": True}, + "maxItems": {"$ref": "#/definitions/nonNegativeInteger"}, + "minItems": {"$ref": "#/definitions/nonNegativeIntegerDefault0"}, + "uniqueItems": {"type": "boolean", "default": False}, + "contains": {"$ref": "#"}, + "maxProperties": {"$ref": "#/definitions/nonNegativeInteger"}, + "minProperties": {"$ref": "#/definitions/nonNegativeIntegerDefault0"}, + "required": {"$ref": "#/definitions/stringArray"}, + "additionalProperties": {"$ref": "#"}, + "definitions": {"type": "object", "additionalProperties": {"$ref": "#"}, "default": {}}, + "properties": {"type": "object", "additionalProperties": {"$ref": "#"}, "default": {}}, + "patternProperties": { + "type": "object", + "additionalProperties": {"$ref": "#"}, + "propertyNames": {"format": "regex"}, + "default": {}, + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [{"$ref": "#"}, {"$ref": "#/definitions/stringArray"}] + }, + }, + "propertyNames": {"$ref": "#"}, + "const": True, + "enum": {"type": "array", "items": True, "minItems": 1, "uniqueItems": True}, + "type": { + "anyOf": [ + {"$ref": "#/definitions/simpleTypes"}, + { + "type": "array", + "items": {"$ref": "#/definitions/simpleTypes"}, + "minItems": 1, + "uniqueItems": True, + }, + ] + }, + "format": {"type": "string"}, + "contentMediaType": {"type": "string"}, + "contentEncoding": {"type": "string"}, + "if": {"$ref": "#"}, + "then": {"$ref": "#"}, + "else": {"$ref": "#"}, + "allOf": {"$ref": "#/definitions/schemaArray"}, + "anyOf": {"$ref": "#/definitions/schemaArray"}, + "oneOf": {"$ref": "#/definitions/schemaArray"}, + "not": {"$ref": "#"}, + }, + "default": True, +} +Draft7Validator.check_schema(JSON_SCHEMA_DRAFT_7_META_SCHEMA) # ensure the schema is valid +JSON_SCHEMA_DRAFT_7_VALIDATOR = Draft7Validator(JSON_SCHEMA_DRAFT_7_META_SCHEMA) + + +def validate_json_schema(schema: dict[str, Any]) -> dict[str, Any]: + """ + Validates that a dictionary is a valid JSON schema. + """ + try: + JSON_SCHEMA_DRAFT_7_VALIDATOR.validate(schema) + except ValidationError as error: + raise ValueError(str(error)) + return schema + + +# Pydantic type with built-in validation for JSON schemas +JSONSchema: TypeAlias = Annotated[dict[str, Any], AfterValidator(validate_json_schema)] diff --git a/src/phoenix/server/api/helpers/prompts/models.py b/src/phoenix/server/api/helpers/prompts/models.py index d287b7f077..84b01fb46c 100644 --- a/src/phoenix/server/api/helpers/prompts/models.py +++ b/src/phoenix/server/api/helpers/prompts/models.py @@ -1,9 +1,11 @@ from enum import Enum from typing import Any, Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, ValidationError, model_validator from typing_extensions import TypeAlias +from phoenix.server.api.helpers.jsonschema import JSONSchema + JSONSerializable = Union[None, bool, int, float, str, dict[str, Any], list[Any]] @@ -122,110 +124,6 @@ def _get_tool_definition_model( return None -# JSON schema -JSONSchemaPrimitiveProperty: TypeAlias = Union[ - "JSONSchemaIntegerProperty", - "JSONSchemaNumberProperty", - "JSONSchemaBooleanProperty", - "JSONSchemaNullProperty", - "JSONSchemaStringProperty", -] -JSONSchemaContainerProperty: TypeAlias = Union[ - "JSONSchemaArrayProperty", - "JSONSchemaObjectProperty", -] -JSONSchemaProperty: TypeAlias = Union[ - "JSONSchemaPrimitiveProperty", - "JSONSchemaContainerProperty", -] - - -class JSONSchemaIntegerProperty(PromptModel): - type: Literal["integer"] - description: str = UNDEFINED - minimum: int = UNDEFINED - maximum: int = UNDEFINED - - @model_validator(mode="after") - def ensure_minimum_lte_maximum(self) -> "JSONSchemaIntegerProperty": - if ( - self.minimum is not UNDEFINED - and self.maximum is not UNDEFINED - and self.minimum > self.maximum - ): - raise ValueError("minimum must be less than or equal to maximum") - return self - - -class JSONSchemaNumberProperty(PromptModel): - type: Literal["number"] - description: str = UNDEFINED - minimum: float = UNDEFINED - maximum: float = UNDEFINED - - @model_validator(mode="after") - def ensure_minimum_lte_maximum(self) -> "JSONSchemaNumberProperty": - if ( - self.minimum is not UNDEFINED - and self.maximum is not UNDEFINED - and self.minimum > self.maximum - ): - raise ValueError("minimum must be less than or equal to maximum") - return self - - -class JSONSchemaBooleanProperty(PromptModel): - type: Literal["boolean"] - description: str = UNDEFINED - - -class JSONSchemaNullProperty(PromptModel): - type: Literal["null"] - description: str = UNDEFINED - - -class JSONSchemaStringProperty(PromptModel): - type: Literal["string"] - description: str = UNDEFINED - enum: list[str] = UNDEFINED - - @field_validator("enum") - def ensure_unique_enum_values(cls, enum_values: list[str]) -> list[str]: - if enum_values is UNDEFINED: - return enum_values - if len(enum_values) != len(set(enum_values)): - raise ValueError("Enum values must be unique") - return enum_values - - -class JSONSchemaArrayProperty(PromptModel): - type: Literal["array"] - description: str = UNDEFINED - items: Union[JSONSchemaProperty, "JSONSchemaAnyOf"] - - -class JSONSchemaObjectProperty(PromptModel): - type: Literal["object"] - description: str = UNDEFINED - properties: dict[str, Union[JSONSchemaProperty, "JSONSchemaAnyOf"]] - required: list[str] = UNDEFINED - additional_properties: bool = Field(UNDEFINED, alias="additionalProperties") - - @model_validator(mode="after") - def ensure_required_fields_are_included_in_properties(self) -> "JSONSchemaObjectProperty": - if self.required is UNDEFINED: - return self - invalid_fields = [field for field in self.required if field not in self.properties] - if invalid_fields: - raise ValueError(f"Required fields {invalid_fields} are not defined in properties") - return self - - -class JSONSchemaAnyOf(PromptModel): - description: str = UNDEFINED - any_of: list[JSONSchemaProperty] = Field(..., alias="anyOf") - - # OpenAI tool definitions class OpenAIFunctionDefinition(PromptModel): """ @@ -234,7 +132,7 @@ class OpenAIFunctionDefinition(PromptModel): name: str description: str = UNDEFINED - parameters: JSONSchemaObjectProperty = UNDEFINED + parameters: JSONSchema = UNDEFINED strict: Optional[bool] = UNDEFINED @@ -247,6 +145,7 @@ class OpenAIToolDefinition(PromptModel): type: Literal["function"] +# Anthropic tool definitions class AnthropicCacheControlEphemeralParam(PromptModel): """ Based on https://github.com/anthropics/anthropic-sdk-python/blob/93cbbbde964e244f02bf1bd2b579c5fabce4e267/src/anthropic/types/cache_control_ephemeral_param.py#L10 @@ -255,13 +154,12 @@ class AnthropicCacheControlEphemeralParam(PromptModel): type: Literal["ephemeral"] -# Anthropic tool definitions class AnthropicToolDefinition(PromptModel): """ Based on https://github.com/anthropics/anthropic-sdk-python/blob/93cbbbde964e244f02bf1bd2b579c5fabce4e267/src/anthropic/types/tool_param.py#L22 """ - input_schema: JSONSchemaObjectProperty + input_schema: JSONSchema name: str cache_control: Optional[AnthropicCacheControlEphemeralParam] = UNDEFINED description: str = UNDEFINED diff --git a/tests/unit/server/api/helpers/test_models.py b/tests/unit/server/api/helpers/test_models.py index d253e84f98..d829d3584a 100644 --- a/tests/unit/server/api/helpers/test_models.py +++ b/tests/unit/server/api/helpers/test_models.py @@ -496,136 +496,6 @@ }, id="pick-tshirt-size-function", ), - pytest.param( - { - "type": "function", - "function": { - "name": "test_primitives", - "description": "Test all primitive types", - "parameters": { - "type": "object", - "properties": { - "string_field": {"type": "string", "description": "A string field"}, - "number_field": {"type": "number", "description": "A number field"}, - "integer_field": {"type": "integer", "description": "An integer field"}, - "boolean_field": {"type": "boolean", "description": "A boolean field"}, - "null_field": {"type": "null", "description": "A null field"}, - }, - "required": [ - "string_field", - "number_field", - "integer_field", - "boolean_field", - "null_field", - ], - "additionalProperties": False, - }, - }, - }, - id="primitive-types-function", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "update_user_profile", - "description": "Updates a user's profile information", - "parameters": { - "type": "object", - "properties": { - "user_id": { - "type": "string", - "description": "The ID of the user to update", - }, - "nickname": { - "description": "Optional nickname that can be null or a string", - "anyOf": [{"type": "string"}, {"type": "null"}], - }, - }, - "required": ["user_id"], - "additionalProperties": False, - }, - }, - }, - id="optional-anyof-parameter", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "categorize_colors", - "description": "Categorize colors into warm, cool, or neutral tones, with null for uncertain cases", # noqa: E501 - "parameters": { - "type": "object", - "properties": { - "colors": { - "type": "array", - "description": "List of color categories, with null for uncertain colors", # noqa: E501 - "items": { - "anyOf": [ - { - "type": "string", - "enum": ["warm", "cool", "neutral"], - "description": "Color category", - }, - {"type": "null"}, - ] - }, - } - }, - "required": ["colors"], - "additionalProperties": False, - }, - }, - }, - id="array-of-optional-enums", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "set_temperature", - "description": "Set temperature within valid range", - "parameters": { - "type": "object", - "properties": { - "temp": { - "type": "integer", - "minimum": 0, - "maximum": 100, - "description": "Temperature in Fahrenheit (0-100)", - } - }, - "required": ["temp"], - "additionalProperties": False, - }, - }, - }, - id="integer-min-max-constraints", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "set_temperature", - "description": "Set temperature within valid range", - "parameters": { - "type": "object", - "properties": { - "temp": { - "type": "number", - "minimum": 0.5, # float min - "maximum": 100, # integer max - "description": "Temperature in Fahrenheit (0-100)", - } - }, - "required": ["temp"], - "additionalProperties": False, - }, - }, - }, - id="number-min-max-constraints", - ), ], ) def test_openai_tool_definition_passes_valid_tool_schemas(tool_definition: dict[str, Any]) -> None: @@ -657,89 +527,6 @@ def test_openai_tool_definition_passes_valid_tool_schemas(tool_definition: dict[ }, id="invalid-data-type", ), - pytest.param( - { - "type": "function", - "function": { - "name": "set_temperature", - "description": "Sets the temperature for the thermostat", - "parameters": { - "type": "object", - "properties": { - "temp": { - "type": "number", - "enum": ["70", "72", "74"], # only string properties can have enums - "description": "The temperature to set in Fahrenheit", - } - }, - "required": ["temp"], - "additionalProperties": False, - }, - }, - }, - id="number-property-with-invalid-enum", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "extra": "extra", # extra properties are not allowed - }, - }, - }, - id="extra-properties", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "update_user", - "description": "Updates user information", - "parameters": { - "type": "object", - "properties": { - "name": {"type": "string"}, - }, - "required": [ - "name", - "email", # email is not in properties - ], - "additionalProperties": False, - }, - }, - }, - id="required-field-not-in-properties", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "set_preferences", - "parameters": { - "type": "object", - "properties": { - "priority": { - "type": "string", - "enum": [ - 0, # integer enum values not allowed - "low", - "medium", - "high", - ], - "description": "The priority level to set", - } - }, - "required": ["priority"], - "additionalProperties": False, - }, - }, - }, - id="string-property-with-priority-enum", - ), pytest.param( { "type": "function", @@ -771,68 +558,15 @@ def test_openai_tool_definition_passes_valid_tool_schemas(tool_definition: dict[ "type": "function", "function": { "name": "set_temperature", - "description": "Set temperature with invalid range", + "description": "Set temperature with invalid schema", "parameters": { "type": "object", - "properties": { - "temp": { - "type": "integer", - "minimum": 100, - "maximum": 0, # min > max - "description": "Temperature in Celsius", - } - }, - "required": ["temp"], - "additionalProperties": False, - }, - }, - }, - id="integer-min-max-range", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "set_count", - "description": "Set an integer count with float bounds", - "parameters": { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1.4, # float not allowed for integer property - "description": "Count value", - } - }, - "required": ["count"], - "additionalProperties": False, - }, - }, - }, - id="integer-float-bounds", - ), - pytest.param( - { - "type": "function", - "function": { - "name": "set_temperature", - "description": "Set temperature with invalid range", - "parameters": { - "type": "object", - "properties": { - "temp": { - "type": "number", - "minimum": 100, - "maximum": 0, # min > max - "description": "Temperature in Celsius", - } - }, - "required": ["temp"], - "additionalProperties": False, + "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, + "required": "name", }, }, }, - id="number-min-max-range", + id="invalid-schema-ref", ), ], ) diff --git a/tests/unit/server/api/mutations/test_prompt_mutations.py b/tests/unit/server/api/mutations/test_prompt_mutations.py index c0b801dc38..a7ce1691c1 100644 --- a/tests/unit/server/api/mutations/test_prompt_mutations.py +++ b/tests/unit/server/api/mutations/test_prompt_mutations.py @@ -295,7 +295,7 @@ async def test_create_chat_prompt_fails_on_name_conflict( assert result.data is None @pytest.mark.parametrize( - "variables,expected_error", + "variables", [ pytest.param( { @@ -316,7 +316,6 @@ async def test_create_chat_prompt_fails_on_name_conflict( }, } }, - "Input should be a valid dictionary", id="invalid-tools", ), pytest.param( @@ -338,7 +337,6 @@ async def test_create_chat_prompt_fails_on_name_conflict( }, } }, - "Input should be a valid dictionary", id="invalid-output-schema", ), pytest.param( @@ -371,7 +369,6 @@ async def test_create_chat_prompt_fails_on_name_conflict( }, } }, - "function.parameters.type", id="with-invalid-openai-tools", ), pytest.param( @@ -416,17 +413,15 @@ async def test_create_chat_prompt_fails_on_name_conflict( }, } }, - "cache_control.type", id="with-invalid-anthropic-tools", ), ], ) async def test_create_chat_prompt_fails_with_invalid_input( - self, gql_client: AsyncGraphQLClient, variables: dict[str, Any], expected_error: str + self, gql_client: AsyncGraphQLClient, variables: dict[str, Any] ) -> None: result = await gql_client.execute(self.CREATE_CHAT_PROMPT_MUTATION, variables) assert len(result.errors) == 1 - assert expected_error in result.errors[0].message assert result.data is None @pytest.mark.parametrize( @@ -646,7 +641,7 @@ async def test_create_chat_prompt_version_fails_with_nonexistent_prompt_id( assert result.data is None @pytest.mark.parametrize( - "variables,expected_error", + "variables", [ pytest.param( { @@ -666,7 +661,6 @@ async def test_create_chat_prompt_version_fails_with_nonexistent_prompt_id( }, } }, - "Input should be a valid dictionary", id="invalid-tools", ), pytest.param( @@ -687,7 +681,6 @@ async def test_create_chat_prompt_version_fails_with_nonexistent_prompt_id( }, } }, - "Input should be a valid dictionary", id="invalid-output-schema", ), pytest.param( @@ -719,7 +712,6 @@ async def test_create_chat_prompt_version_fails_with_nonexistent_prompt_id( }, } }, - "function.parameters.type", id="with-invalid-openai-tools", ), pytest.param( @@ -763,7 +755,6 @@ async def test_create_chat_prompt_version_fails_with_nonexistent_prompt_id( }, } }, - "cache_control.type", id="with-invalid-anthropic-tools", ), ], @@ -773,7 +764,6 @@ async def test_create_chat_prompt_version_fails_with_invalid_input( db: DbSessionFactory, gql_client: AsyncGraphQLClient, variables: dict[str, Any], - expected_error: str, ) -> None: # Create initial prompt create_prompt_result = await gql_client.execute( @@ -799,5 +789,4 @@ async def test_create_chat_prompt_version_fails_with_invalid_input( # Try to create invalid prompt version result = await gql_client.execute(self.CREATE_CHAT_PROMPT_VERSION_MUTATION, variables) assert len(result.errors) == 1 - assert expected_error in result.errors[0].message assert result.data is None From 6629abad8213f5fa04262951f9dc819b15e0a903 Mon Sep 17 00:00:00 2001 From: Alexander Song Date: Fri, 10 Jan 2025 01:17:11 -0800 Subject: [PATCH 2/6] bounds on jsonschema --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 98098ade73..163f27c953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ dependencies = [ "pydantic>=1.0,!=2.0.*,<3", # exclude 2.0.* since it does not support the `json_encoders` configuration setting "authlib", "websockets", - "jsonschema", + "jsonschema>=4.0.0,<=4.23.0", # the upper bound is to keep us off the bleeding edge in case there's a regression since this controls what gets written to the database ] dynamic = ["version"] From dd3ed19254f4fc842f5f926b12a94c6462c4a243 Mon Sep 17 00:00:00 2001 From: Alexander Song Date: Fri, 10 Jan 2025 13:07:20 -0800 Subject: [PATCH 3/6] ensure object type --- src/phoenix/server/api/helpers/jsonschema.py | 10 +++++++--- src/phoenix/server/api/helpers/prompts/models.py | 6 +++--- tests/unit/server/api/helpers/test_models.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/phoenix/server/api/helpers/jsonschema.py b/src/phoenix/server/api/helpers/jsonschema.py index 4823505b07..abfb74b263 100644 --- a/src/phoenix/server/api/helpers/jsonschema.py +++ b/src/phoenix/server/api/helpers/jsonschema.py @@ -101,16 +101,20 @@ JSON_SCHEMA_DRAFT_7_VALIDATOR = Draft7Validator(JSON_SCHEMA_DRAFT_7_META_SCHEMA) -def validate_json_schema(schema: dict[str, Any]) -> dict[str, Any]: +def validate_json_schema_object_definition(schema: dict[str, Any]) -> dict[str, Any]: """ - Validates that a dictionary is a valid JSON schema. + Validates that a dictionary is a valid JSON schema object property. """ try: JSON_SCHEMA_DRAFT_7_VALIDATOR.validate(schema) except ValidationError as error: raise ValueError(str(error)) + if schema.get("type") != "object": + raise ValueError("The 'type' property must be 'object'") return schema # Pydantic type with built-in validation for JSON schemas -JSONSchema: TypeAlias = Annotated[dict[str, Any], AfterValidator(validate_json_schema)] +JSONSchemaObjectDefinition: TypeAlias = Annotated[ + dict[str, Any], AfterValidator(validate_json_schema_object_definition) +] diff --git a/src/phoenix/server/api/helpers/prompts/models.py b/src/phoenix/server/api/helpers/prompts/models.py index 84b01fb46c..7b5816e1d6 100644 --- a/src/phoenix/server/api/helpers/prompts/models.py +++ b/src/phoenix/server/api/helpers/prompts/models.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError, model_validator from typing_extensions import TypeAlias -from phoenix.server.api.helpers.jsonschema import JSONSchema +from phoenix.server.api.helpers.jsonschema import JSONSchemaObjectDefinition JSONSerializable = Union[None, bool, int, float, str, dict[str, Any], list[Any]] @@ -132,7 +132,7 @@ class OpenAIFunctionDefinition(PromptModel): name: str description: str = UNDEFINED - parameters: JSONSchema = UNDEFINED + parameters: JSONSchemaObjectDefinition = UNDEFINED strict: Optional[bool] = UNDEFINED @@ -159,7 +159,7 @@ class AnthropicToolDefinition(PromptModel): Based on https://github.com/anthropics/anthropic-sdk-python/blob/93cbbbde964e244f02bf1bd2b579c5fabce4e267/src/anthropic/types/tool_param.py#L22 """ - input_schema: JSONSchema + input_schema: JSONSchemaObjectDefinition name: str cache_control: Optional[AnthropicCacheControlEphemeralParam] = UNDEFINED description: str = UNDEFINED diff --git a/tests/unit/server/api/helpers/test_models.py b/tests/unit/server/api/helpers/test_models.py index d829d3584a..d5fa8ac662 100644 --- a/tests/unit/server/api/helpers/test_models.py +++ b/tests/unit/server/api/helpers/test_models.py @@ -568,6 +568,19 @@ def test_openai_tool_definition_passes_valid_tool_schemas(tool_definition: dict[ }, id="invalid-schema-ref", ), + pytest.param( + { + "type": "function", + "function": { + "name": "get_status", + "description": "Get system status", + "parameters": { + "type": "string", + }, + }, + }, + id="non-object-parameters", + ), ], ) def test_openai_tool_definition_fails_invalid_tool_schemas(tool_definition: dict[str, Any]) -> None: From a5f2d7d09609d556850d29c792f2000b08bfab3f Mon Sep 17 00:00:00 2001 From: Alexander Song Date: Fri, 10 Jan 2025 13:44:28 -0800 Subject: [PATCH 4/6] retrigger pipeline From 7d4cdec5314643963e902c8bf013ca1bba4069ee Mon Sep 17 00:00:00 2001 From: Alexander Song Date: Fri, 10 Jan 2025 13:48:35 -0800 Subject: [PATCH 5/6] retrigger --- tests/unit/server/api/helpers/test_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/server/api/helpers/test_models.py b/tests/unit/server/api/helpers/test_models.py index d5fa8ac662..d5e8033a65 100644 --- a/tests/unit/server/api/helpers/test_models.py +++ b/tests/unit/server/api/helpers/test_models.py @@ -658,7 +658,7 @@ def test_openai_tool_definition_fails_invalid_tool_schemas(tool_definition: dict "b": {"type": "number", "description": "blue value [0.0, 1.0]"}, "name": { "type": "string", - "description": 'Human-readable color name in snake_case, e.g. "olive_green" or "turquoise"', # noqa: E501 + "description": 'Human-readable color name in snake_case, e.g., "olive_green" or "turquoise"', # noqa: E501 }, }, "required": ["r", "g", "b", "name"], From db2fc39ed91abbce6194dbb95191f083126db4f3 Mon Sep 17 00:00:00 2001 From: Xander Song Date: Fri, 10 Jan 2025 13:36:56 -0800 Subject: [PATCH 6/6] fix: pin upper bound on litellm to prevent windows break (#6004) --- pyproject.toml | 2 +- requirements/unit-tests.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 163f27c953..87185ea419 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,7 @@ dev = [ "arize[AutoEmbeddings, LLM_Evaluation]", "llama-index>=0.10.3", "langchain>=0.0.334", - "litellm>=1.0.3", + "litellm>=1.0.3,<1.57.5", # windows compatibility broken on 1.57.5 (https://github.com/BerriAI/litellm/issues/7677) "google-cloud-aiplatform>=1.3", "anthropic", "prometheus_client", diff --git a/requirements/unit-tests.txt b/requirements/unit-tests.txt index e1b5482d38..5621da58d8 100644 --- a/requirements/unit-tests.txt +++ b/requirements/unit-tests.txt @@ -7,7 +7,7 @@ asyncpg grpc-interceptor[testing] httpx<0.28 httpx-ws -litellm>=1.0.3 +litellm>=1.0.3,<1.57.5 nest-asyncio # for executor testing numpy openai>=1.0.0