From 1ba36103068e2a3179965ddd8f1aca0822b48050 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Fri, 2 Aug 2024 03:54:22 +0800 Subject: [PATCH 01/17] chore(schema): add ai task schema --- ai-tasks.json | 310 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 ai-tasks.json diff --git a/ai-tasks.json b/ai-tasks.json new file mode 100644 index 00000000..02ca165d --- /dev/null +++ b/ai-tasks.json @@ -0,0 +1,310 @@ +{ + "TASK_CHAT": { + "title": "Chat", + "instillShortDescription": "Generate response base on conversation input", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Chat input", + "description": "Input schema of the chat task", + "instillShortDescription": "Input schema of the chat task", + "type": "object", + "properties": { + "data": { + "description": "Input data", + "instillShortDescription": "Input data", + "type": "object", + "properties": { + "model": { + "description": "The model to be used.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "messages": { + "title": "Chat Messages", + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { + "description": "The message content", + "instillShortDescription": "The message content", + "title": "Content", + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "text": { + "description": "Text message", + "instillShortDescription": "Text message", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + } + }, + "required": [ + "text" + ] + }, + { + "type": "object", + "properties": { + "datauri": { + "description": "Image message with either url or base64 encoded string", + "instillShortDescription": "Image message with either url or base64 encoded string", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + } + }, + "required": [ + "datauri" + ] + } + ] + } + }, + "role": { + "description": "The message role, i.e. 'system', 'user' or 'assistant'", + "instillShortDescription": "The message role, i.e. 'system', 'user' or 'assistant'", + "instillAcceptFormats": [ + "string" + ], + "title": "Role", + "type": "string", + "enum": [ + "system", + "user", + "assistant" + ] + }, + "name": { + "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + "instillShortDescription": "An optional name for the participant. Provides the model information to differentiate between participants of the same role.", + "instillAcceptFormats": [ + "string" + ], + "title": "Role", + "type": "string" + } + }, + "required": [ + "content", + "role" + ] + } + } + }, + "required": [ + "model", + "messages" + ] + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": { + "max-tokens": { + "title": "Max new tokens", + "type": "integer", + "description": "The maximum number of tokens for model to generate", + "instillShortDescription": "The maximum number of tokens for model to generate", + "instillAcceptFormats": [ + "integer" + ], + "default": 50 + }, + "seed": { + "title": "Seed", + "type": "integer", + "description": "The seed, default is 0", + "instillShortDescription": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0 + }, + "n": { + "title": "Number of choices", + "type": "integer", + "description": "How many chat completion choices to generate for each input message.", + "instillShortDescription": "How many chat completion choices to generate for each input message.", + "instillAcceptFormats": [ + "integer" + ], + "default": 1 + }, + "temperature": { + "title": "Temperature", + "type": "number", + "description": "The temperature for sampling", + "instillShortDescription": "The temperature for sampling", + "instillAcceptFormats": [ + "number" + ], + "default": 0.7 + }, + "top-p": { + "title": "Top P", + "type": "integer", + "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both.", + "instillShortDescription": "Nucleus sampling", + "instillAcceptFormats": [ + "integer" + ], + "default": 1 + }, + "stream": { + "title": "Stream", + "type": "boolean", + "description": "If set, partial message deltas will be sent. Tokens will be sent as data-only server-sent events as they become available.", + "instillShortDescription": "If set, partial message deltas will be sent", + "instillAcceptFormats": [ + "boolean" + ], + "default": false + } + } + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Chat output", + "description": "Output schema of the chat task", + "instillShortDescription": "Output schema of the chat task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "properties": { + "choices": { + "title": "Choices", + "type": "array", + "description": "List of chat completion choices", + "instillShortDescription": "List of chat completion choices", + "instillFormat": "array", + "items": { + "type": "object", + "properties": { + "finish-reason": { + "title": "Finish reason", + "type": "string", + "description": "The reason the model stopped generating tokens.", + "instillShortDescription": "The reason the model stopped generating tokens.", + "instillFormat": "string", + "enum": [ + "stop", + "length" + ] + }, + "index": { + "title": "Index", + "type": "integer", + "description": "The index of the choice in the list of choices.", + "instillShortDescription": "The index of the choice in the list of choices.", + "instillFormat": "integer" + }, + "message": { + "title": "Message", + "type": "object", + "description": "A chat message generated by the model.", + "instillShortDescription": "A chat message generated by the model.", + "properties": { + "content": { + "title": "Content", + "type": "string", + "description": "The contents of the message.", + "instillShortDescription": "The contents of the message.", + "instillFormat": "string" + }, + "role": { + "title": "Role", + "type": "string", + "description": "The role of the author of this message.", + "instillShortDescription": "The role of the author of this message.", + "instillFormat": "string" + } + } + }, + "created": { + "title": "Created", + "type": "integer", + "description": "The Unix timestamp (in seconds) of when the chat completion was created.", + "instillShortDescription": "The Unix timestamp (in seconds) of when the chat completion was created.", + "instillFormat": "integer" + } + }, + "required": [ + "finish_reason", + "index", + "message", + "created" + ] + } + } + }, + "required": [ + "choices" + ] + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": { + "completion-tokens": { + "title": "Completion tokens", + "type": "integer", + "description": "Number of tokens in the generated response.", + "instillShortDescription": "Number of tokens in the generated response.", + "instillFormat": "integer" + }, + "prompt-tokens": { + "title": "Prompt tokens", + "type": "integer", + "description": "Number of tokens in the prompt.", + "instillShortDescription": "Number of tokens in the prompt.", + "instillFormat": "integer" + }, + "total-tokens": { + "title": "Total tokens", + "type": "integer", + "description": "Total number of tokens used in the request (prompt + completion).", + "instillShortDescription": "Total number of tokens used in the request (prompt + completion).", + "instillFormat": "integer" + } + }, + "required": [ + "completion-tokens", + "prompt-tokens", + "total-tokens" + ] + } + } + } + }, + "required": [ + "data" + ] + } + } +} \ No newline at end of file From 8aaca5021d0838af762cba2d8ef4d4f8e9db560e Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Tue, 6 Aug 2024 05:17:12 +0800 Subject: [PATCH 02/17] chore: update schema --- ai-tasks.json => schema/ai-tasks.json | 230 +++++++++++++++++- schema/schema.json | 320 ++++++++++++++++++++++++++ 2 files changed, 548 insertions(+), 2 deletions(-) rename ai-tasks.json => schema/ai-tasks.json (64%) create mode 100644 schema/schema.json diff --git a/ai-tasks.json b/schema/ai-tasks.json similarity index 64% rename from ai-tasks.json rename to schema/ai-tasks.json index 02ca165d..161abe08 100644 --- a/ai-tasks.json +++ b/schema/ai-tasks.json @@ -1,4 +1,64 @@ { + "$defs": { + "vision-input": { + "description": "Input data", + "instillShortDescription": "Input data", + "type": "object", + "properties": { + "model": { + "description": "The model to be used.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "image": { + "title": "Input image", + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "image-url": { + "description": "Image url", + "instillShortDescription": "Image url", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + } + }, + "required": [ + "image-url" + ] + }, + { + "type": "object", + "properties": { + "image-base64": { + "description": "Image base64 encoded string", + "instillShortDescription": "Image base64 encoded string", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + } + }, + "required": [ + "image-base64" + ] + } + ] + } + }, + "required": [ + "model", + "image" + ] + } + }, "TASK_CHAT": { "title": "Chat", "instillShortDescription": "Generate response base on conversation input", @@ -154,11 +214,11 @@ }, "top-p": { "title": "Top P", - "type": "integer", + "type": "number", "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both.", "instillShortDescription": "Nucleus sampling", "instillAcceptFormats": [ - "integer" + "number" ], "default": 1 }, @@ -306,5 +366,171 @@ "data" ] } + }, + "TASK_CLASSIFICATION": { + "title": "Classification", + "instillShortDescription": "Classify images into predefined categories.", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Classification input", + "description": "Input schema of the classification task", + "instillShortDescription": "Input schema of the classification task", + "type": "object", + "properties": { + "data": { + "$ref": "#/$defs/vision-input", + "type": "object" + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Classification output", + "description": "Output schema of the classification task", + "instillShortDescription": "Output schema of the classification task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "properties": { + "category": { + "title": "Category", + "type": "string", + "description": "The detected category.", + "instillShortDescription": "The detected category.", + "instillFormat": "string" + }, + "score": { + "title": "Score", + "type": "number", + "description": "The score of the detected category.", + "instillShortDescription": "The score of the detected category.", + "instillFormat": "number" + } + }, + "required": [ + "category", + "score" + ] + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] + } + } + } + }, + "required": [ + "data" + ] + } + }, + "TASK_DETECTION": { + "title": "Detection", + "instillShortDescription": "Detect and localize multiple objects in images.", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Detection input", + "description": "Input schema of the detection task", + "instillShortDescription": "Input schema of the detection task", + "type": "object", + "properties": { + "data": { + "$ref": "#/$defs/vision-input", + "type": "object" + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Detection output", + "description": "Output schema of the detection task", + "instillShortDescription": "Output schema of the detection task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "properties": { + "objects": { + "title": "Objects", + "type": "array", + "description": "The detected objects.", + "instillShortDescription": "The detected objects.", + "instillFormat": "array", + "items": { + "type": "object", + "properties": { + "bounding-box": {}, + "category": { + "title": "Category", + "type": "string", + "description": "The detected category.", + "instillShortDescription": "The detected category.", + "instillFormat": "string" + }, + "score": { + "title": "Score", + "type": "number", + "description": "The score of the detected category.", + "instillShortDescription": "The score of the detected category.", + "instillFormat": "number" + } + } + } + }, + "required": [ + "objects" + ] + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] + } + } + } + }, + "required": [ + "data" + ] + } + } } } \ No newline at end of file diff --git a/schema/schema.json b/schema/schema.json new file mode 100644 index 00000000..6dba73a1 --- /dev/null +++ b/schema/schema.json @@ -0,0 +1,320 @@ +{ + "$defs": { + "instill-types": { + "bounding-box": { + "additionalProperties": false, + "description": "The detected bounding box in (left, top, width, height) format.", + "instillFormat": "structured/bounding-box", + "properties": { + "height": { + "description": "Bounding box height value", + "instillFormat": "number", + "title": "Height", + "type": "number" + }, + "left": { + "description": "Bounding box left x-axis value", + "instillFormat": "number", + "title": "Left", + "type": "number" + }, + "top": { + "description": "Bounding box top y-axis value", + "instillFormat": "number", + "title": "Top", + "type": "number" + }, + "width": { + "description": "Bounding box width value", + "instillFormat": "number", + "title": "Width", + "type": "number" + } + }, + "required": [ + "left", + "top", + "width", + "height" + ], + "title": "Bounding Box", + "type": "object" + }, + "classification": { + "additionalProperties": false, + "properties": { + "category": { + "description": "The predicted category of the input.", + "instillFormat": "string", + "title": "Category", + "type": "string" + }, + "score": { + "description": "The confidence score of the predicted category of the input.", + "instillFormat": "number", + "title": "Score", + "type": "number" + } + }, + "required": [ + "category", + "score" + ], + "type": "object" + }, + "detection": { + "additionalProperties": false, + "properties": { + "objects": { + "description": "A list of detected objects.", + "instillFormat": "array:structured/detection-object", + "items": { + "additionalProperties": false, + "instillFormat": "structured/detection-object", + "properties": { + "bounding-box": { + "$ref": "#/$defs/instill-types/bounding-box", + "title": "Bounding box" + }, + "category": { + "description": "The predicted category of the bounding box.", + "instillFormat": "string", + "title": "Category", + "type": "string" + }, + "score": { + "description": "The confidence score of the predicted category of the bounding box.", + "instillFormat": "number", + "title": "Score", + "type": "number" + } + }, + "required": [ + "bounding-box", + "category", + "score" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "embedding": { + "instillFormat": "array:number", + "items": { + "instillFormat": "number", + "title": "Embedding", + "type": "number" + }, + "title": "Embedding", + "type": "array" + }, + "instance-segmentation": { + "additionalProperties": false, + "properties": { + "objects": { + "description": "A list of detected instance bounding boxes.", + "instillFormat": "array:structured/instance-segmentation-object", + "items": { + "instillFormat": "structured/instance-segmentation-object", + "properties": { + "bounding-box": { + "$ref": "#/$defs/instill-types/bounding-box", + "title": "Bounding Box" + }, + "category": { + "description": "The predicted category of the bounding box.", + "instillFormat": "string", + "title": "Category", + "type": "string" + }, + "rle": { + "description": "Run Length Encoding (RLE) of instance mask within the bounding box.", + "instillFormat": "string", + "title": "RLE", + "type": "string" + }, + "score": { + "description": "The confidence score of the predicted instance object.", + "instillFormat": "number", + "title": "Score", + "type": "number" + } + }, + "required": [ + "rle", + "bounding-box", + "category", + "score" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "keypoint": { + "additionalProperties": false, + "properties": { + "objects": { + "description": "A list of keypoint objects, a keypoint object includes all the pre-defined keypoints of a detected object.", + "instillFormat": "array:structured/keypoint-object", + "items": { + "instillFormat": "structured/keypoint-object", + "properties": { + "bounding-box": { + "$ref": "#/$defs/instill-types/bounding-box", + "title": "Bounding Box" + }, + "keypoints": { + "description": "A keypoint group is composed of a list of pre-defined keypoints of a detected object.", + "items": { + "properties": { + "v": { + "description": "visibility score of the keypoint.", + "instillFormat": "number", + "type": "number" + }, + "x": { + "description": "x coordinate of the keypoint.", + "instillFormat": "number", + "type": "number" + }, + "y": { + "description": "y coordinate of the keypoint.", + "instillFormat": "number", + "type": "number" + } + }, + "required": [ + "x", + "y", + "v" + ], + "title": "Keypoints", + "type": "object" + }, + "title": "Keypoints", + "type": "array" + }, + "score": { + "description": "The confidence score of the predicted object.", + "instillFormat": "number", + "title": "Score", + "type": "number" + } + }, + "required": [ + "keypoints", + "score", + "bounding-box" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "ocr": { + "additionalProperties": false, + "properties": { + "objects": { + "description": "A list of detected bounding boxes.", + "instillFormat": "array:structured/ocr-object", + "items": { + "instillFormat": "structured/ocr-object", + "properties": { + "bounding-box": { + "$ref": "#/$defs/instill-types/bounding-box", + "title": "Bounding Box" + }, + "score": { + "description": "The confidence score of the predicted object.", + "instillFormat": "number", + "title": "Score", + "type": "number" + }, + "text": { + "description": "Text string recognised per bounding box.", + "instillFormat": "string", + "title": "Text", + "type": "string" + } + }, + "required": [ + "bounding-box", + "text", + "score" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "semantic-segmentation": { + "additionalProperties": false, + "properties": { + "stuffs": { + "description": "A list of RLE binary masks.", + "instillFormat": "array:structured/semantic-segmentation-stuff", + "items": { + "instillFormat": "structured/semantic-segmentation-stuff", + "properties": { + "category": { + "description": "Category text string corresponding to each stuff mask.", + "instillFormat": "string", + "title": "Category", + "type": "string" + }, + "rle": { + "description": "Run Length Encoding (RLE) of each stuff mask within the image.", + "instillFormat": "string", + "title": "RLE", + "type": "string" + } + }, + "required": [ + "rle", + "category" + ], + "title": "Object", + "type": "object" + }, + "title": "Stuffs", + "type": "array" + } + }, + "required": [ + "stuffs" + ], + "type": "object" + } + } + } +} \ No newline at end of file From a9b676c93d23edd39908de24533a0535a4a978d0 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Tue, 6 Aug 2024 05:35:02 +0800 Subject: [PATCH 03/17] chore: update schema --- schema/ai-tasks.json | 316 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 249 insertions(+), 67 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 161abe08..94dc20e8 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -403,26 +403,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "properties": { - "category": { - "title": "Category", - "type": "string", - "description": "The detected category.", - "instillShortDescription": "The detected category.", - "instillFormat": "string" - }, - "score": { - "title": "Score", - "type": "number", - "description": "The score of the detected category.", - "instillShortDescription": "The score of the detected category.", - "instillFormat": "number" - } - }, - "required": [ - "category", - "score" - ] + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/classification" }, "metadata": { "description": "Output metadata", @@ -480,57 +461,258 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/detection" + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", "properties": { - "objects": { - "title": "Objects", - "type": "array", - "description": "The detected objects.", - "instillShortDescription": "The detected objects.", - "instillFormat": "array", - "items": { - "type": "object", - "properties": { - "bounding-box": {}, - "category": { - "title": "Category", - "type": "string", - "description": "The detected category.", - "instillShortDescription": "The detected category.", - "instillFormat": "string" - }, - "score": { - "title": "Score", - "type": "number", - "description": "The score of the detected category.", - "instillShortDescription": "The score of the detected category.", - "instillFormat": "number" - } - } - } - }, - "required": [ - "objects" - ] - }, - "metadata": { - "description": "Output metadata", - "instillShortDescription": "Output metadata", - "type": "object", - "properties": { - "usage": { - "description": "Usage statistics for the request.", - "instillShortDescription": "Usage statistics for the request.", - "type": "object", - "properties": {}, - "required": [] - } + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] + } + } + } + }, + "required": [ + "data" + ] + } + }, + "TASK_KEYPOINT": { + "title": "Keypoint", + "instillShortDescription": "Detect and localize multiple keypoints of objects in images.", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Keypoint input", + "description": "Input schema of the keypoint task", + "instillShortDescription": "Input schema of the keypoint task", + "type": "object", + "properties": { + "data": { + "$ref": "#/$defs/vision-input", + "type": "object" + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Keypoint output", + "description": "Output schema of the keypoint task", + "instillShortDescription": "Output schema of the keypoint task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/keypoint" + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] + } + } + } + }, + "required": [ + "data" + ] + } + }, + "TASK_OCR": { + "title": "OCR", + "instillShortDescription": "Detect and recognize text in images.", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "OCR input", + "description": "Input schema of the OCR task", + "instillShortDescription": "Input schema of the OCR task", + "type": "object", + "properties": { + "data": { + "$ref": "#/$defs/vision-input", + "type": "object" + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "OCR output", + "description": "Output schema of the OCR task", + "instillShortDescription": "Output schema of the OCR task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/ocr" + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] + } + } + } + }, + "required": [ + "data" + ] + } + }, + "TASK_SEMANTIC_SEGMENTATION": { + "title": "Semantic Segmentation", + "instillShortDescription": "Classify image pixels into predefined categories.", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Semantic segmentation input", + "description": "Input schema of the semantic segmentation task", + "instillShortDescription": "Input schema of the semantic segmentation task", + "type": "object", + "properties": { + "data": { + "$ref": "#/$defs/vision-input", + "type": "object" + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Semantic segmentation output", + "description": "Output schema of the semantic segmentation task", + "instillShortDescription": "Output schema of the semantic segmentation task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/semantic-segmentation" + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] } } + } + }, + "required": [ + "data" + ] + } + }, + "TASK_INSTANCE_SEGMENTATION": { + "title": "Instance Segmentation", + "instillShortDescription": "Detect, localize and delineate multiple objects in images.", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Instance segmentation input", + "description": "Input schema of the instance segmentation task", + "instillShortDescription": "Input schema of the instance segmentation task", + "type": "object", + "properties": { + "data": { + "$ref": "#/$defs/vision-input", + "type": "object" }, - "required": [ - "data" - ] - } + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Instance segmentation output", + "description": "Output schema of the Instance segmentation task", + "instillShortDescription": "Output schema of the Instance segmentation task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/instance-segmentation" + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] + } + } + } + }, + "required": [ + "data" + ] } } } \ No newline at end of file From 3b971e8d80a374b7a879703ed8197548c03492eb Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Tue, 6 Aug 2024 05:43:33 +0800 Subject: [PATCH 04/17] chore: move vision inputs --- schema/ai-tasks.json | 60 -------------------------------------------- schema/schema.json | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 94dc20e8..f6fed102 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -1,64 +1,4 @@ { - "$defs": { - "vision-input": { - "description": "Input data", - "instillShortDescription": "Input data", - "type": "object", - "properties": { - "model": { - "description": "The model to be used.", - "instillShortDescription": "The model to be used.", - "instillAcceptFormats": [ - "string" - ], - "title": "Model Name", - "type": "string" - }, - "image": { - "title": "Input image", - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "image-url": { - "description": "Image url", - "instillShortDescription": "Image url", - "instillAcceptFormats": [ - "string" - ], - "type": "string" - } - }, - "required": [ - "image-url" - ] - }, - { - "type": "object", - "properties": { - "image-base64": { - "description": "Image base64 encoded string", - "instillShortDescription": "Image base64 encoded string", - "instillAcceptFormats": [ - "string" - ], - "type": "string" - } - }, - "required": [ - "image-base64" - ] - } - ] - } - }, - "required": [ - "model", - "image" - ] - } - }, "TASK_CHAT": { "title": "Chat", "instillShortDescription": "Generate response base on conversation input", diff --git a/schema/schema.json b/schema/schema.json index 6dba73a1..b0070a34 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -1,6 +1,64 @@ { "$defs": { "instill-types": { + "vision-input": { + "description": "Input data", + "instillShortDescription": "Input data", + "type": "object", + "properties": { + "model": { + "description": "The model to be used.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "image": { + "title": "Input image", + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "image-url": { + "description": "Image url", + "instillShortDescription": "Image url", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + } + }, + "required": [ + "image-url" + ] + }, + { + "type": "object", + "properties": { + "image-base64": { + "description": "Image base64 encoded string", + "instillShortDescription": "Image base64 encoded string", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + } + }, + "required": [ + "image-base64" + ] + } + ] + } + }, + "required": [ + "model", + "image" + ] + }, "bounding-box": { "additionalProperties": false, "description": "The detected bounding box in (left, top, width, height) format.", From cbd21cdbd34df818a46982e23bb15ab66129bf6e Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Tue, 6 Aug 2024 05:45:35 +0800 Subject: [PATCH 05/17] chore: update vision inputs --- schema/ai-tasks.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index f6fed102..3b953cdd 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -318,7 +318,7 @@ "type": "object", "properties": { "data": { - "$ref": "#/$defs/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -343,7 +343,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/classification" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/classification" }, "metadata": { "description": "Output metadata", @@ -376,7 +376,7 @@ "type": "object", "properties": { "data": { - "$ref": "#/$defs/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -401,7 +401,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/detection" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/detection" }, "metadata": { "description": "Output metadata", @@ -434,7 +434,7 @@ "type": "object", "properties": { "data": { - "$ref": "#/$defs/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -459,7 +459,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/keypoint" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/keypoint" }, "metadata": { "description": "Output metadata", @@ -492,7 +492,7 @@ "type": "object", "properties": { "data": { - "$ref": "#/$defs/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -517,7 +517,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/ocr" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/ocr" }, "metadata": { "description": "Output metadata", @@ -550,7 +550,7 @@ "type": "object", "properties": { "data": { - "$ref": "#/$defs/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -575,7 +575,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/semantic-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/semantic-segmentation" }, "metadata": { "description": "Output metadata", @@ -608,7 +608,7 @@ "type": "object", "properties": { "data": { - "$ref": "#/$defs/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -633,7 +633,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/c74ff8f/schema/schema.json#/$defs/instill-types/instance-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/instance-segmentation" }, "metadata": { "description": "Output metadata", From 2fc9330e33fbb6c78ad5bdab9e78f93038d94e83 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Thu, 8 Aug 2024 15:36:50 +0800 Subject: [PATCH 06/17] chore: add completion task --- schema/ai-tasks.json | 224 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 3b953cdd..22a49845 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -307,6 +307,230 @@ ] } }, + "TASK_COMPLETION": { + "title": "Completion", + "instillShortDescription": "Generate text response base on input", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Completion input", + "description": "Input schema of the completion task", + "instillShortDescription": "Input schema of the completion task", + "type": "object", + "properties": { + "data": { + "description": "Input data", + "instillShortDescription": "Input data", + "type": "object", + "properties": { + "model": { + "description": "The model to be used.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "prompt": { + "title": "Input prompt", + "type": "string", + "description": "The input prompt to generate text on.", + "instillShortDescription": "The input prompt to generate text on.", + "instillAcceptFormats": [ + "string" + ] + } + }, + "required": [ + "model", + "prompt" + ] + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": { + "max-tokens": { + "title": "Max new tokens", + "type": "integer", + "description": "The maximum number of tokens for model to generate", + "instillShortDescription": "The maximum number of tokens for model to generate", + "instillAcceptFormats": [ + "integer" + ], + "default": 50 + }, + "seed": { + "title": "Seed", + "type": "integer", + "description": "The seed, default is 0", + "instillShortDescription": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0 + }, + "n": { + "title": "Number of choices", + "type": "integer", + "description": "How many chat completion choices to generate for each input message.", + "instillShortDescription": "How many chat completion choices to generate for each input message.", + "instillAcceptFormats": [ + "integer" + ], + "default": 1 + }, + "temperature": { + "title": "Temperature", + "type": "number", + "description": "The temperature for sampling", + "instillShortDescription": "The temperature for sampling", + "instillAcceptFormats": [ + "number" + ], + "default": 0.7 + }, + "top-p": { + "title": "Top P", + "type": "number", + "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or temperature but not both.", + "instillShortDescription": "Nucleus sampling", + "instillAcceptFormats": [ + "number" + ], + "default": 1 + }, + "stream": { + "title": "Stream", + "type": "boolean", + "description": "If set, partial message deltas will be sent. Tokens will be sent as data-only server-sent events as they become available.", + "instillShortDescription": "If set, partial message deltas will be sent", + "instillAcceptFormats": [ + "boolean" + ], + "default": false + } + } + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Chat output", + "description": "Output schema of the chat task", + "instillShortDescription": "Output schema of the chat task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "properties": { + "choices": { + "title": "Choices", + "type": "array", + "description": "List of chat completion choices", + "instillShortDescription": "List of chat completion choices", + "instillFormat": "array", + "items": { + "type": "object", + "properties": { + "finish-reason": { + "title": "Finish reason", + "type": "string", + "description": "The reason the model stopped generating tokens.", + "instillShortDescription": "The reason the model stopped generating tokens.", + "instillFormat": "string", + "enum": [ + "stop", + "length" + ] + }, + "index": { + "title": "Index", + "type": "integer", + "description": "The index of the choice in the list of choices.", + "instillShortDescription": "The index of the choice in the list of choices.", + "instillFormat": "integer" + }, + "content": { + "title": "Content", + "type": "string", + "description": "The contents generated by the model.", + "instillShortDescription": "The contents generated by the model.", + "instillFormat": "string" + }, + "created": { + "title": "Created", + "type": "integer", + "description": "The Unix timestamp (in seconds) of when the chat completion was created.", + "instillShortDescription": "The Unix timestamp (in seconds) of when the chat completion was created.", + "instillFormat": "integer" + } + }, + "required": [ + "finish_reason", + "index", + "content", + "created" + ] + } + } + }, + "required": [ + "choices" + ] + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": { + "completion-tokens": { + "title": "Completion tokens", + "type": "integer", + "description": "Number of tokens in the generated response.", + "instillShortDescription": "Number of tokens in the generated response.", + "instillFormat": "integer" + }, + "prompt-tokens": { + "title": "Prompt tokens", + "type": "integer", + "description": "Number of tokens in the prompt.", + "instillShortDescription": "Number of tokens in the prompt.", + "instillFormat": "integer" + }, + "total-tokens": { + "title": "Total tokens", + "type": "integer", + "description": "Total number of tokens used in the request (prompt + completion).", + "instillShortDescription": "Total number of tokens used in the request (prompt + completion).", + "instillFormat": "integer" + } + }, + "required": [ + "completion-tokens", + "prompt-tokens", + "total-tokens" + ] + } + } + } + }, + "required": [ + "data" + ] + } + }, "TASK_CLASSIFICATION": { "title": "Classification", "instillShortDescription": "Classify images into predefined categories.", From c2194a43eb3fd41bc57eb38ac6a5a7faa511d58c Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Thu, 8 Aug 2024 17:34:21 +0800 Subject: [PATCH 07/17] chore: add system message in completion --- schema/ai-tasks.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 22a49845..b2a4e8b6 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -331,6 +331,15 @@ "title": "Model Name", "type": "string" }, + "system-message": { + "title": "System message", + "type": "string", + "description": "The contents of the system message.", + "instillShortDescription": "The contents of the system message.", + "instillAcceptFormats": [ + "string" + ] + }, "prompt": { "title": "Input prompt", "type": "string", From 5bbf763b0d3d1d332cec741b162f5b1afd19482d Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Fri, 9 Aug 2024 03:55:11 +0800 Subject: [PATCH 08/17] chore: add text to image --- schema/ai-tasks.json | 177 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 2 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index b2a4e8b6..7f7e3f10 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -249,7 +249,7 @@ } }, "required": [ - "finish_reason", + "finish-reason", "index", "message", "created" @@ -482,7 +482,7 @@ } }, "required": [ - "finish_reason", + "finish-reason", "index", "content", "created" @@ -540,6 +540,179 @@ ] } }, + "TASK_TEXT_TO_IMAGE": { + "title": "Completion", + "instillShortDescription": "Generate text response base on input", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Completion input", + "description": "Input schema of the completion task", + "instillShortDescription": "Input schema of the completion task", + "type": "object", + "properties": { + "data": { + "description": "Input data", + "instillShortDescription": "Input data", + "type": "object", + "properties": { + "model": { + "description": "The model to be used.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "prompt": { + "title": "Input prompt", + "type": "string", + "description": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.", + "instillShortDescription": "Descriptive prompt.", + "instillAcceptFormats": [ + "string" + ] + } + }, + "required": [ + "model", + "prompt" + ] + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": { + "aspect-ratio": { + "title": "Aspect ratio", + "type": "string", + "description": "Controls the aspect ratio of the generated image. Defaults to 1:1.", + "instillShortDescription": "Controls the aspect ratio of the generated image. Defaults to 1:1.", + "instillAcceptFormats": [ + "string" + ], + "default": "1:1", + "enum": [ + "16:9", + "1:1", + "21:9", + "2:3", + "3:2", + "4:5", + "5:4", + "9:16", + "9:21" + ] + }, + "negative-prompt": { + "title": "Aspect ratio", + "type": "string", + "description": "Keywords of what you do not wish to see in the output image.", + "instillShortDescription": "Keywords of what you do not wish to see in the output image.", + "instillAcceptFormats": [ + "string" + ], + "default": "1:1" + }, + "n": { + "title": "Number of choices", + "type": "integer", + "description": "How many samples to generate for each input prompt.", + "instillShortDescription": "How many samples to generate for each input prompt.", + "instillAcceptFormats": [ + "integer" + ], + "default": 1 + }, + "seed": { + "title": "Seed", + "type": "integer", + "description": "The seed, default is 0", + "instillShortDescription": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0 + } + } + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Chat output", + "description": "Output schema of the chat task", + "instillShortDescription": "Output schema of the chat task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "properties": { + "choices": { + "title": "Choices", + "type": "array", + "description": "List of generated sample images", + "instillShortDescription": "List of generated sample images", + "instillFormat": "array", + "items": { + "type": "object", + "properties": { + "finish-reason": { + "title": "Finish reason", + "type": "string", + "description": "The reason the model stopped generating tokens.", + "instillShortDescription": "The reason the model stopped generating tokens.", + "instillFormat": "string", + "enum": [ + "content_filtered", + "success" + ] + }, + "image": { + "title": "Image", + "type": "string", + "description": "The generated image, encoded to base64.", + "instillShortDescription": "The generated image, encoded to base64.", + "instillFormat": "string" + } + }, + "required": [ + "finish-reason", + "image" + ] + } + } + }, + "required": [ + "choices" + ] + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": { + "usage": { + "description": "Usage statistics for the request.", + "instillShortDescription": "Usage statistics for the request.", + "type": "object", + "properties": {}, + "required": [] + } + } + } + }, + "required": [ + "data" + ] + } + }, "TASK_CLASSIFICATION": { "title": "Classification", "instillShortDescription": "Classify images into predefined categories.", From 8eb267e448f4e806ba7d57a68f15c67644730418 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Mon, 12 Aug 2024 20:41:11 +0800 Subject: [PATCH 09/17] chore: update oneof --- schema/ai-tasks.json | 66 +++++++++++++++++++++++++++++++++++++++----- schema/schema.json | 55 ++++++++++++++---------------------- 2 files changed, 79 insertions(+), 42 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 7f7e3f10..28c09654 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -40,32 +40,84 @@ "type": "object", "properties": { "text": { - "description": "Text message", - "instillShortDescription": "Text message", + "title": "Text message", + "description": "Text message.", + "instillShortDescription": "Text message.", "instillAcceptFormats": [ "string" ], "type": "string" + }, + "type": { + "title": "Content type", + "description": "Input content type.", + "instillShortDescription": "Input content type.", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "text" } }, "required": [ - "text" + "text", + "type" ] }, { "type": "object", "properties": { - "datauri": { - "description": "Image message with either url or base64 encoded string", - "instillShortDescription": "Image message with either url or base64 encoded string", + "image-url": { + "title": "Image url", + "description": "Image message url.", + "instillShortDescription": "Image message url.", "instillAcceptFormats": [ "string" ], "type": "string" + }, + "type": { + "title": "Content type", + "description": "Input content type", + "instillShortDescription": "Input content type", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "image-url" + } + }, + "required": [ + "image-url", + "type" + ] + }, + { + "type": "object", + "properties": { + "image-base64": { + "title": "Image base64", + "description": "Image base64 encoded string.", + "instillShortDescription": "Image base64 encoded string.", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + }, + "type": { + "title": "Content type", + "description": "Input content type", + "instillShortDescription": "Input content type", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "image-base64" } }, "required": [ - "datauri" + "image-base64", + "type" ] } ] diff --git a/schema/schema.json b/schema/schema.json index b0070a34..e2c8f88a 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -17,46 +17,31 @@ }, "image": { "title": "Input image", - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "image-url": { - "description": "Image url", - "instillShortDescription": "Image url", - "instillAcceptFormats": [ - "string" - ], - "type": "string" - } - }, - "required": [ - "image-url" - ] - }, - { - "type": "object", - "properties": { - "image-base64": { - "description": "Image base64 encoded string", - "instillShortDescription": "Image base64 encoded string", - "instillAcceptFormats": [ - "string" - ], - "type": "string" - } - }, - "required": [ - "image-base64" - ] - } + "description": "Input image data.", + "instillShortDescription": "Input image data.", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + }, + "type": { + "title": "Input image type", + "description": "Image url or image base64.", + "instillShortDescription": "Image url or image base64.", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "enum": [ + "url", + "base64" ] } }, "required": [ "model", - "image" + "image", + "type" ] }, "bounding-box": { From 9cad71a79cad10cb7ef0cd9ae99c20eda1a246c0 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Mon, 12 Aug 2024 20:42:26 +0800 Subject: [PATCH 10/17] chore: update ref commit --- schema/ai-tasks.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 28c09654..74fd59a5 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -776,7 +776,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -801,7 +801,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/classification" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/classification" }, "metadata": { "description": "Output metadata", @@ -834,7 +834,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -859,7 +859,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/detection" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/detection" }, "metadata": { "description": "Output metadata", @@ -892,7 +892,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -917,7 +917,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/keypoint" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/keypoint" }, "metadata": { "description": "Output metadata", @@ -950,7 +950,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -975,7 +975,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/ocr" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/ocr" }, "metadata": { "description": "Output metadata", @@ -1008,7 +1008,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1033,7 +1033,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/semantic-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/semantic-segmentation" }, "metadata": { "description": "Output metadata", @@ -1066,7 +1066,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1091,7 +1091,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/8d508b1/schema/schema.json#/$defs/instill-types/instance-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/instance-segmentation" }, "metadata": { "description": "Output metadata", From 21b6868d898f2ca3f302ec8501b8b5e7bede351a Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Tue, 20 Aug 2024 23:55:30 +0800 Subject: [PATCH 11/17] chore: add enbedding task --- schema/ai-tasks.json | 306 +++++++++++++++++++++++++++++++++++++++++++ schema/schema.json | 3 +- 2 files changed, 308 insertions(+), 1 deletion(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 74fd59a5..2672916a 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -1,4 +1,258 @@ { + "TASK_EMBEDDING": { + "title": "Embedding", + "instillShortDescription": "Generate embeddings based on input data", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Embedding input", + "description": "Input schema of the embedding task", + "instillShortDescription": "Input schema of the embedding task", + "type": "object", + "properties": { + "data": { + "description": "Input data", + "instillShortDescription": "Input data", + "type": "object", + "properties": { + "model": { + "description": "The model to be used for generating embeddings.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "input": { + "title": "Embedding Input", + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { + "description": "The content to be embedded.", + "instillShortDescription": "The content to be embedded.", + "title": "Content", + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "text": { + "title": "Text content", + "description": "Text content to be embedded", + "instillShortDescription": "Text content", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + }, + "type": { + "title": "Content type", + "description": "Input content type.", + "instillShortDescription": "Input content type.", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "text" + } + }, + "required": [ + "text", + "type" + ] + }, + { + "type": "object", + "properties": { + "image-url": { + "title": "Image url", + "description": "Image content with URL.", + "instillShortDescription": "Image content url.", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + }, + "type": { + "title": "Content type", + "description": "Input content type", + "instillShortDescription": "Input content type", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "image-url" + } + }, + "required": [ + "image-url", + "type" + ] + }, + { + "type": "object", + "properties": { + "image-base64": { + "title": "Image base64", + "description": "Image content with base64 encoded string.", + "instillShortDescription": "Image content with base64 encoded string.", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + }, + "type": { + "title": "Content type", + "description": "Input content type", + "instillShortDescription": "Input content type", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "image-base64" + } + }, + "required": [ + "image-base64", + "type" + ] + } + ] + } + } + }, + "required": [ + "content" + ] + } + } + }, + "required": [ + "model", + "input" + ] + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": { + "format": { + "title": "Data format", + "type": "string", + "description": "The data format of the embeddings. Defaults to float.", + "instillShortDescription": "Data format", + "instillAcceptFormats": [ + "string" + ], + "enum": [ + "float", + "base64" + ], + "default": "float" + }, + "dimensions": { + "title": "Dimensions", + "type": "integer", + "description": "Number of dimensions in the output embedding vectors.", + "instillShortDescription": "Number of dimensions", + "instillAcceptFormats": [ + "integer" + ], + "default": 512 + }, + "input-type": { + "title": "Input type", + "type": "string", + "description": "The type of input data to be embedded (e.g., query, document).", + "instillShortDescription": "Type of input data", + "instillAcceptFormats": [ + "string" + ] + }, + "truncate": { + "title": "Truncate", + "type": "string", + "description": "How to handle inputs longer than the max token length. Defaults to 'End'.", + "instillShortDescription": "Truncation handling", + "instillAcceptFormats": [ + "string" + ], + "enum": [ + "None", + "End", + "Start" + ], + "default": "End" + } + } + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Embedding output", + "description": "Output schema of the embedding task", + "instillShortDescription": "Output schema of the embedding task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "properties": { + "embeddings": { + "title": "Embeddings", + "type": "array", + "description": "List of generated embeddings.", + "instillShortDescription": "List of embeddings", + "instillFormat": "array", + "items": { + "type": "object", + "properties": { + "index": { + "title": "Index", + "type": "integer", + "description": "The index of the embedding vector in the array.", + "instillShortDescription": "Index in the array", + "instillFormat": "integer" + }, + "vector": { + "title": "Embedding Vector", + "type": "array", + "description": "The embedding vector.", + "instillShortDescription": "Embedding vector", + "instillFormat": "array" + }, + "created": { + "title": "Created", + "type": "integer", + "description": "The Unix timestamp (in seconds) of when the embedding was created.", + "instillShortDescription": "Timestamp of creation", + "instillFormat": "integer" + } + }, + "required": [ + "index", + "vector", + "created" + ] + } + } + }, + "required": [ + "embeddings" + ] + } + } + } + }, "TASK_CHAT": { "title": "Chat", "instillShortDescription": "Generate response base on conversation input", @@ -1112,5 +1366,57 @@ "data" ] } + }, + "TASK_CUSTOM": { + "title": "Custom", + "instillShortDescription": "Custom, with arbitrary input/output data", + "input": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Custom input", + "description": "Input schema of the custom task", + "instillShortDescription": "Input schema of the custom task", + "type": "object", + "properties": { + "data": { + "description": "Input data", + "instillShortDescription": "Input data", + "type": "object", + "properties": {} + }, + "parameter": { + "description": "Input parameter", + "instillShortDescription": "Input parameter", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + }, + "output": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Custom output", + "description": "Output schema of the custom task", + "instillShortDescription": "Output schema of the custom task", + "type": "object", + "properties": { + "data": { + "description": "Output data", + "instillShortDescription": "Output data", + "type": "object", + "properties": {} + }, + "metadata": { + "description": "Output metadata", + "instillShortDescription": "Output metadata", + "type": "object", + "properties": {} + } + }, + "required": [ + "data" + ] + } } } \ No newline at end of file diff --git a/schema/schema.json b/schema/schema.json index e2c8f88a..79472600 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -20,7 +20,8 @@ "description": "Input image data.", "instillShortDescription": "Input image data.", "instillAcceptFormats": [ - "string" + "string", + "image/*" ], "type": "string" }, From 2b7a3558a472b1747d772168ddee937ddbbae2fe Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Thu, 29 Aug 2024 07:20:39 +0800 Subject: [PATCH 12/17] chore: update image schema --- schema/ai-tasks.json | 4 +- schema/schema.json | 109 +++++++++++++++++++++++++++++-------------- 2 files changed, 76 insertions(+), 37 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 2672916a..5ab9af16 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -100,7 +100,7 @@ "description": "Image content with base64 encoded string.", "instillShortDescription": "Image content with base64 encoded string.", "instillAcceptFormats": [ - "string" + "image/*" ], "type": "string" }, @@ -354,7 +354,7 @@ "description": "Image base64 encoded string.", "instillShortDescription": "Image base64 encoded string.", "instillAcceptFormats": [ - "string" + "image/*" ], "type": "string" }, diff --git a/schema/schema.json b/schema/schema.json index 79472600..c960c27c 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -5,44 +5,83 @@ "description": "Input data", "instillShortDescription": "Input data", "type": "object", - "properties": { - "model": { - "description": "The model to be used.", - "instillShortDescription": "The model to be used.", - "instillAcceptFormats": [ - "string" - ], - "title": "Model Name", - "type": "string" - }, - "image": { - "title": "Input image", - "description": "Input image data.", - "instillShortDescription": "Input image data.", - "instillAcceptFormats": [ - "string", - "image/*" - ], - "type": "string" + "oneOf": [ + { + "type": "object", + "properties": { + "model": { + "description": "The model to be used.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "image-url": { + "title": "Input image url", + "description": "Input image url.", + "instillShortDescription": "Input image url.", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + }, + "type": { + "title": "Input image type", + "description": "Image url type.", + "instillShortDescription": "Image url type.", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "image-url" + } + }, + "required": [ + "model", + "image-url", + "type" + ] }, - "type": { - "title": "Input image type", - "description": "Image url or image base64.", - "instillShortDescription": "Image url or image base64.", - "instillAcceptFormats": [ - "string" - ], - "type": "string", - "enum": [ - "url", - "base64" + { + "type": "object", + "properties": { + "model": { + "description": "The model to be used.", + "instillShortDescription": "The model to be used.", + "instillAcceptFormats": [ + "string" + ], + "title": "Model Name", + "type": "string" + }, + "image-base64": { + "title": "Input image file", + "description": "Input image file.", + "instillShortDescription": "Input image file.", + "instillAcceptFormats": [ + "string" + ], + "type": "string" + }, + "type": { + "title": "Input image type", + "description": "Image base64 type.", + "instillShortDescription": "Image base64 type.", + "instillAcceptFormats": [ + "string" + ], + "type": "string", + "const": "image-base64" + } + }, + "required": [ + "model", + "image-base64", + "type" ] } - }, - "required": [ - "model", - "image", - "type" ] }, "bounding-box": { From 1b654dd1d3eaf504eda08bd7b9d04ff7de4ed8d2 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Thu, 29 Aug 2024 07:21:24 +0800 Subject: [PATCH 13/17] chore: update ref commit --- schema/ai-tasks.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 5ab9af16..7d657649 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -1030,7 +1030,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1055,7 +1055,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/classification" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/classification" }, "metadata": { "description": "Output metadata", @@ -1088,7 +1088,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1113,7 +1113,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/detection" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/detection" }, "metadata": { "description": "Output metadata", @@ -1146,7 +1146,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1171,7 +1171,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/keypoint" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/keypoint" }, "metadata": { "description": "Output metadata", @@ -1204,7 +1204,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1229,7 +1229,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/ocr" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/ocr" }, "metadata": { "description": "Output metadata", @@ -1262,7 +1262,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1287,7 +1287,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/semantic-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/semantic-segmentation" }, "metadata": { "description": "Output metadata", @@ -1320,7 +1320,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1345,7 +1345,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/f67bf97/schema/schema.json#/$defs/instill-types/instance-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/instance-segmentation" }, "metadata": { "description": "Output metadata", From 10fb18a1b1c0797e8d22439ce8966f0e30358b12 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Thu, 29 Aug 2024 11:24:10 +0800 Subject: [PATCH 14/17] chore: update schema --- schema/schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/schema.json b/schema/schema.json index c960c27c..c046551b 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -28,7 +28,7 @@ "type": "string" }, "type": { - "title": "Input image type", + "title": "URL", "description": "Image url type.", "instillShortDescription": "Image url type.", "instillAcceptFormats": [ @@ -61,12 +61,12 @@ "description": "Input image file.", "instillShortDescription": "Input image file.", "instillAcceptFormats": [ - "string" + "image/*" ], "type": "string" }, "type": { - "title": "Input image type", + "title": "file", "description": "Image base64 type.", "instillShortDescription": "Image base64 type.", "instillAcceptFormats": [ From 52442a23a345fe83d5d1202526d553bffdabe0ce Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Thu, 29 Aug 2024 11:53:52 +0800 Subject: [PATCH 15/17] chore: update ref commit --- schema/ai-tasks.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 7d657649..43e396e7 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -1030,7 +1030,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1055,7 +1055,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/classification" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/classification" }, "metadata": { "description": "Output metadata", @@ -1088,7 +1088,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1113,7 +1113,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/detection" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/detection" }, "metadata": { "description": "Output metadata", @@ -1146,7 +1146,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1171,7 +1171,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/keypoint" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/keypoint" }, "metadata": { "description": "Output metadata", @@ -1204,7 +1204,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1229,7 +1229,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/ocr" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/ocr" }, "metadata": { "description": "Output metadata", @@ -1262,7 +1262,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1287,7 +1287,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/semantic-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/semantic-segmentation" }, "metadata": { "description": "Output metadata", @@ -1320,7 +1320,7 @@ "type": "object", "properties": { "data": { - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/vision-input", + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/vision-input", "type": "object" }, "parameter": { @@ -1345,7 +1345,7 @@ "description": "Output data", "instillShortDescription": "Output data", "type": "object", - "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/6530ff4/schema/schema.json#/$defs/instill-types/instance-segmentation" + "$ref": "https://raw.githubusercontent.com/instill-ai/instill-core/62743c4/schema/schema.json#/$defs/instill-types/instance-segmentation" }, "metadata": { "description": "Output metadata", From 74939dd0df80822a6ee2f66468bca639a8bfbbd4 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Wed, 4 Sep 2024 00:30:52 +0800 Subject: [PATCH 16/17] chore: update schema --- schema/ai-tasks.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/schema/ai-tasks.json b/schema/ai-tasks.json index 43e396e7..36774a6d 100644 --- a/schema/ai-tasks.json +++ b/schema/ai-tasks.json @@ -35,6 +35,7 @@ "title": "Content", "type": "array", "items": { + "type": "object", "oneOf": [ { "type": "object", @@ -289,6 +290,7 @@ "title": "Content", "type": "array", "items": { + "type": "object", "oneOf": [ { "type": "object", @@ -397,7 +399,7 @@ "instillAcceptFormats": [ "string" ], - "title": "Role", + "title": "Name", "type": "string" } }, From 2ca1302b870682475d00b44887558fd23bd55078 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Wed, 4 Sep 2024 20:38:58 +0800 Subject: [PATCH 17/17] chore: update test model_hub --- model-hub/model_hub_test.json | 44 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/model-hub/model_hub_test.json b/model-hub/model_hub_test.json index 72e26189..d459f870 100644 --- a/model-hub/model_hub_test.json +++ b/model-hub/model_hub_test.json @@ -25,19 +25,6 @@ "version": "latest", "configuration": {} }, - { - "id": "dummy-image-to-image", - "owner_type": "users", - "owner_id": "admin", - "description": "A image to image model for integration-test", - "model_definition": "model-definitions/container", - "visibility": "VISIBILITY_PUBLIC", - "task": "TASK_IMAGE_TO_IMAGE", - "region": "REGION_LOCAL", - "hardware": "CPU", - "version": "latest", - "configuration": {} - }, { "id": "dummy-instance-segmentation", "owner_type": "users", @@ -78,26 +65,26 @@ "configuration": {} }, { - "id": "dummy-text-generation", + "id": "dummy-completion", "owner_type": "users", "owner_id": "admin", - "description": "A text generation model for integration-test", + "description": "A completion model for integration-test", "model_definition": "model-definitions/container", "visibility": "VISIBILITY_PUBLIC", - "task": "TASK_TEXT_GENERATION", + "task": "TASK_COMPLETION", "region": "REGION_LOCAL", "hardware": "CPU", "version": "latest", "configuration": {} }, { - "id": "dummy-text-generation-chat", + "id": "dummy-chat", "owner_type": "users", "owner_id": "admin", - "description": "A text generation chat model for integration-test", + "description": "A chat model for integration-test", "model_definition": "model-definitions/container", "visibility": "VISIBILITY_PUBLIC", - "task": "TASK_TEXT_GENERATION_CHAT", + "task": "TASK_CHAT", "region": "REGION_LOCAL", "hardware": "CPU", "version": "latest", @@ -117,13 +104,26 @@ "configuration": {} }, { - "id": "dummy-visual-question-answering", + "id": "dummy-multimodal-chat", + "owner_type": "users", + "owner_id": "admin", + "description": "A multimodal chat model for integration-test", + "model_definition": "model-definitions/container", + "visibility": "VISIBILITY_PUBLIC", + "task": "TASK_CHAT", + "region": "REGION_LOCAL", + "hardware": "CPU", + "version": "latest", + "configuration": {} + }, + { + "id": "dummy-text-embedding", "owner_type": "users", "owner_id": "admin", - "description": "A visual question answering model for integration-test", + "description": "A embedding model for integration-test", "model_definition": "model-definitions/container", "visibility": "VISIBILITY_PUBLIC", - "task": "TASK_VISUAL_QUESTION_ANSWERING", + "task": "TASK_EMBEDDING", "region": "REGION_LOCAL", "hardware": "CPU", "version": "latest",