Skip to content

Commit

Permalink
Add docs for usage of Seldon CLI to load custom HF models
Browse files Browse the repository at this point in the history
  • Loading branch information
@vtaskow
vtaskow committed Aug 23, 2023
1 parent f536b48 commit d5c5b58
Show file tree
Hide file tree
Showing 3 changed files with 277 additions and 19 deletions.
195 changes: 180 additions & 15 deletions samples/huggingface.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 45,
"id": "8b4be911",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -51,7 +51,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 46,
"id": "checked-cream",
"metadata": {},
"outputs": [
Expand All @@ -69,7 +69,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 48,
"id": "interracial-secret",
"metadata": {},
"outputs": [
Expand All @@ -87,7 +87,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 49,
"id": "upper-cholesterol",
"metadata": {},
"outputs": [
Expand All @@ -98,7 +98,7 @@
"{\r\n",
"\t\"model_name\": \"text-gen_1\",\r\n",
"\t\"model_version\": \"1\",\r\n",
"\t\"id\": \"bde3e68b-2710-4763-828c-21e58416b45c\",\r\n",
"\t\"id\": \"95270c33-e2e3-4799-b243-06e834e730b8\",\r\n",
"\t\"parameters\": {},\r\n",
"\t\"outputs\": [\r\n",
"\t\t{\r\n",
Expand All @@ -109,10 +109,10 @@
"\t\t\t],\r\n",
"\t\t\t\"datatype\": \"BYTES\",\r\n",
"\t\t\t\"parameters\": {\r\n",
"\t\t\t\t\"content_type\": \"str\"\r\n",
"\t\t\t\t\"content_type\": \"hg_jsonlist\"\r\n",
"\t\t\t},\r\n",
"\t\t\t\"data\": [\r\n",
"\t\t\t\t\"[{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away, you can see the galaxy and the universe together.\\\\n\\\\n\\\\n\\\\nIn the same space as Earth, there are two locations (including Earth) or regions. These regions are located on both of the\\\"}]\"\r\n",
"\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away?\\\\nI hope the experience will be filled with awe and excitement for our current hero's creation. I would love for you to give you an opportunity to enjoy the game and share some of the gameplay ideas\\\"}\"\r\n",
"\t\t\t]\r\n",
"\t\t}\r\n",
"\t]\r\n",
Expand All @@ -127,7 +127,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 50,
"id": "right-talent",
"metadata": {},
"outputs": [],
Expand All @@ -138,17 +138,17 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 51,
"id": "4a7d7108",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"b'[{\"generated_text\": \"Once upon a time\\\\n\\\\n\\\\nThe Great Depression was a devastating economic collapse. The depression was not only due to soaring interest rates, but to an epidemic of illegal foreign trade. It was the worst recession of all time because of the government\\'s actions\"}]'"
"b'{\"generated_text\": \"Once upon a time\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\\\\n\"}'"
]
},
"execution_count": 18,
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -170,9 +170,61 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 52,
"id": "ceramic-illness",
"metadata": {},
"outputs": [],
"source": [
"!seldon model unload text-gen"
]
},
{
"cell_type": "markdown",
"id": "6ff63bb7",
"metadata": {},
"source": [
"### Custom Text Generation Model\n"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "6d48e38c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"apiVersion: mlops.seldon.io/v1alpha1\r\n",
"kind: Model\r\n",
"metadata:\r\n",
" name: custom-text-gen\r\n",
"spec:\r\n",
" storageUri: \"gs://viktor-models/scv2/samples/mlserver_1.3.5/text-generation-huggingface\" # change bucket name\r\n",
" requirements:\r\n",
" - huggingface\r\n",
" memory: 3Gi\r\n"
]
}
],
"source": [
"!cat ./models/hf-custom-text-gen.yaml"
]
},
{
"cell_type": "markdown",
"id": "62ec637f",
"metadata": {},
"source": [
"Load the model"
]
},
{
"cell_type": "code",
"execution_count": 54,
"id": "abef8088",
"metadata": {},
"outputs": [
{
"name": "stdout",
Expand All @@ -183,13 +235,126 @@
}
],
"source": [
"!seldon model unload text-gen"
"!seldon model load -f ./models/hf-custom-text-gen.yaml"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "f25c54d4",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{}\r\n"
]
}
],
"source": [
"!seldon model status custom-text-gen -w ModelAvailable | jq -M ."
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "21f26bbd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\r\n",
"\t\"model_name\": \"custom-text-gen_1\",\r\n",
"\t\"model_version\": \"1\",\r\n",
"\t\"id\": \"726a4b73-25b6-4f6c-8473-02ab5eaa72fd\",\r\n",
"\t\"parameters\": {},\r\n",
"\t\"outputs\": [\r\n",
"\t\t{\r\n",
"\t\t\t\"name\": \"output\",\r\n",
"\t\t\t\"shape\": [\r\n",
"\t\t\t\t1,\r\n",
"\t\t\t\t1\r\n",
"\t\t\t],\r\n",
"\t\t\t\"datatype\": \"BYTES\",\r\n",
"\t\t\t\"parameters\": {\r\n",
"\t\t\t\t\"content_type\": \"hg_jsonlist\"\r\n",
"\t\t\t},\r\n",
"\t\t\t\"data\": [\r\n",
"\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away that the vast majority of our current civilization is of the same technological, racial, and culture, humanity may have found a way out of those limitations. This may be where the future is not at stake.\\\"}\"\r\n",
"\t\t\t]\r\n",
"\t\t}\r\n",
"\t]\r\n",
"}\r\n"
]
}
],
"source": [
"!seldon model infer custom-text-gen \\\n",
" '{\"inputs\": [{\"name\": \"args\",\"shape\": [1],\"datatype\": \"BYTES\",\"data\": [\"Once upon a time in a galaxy far away\"]}]}'"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "30e8bb29",
"metadata": {},
"outputs": [],
"source": [
"res = !seldon model infer custom-text-gen --inference-mode grpc \\\n",
" '{\"inputs\":[{\"name\":\"args\",\"contents\":{\"bytes_contents\":[\"T25jZSB1cG9uIGEgdGltZQo=\"]},\"datatype\":\"BYTES\",\"shape\":[1]}]}' "
]
},
{
"cell_type": "code",
"execution_count": 58,
"id": "eab4929b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"b'{\"generated_text\": \"Once upon a time\\\\n\\\\nIn the past, there were certain classes of adventurers and travelers, who sought to enter a single location and do one thing a little different than everybody else. Then there were those who were good folk, and those who had\"}'"
]
},
"execution_count": 58,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import json\n",
"import base64\n",
"r = json.loads(res[0])\n",
"base64.b64decode(r[\"outputs\"][0][\"contents\"][\"bytesContents\"][0])"
]
},
{
"cell_type": "markdown",
"id": "877e266c",
"metadata": {},
"source": [
"Unload the model"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "e1829439",
"metadata": {},
"outputs": [],
"source": [
"!seldon model unload custom-text-gen"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3a29f9cb",
"id": "bb46ff9d",
"metadata": {},
"outputs": [],
"source": []
Expand All @@ -211,7 +376,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
"version": "3.10.12"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit d5c5b58

Please sign in to comment.