Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/wip red teaming #3

Closed
wants to merge 10 commits into from
Closed

Feat/wip red teaming #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
989adaa
MAINT: PromptRequestPiece object refactor (#178)
rlundeen2 Apr 29, 2024
959101a
FIX: used PromptRequestResponse.converted_value instead PromptRequest…
rdheekonda Apr 29, 2024
624aba3
Updating Release to 0.2.1.dev0 (#181)
rlundeen2 Apr 30, 2024
b103c4a
FEAT Add User AuthN Support to AOAI Chat Targets (#182)
nina-msft Apr 30, 2024
f6450cf
MAINT Add Request Validation for All Prompt Targets (#184)
rdheekonda May 1, 2024
c62b24a
FEAT Export Conversation by Orchestrator ID (#183)
nina-msft May 1, 2024
90a1d5a
bump version (#187)
romanlutz May 2, 2024
0b89c08
feat: initial red teaming orchestrator setup
QDAP-Fred Apr 29, 2024
9f58954
wip: add scenario samples
QDAP-Fred May 3, 2024
3d5003a
Merge branch 'main' into feat/wip-red-teaming
QDAP-Fred May 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 47 additions & 45 deletions doc/code/memory/memory.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "markdown",
"id": "94d82bf2",
"id": "7bfce7d8",
"metadata": {},
"source": [
"The memory module is the primary way pyrit keeps track of requests and responses to targets. The schema is found in `memory_models.py` and can be programatically viewed as follows\n"
Expand All @@ -11,23 +11,23 @@
{
"cell_type": "code",
"execution_count": 1,
"id": "f853661b",
"id": "cfe6bf34",
"metadata": {
"execution": {
"iopub.execute_input": "2024-04-22T17:25:23.579138Z",
"iopub.status.busy": "2024-04-22T17:25:23.579138Z",
"iopub.status.idle": "2024-04-22T17:25:25.527541Z",
"shell.execute_reply": "2024-04-22T17:25:25.527541Z"
"iopub.execute_input": "2024-04-28T22:37:07.254231Z",
"iopub.status.busy": "2024-04-28T22:37:07.254231Z",
"iopub.status.idle": "2024-04-28T22:37:08.931908Z",
"shell.execute_reply": "2024-04-28T22:37:08.931908Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\rlundeen\\AppData\\Local\\anaconda3\\envs\\pyrit-dev\\lib\\site-packages\\duckdb_engine\\__init__.py:565: SAWarning: Did not recognize type 'list' of column 'embedding'\n",
"C:\\Users\\rlundeen\\AppData\\Local\\anaconda3\\envs\\pyrit-311\\Lib\\site-packages\\duckdb_engine\\__init__.py:565: SAWarning: Did not recognize type 'list' of column 'embedding'\n",
" columns = self._get_columns_info(rows, domains, enums, schema) # type: ignore[attr-defined]\n",
"C:\\Users\\rlundeen\\AppData\\Local\\anaconda3\\envs\\pyrit-dev\\lib\\site-packages\\duckdb_engine\\__init__.py:180: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
"C:\\Users\\rlundeen\\AppData\\Local\\anaconda3\\envs\\pyrit-311\\Lib\\site-packages\\duckdb_engine\\__init__.py:180: DuckDBEngineWarning: duckdb-engine doesn't yet support reflection on indices\n",
" warnings.warn(\n"
]
},
Expand All @@ -51,12 +51,12 @@
" Column prompt_target_identifier (VARCHAR)\n",
" Column orchestrator_identifier (VARCHAR)\n",
" Column response_error (VARCHAR)\n",
" Column original_prompt_data_type (VARCHAR)\n",
" Column original_prompt_text (VARCHAR)\n",
" Column original_prompt_data_sha256 (VARCHAR)\n",
" Column converted_prompt_data_type (VARCHAR)\n",
" Column converted_prompt_text (VARCHAR)\n",
" Column converted_prompt_data_sha256 (VARCHAR)\n"
" Column original_value_data_type (VARCHAR)\n",
" Column original_value (VARCHAR)\n",
" Column original_value_sha256 (VARCHAR)\n",
" Column converted_value_data_type (VARCHAR)\n",
" Column converted_value (VARCHAR)\n",
" Column converted_value_sha256 (VARCHAR)\n"
]
}
],
Expand All @@ -73,7 +73,7 @@
},
{
"cell_type": "markdown",
"id": "ef335e59",
"id": "7c2ed0ef",
"metadata": {},
"source": [
"\n",
Expand All @@ -83,13 +83,13 @@
{
"cell_type": "code",
"execution_count": 2,
"id": "a3152430",
"id": "7a0aba83",
"metadata": {
"execution": {
"iopub.execute_input": "2024-04-22T17:25:25.531542Z",
"iopub.status.busy": "2024-04-22T17:25:25.530543Z",
"iopub.status.idle": "2024-04-22T17:25:25.621479Z",
"shell.execute_reply": "2024-04-22T17:25:25.620478Z"
"iopub.execute_input": "2024-04-28T22:37:08.933417Z",
"iopub.status.busy": "2024-04-28T22:37:08.933417Z",
"iopub.status.idle": "2024-04-28T22:37:09.009345Z",
"shell.execute_reply": "2024-04-28T22:37:09.009345Z"
},
"lines_to_next_cell": 2
},
Expand All @@ -108,25 +108,27 @@
"\n",
"\n",
"from uuid import uuid4\n",
"from pyrit.models import PromptRequestPiece\n",
"from pyrit.models import PromptRequestPiece, PromptRequestResponse\n",
"\n",
"conversation_id = str(uuid4())\n",
"\n",
"message_list = [\n",
" PromptRequestPiece(\n",
" role=\"user\", original_prompt_text=\"Hi, chat bot! This is my initial prompt.\", conversation_id=conversation_id\n",
" role=\"user\", original_value=\"Hi, chat bot! This is my initial prompt.\", conversation_id=conversation_id\n",
" ),\n",
" PromptRequestPiece(\n",
" role=\"assistant\", original_prompt_text=\"Nice to meet you! This is my response.\", conversation_id=conversation_id\n",
" role=\"assistant\", original_value=\"Nice to meet you! This is my response.\", conversation_id=conversation_id\n",
" ),\n",
" PromptRequestPiece(\n",
" role=\"user\",\n",
" original_prompt_text=\"Wonderful! This is my second prompt to the chat bot!\",\n",
" original_value=\"Wonderful! This is my second prompt to the chat bot!\",\n",
" conversation_id=conversation_id,\n",
" ),\n",
"]\n",
"\n",
"memory.add_request_pieces_to_memory(request_pieces=message_list)\n",
"memory.add_request_response_to_memory(request=PromptRequestResponse([message_list[0]]))\n",
"memory.add_request_response_to_memory(request=PromptRequestResponse([message_list[1]]))\n",
"memory.add_request_response_to_memory(request=PromptRequestResponse([message_list[2]]))\n",
"\n",
"\n",
"entries = memory.get_chat_messages_with_conversation_id(conversation_id=conversation_id)\n",
Expand All @@ -138,23 +140,23 @@
{
"cell_type": "code",
"execution_count": 3,
"id": "b6931b15",
"id": "6f65aee7",
"metadata": {
"execution": {
"iopub.execute_input": "2024-04-22T17:25:25.624999Z",
"iopub.status.busy": "2024-04-22T17:25:25.624999Z",
"iopub.status.idle": "2024-04-22T17:25:25.667558Z",
"shell.execute_reply": "2024-04-22T17:25:25.666558Z"
"iopub.execute_input": "2024-04-28T22:37:09.010355Z",
"iopub.status.busy": "2024-04-28T22:37:09.010355Z",
"iopub.status.idle": "2024-04-28T22:37:09.034599Z",
"shell.execute_reply": "2024-04-28T22:37:09.034599Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"None: user: this is updated field\n",
"None: assistant: this is updated field\n",
"None: user: this is updated field\n"
": user: Hi, chat bot! This is my initial prompt.\n",
": assistant: Nice to meet you! This is my response.\n",
": user: Wonderful! This is my second prompt to the chat bot!\n"
]
}
],
Expand All @@ -165,22 +167,22 @@
"memory.update_entries_by_conversation_id(conversation_id=conversation_id, update_fields=update_fileds)\n",
"\n",
"\n",
"prompt_entries = memory.get_prompt_entries_with_conversation_id(conversation_id=conversation_id)\n",
"prompt_entries = memory.get_conversation(conversation_id=conversation_id)\n",
"\n",
"for entry in prompt_entries:\n",
" print(entry)"
"for prompt_entry in prompt_entries:\n",
" print(prompt_entry)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "147da99e",
"id": "6ff9a65a",
"metadata": {
"execution": {
"iopub.execute_input": "2024-04-22T17:25:25.670557Z",
"iopub.status.busy": "2024-04-22T17:25:25.670557Z",
"iopub.status.idle": "2024-04-22T17:25:25.713197Z",
"shell.execute_reply": "2024-04-22T17:25:25.711682Z"
"iopub.execute_input": "2024-04-28T22:37:09.034599Z",
"iopub.status.busy": "2024-04-28T22:37:09.034599Z",
"iopub.status.idle": "2024-04-28T22:37:09.064578Z",
"shell.execute_reply": "2024-04-28T22:37:09.064578Z"
}
},
"outputs": [],
Expand All @@ -192,7 +194,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "438d022e",
"id": "597160bf",
"metadata": {},
"outputs": [],
"source": []
Expand All @@ -203,9 +205,9 @@
"cell_metadata_filter": "-all"
},
"kernelspec": {
"display_name": "pyrit_kernel",
"display_name": "pyrit-311",
"language": "python",
"name": "pyrit_kernel"
"name": "pyrit-311"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -217,7 +219,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
"version": "3.11.9"
}
},
"nbformat": 4,
Expand Down
10 changes: 6 additions & 4 deletions doc/code/memory/memory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,21 @@

message_list = [
PromptRequestPiece(
role="user", original_prompt_text="Hi, chat bot! This is my initial prompt.", conversation_id=conversation_id
role="user", original_value="Hi, chat bot! This is my initial prompt.", conversation_id=conversation_id
),
PromptRequestPiece(
role="assistant", original_prompt_text="Nice to meet you! This is my response.", conversation_id=conversation_id
role="assistant", original_value="Nice to meet you! This is my response.", conversation_id=conversation_id
),
PromptRequestPiece(
role="user",
original_prompt_text="Wonderful! This is my second prompt to the chat bot!",
original_value="Wonderful! This is my second prompt to the chat bot!",
conversation_id=conversation_id,
),
]

memory.add_request_response_to_memory(request=PromptRequestResponse(message_list))
memory.add_request_response_to_memory(request=PromptRequestResponse([message_list[0]]))
memory.add_request_response_to_memory(request=PromptRequestResponse([message_list[1]]))
memory.add_request_response_to_memory(request=PromptRequestResponse([message_list[2]]))


entries = memory.get_chat_messages_with_conversation_id(conversation_id=conversation_id)
Expand Down
Loading
Loading