diff --git a/README.md b/README.md index 113d9a7..c373837 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@
-# vina +vina -Ai generated visual novel +AI-powered visual novel generator [![crates.io](https://img.shields.io/crates/v/vina.svg)](https://crates.io/crates/vina) [![docs.rs](https://docs.rs/vina/badge.svg)](https://docs.rs/vina) @@ -10,4 +10,53 @@ Ai generated visual novel
+**VinA** is a visual novel generator. Once you specify a prompt on the type of +story you want, we generate an entire plot, detailed characters with +personalities, locations, music, and more. The result is a fully playable +and polished visual novel you can play. + +## Example + +With the following prompt: +``` +Write a sci-fi story about a hackathon project gone haywire, where twofriends are working +together on a coding project over the weekend. Then, they are sucked into their laptop and +have to find a way back to reality. They overcome an obstacle and successfully return back home. +``` + +We get this visual novel. + +## Features + +Dynamic facial expressions depending on the dialogue +
+ + +
+
+ + +
+ +Generated background images for each scene +
+ + +
+ +## Usage + +To run **VinA** for yourself, you need the following: +- An OpenAI API key, find out how to get one [here](https://platform.openai.com/docs/api-reference/authentication) +- An instance of Automatic1111's stable diffusion web UI, ensure the instance you are using has API support. More info [here](https://github.com/AUTOMATIC1111/stable-diffusion-webui) +- RenPy, installation instructions are [here](https://renpy.org/doc/html/quickstart.html) + +The following environment variables should be set: +- `REN_PATH`: path to renpy executable +- `OPENAI_KEY`: your openai API key +- `NOVELAI_URL`: url to your instance of the stable diffusion web UI + +## What's with the name? + +**VinA** is an anagram of the much less creative name, 'AI VN'. diff --git a/crates/vina_story/src/api.rs b/crates/vina_story/src/api.rs index 600df0e..4ea650c 100644 --- a/crates/vina_story/src/api.rs +++ b/crates/vina_story/src/api.rs @@ -101,6 +101,12 @@ impl ApiClient { /// Parse a function call pub fn parse_fncall(msg: &Value) -> anyhow::Result { + let fn_args = parse_fncall_raw(msg)?; + let downcasted = serde_json::from_value(fn_args)?; + Ok(downcasted) +} + +pub fn parse_fncall_raw(msg: &Value) -> anyhow::Result { let fn_call = &msg["function_call"]; let fn_name = fn_call["name"].as_str().unwrap(); @@ -108,10 +114,7 @@ pub fn parse_fncall(msg: &Value) -> anyhow::Result { let fn_args = fn_call["arguments"].as_str().unwrap(); let mut fn_args: Value = serde_json::from_str(fn_args).unwrap(); let fn_args = fn_args["inner"].take(); - - let downcasted = serde_json::from_value(fn_args)?; - - Ok(downcasted) + Ok(fn_args) } /// Parse text content @@ -177,7 +180,9 @@ pub fn get_scenes_fn() -> Value { "description:": "Descriptive title of the scene based on it's contents", }, "music": { - "type": "string", + "enum": [ + "Funky", "Calm", "Dark", "Inspirational", "Bright", "Dramatic", "Happy", "Romantic", "Angry", "Sad" + ], "description": "Genre of music that should be played in this scene", }, "location": { @@ -196,41 +201,48 @@ pub fn get_scenes_fn() -> Value { "type": "string", "description": "Landmarks and objects of focus that are present in the scene. Omit any descriptions of people.", }, - "mood": { - "type": "string", - "description": "Information about the mood. Omit any descriptions of people.", - }, "time_of_day": { "type": "string", "description": "What time of day it is", }, } }, - "script": { - "type": "array", - "items": { - "type": "object", - "description": "A line in the script, contains information like the speaker, choose a facial expression from this list: smiling, crying, nervous, excited, blushing to match what is being said, and also what is being said", - "properties": { - "speaker": { - "type": "string", - "description": "Name of the speaker" - }, - "facial_expression": { - "type": "string", - "description": "Use an emotion from this list: smiling, crying, nervous, excited, blushing to match the dialogue spoken" - }, - "content": { - "type": "string", - "description": "What the speaker actually says" - } - } - } + } + } + }, + }, + "required": ["inner"], + } + }) +} +pub fn get_script_fn() -> Value { + json!({ + "name": "get_script_fn", + "description": "Script to be used in scene", + "parameters": { + "type": "object", + "properties": { + "inner": { + "type": "array", + "items": { + "type": "object", + "description": "A line in the script, contains information like the speaker, what is being said, and facial expression", + "properties": { + "speaker": { + "type": "string", + "description": "Name of the speaker" }, + "facial_expression": { + "type": "string", + "description": "Use an emotion from this list: smiling, crying, nervous, excited, blushing to match the dialogue spoken" + }, + "content": { + "type": "string", + "description": "What the speaker says" + } } } - }, }, diff --git a/crates/vina_story/src/content.rs b/crates/vina_story/src/content.rs index bcd811d..c194e86 100644 --- a/crates/vina_story/src/content.rs +++ b/crates/vina_story/src/content.rs @@ -53,8 +53,6 @@ pub struct Location { pub description: String, /// Concrete objects and landmarks in the scene pub landmarks: String, - /// Information on the mood and time of day - pub mood: String, /// Time of day pub time_of_day: String, } diff --git a/crates/vina_story/src/lib.rs b/crates/vina_story/src/lib.rs index 1e229ea..bec3e97 100644 --- a/crates/vina_story/src/lib.rs +++ b/crates/vina_story/src/lib.rs @@ -4,7 +4,8 @@ pub mod api; pub mod content; pub mod music; -use content::Location; +use content::{Dialogue, Location}; +use serde_json::{json, Value}; use crate::{ api::*, @@ -15,10 +16,11 @@ pub fn generate_story(token: &str, prompt: &str) -> anyhow::Result { // Client to generate details of the story let mut story_client = ApiClient::new(token); - story_client.run_prompt(prompt, None).unwrap(); + let res = story_client.run_prompt(prompt, None).unwrap(); + let game_name = parse_content(res)?; story_client - .run_prompt("Generate a title for this story", None) + .run_prompt("Generate a short game title for this story", None) .unwrap(); let res = story_client.run_prompt("Limit the number of characters to a maximum of 3. Give me each of the characters in the story, along with detailed personality, clothing, and physical appearance details (include age, race, gender).", Some(get_characters_fn())).unwrap(); @@ -26,11 +28,33 @@ pub fn generate_story(token: &str, prompt: &str) -> anyhow::Result { let characters: Vec = parse_fncall(&res).unwrap(); // println!("CHARACTERS {:?}", characters); - let res = story_client.run_prompt("Limit the number of locations to a maximum of 5. Separate the story into multiple scenes, and for each scene give me a long and detailed description of the setting of the scene, omit any descriptions of people, include the name of the location, physical location it takes place in, objects and landmarks in the scene, mood, and time of day. Also create a title each scene that corresponds to the contents of the scene. Furthermore, for each scene, write me a script and return the result in a list with each element as a character's dialogue, and use a facial expression from this list: smiling, crying, nervous, excited, blushing to match the dialogue spoken. Also For each scene, tell me the music genre from this list Funky, Calm, Dark, Inspirational, Bright, Dramatic, Happy, Romantic, Angry, Sad", Some(get_scenes_fn())).unwrap(); + let res = story_client.run_prompt("Separate the story into multiple scenes, and for each scene give me a long and detailed description of the setting of the scene, omit any descriptions of people, include the name of the location, physical location it takes place in, objects and landmarks in the scene, mood, and time of day. Also create a title each scene that corresponds to the contents of the scene. Furthermore, for each scene, write me a script and return the result in a list with each element as a character's dialogue, and use a facial expression from this list: smiling, crying, nervous, excited, blushing to match the dialogue spoken. Also For each scene, tell me the music genre from this list Funky, Calm, Dark, Inspirational, Bright, Dramatic, Happy, Romantic, Angry, Sad", Some(get_scenes_fn())).unwrap(); - let scenes: Vec = parse_fncall(&res).unwrap(); + let raw_scenes: Value = parse_fncall_raw(&res).unwrap(); // println!("SCENES {:?}", scenes); + let mut val_scenes: Vec = vec![]; + for (i, raw_scene) in raw_scenes.as_array().unwrap().iter().enumerate() { + let scene_number = i + 1; + + let prompt = format!( + r#"For scene {scene_number}, write me a script with a lot of speaking. Prioritize number of lines of dialogue. When writing each line of dialogue, take into account the personality and mood of the character as well as the setting. Do not use a narrator. Ensure that the script transitions smoothly into the next scene. Return the result in a list. Also include facial expression from this list: smiling, crying, nervous, excited, blushing to match the dialogue spoken. Output as json."# + ); + let res = story_client + .run_prompt(&prompt, Some(get_script_fn())) + .unwrap(); + + let script: Vec = parse_fncall(&res).unwrap(); + + // construct finished scene + let mut obj_scene = raw_scene.as_object().unwrap().clone(); + obj_scene.insert(String::from("script"), json! {script}); + val_scenes.push(Value::Object(obj_scene)); + } + // println!("BUILT SCENE {val_scenes:?}"); + + let scenes: Vec = serde_json::from_value(Value::Array(val_scenes)).unwrap(); + let game = Game { name: String::from("VinaGame"), synopsis: String::new(), @@ -51,8 +75,8 @@ pub fn generate_location_prompt(token: &str, location: &Location) -> anyhow::Res generate_prompt( token, &format!( - "{}. {}. {}. {}", - location.description, location.landmarks, location.mood, location.time_of_day + "{}. {}. {}", + location.description, location.landmarks, location.time_of_day ), ) } diff --git a/media/alex_anger.png b/media/alex_anger.png new file mode 100644 index 0000000..a90bdc1 Binary files /dev/null and b/media/alex_anger.png differ diff --git a/media/alex_base.png b/media/alex_base.png new file mode 100644 index 0000000..11b6b6a Binary files /dev/null and b/media/alex_base.png differ diff --git a/media/bg0.png b/media/bg0.png new file mode 100644 index 0000000..0b7db59 Binary files /dev/null and b/media/bg0.png differ diff --git a/media/bg1.png b/media/bg1.png new file mode 100644 index 0000000..cdda579 Binary files /dev/null and b/media/bg1.png differ diff --git a/media/hackathon_bg0.png b/media/hackathon_bg0.png new file mode 100644 index 0000000..a2a9911 Binary files /dev/null and b/media/hackathon_bg0.png differ diff --git a/media/hackathon_bg1.png b/media/hackathon_bg1.png new file mode 100644 index 0000000..0ba4ae8 Binary files /dev/null and b/media/hackathon_bg1.png differ diff --git a/media/hackathon_jessie.png b/media/hackathon_jessie.png new file mode 100644 index 0000000..4176c4d Binary files /dev/null and b/media/hackathon_jessie.png differ diff --git a/media/hackathon_peyton.png b/media/hackathon_peyton.png new file mode 100644 index 0000000..89a263c Binary files /dev/null and b/media/hackathon_peyton.png differ diff --git a/media/lisa_base.png b/media/lisa_base.png new file mode 100644 index 0000000..61d676c Binary files /dev/null and b/media/lisa_base.png differ diff --git a/media/lisa_cry.png b/media/lisa_cry.png new file mode 100644 index 0000000..e5c511b Binary files /dev/null and b/media/lisa_cry.png differ diff --git a/media/logo.png b/media/logo.png new file mode 100644 index 0000000..bbc872c Binary files /dev/null and b/media/logo.png differ