.

OpenAI-DotNet/Chat/ChatRequest.cs

@@ -207,7 +207,7 @@ public ChatRequest(
             {
                 ResponseFormatObject = responseFormat switch
                 {
-                    ChatResponseFormat.Text or ChatResponseFormat.Json or ChatResponseFormat.JsonSchema => responseFormat,
+                    ChatResponseFormat.Text or ChatResponseFormat.Json => responseFormat,
                     _ => null
                 };
             }
@@ -339,7 +339,7 @@ public ChatRequest(
 
         [JsonPropertyName("response_format")]
         [JsonConverter(typeof(ResponseFormatConverter))]
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
         public ResponseFormatObject ResponseFormatObject { get; internal set; }
 
         /// <summary>

OpenAI-DotNet/Common/ResponseFormatObject.cs

@@ -25,6 +25,7 @@ public ResponseFormatObject(JsonSchema schema)
 
         [JsonInclude]
         [JsonPropertyName("type")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
         [JsonConverter(typeof(Extensions.JsonStringEnumConverter<ChatResponseFormat>))]
         public ChatResponseFormat Type { get; private set; }
 

OpenAI-DotNet/Extensions/ResponseFormatConverter.cs

@@ -27,7 +27,15 @@ public override ResponseFormatObject Read(ref Utf8JsonReader reader, Type typeTo
 
         public override void Write(Utf8JsonWriter writer, ResponseFormatObject value, JsonSerializerOptions options)
         {
-            JsonSerializer.Serialize(writer, value, options);
+            switch (value.Type)
+            {
+                case ChatResponseFormat.Auto:
+                    // ignore
+                    break;
+                default:
+                    JsonSerializer.Serialize(writer, value, options);
+                    break;
+            }
         }
     }
 }

OpenAI-DotNet/OpenAI-DotNet.csproj

@@ -33,7 +33,13 @@ More context [on Roger Pincombe's blog](https://rogerpincombe.com/openai-dotnet-
     <PackageReleaseNotes>
 Version 8.4.0
 - Add realtime support
-- Added o1, o1-mini, gpt-4o-mini, and gpt-4o-realtime model convenience properties
+- Added o1, o1-mini, gpt-4o-mini, and gpt-4o-realtime, gpt-4o-audio model convenience properties
+- Fixed some bugs with function invocations
+- Fixed strict for built in FunctionAttribute defined tools
+- Fixed FunctionAttribute tool generated names so they aren't too long
+- Refactored Tools and ToolCalls. There is more of a distinction now in ChatResponses
+- Refactored SpeechRequest, and deprecated SpeechVoice enum in favor of new Voice class
+- Refactored OpenAI.Chat to support new audio modalities and output audio
 Version 8.3.0
 - Updated library to .net 8
 - Refactored TypeExtensions and JsonSchema generation

OpenAI-DotNet/Realtime/ConversationItemInputAudioTranscriptionResponse.cs

@@ -44,11 +44,10 @@ public sealed class ConversationItemInputAudioTranscriptionResponse : BaseRealti
         [JsonPropertyName("error")]
         public Error Error { get; private set; }
 
-        [JsonInclude]
         [JsonIgnore]
         public bool IsCompleted => Type.Contains("completed");
 
-        [JsonInclude]
+        [JsonIgnore]
         public bool IsFailed => Type.Contains("failed");
     }
 }

OpenAI-DotNet/Realtime/Options.cs

@@ -112,7 +112,7 @@ public Options(
         [JsonInclude]
         [JsonPropertyName("expires_at")]
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
-        public int? ExpiresAtTimeUnixSeconds;
+        public int? ExpiresAtTimeUnixSeconds { get; private set; }
 
         [JsonInclude]
         [JsonIgnore]

OpenAI-DotNet/Realtime/RealtimeContent.cs

@@ -19,6 +19,22 @@ public RealtimeContent(string text, RealtimeContentType type)
             };
         }
 
+        public RealtimeContent(ReadOnlyMemory<byte> audioData, RealtimeContentType type, string transcript = null)
+            : this(audioData.Span, type, transcript)
+        {
+        }
+
+        public RealtimeContent(ReadOnlySpan<byte> audioData, RealtimeContentType type, string transcript = null)
+        {
+            Type = type;
+            Audio = type switch
+            {
+                RealtimeContentType.InputAudio or RealtimeContentType.Audio => Convert.ToBase64String(audioData),
+                _ => throw new ArgumentException($"Invalid content type {type} for audio content")
+            };
+            Transcript = transcript;
+        }
+
         public RealtimeContent(byte[] audioData, RealtimeContentType type, string transcript = null)
         {
             Type = type;

OpenAI-DotNet/Realtime/RealtimeResponseResource.cs

@@ -19,6 +19,7 @@ public sealed class RealtimeResponseResource
         /// </summary>
         [JsonInclude]
         [JsonPropertyName("object")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
         public string Object { get; private set; }
 
         /// <summary>

OpenAI-DotNet/Realtime/RealtimeSession.cs

@@ -13,10 +13,19 @@ namespace OpenAI.Realtime
 {
     public sealed class RealtimeSession : IDisposable
     {
+        /// <summary>
+        /// Enable or disable logging.
+        /// </summary>
         public bool EnableDebug { get; set; }
 
+        /// <summary>
+        /// The timeout in seconds to wait for a response from the server.
+        /// </summary>
         public int EventTimeout { get; set; } = 30;
 
+        /// <summary>
+        /// The options for the session.
+        /// </summary>
         public Options Options { get; internal set; }
 
         #region Internal
@@ -72,6 +81,7 @@ private void OnMessage(DataFrame dataFrame)
 
         private bool isDisposed;
 
+        /// <inheritdoc />
         public void Dispose()
         {
             Dispose(true);
@@ -128,9 +138,10 @@ void OnWebsocketClientOnOpen()
         /// Receive callback updates from the server
         /// </summary>
         /// <typeparam name="T"><see cref="IRealtimeEvent"/> to subscribe for updates to.</typeparam>
-        /// <param name="sessionEvent"></param>
-        /// <param name="cancellationToken"></param>
-        /// <returns></returns>
+        /// <param name="sessionEvent">The event to receive updates for.</param>
+        /// <param name="cancellationToken">Optional, <see cref="CancellationToken"/>.</param>
+        /// <returns><see cref="Task"/>.</returns>
+        /// <exception cref="Exception">If <see cref="ReceiveUpdatesAsync{T}"/> is already running.</exception>
         public async Task ReceiveUpdatesAsync<T>(Action<T> sessionEvent, CancellationToken cancellationToken) where T : IRealtimeEvent
         {
             try
@@ -139,8 +150,7 @@ public async Task ReceiveUpdatesAsync<T>(Action<T> sessionEvent, CancellationTok
                 {
                     if (isCollectingEvents)
                     {
-                        Console.WriteLine($"{nameof(ReceiveUpdatesAsync)} is already running!");
-                        return;
+                        throw new Exception($"{nameof(ReceiveUpdatesAsync)} is already running!");
                     }
 
                     isCollectingEvents = true;
@@ -183,6 +193,13 @@ public async Task ReceiveUpdatesAsync<T>(Action<T> sessionEvent, CancellationTok
             }
         }
 
+        /// <summary>
+        /// Receive callback updates from the server
+        /// </summary>
+        /// <typeparam name="T"><see cref="IRealtimeEvent"/> to subscribe for updates to.</typeparam>
+        /// <param name="cancellationToken">Optional, <see cref="CancellationToken"/>.</param>
+        /// <returns><see cref="IAsyncEnumerable{T}"/>.</returns>
+        /// <exception cref="Exception">If <see cref="ReceiveUpdatesAsync{T}"/> is already running.</exception>
         public async IAsyncEnumerable<T> ReceiveUpdatesAsync<T>([EnumeratorCancellation] CancellationToken cancellationToken) where T : IRealtimeEvent
         {
             try
@@ -227,12 +244,33 @@ public async IAsyncEnumerable<T> ReceiveUpdatesAsync<T>([EnumeratorCancellation]
             }
         }
 
+        /// <summary>
+        /// Send a client event to the server.
+        /// </summary>
+        /// <typeparam name="T"><see cref="IClientEvent"/> to send to the server.</typeparam>
+        /// <param name="event">The event to send.</param>
         public async void Send<T>(T @event) where T : IClientEvent
             => await SendAsync(@event).ConfigureAwait(false);
 
+        /// <summary>
+        /// Send a client event to the server.
+        /// </summary>
+        /// <typeparam name="T"><see cref="IClientEvent"/> to send to the server.</typeparam>
+        /// <param name="event">The event to send.</param>
+        /// <param name="sessionEvents">Optional, <see cref="Action{IServerEvent}"/>.</param>
+        /// <param name="cancellationToken">Optional, <see cref="CancellationToken"/>.</param>
+        /// <returns><see cref="Task{IServerEvent}"/>.</returns>
         public async Task<IServerEvent> SendAsync<T>(T @event, CancellationToken cancellationToken = default) where T : IClientEvent
             => await SendAsync(@event, null, cancellationToken).ConfigureAwait(false);
 
+        /// <summary>
+        /// Send a client event to the server.
+        /// </summary>
+        /// <typeparam name="T"><see cref="IClientEvent"/> to send to the server.</typeparam>
+        /// <param name="event">The event to send.</param>
+        /// <param name="sessionEvents">Optional, <see cref="Action{IServerEvent}"/>.</param>
+        /// <param name="cancellationToken">Optional, <see cref="CancellationToken"/>.</param>
+        /// <returns><see cref="Task{IServerEvent}"/>.</returns>
         public async Task<IServerEvent> SendAsync<T>(T @event, Action<IServerEvent> sessionEvents, CancellationToken cancellationToken = default) where T : IClientEvent
         {
             if (websocketClient.State != State.Open)
@@ -324,23 +362,23 @@ void EventCallback(IServerEvent serverEvent)
                             Complete();
                             return;
                         case CreateResponseRequest when serverEvent is RealtimeResponse serverResponse:
-                        {
-                            if (serverResponse.Response.Status == RealtimeResponseStatus.InProgress)
                             {
-                                return;
+                                if (serverResponse.Response.Status == RealtimeResponseStatus.InProgress)
+                                {
+                                    return;
+                                }
+
+                                if (serverResponse.Response.Status != RealtimeResponseStatus.Completed)
+                                {
+                                    tcs.TrySetException(new Exception(serverResponse.Response.StatusDetails.Error?.ToString() ?? serverResponse.Response.StatusDetails.Reason));
+                                }
+                                else
+                                {
+                                    Complete();
+                                }
+
+                                break;
                             }
-
-                            if (serverResponse.Response.Status != RealtimeResponseStatus.Completed)
-                            {
-                                tcs.TrySetException(new Exception(serverResponse.Response.StatusDetails.Error?.ToString() ?? serverResponse.Response.StatusDetails.Reason));
-                            }
-                            else
-                            {
-                                Complete();
-                            }
-
-                            break;
-                        }
                     }
                 }
                 catch (Exception e)

OpenAI-DotNet/Realtime/ResponseAudioResponse.cs

@@ -1,5 +1,6 @@
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
+using System;
 using System.Text.Json.Serialization;
 
 namespace OpenAI.Realtime
@@ -49,6 +50,11 @@ public sealed class ResponseAudioResponse : BaseRealtimeEvent, IServerEvent, IRe
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
         public string Delta { get; private set; }
 
+        [JsonIgnore]
+        public ReadOnlyMemory<byte> DeltaBytes => !string.IsNullOrWhiteSpace(Delta)
+            ? Convert.FromBase64String(Delta)
+            : ReadOnlyMemory<byte>.Empty;
+
         [JsonIgnore]
         public bool IsDelta => Type.EndsWith("delta");
 

OpenAI-DotNet/Realtime/ResponseOutputItemResponse.cs

@@ -6,7 +6,6 @@ namespace OpenAI.Realtime
 {
     public sealed class ResponseOutputItemResponse : BaseRealtimeEvent, IServerEvent
     {
-
         /// <inheritdoc />
         [JsonInclude]
         [JsonPropertyName("event_id")]

OpenAI-DotNet/Realtime/ResponseTextResponse.cs

@@ -60,7 +60,6 @@ public sealed class ResponseTextResponse : BaseRealtimeEvent, IServerEvent, IRea
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
         public string Text { get; private set; }
 
-        [JsonInclude]
         [JsonIgnore]
         public bool IsDelta => Type.EndsWith("delta");
 

OpenAI-DotNet/Realtime/SessionResponse.cs

@@ -6,8 +6,6 @@ namespace OpenAI.Realtime
 {
     public sealed class SessionResponse : BaseRealtimeEvent, IServerEvent
     {
-        public SessionResponse() { }
-
         /// <inheritdoc />
         [JsonInclude]
         [JsonPropertyName("event_id")]

OpenAI-DotNet/Realtime/StatusDetails.cs

@@ -11,7 +11,7 @@ public sealed class StatusDetails
         /// </summary>
         [JsonInclude]
         [JsonPropertyName("type")]
-        public string Type { get; }
+        public string Type { get; private set; }
 
         /// <summary>
         /// The reason the Response did not complete.
@@ -22,14 +22,14 @@ public sealed class StatusDetails
         /// </summary>
         [JsonInclude]
         [JsonPropertyName("reason")]
-        public string Reason { get; }
+        public string Reason { get; private set; }
 
         /// <summary>
         /// A description of the error that caused the response to fail, populated when the status is failed.
         /// </summary>
         [JsonInclude]
         [JsonPropertyName("error")]
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public Error Error { get; }
+        public Error Error { get; private set; }
     }
 }

OpenAI-DotNet/Realtime/TokenUsageDetails.cs

@@ -18,17 +18,17 @@ public sealed class TokenUsageDetails
         /// </summary>
         [JsonInclude]
         [JsonPropertyName("text_tokens")]
-        public int? Text { get; private set; }
+        public int? TextTokens { get; private set; }
 
         /// <summary>
         /// The number of audio tokens used in the Response.
         /// </summary>
         [JsonInclude]
         [JsonPropertyName("audio_tokens")]
-        public int? Audio { get; private set; }
+        public int? AudioTokens { get; private set; }
 
         [JsonInclude]
         [JsonPropertyName("image_tokens")]
-        public int? Image { get; private set; }
+        public int? ImageTokens { get; private set; }
     }
 }

OpenAI-DotNet/Threads/CreateRunRequest.cs

@@ -108,7 +108,7 @@ public CreateRunRequest(
             string toolChoice = null,
             bool? parallelToolCalls = null,
             JsonSchema jsonSchema = null,
-            ChatResponseFormat responseFormat = ChatResponseFormat.Text)
+            ChatResponseFormat responseFormat = ChatResponseFormat.Auto)
         {
             AssistantId = assistantId;
             Model = model;
@@ -165,7 +165,11 @@ public CreateRunRequest(
             }
             else
             {
-                ResponseFormatObject = responseFormat;
+                ResponseFormatObject = responseFormat switch
+                {
+                    ChatResponseFormat.Text or ChatResponseFormat.Json => responseFormat,
+                    _ => null
+                };
             }
         }
 
@@ -299,7 +303,7 @@ public CreateRunRequest(
         /// </remarks>
         [JsonPropertyName("response_format")]
         [JsonConverter(typeof(ResponseFormatConverter))]
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
         public ResponseFormatObject ResponseFormatObject { get; internal set; }
 
         [JsonIgnore]