Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Agent: Update WebRTC data channel messages #505

Merged
merged 10 commits into from
Nov 20, 2023
Merged

Agent: Update WebRTC data channel messages #505

Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5f3c021
Agent: Interact over WebRTC if URL param is set
juberti Nov 13, 2023
98e3f22
Merge branch 'main' into juberti/webrtc-chat
juberti Nov 13, 2023
0467ae6
fix url
juberti Nov 14, 2023
73c6b91
Merge branch 'main' into juberti/webrtc-chat
juberti Nov 14, 2023
682b854
Update websocket host/port
juberti Nov 16, 2023
fff56ee
Merge branch 'main' into juberti/webrtc-chat
juberti Nov 16, 2023
9b393bb
cr
juberti Nov 17, 2023
3c707e5
Merge branch 'main' into juberti/webrtc-chat
juberti Nov 18, 2023
7dd9213
add interrupt message
juberti Nov 20, 2023
6ea0e40
Merge branch 'main' into juberti/webrtc-chat
juberti Nov 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions packages/voice/src/app/agent/chat.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,8 +585,8 @@ export class WebRtcChatManager implements ChatManager {
console.log('[chat] got mic stream');
this.inAnalyzer = new StreamAnalyzer(this.audioContext, this.localAudioTrack!.mediaStream!);
this.pinger = setInterval(() => {
var obj = { type: 'ping', timestamp: performance.now() };
this.room?.localParticipant.publishData(this.textEncoder.encode(JSON.stringify(obj)), DataPacket_Kind.RELIABLE);
const obj = { type: 'ping', timestamp: performance.now() };
this.sendData(obj);
}, 5000);
this.maybePublishLocalAudio();
this.changeState(ChatManagerState.LISTENING);
Expand All @@ -608,7 +608,9 @@ export class WebRtcChatManager implements ChatManager {
this.changeState(ChatManagerState.IDLE);
}
interrupt() {
throw new Error('Method not implemented.');
console.log('[chat] interrupting');
const obj = { type: 'interrupt' };
this.sendData(obj);
}
private changeState(state: ChatManagerState) {
if (state != this._state) {
Expand All @@ -624,6 +626,9 @@ export class WebRtcChatManager implements ChatManager {
this.room.localParticipant.publishTrack(this.localAudioTrack, opts);
}
}
private sendData(obj: any) {
this.room?.localParticipant.publishData(this.textEncoder.encode(JSON.stringify(obj)), DataPacket_Kind.RELIABLE);
}
private handleSocketOpen() {
console.log('[chat] socket opened');
const obj = {
Expand Down Expand Up @@ -678,6 +683,13 @@ export class WebRtcChatManager implements ChatManager {
if (data.type === 'pong') {
const elapsed_ms = performance.now() - data.timestamp;
console.debug(`[chat] worker RTT: ${elapsed_ms.toFixed(0)} ms`);
} else if (data.type === 'state') {
const newState = data.state;
this.changeState(newState);
} else if (data.type === 'transcript') {
console.log(`[chat] transcript: ${data.text}`);
} else if (data.type === 'output') {
console.log(`[chat] output: ${data.text}`);
}
}
}
Expand Down