diff --git a/.gitignore b/.gitignore index 93b96e5..2faf15e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,8 @@ config/ logs/ package*.json src/main/bundles -src/main/frontend +src/main/frontend/generated +src/main/frontend/index.html tsconfig.json types.d.ts node_modules diff --git a/src/main/frontend/flow/audio-recorder.js b/src/main/frontend/flow/audio-recorder.js new file mode 100644 index 0000000..c918dcf --- /dev/null +++ b/src/main/frontend/flow/audio-recorder.js @@ -0,0 +1,63 @@ +window.audioRecorder = { + mediaRecorder: null, + audioChunks: [], + + startRecording: async function() { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + this.mediaRecorder = new MediaRecorder(stream); + this.audioChunks = []; + + this.mediaRecorder.ondataavailable = (event) => { + if (event.data.size > 0) { + this.audioChunks.push(event.data); + } + }; + + this.mediaRecorder.start(); + } catch (error) { + console.error('Error accessing microphone:', error); + } + }, + + stopRecording: async function() { + return new Promise((resolve, reject) => { + if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') { + this.mediaRecorder.stop(); + this.mediaRecorder.onstop = async () => { + try { + const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' }); + const base64Data = await this.blobToBase64(audioBlob); + + // Stop all tracks + this.mediaRecorder.stream.getTracks().forEach(track => track.stop()); + + resolve({ audioData: base64Data }); + } catch (error) { + reject(error); + } + }; + } else { + reject(new Error('No recording in progress')); + } + }); + }, + + blobToBase64: function(blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => { + const base64String = reader.result + .replace('data:audio/webm;base64,', ''); + resolve(base64String); + }; + reader.onerror = reject; + reader.readAsDataURL(blob); + }); + }, + + playAudioResponse: function(base64Audio) { + const audio = new Audio(`data:audio/wav;base64,${base64Audio}`); + audio.play(); + } +}; \ No newline at end of file diff --git a/src/main/java/org/cftoolsuite/client/ProfilesClient.java b/src/main/java/org/cftoolsuite/client/ProfilesClient.java new file mode 100644 index 0000000..d34795f --- /dev/null +++ b/src/main/java/org/cftoolsuite/client/ProfilesClient.java @@ -0,0 +1,55 @@ +package org.cftoolsuite.client; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClient; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +@Component +public class ProfilesClient { + + private static final Logger log = LoggerFactory.getLogger(ProfilesClient.class); + + private final RestClient client; + + public ProfilesClient(@Value("${document.service.url}") String sanfordUrl) { + client = RestClient.builder() + .baseUrl(sanfordUrl) + .build(); + } + + public Set getProfiles() { + Set result = new HashSet<>(); + ResponseEntity> response = + client + .get() + .uri("/actuator/info") + .retrieve() + .toEntity(new ParameterizedTypeReference>() {}); + if (response.getStatusCode().is2xxSuccessful()) { + Map body = response.getBody(); + if (body != null && body.containsKey("active-profiles")) { + String profiles = (String) body.get("active-profiles"); + result = Stream.of(profiles.trim().split(",")) + .collect(Collectors.toSet()); + } else { + log.warn("Could not determine active profiles."); + } + } + return result; + } + + public boolean contains(String profile) { + return getProfiles().contains(profile); + } +} diff --git a/src/main/java/org/cftoolsuite/client/SanfordClient.java b/src/main/java/org/cftoolsuite/client/SanfordClient.java index 2ba209b..bc98bc6 100644 --- a/src/main/java/org/cftoolsuite/client/SanfordClient.java +++ b/src/main/java/org/cftoolsuite/client/SanfordClient.java @@ -1,8 +1,10 @@ package org.cftoolsuite.client; +import java.io.IOException; import java.util.List; import org.cftoolsuite.domain.FileMetadata; +import org.cftoolsuite.domain.chat.AudioResponse; import org.cftoolsuite.domain.chat.Inquiry; import org.cftoolsuite.domain.crawl.CrawlRequest; import org.cftoolsuite.domain.crawl.CrawlResponse; @@ -33,6 +35,9 @@ public interface SanfordClient { @PostMapping("/api/fetch") public ResponseEntity fetchUrls(@RequestBody FetchRequest request); + @PostMapping("/api/converse") + public ResponseEntity converse(@RequestParam("file") MultipartFile file) throws IOException; + @PostMapping("/api/chat") public ResponseEntity chat(@RequestBody Inquiry inquiry); diff --git a/src/main/java/org/cftoolsuite/domain/chat/AudioResponse.java b/src/main/java/org/cftoolsuite/domain/chat/AudioResponse.java new file mode 100644 index 0000000..4ef32f0 --- /dev/null +++ b/src/main/java/org/cftoolsuite/domain/chat/AudioResponse.java @@ -0,0 +1,9 @@ +package org.cftoolsuite.domain.chat; + +import java.util.Base64; + +public record AudioResponse(String text, String audioBase64) { + public AudioResponse(String text, byte[] audio) { + this(text, Base64.getEncoder().encodeToString(audio)); + } +} diff --git a/src/main/java/org/cftoolsuite/ui/MainLayout.java b/src/main/java/org/cftoolsuite/ui/MainLayout.java index 01c5f5d..47c7a5f 100644 --- a/src/main/java/org/cftoolsuite/ui/MainLayout.java +++ b/src/main/java/org/cftoolsuite/ui/MainLayout.java @@ -1,37 +1,26 @@ package org.cftoolsuite.ui; -import com.vaadin.flow.component.html.Image; -import com.vaadin.flow.server.StreamResource; -import org.cftoolsuite.ui.view.ChatView; -import org.cftoolsuite.ui.view.CrawlView; -import org.cftoolsuite.ui.view.DeleteView; -import org.cftoolsuite.ui.view.DownloadView; -import org.cftoolsuite.ui.view.FetchView; -import org.cftoolsuite.ui.view.HomeView; -import org.cftoolsuite.ui.view.ListView; -import org.cftoolsuite.ui.view.SearchView; -import org.cftoolsuite.ui.view.SummarizeView; -import org.cftoolsuite.ui.view.UploadView; - import com.vaadin.flow.component.Component; -import com.vaadin.flow.component.accordion.Accordion; import com.vaadin.flow.component.applayout.AppLayout; import com.vaadin.flow.component.applayout.DrawerToggle; -import com.vaadin.flow.component.details.DetailsVariant; import com.vaadin.flow.component.html.Div; +import com.vaadin.flow.component.html.Image; import com.vaadin.flow.component.html.Span; import com.vaadin.flow.component.icon.Icon; import com.vaadin.flow.component.icon.VaadinIcon; import com.vaadin.flow.component.tabs.Tab; import com.vaadin.flow.component.tabs.Tabs; import com.vaadin.flow.router.RouterLink; +import com.vaadin.flow.server.StreamResource; +import org.cftoolsuite.client.ProfilesClient; +import org.cftoolsuite.ui.view.*; public class MainLayout extends AppLayout { private static final long serialVersionUID = 1L; - public MainLayout() { + public MainLayout(ProfilesClient modeClient) { Tab homeTab = createTab(VaadinIcon.HOME.create(), "Home", HomeView.class); Tabs actionTabs = createTabs(); @@ -39,13 +28,18 @@ public MainLayout() { Tab uploadTab = createTab(VaadinIcon.UPLOAD.create(), "Upload documents", UploadView.class); Tab crawlTab = createTab(VaadinIcon.SITEMAP.create(), "Crawl websites for documents", CrawlView.class); Tab fetchTab = createTab(VaadinIcon.CROSSHAIRS.create(), "Fetch documents", FetchView.class); + Tab converseTab = createTab(VaadinIcon.MEGAPHONE.create(), "Converse with AI bot about documents", ConverseView.class); Tab chatTab = createTab(VaadinIcon.CHAT.create(), "Chat with AI bot about documents", ChatView.class); Tab listTab = createTab(VaadinIcon.LIST.create(), "List document metadata", ListView.class); Tab searchTab = createTab(VaadinIcon.SEARCH.create(), "Search for document metadata", SearchView.class); Tab summaryTab = createTab(VaadinIcon.BULLETS.create(), "Summarize a document", SummarizeView.class); Tab downloadTab = createTab(VaadinIcon.DOWNLOAD.create(), "Download a document", DownloadView.class); Tab deleteTab = createTab(VaadinIcon.TRASH.create(), "Delete a document", DeleteView.class); - actionTabs.add(uploadTab, crawlTab, fetchTab, chatTab, listTab, searchTab, summaryTab, downloadTab, deleteTab); + actionTabs.add(uploadTab, crawlTab, fetchTab); + if (modeClient.contains("openai")) { + actionTabs.add(converseTab); + } + actionTabs.add(chatTab, listTab, searchTab, summaryTab, downloadTab, deleteTab); addToNavbar(true, homeTab, new DrawerToggle()); addToDrawer(getLogoImage(), actionTabs); } diff --git a/src/main/java/org/cftoolsuite/ui/view/BaseView.java b/src/main/java/org/cftoolsuite/ui/view/BaseView.java index dec112f..4ca326f 100644 --- a/src/main/java/org/cftoolsuite/ui/view/BaseView.java +++ b/src/main/java/org/cftoolsuite/ui/view/BaseView.java @@ -26,7 +26,7 @@ public BaseView(SanfordClient sanfordClient, AppProperties appProperties) { protected abstract void setupUI(); - protected abstract void clearAllFields(); + protected void clearAllFields() {} protected void showNotification(String message, NotificationVariant variant) { Notification notification = new Notification(message, 5000, Position.TOP_STRETCH); diff --git a/src/main/java/org/cftoolsuite/ui/view/ConverseView.java b/src/main/java/org/cftoolsuite/ui/view/ConverseView.java new file mode 100644 index 0000000..a8b37d0 --- /dev/null +++ b/src/main/java/org/cftoolsuite/ui/view/ConverseView.java @@ -0,0 +1,184 @@ +package org.cftoolsuite.ui.view; + +import com.vaadin.flow.component.UI; +import com.vaadin.flow.component.button.Button; +import com.vaadin.flow.component.dependency.JavaScript; +import com.vaadin.flow.component.html.Div; +import com.vaadin.flow.component.html.H2; +import com.vaadin.flow.component.html.Span; +import com.vaadin.flow.component.icon.VaadinIcon; +import com.vaadin.flow.component.notification.NotificationVariant; +import com.vaadin.flow.component.page.PendingJavaScriptResult; +import com.vaadin.flow.router.PageTitle; +import com.vaadin.flow.router.Route; +import elemental.json.JsonObject; +import org.apache.commons.io.FileUtils; +import org.cftoolsuite.client.SanfordClient; +import org.cftoolsuite.domain.AppProperties; +import org.cftoolsuite.domain.CustomMultipartFile; +import org.cftoolsuite.ui.MainLayout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.util.Base64; + +@PageTitle("sanford-ui » Converse") +@Route(value = "converse", layout = MainLayout.class) +@JavaScript("./flow/audio-recorder.js") +public class ConverseView extends BaseView { + + private static final Logger log = LoggerFactory.getLogger(ConverseView.class); + + private boolean isRecording = false; + private Button recordButton; + private Div responseDiv; + private Div loadingDiv; + + public ConverseView(SanfordClient sanfordClient, AppProperties appProperties) { + super(sanfordClient, appProperties); + } + + @Override + protected void setupUI() { + addClassName("box-border"); + setAlignItems(Alignment.CENTER); + setSpacing(true); + setPadding(true); + + setupRobotEmoji(); + setupTitle(); + setupMainContainer(); + } + + private void setupRobotEmoji() { + Span robotEmoji = new Span("🤖"); + robotEmoji.getStyle().set("font-size", "100px"); + add(robotEmoji); + } + + private void setupTitle() { + H2 title = new H2("Converse"); + add(title); + } + + private void setupMainContainer() { + Div container = new Div(); + container.addClassNames("max-w-screen-sm", "w-full", "space-y-4"); + container.getStyle() + .set("display", "flex") + .set("flex-direction", "column") + .set("gap", "var(--lumo-space-l)"); + + setupRecordButton(); + setupLoadingIndicator(); + setupResponseDiv(); + + Div buttonContainer = new Div(recordButton); + buttonContainer.getStyle() + .set("display", "flex") + .set("justify-content", "center") + .set("gap", "var(--lumo-space-m)"); + + container.add(buttonContainer, loadingDiv, responseDiv); + add(container); + } + + private void setupRecordButton() { + recordButton = new Button(); + updateRecordButtonState(); + recordButton.addClickListener(e -> toggleRecording()); + } + + private void setupLoadingIndicator() { + loadingDiv = new Div(); + loadingDiv.setText("Hold on, I'm thinking..."); + loadingDiv.getStyle() + .set("display", "none") + .set("justify-content", "center"); + } + + private void setupResponseDiv() { + responseDiv = new Div(); + responseDiv.getStyle().set("display", "none"); + } + + private void updateRecordButtonState() { + recordButton.setIcon(isRecording ? VaadinIcon.STOP.create() : VaadinIcon.MICROPHONE.create()); + recordButton.setText(isRecording ? "Stop Recording" : "Start Recording"); + recordButton.setThemeName(isRecording ? "error" : "primary"); + } + + private void toggleRecording() { + isRecording = !isRecording; + updateRecordButtonState(); + + if (isRecording) { + getUI().ifPresent(ui -> + ui.getPage().executeJs("return window.audioRecorder.startRecording()")); + } else { + getUI().ifPresent(ui -> { + PendingJavaScriptResult result = ui.getPage() + .executeJs("return window.audioRecorder.stopRecording()"); + result.then(JsonObject.class, this::handleRecordingResult); + }); + } + } + + private void handleRecordingResult(JsonObject result) { + try { + String base64Audio = result.getString("audioData"); + byte[] audioData = Base64.getDecoder().decode(base64Audio); + + // Create temporary file + File tempFile = File.createTempFile("audio-", ".webm"); + FileUtils.writeByteArrayToFile(tempFile, audioData); + + // Convert to MultipartFile + MultipartFile multipartFile = new CustomMultipartFile( + "file", + "recording.webm", + "audio/webm", + FileUtils.readFileToByteArray(tempFile) + ); + + // Show loading indicator + UI.getCurrent().access(() -> loadingDiv.getStyle().set("display", "flex")); + + // Make API call using Feign client + var response = sanfordClient.converse(multipartFile); + if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { + // Update UI with response + UI.getCurrent().access(() -> { + loadingDiv.getStyle().set("display", "none"); + responseDiv.getStyle().set("display", "block"); + + if (response.getBody() != null) { + responseDiv.setText(response.getBody().text()); + + // Play audio response using our JavaScript function + UI.getCurrent().getPage().executeJs( + "window.audioRecorder.playAudioResponse($0)", response.getBody().audioBase64() + ); + } + }); + } else { + UI.getCurrent().access(() -> { + loadingDiv.getStyle().set("display", "none"); + }); + showNotification("Error conversing with chatbot", NotificationVariant.LUMO_ERROR); + } + + // Cleanup temp file + tempFile.delete(); + + } catch (Exception e) { + log.error("Error conversing with chatbot", e); + UI.getCurrent().access(() -> { + loadingDiv.getStyle().set("display", "none"); + }); + showNotification("Error conversing with chatbot: " + e.getMessage(), NotificationVariant.LUMO_ERROR); + } + } +}