From 2036a6d9d5a596ae1b37f9def55416166da3fd2b Mon Sep 17 00:00:00 2001 From: Jean-Francois Arbour Date: Wed, 21 Feb 2024 02:55:09 -0500 Subject: [PATCH] refactor code into modules --- .../data => data}/laptop_en_linux_v3_0_0.ppn | Bin .../speech-command-cheetah-v1.pv | Bin {listener/data => data}/transcript_begin.wav | Bin .../data => data}/transcript_success.wav | Bin echo_crafter/config/__init__.py | 28 ++++ .../listener}/__init__.py | 0 .../listener/listener_with_wake_word.py | 40 ++++++ .../listener}/socket_read.py | 23 +--- echo_crafter/listener/utils/__init__.py | 7 + echo_crafter/listener/utils/porcupine.py | 88 ++++++++++++ echo_crafter/listener/utils/sockets.py | 12 ++ echo_crafter/logger/__init__.py | 45 +++++++ .../prompts}/__init__.py | 0 .../prompts}/examples/createlink.sh | 0 .../prompts}/examples/fzf_prompt_history.sh | 0 .../prompts}/examples/lastprompt.py | 0 .../prompts}/examples/make_shebang.sh | 0 .../prompts}/examples/prompt_history.py | 0 .../prompts}/prompt_elisp.py | 0 .../prompts}/prompt_python.py | 0 .../prompts}/prompt_question.py | 0 .../prompts}/prompt_shell.py | 0 listener/listener_with_wake_word.py | 126 ------------------ scripts/restart_daemons.py | 64 +++++++++ setup.py | 14 ++ 25 files changed, 304 insertions(+), 143 deletions(-) rename {listener/data => data}/laptop_en_linux_v3_0_0.ppn (100%) rename {listener/data => data}/speech-command-cheetah-v1.pv (100%) rename {listener/data => data}/transcript_begin.wav (100%) rename {listener/data => data}/transcript_success.wav (100%) create mode 100644 echo_crafter/config/__init__.py rename {listener => echo_crafter/listener}/__init__.py (100%) create mode 100644 echo_crafter/listener/listener_with_wake_word.py rename {listener => echo_crafter/listener}/socket_read.py (73%) create mode 100644 echo_crafter/listener/utils/__init__.py create mode 100644 echo_crafter/listener/utils/porcupine.py create mode 100644 echo_crafter/listener/utils/sockets.py create mode 100644 echo_crafter/logger/__init__.py rename {make-prompt => echo_crafter/prompts}/__init__.py (100%) rename {make-prompt => echo_crafter/prompts}/examples/createlink.sh (100%) rename {make-prompt => echo_crafter/prompts}/examples/fzf_prompt_history.sh (100%) rename {make-prompt => echo_crafter/prompts}/examples/lastprompt.py (100%) rename {make-prompt => echo_crafter/prompts}/examples/make_shebang.sh (100%) rename {make-prompt => echo_crafter/prompts}/examples/prompt_history.py (100%) rename {make-prompt => echo_crafter/prompts}/prompt_elisp.py (100%) rename {make-prompt => echo_crafter/prompts}/prompt_python.py (100%) rename {make-prompt => echo_crafter/prompts}/prompt_question.py (100%) rename {make-prompt => echo_crafter/prompts}/prompt_shell.py (100%) delete mode 100644 listener/listener_with_wake_word.py create mode 100755 scripts/restart_daemons.py create mode 100644 setup.py diff --git a/listener/data/laptop_en_linux_v3_0_0.ppn b/data/laptop_en_linux_v3_0_0.ppn similarity index 100% rename from listener/data/laptop_en_linux_v3_0_0.ppn rename to data/laptop_en_linux_v3_0_0.ppn diff --git a/listener/data/speech-command-cheetah-v1.pv b/data/speech-command-cheetah-v1.pv similarity index 100% rename from listener/data/speech-command-cheetah-v1.pv rename to data/speech-command-cheetah-v1.pv diff --git a/listener/data/transcript_begin.wav b/data/transcript_begin.wav similarity index 100% rename from listener/data/transcript_begin.wav rename to data/transcript_begin.wav diff --git a/listener/data/transcript_success.wav b/data/transcript_success.wav similarity index 100% rename from listener/data/transcript_success.wav rename to data/transcript_success.wav diff --git a/echo_crafter/config/__init__.py b/echo_crafter/config/__init__.py new file mode 100644 index 0000000..728aa0c --- /dev/null +++ b/echo_crafter/config/__init__.py @@ -0,0 +1,28 @@ +import os +from pathlib import Path +from typing import TypedDict + +PROJECT_ROOT=Path(os.getenv('EC_ROOT') or '') +DATA_DIR=Path(os.getenv('EC_DATA_DIR') or '') + +class _Config(TypedDict): + CHEETAH_MODEL_FILE: str + PICOVOICE_API_KEY: str + FRAME_LENGTH: int + ENDPOINT_DURATION_SEC: float + TRANSCRIPT_BEGIN_WAV: str + TRANSCRIPT_SUCCESS_WAV: str + SOCKET_PATH: str + + +Config: _Config = dict( + CHEETAH_MODEL_FILE=str(DATA_DIR/"speech-command-cheetah-v1.pv"), + PICOVOICE_API_KEY=os.environ.get('PICOVOICE_API_KEY', ''), + FRAME_LENGTH=512, + ENDPOINT_DURATION_SEC=1.3, + TRANSCRIPT_BEGIN_WAV=str(DATA_DIR/"transcript_begin.wav"), + TRANSCRIPT_SUCCESS_WAV=str(DATA_DIR/"transcript_success.wav"), + SOCKET_PATH=str(Path(os.getenv('EC_SOCKET_FILE') or '/tmp/echo-crafter.sock')) +) + +__all__ = ['Config'] diff --git a/listener/__init__.py b/echo_crafter/listener/__init__.py similarity index 100% rename from listener/__init__.py rename to echo_crafter/listener/__init__.py diff --git a/echo_crafter/listener/listener_with_wake_word.py b/echo_crafter/listener/listener_with_wake_word.py new file mode 100644 index 0000000..1693edf --- /dev/null +++ b/echo_crafter/listener/listener_with_wake_word.py @@ -0,0 +1,40 @@ +from echo_crafter.listener.utils import ( + socket_connection, + Microphone +) +from echo_crafter.logger import setup_logger +from echo_crafter.config import Config + +import subprocess +import traceback + + +def play_sound(wav_file): + """Play a ding sound to indicate that the wake word was detected.""" + subprocess.Popen(["aplay", "-q", str(wav_file)]) + + +def main(): + """Upon detection of a wake word, transcribe speech until endpoint is detected.""" + logger = setup_logger() + + with Microphone() as mic: + try: + while True: + mic.wait_for_wake_word() + play_sound(Config['TRANSCRIPT_BEGIN_WAV']) + + with socket_connection(Config['SOCKET_PATH']) as client: + mic.process_and_transmit_utterance(client) + play_sound(Config['TRANSCRIPT_SUCCESS_WAV']) + + except KeyboardInterrupt: + pass + + except Exception as e: + logger.error(f"An error occured:\n{e}") + logger.error(traceback.format_exc()) + + +if __name__ == '__main__': + main() diff --git a/listener/socket_read.py b/echo_crafter/listener/socket_read.py similarity index 73% rename from listener/socket_read.py rename to echo_crafter/listener/socket_read.py index 0893fb2..dcf5519 100644 --- a/listener/socket_read.py +++ b/echo_crafter/listener/socket_read.py @@ -1,26 +1,18 @@ -#!/usr/bin/env python3 +from echo_crafter.config import Config -import os from pathlib import Path import socket import subprocess -SOCKET_PATH = Path(os.getenv('XDG_RUNTIME_DIR')) / "transcription" -n_connections = 0 - - def handle_partial_transcript(partial_transcript): """Send partial transcript to the active window.""" - subprocess.run( + subprocess.Popen( ['xdotool', 'type', '--clearmodifiers', '--delay', '0', partial_transcript] ) -def handle_client(client_socket, client_address): +def handle_client(client_socket, _): """Handle a client connection.""" - global n_connections - n_connections += 1 - try: while True: partial_transcript = client_socket.recv(1024) @@ -35,11 +27,11 @@ def handle_client(client_socket, client_address): handle_partial_transcript(partial_transcript_s) finally: client_socket.close() - n_connections -= 1 def main(): """Listen for connections and handle them.""" + SOCKET_PATH = Path(Config['SOCKET_PATH']) try: SOCKET_PATH.unlink() except OSError: @@ -47,16 +39,13 @@ def main(): raise with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server_socket: - server_socket.bind(str(SOCKET_PATH)) + server_socket.bind(Config['SOCKET_PATH']) server_socket.listen(1) try: while True: client_socket, client_address = server_socket.accept() - if n_connections > 0: - client_socket.close() - else: - handle_client(client_socket, client_address) + handle_client(client_socket, client_address) finally: SOCKET_PATH.unlink() diff --git a/echo_crafter/listener/utils/__init__.py b/echo_crafter/listener/utils/__init__.py new file mode 100644 index 0000000..4678372 --- /dev/null +++ b/echo_crafter/listener/utils/__init__.py @@ -0,0 +1,7 @@ +from .porcupine import Microphone +from .sockets import socket_connection + +__all__ = [ + 'socket_connection', + 'Microphone' +] diff --git a/echo_crafter/listener/utils/porcupine.py b/echo_crafter/listener/utils/porcupine.py new file mode 100644 index 0000000..a75d412 --- /dev/null +++ b/echo_crafter/listener/utils/porcupine.py @@ -0,0 +1,88 @@ +from echo_crafter.config import Config + +from contextlib import contextmanager +import pvcheetah +import pvporcupine +import pvrecorder + +@contextmanager +def porcupine_context_manager(): + """Create a Porcupine instance and yield it. Delete the instance upon exit.""" + porcupine_instance = None + try: + porcupine_instance = pvporcupine.create( + keywords=['computer'], + sensitivities=[0.1], + access_key=Config['PICOVOICE_API_KEY'] + ) + yield porcupine_instance + finally: + if porcupine_instance is not None: + porcupine_instance.delete() + + +@contextmanager +def cheetah_context_manager(): + """Create a Cheetah instance and yield it. Delete the instance upon exit.""" + cheetah_instance = None + try: + cheetah_instance = pvcheetah.create( + access_key=Config['PICOVOICE_API_KEY'], + endpoint_duration_sec=Config['ENDPOINT_DURATION_SEC'], + model_path=Config['CHEETAH_MODEL_FILE'] + ) + yield cheetah_instance + finally: + if cheetah_instance is not None: + cheetah_instance.delete() + + +@contextmanager +def recorder_context_manager(): + """Create a PvRecorder instance and yield it. Delete the instance upon exit.""" + recorder_instance = None + try: + recorder_instance = pvrecorder.PvRecorder( + frame_length=Config['FRAME_LENGTH'] + ) + recorder_instance.start() + yield recorder_instance + finally: + if recorder_instance is not None: + recorder_instance.delete() + + +class _Microphone: + """A context manager for the recorder_context_manager.""" + + def __init__(self, recorder, porcupine, cheetah): + self.recorder = recorder + self.porcupine = porcupine + self.cheetah = cheetah + + def wait_for_wake_word(self): + """Wait for the wake word to be detected.""" + while True: + pcm_frame = self.recorder.read() + keyword_index = self.porcupine.process(pcm_frame) + if keyword_index >= 0: + break + + def process_and_transmit_utterance(self, client): + """Process the utterance and transmit the partial transcript to the client.""" + while True: + pcm_frame = self.recorder.read() + partial_transcript, is_endpoint = self.cheetah.process(pcm_frame) + client.sendall((partial_transcript).encode()) + if is_endpoint: + partial_transcript = self.cheetah.flush() + client.sendall((partial_transcript + 'STOP').encode()) + break + + +@contextmanager +def Microphone(): + """Create a Microphone instance and yield it. Delete the instance upon exit.""" + with recorder_context_manager() as recorder, porcupine_context_manager() as porcupine, cheetah_context_manager() as cheetah: + mic = _Microphone(recorder, porcupine, cheetah) + yield mic diff --git a/echo_crafter/listener/utils/sockets.py b/echo_crafter/listener/utils/sockets.py new file mode 100644 index 0000000..701f5ab --- /dev/null +++ b/echo_crafter/listener/utils/sockets.py @@ -0,0 +1,12 @@ +import socket +import contextlib + +# Define a context manager for the socket connection +@contextlib.contextmanager +def socket_connection(socket_path): + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + client.connect(socket_path) + yield client + finally: + client.close() diff --git a/echo_crafter/logger/__init__.py b/echo_crafter/logger/__init__.py new file mode 100644 index 0000000..6a1dd6e --- /dev/null +++ b/echo_crafter/logger/__init__.py @@ -0,0 +1,45 @@ +"""Logging module for the Echo Crafter application.""" + +import logging +import json +import time +import os +from pathlib import Path + +LOG_FILE = Path(os.environ.get('EC_LOG_DIR', '')) / "transcripts.jsonl" + +class CustomRecord(logging.LogRecord): + """Custom LogRecord class with a timestamp attribute.""" + + def __init__(self, name, level, pathname, lineno, msg, args, + exc_info, func=None, sinfo=None, **kwargs): + """Initialize a CustomRecord instance.""" + super().__init__(name, level, pathname, lineno, msg, args, + exc_info, func=func, sinfo=sinfo) + self.timestamp = time.time() + self.intent = kwargs.get('intent', '') + self.slots = kwargs.get('slots', {}) + + +class JsonFormatter(logging.Formatter): + """JSON formatter for log records.""" + + def format(self, record): + """Format a log record as a JSON string.""" + log_dict = record.__dict__.copy() + log_dict['msg'] = record.getMessage() + return json.dumps(log_dict) + + +def setup_logger(name=__name__, level=logging.INFO): + """Set up a logger with a JSON formatter.""" + logger = logging.getLogger(name) + logger.setLevel(level) + logging.setLogRecordFactory(CustomRecord) + handler = logging.FileHandler(LOG_FILE, encoding='utf-8') + handler.setFormatter(JsonFormatter()) + logger.addHandler(handler) + return logger + + +__all__ = ['setup_logger'] diff --git a/make-prompt/__init__.py b/echo_crafter/prompts/__init__.py similarity index 100% rename from make-prompt/__init__.py rename to echo_crafter/prompts/__init__.py diff --git a/make-prompt/examples/createlink.sh b/echo_crafter/prompts/examples/createlink.sh similarity index 100% rename from make-prompt/examples/createlink.sh rename to echo_crafter/prompts/examples/createlink.sh diff --git a/make-prompt/examples/fzf_prompt_history.sh b/echo_crafter/prompts/examples/fzf_prompt_history.sh similarity index 100% rename from make-prompt/examples/fzf_prompt_history.sh rename to echo_crafter/prompts/examples/fzf_prompt_history.sh diff --git a/make-prompt/examples/lastprompt.py b/echo_crafter/prompts/examples/lastprompt.py similarity index 100% rename from make-prompt/examples/lastprompt.py rename to echo_crafter/prompts/examples/lastprompt.py diff --git a/make-prompt/examples/make_shebang.sh b/echo_crafter/prompts/examples/make_shebang.sh similarity index 100% rename from make-prompt/examples/make_shebang.sh rename to echo_crafter/prompts/examples/make_shebang.sh diff --git a/make-prompt/examples/prompt_history.py b/echo_crafter/prompts/examples/prompt_history.py similarity index 100% rename from make-prompt/examples/prompt_history.py rename to echo_crafter/prompts/examples/prompt_history.py diff --git a/make-prompt/prompt_elisp.py b/echo_crafter/prompts/prompt_elisp.py similarity index 100% rename from make-prompt/prompt_elisp.py rename to echo_crafter/prompts/prompt_elisp.py diff --git a/make-prompt/prompt_python.py b/echo_crafter/prompts/prompt_python.py similarity index 100% rename from make-prompt/prompt_python.py rename to echo_crafter/prompts/prompt_python.py diff --git a/make-prompt/prompt_question.py b/echo_crafter/prompts/prompt_question.py similarity index 100% rename from make-prompt/prompt_question.py rename to echo_crafter/prompts/prompt_question.py diff --git a/make-prompt/prompt_shell.py b/echo_crafter/prompts/prompt_shell.py similarity index 100% rename from make-prompt/prompt_shell.py rename to echo_crafter/prompts/prompt_shell.py diff --git a/listener/listener_with_wake_word.py b/listener/listener_with_wake_word.py deleted file mode 100644 index 20944e8..0000000 --- a/listener/listener_with_wake_word.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 - -from contextlib import contextmanager -import os -from pathlib import Path -import pvcheetah -import pvporcupine -import pvrecorder -import socket -import subprocess -import sys - -PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT')) -CHEETAH_MODEL_FILE = PROJECT_ROOT / "listener/data/speech-command-cheetah-v1.pv" -PORCUPINE_LAPTOP_KEYWORD_FILE = PROJECT_ROOT / "listener/data/laptop_en_linux_v3_0_0.ppn" -TRANSCRIPT_BEGIN_WAV = PROJECT_ROOT / "listener/data/transcript_begin.wav" -TRANSCRIPT_SUCCESS_WAV = PROJECT_ROOT / "listener/data/transcript_success.wav" -SOCKET_PATH = Path(os.getenv('XDG_RUNTIME_DIR')) / "transcription" - - -@contextmanager -def porcupine_context_manager(keyword_paths, sensitivities): - """Create a Porcupine instance and yield it. Delete the instance upon exit.""" - porcupine_instance = None - try: - porcupine_instance = pvporcupine.create( - keyword_paths=keyword_paths, - sensitivities=sensitivities, - access_key=os.getenv('PICOVOICE_API_KEY') - ) - yield porcupine_instance - finally: - if porcupine_instance: - porcupine_instance.delete() - - -@contextmanager -def cheetah_context_manager(): - """Create a Cheetah instance and yield it. Delete the instance upon exit.""" - cheetah_instance = None - try: - cheetah_instance = pvcheetah.create( - access_key=os.getenv('PICOVOICE_API_KEY'), - endpoint_duration_sec=1.3, - model_path=str(CHEETAH_MODEL_FILE) - ) - yield cheetah_instance - finally: - if cheetah_instance: - cheetah_instance.delete() - - -@contextmanager -def recorder_context_manager(device_index, frame_length): - """Create a PvRecorder instance and yield it. Delete the instance upon exit.""" - recorder_instance = None - try: - recorder_instance = pvrecorder.PvRecorder( - device_index=device_index, - frame_length=frame_length - ) - recorder_instance.start() - yield recorder_instance - finally: - if recorder_instance: - recorder_instance.stop() - recorder_instance.delete() - - -def play_sound(wav_file): - """Play a ding sound to indicate that the wake word was detected.""" - subprocess.Popen(["aplay", str(wav_file)]) - - -def main(): - """Upon detection of a wake word, transcribe speech until endpoint is detected.""" - keywords = ["computer"] - keyword_paths = [PORCUPINE_LAPTOP_KEYWORD_FILE] - keyword_paths.extend(pvporcupine.KEYWORD_PATHS[w] for w in keywords) - keywords = [Path(w).stem.split('_')[0] for w in keyword_paths] - sensitivities = [0.1] * len(keyword_paths) - device_index = -1 - frame_length = 512 - client = None - - try: - with porcupine_context_manager(keyword_paths, sensitivities) as porcupine, \ - cheetah_context_manager() as cheetah, \ - recorder_context_manager(device_index, frame_length) as recorder: - - wake_word_detected = False - - while True: - pcm_frame = recorder.read() - - if not wake_word_detected: - keyword_index = porcupine.process(pcm_frame) - if keyword_index >= 0: - wake_word_detected = True - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.connect(str(SOCKET_PATH)) - play_sound(TRANSCRIPT_BEGIN_WAV) - - else: - partial_transcript, is_endpoint = cheetah.process(pcm_frame) - client.sendall((partial_transcript).encode()) - if is_endpoint: - partial_transcript = cheetah.flush() - client.sendall((partial_transcript + 'STOP').encode()) - client.close() - client = None - wake_word_detected = False - play_sound(TRANSCRIPT_SUCCESS_WAV) - - except KeyboardInterrupt: - pass - except Exception as e: - print(f"An error occured: {e}", file=sys.stderr) - finally: - if client is not None: - client.sendall(b'STOP') - client.close() - - -if __name__ == '__main__': - main() diff --git a/scripts/restart_daemons.py b/scripts/restart_daemons.py new file mode 100755 index 0000000..0570839 --- /dev/null +++ b/scripts/restart_daemons.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +"""This script is used to restart the listener and socket_read processes.""" + +from pathlib import Path +from psutil import Process, process_iter, NoSuchProcess, AccessDenied, ZombieProcess +from shlex import join as shlex_join +from typing import List, Sequence +import subprocess + +def search_processes(search_strings: Sequence[str]) -> List[Process]: + """ + Search for processes that contain the `search_string` in their command line. + + This is a python equivalent of running `pgrep -f SEARCH_STRING` from the shell. + + :param search_string: The string to search for in the command line of the processes. + """ + matching_processes = [] + for proc in process_iter(attrs=['cmdline']): + try: + # The `info` attribute is not part of the `psutil.Process` class, but it is added within the `process_iter` function. + cmdline = shlex_join(proc.info['cmdline']) # type: ignore + if any(search_string in cmdline for search_string in search_strings): + matching_processes.append(proc) + except (NoSuchProcess, AccessDenied, ZombieProcess, TypeError): + pass + + return matching_processes + +def get_corresponding_filename(process: Process) -> str: + """ + Get the filename of the process. + + This is a python equivalent of running `readlink /proc/PID/exe` from the shell. + + :param process: The process to get the filename of. + """ + try: + return process.info['cmdline'][1].split('/')[-1] # type: ignore + except (NoSuchProcess, AccessDenied, ZombieProcess): + return '' + +def main(): + """Restart the listener and socket_read processes.""" + matching_processes = search_processes(('listener_with_wake_word', 'socket_read')) + matching_processes_filenames = [get_corresponding_filename(proc) for proc in matching_processes] + + for proc, filename in zip(matching_processes, matching_processes_filenames): + try: + proc.terminate() + except NoSuchProcess: + pass + except AccessDenied: + print(f"No permission to terminate process with PID {proc.pid}.") + print(f"Terminated {filename}...") + + for filename in ('listener_with_wake_word', 'socket_read'): + subprocess.Popen(['python', f'echo_crafter/listener/{filename}.py'], cwd=Path(__file__).parent.parent) # type: ignore + print(f"Started {filename}.py...") + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7feab56 --- /dev/null +++ b/setup.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +from setuptools import setup, find_packages + +setup( + name='echo-crafter', + version='0.1.0', + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'microphone_listen = echo_crafter.listener.listener_with_wake_word:main', + 'transcripts_collect = echo_crafter.listener.socket_read:main' + ] + } +)