Skip to content

Commit

Permalink
feat: volume change event support
Browse files Browse the repository at this point in the history
  • Loading branch information
jamsch committed Oct 5, 2024
1 parent 42f52a8 commit 80e8e1e
Show file tree
Hide file tree
Showing 14 changed files with 419 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ class ExpoSpeechRecognitionModule : Module() {
"start",
// Called when there's results (as a string array, not API compliant)
"results",
// Fired when the input volume changes
"volumechange",
)

Function("getDefaultRecognitionService") {
Expand Down Expand Up @@ -325,26 +327,32 @@ class ExpoSpeechRecognitionModule : Module() {
promise: Promise,
) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {
promise.resolve(mapOf(
"locales" to mutableListOf<String>(),
"installedLocales" to mutableListOf<String>(),
))
promise.resolve(
mapOf(
"locales" to mutableListOf<String>(),
"installedLocales" to mutableListOf<String>(),
),
)
return
}

if (options.androidRecognitionServicePackage == null && !SpeechRecognizer.isOnDeviceRecognitionAvailable(appContext)) {
promise.resolve(mapOf(
"locales" to mutableListOf<String>(),
"installedLocales" to mutableListOf<String>(),
))
promise.resolve(
mapOf(
"locales" to mutableListOf<String>(),
"installedLocales" to mutableListOf<String>(),
),
)
return
}

if (options.androidRecognitionServicePackage != null && !SpeechRecognizer.isRecognitionAvailable(appContext)) {
promise.resolve(mapOf(
"locales" to mutableListOf<String>(),
"installedLocales" to mutableListOf<String>(),
))
promise.resolve(
mapOf(
"locales" to mutableListOf<String>(),
"installedLocales" to mutableListOf<String>(),
),
)
return
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,17 @@ class SpeechRecognitionOptions : Record {

@Field
val iosCategory: Map<String, Any>? = null

@Field
val volumeChangeEventOptions: VolumeChangeEventOptions? = null
}

class VolumeChangeEventOptions : Record {
@Field
val enabled: Boolean? = false

@Field
val intervalMillis: Int? = null
}

class RecordingOptions : Record {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ class ExpoSpeechService(
private var speech: SpeechRecognizer? = null
private val mainHandler = Handler(Looper.getMainLooper())

private lateinit var options: SpeechRecognitionOptions
private var lastVolumeChangeEventTime: Long = 0L

/** Audio recorder for persisting audio */
private var audioRecorder: ExpoAudioRecorder? = null

Expand Down Expand Up @@ -108,6 +111,7 @@ class ExpoSpeechService(

/** Starts speech recognition */
fun start(options: SpeechRecognitionOptions) {
this.options = options
mainHandler.post {
log("Start recognition.")

Expand All @@ -119,6 +123,7 @@ class ExpoSpeechService(
delayedFileStreamer = null
recognitionState = RecognitionState.STARTING
soundState = SoundState.INACTIVE
lastVolumeChangeEventTime = 0L
try {
val intent = createSpeechIntent(options)
speech = createSpeechRecognizer(options)
Expand Down Expand Up @@ -454,6 +459,21 @@ class ExpoSpeechService(
}

override fun onRmsChanged(rmsdB: Float) {
if (options.volumeChangeEventOptions?.enabled != true) {
return
}

val intervalMs = options.volumeChangeEventOptions?.intervalMillis

if (intervalMs == null) {
sendEvent("volumechange", mapOf("rmsdB" to rmsdB))
} else {
val currentTime = System.currentTimeMillis()
if (currentTime - lastVolumeChangeEventTime >= intervalMs) {
sendEvent("volumechange", mapOf("rmsdB" to rmsdB))
lastVolumeChangeEventTime = currentTime
}
}
/*
val isSilent = rmsdB <= 0
Expand Down
22 changes: 21 additions & 1 deletion example/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import {
AndroidOutputFormat,
IOSOutputFormat,
} from "expo-av/build/Audio";
import { VolumeMeteringAvatar } from "./components/VolumeMeteringAvatar";

const speechRecognitionServices = getSpeechRecognitionServices();

Expand All @@ -72,6 +73,10 @@ export default function App() {
requiresOnDeviceRecognition: false,
addsPunctuation: true,
contextualStrings: ["Carlsen", "Ian Nepomniachtchi", "Praggnanandhaa"],
volumeChangeEventOptions: {
enabled: false,
intervalMillis: 300,
},
});

useSpeechRecognitionEvent("result", (ev) => {
Expand Down Expand Up @@ -140,6 +145,10 @@ export default function App() {
<SafeAreaView style={styles.container}>
<StatusBar style="dark" translucent={false} />

{settings.volumeChangeEventOptions?.enabled ? (
<VolumeMeteringAvatar />
) : null}

<View style={styles.card}>
<Text style={styles.text}>
{error ? JSON.stringify(error) : "Error messages go here"}
Expand Down Expand Up @@ -510,6 +519,17 @@ function GeneralSettings(props: {
checked={Boolean(settings.continuous)}
onPress={() => handleChange("continuous", !settings.continuous)}
/>

<CheckboxButton
title="Volume events"
checked={Boolean(settings.volumeChangeEventOptions?.enabled)}
onPress={() =>
handleChange("volumeChangeEventOptions", {
enabled: !settings.volumeChangeEventOptions?.enabled,
intervalMillis: settings.volumeChangeEventOptions?.intervalMillis,
})
}
/>
</View>

<View style={styles.textOptionContainer}>
Expand Down Expand Up @@ -714,7 +734,7 @@ function AndroidSettings(props: {
onPress={() =>
handleChange("androidIntentOptions", {
...settings.androidIntentOptions,
[key]: !settings.androidIntentOptions?.[key] ?? false,
[key]: !settings.androidIntentOptions?.[key],
})
}
/>
Expand Down
Binary file added example/assets/avatar.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
1 change: 1 addition & 0 deletions example/babel.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ module.exports = function (api) {
return {
presets: ["babel-preset-expo"],
plugins: [
"react-native-reanimated/plugin",
[
"module-resolver",
{
Expand Down
140 changes: 140 additions & 0 deletions example/components/VolumeMeteringAvatar.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
import { useSpeechRecognitionEvent } from "expo-speech-recognition";
import { Image, StyleSheet, View } from "react-native";
import Animated, {
Extrapolation,
interpolate,
useAnimatedStyle,
useSharedValue,
withTiming,
Easing,
withSpring,
withDelay,
withSequence,
} from "react-native-reanimated";
const avatar = require("../assets/avatar.png");

const minScale = 1;
const maxScale = 1.5;

/**
* This is an example component that uses the `volumechange` event to animate the volume metering of a user's voice.
*/
export function VolumeMeteringAvatar() {
const haloScale = useSharedValue(minScale);
const pulseScale = useSharedValue(minScale);
const pulseOpacity = useSharedValue(0);

const haloAnimatedStyle = useAnimatedStyle(() => ({
transform: [{ scale: haloScale.value }],
}));

const pulseAnimatedStyle = useAnimatedStyle(() => ({
opacity: pulseOpacity.value,
transform: [{ scale: pulseScale.value }],
}));

const reset = () => {
haloScale.value = minScale;
pulseScale.value = minScale;
pulseOpacity.value = 0;
};

useSpeechRecognitionEvent("start", reset);
useSpeechRecognitionEvent("end", reset);

useSpeechRecognitionEvent("volumechange", (event) => {
// Don't animate anything if the volume is too low
if (event.rmsdB <= 1) {
return;
}
const newScale = interpolate(
event.rmsdB,
[-2, 10], // The rmsDB range is between -2 and 10
[minScale, maxScale],
Extrapolation.CLAMP,
);

// Animate the halo scaling
haloScale.value = withSequence(
withSpring(newScale, {
damping: 15,
stiffness: 150,
}),
withTiming(minScale, {
duration: 1000,
easing: Easing.linear,
}),
);

// Animate the pulse (scale and fade out)
if (pulseScale.value < newScale) {
pulseScale.value = withSequence(
withTiming(maxScale, {
duration: 1000,
easing: Easing.out(Easing.quad),
}),
withTiming(minScale, {
duration: 400,
easing: Easing.linear,
}),
);
pulseOpacity.value = withSequence(
withTiming(1, { duration: 800 }),
withDelay(300, withTiming(0, { duration: 200 })),
);
}
});

return (
<View style={styles.container}>
<View style={styles.pulseContainer}>
<Animated.View style={[styles.pulse, pulseAnimatedStyle]} />
</View>
<View style={styles.pulseContainer}>
<Animated.View style={[styles.halo, haloAnimatedStyle]} />
</View>
<View style={[styles.centered]}>
<Image source={avatar} style={styles.avatar} />
</View>
</View>
);
}

const styles = StyleSheet.create({
container: {
position: "relative",
marginVertical: 20,
},
pulseContainer: {
position: "absolute",
top: 0,
bottom: 0,
left: 0,
right: 0,
justifyContent: "center",
alignItems: "center",
},
pulse: {
borderWidth: 1,
borderColor: "#539bf5",
width: 96,
height: 96,
borderRadius: 96,
},
halo: {
backgroundColor: "#6b7280",
width: 96,
height: 96,
borderRadius: 96,
},
centered: {
justifyContent: "center",
alignItems: "center",
},
avatar: {
width: 96,
height: 96,
borderRadius: 96,
overflow: "hidden",
},
});
29 changes: 27 additions & 2 deletions example/ios/Podfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ PODS:
- ReactCommon/turbomodule/bridging
- ReactCommon/turbomodule/core
- Yoga
- ExpoSpeechRecognition (0.2.17):
- ExpoSpeechRecognition (0.2.21):
- ExpoModulesCore
- EXSplashScreen (0.27.5):
- DoubleConversion
Expand Down Expand Up @@ -1226,6 +1226,27 @@ PODS:
- React-logger (= 0.74.5)
- React-perflogger (= 0.74.5)
- React-utils (= 0.74.5)
- RNReanimated (3.10.1):
- DoubleConversion
- glog
- hermes-engine
- RCT-Folly (= 2024.01.01.00)
- RCTRequired
- RCTTypeSafety
- React-Codegen
- React-Core
- React-debug
- React-Fabric
- React-featureflags
- React-graphics
- React-ImageManager
- React-NativeModulesApple
- React-RCTFabric
- React-rendererdebug
- React-utils
- ReactCommon/turbomodule/bridging
- ReactCommon/turbomodule/core
- Yoga
- SocketRocket (0.7.0)
- Yoga (0.0.0)

Expand Down Expand Up @@ -1295,6 +1316,7 @@ DEPENDENCIES:
- React-runtimescheduler (from `../node_modules/react-native/ReactCommon/react/renderer/runtimescheduler`)
- React-utils (from `../node_modules/react-native/ReactCommon/react/utils`)
- ReactCommon/turbomodule/core (from `../node_modules/react-native/ReactCommon`)
- RNReanimated (from `../node_modules/react-native-reanimated`)
- Yoga (from `../node_modules/react-native/ReactCommon/yoga`)

SPEC REPOS:
Expand Down Expand Up @@ -1429,6 +1451,8 @@ EXTERNAL SOURCES:
:path: "../node_modules/react-native/ReactCommon/react/utils"
ReactCommon:
:path: "../node_modules/react-native/ReactCommon"
RNReanimated:
:path: "../node_modules/react-native-reanimated"
Yoga:
:path: "../node_modules/react-native/ReactCommon/yoga"

Expand All @@ -1443,7 +1467,7 @@ SPEC CHECKSUMS:
ExpoFont: 00756e6c796d8f7ee8d211e29c8b619e75cbf238
ExpoKeepAwake: 3b8815d9dd1d419ee474df004021c69fdd316d08
ExpoModulesCore: a113755f96c40590671f01cfcdce8ebdf0cf5f83
ExpoSpeechRecognition: 66f2525786fd2fe299eb001e84b0176fd9c4252b
ExpoSpeechRecognition: 80eefd4f4bd7541f5fad24744b19a4b36a365414
EXSplashScreen: a7e8d13c476f9937e39d654af4235758b567a1be
FBLazyVector: ac12dc084d1c8ec4cc4d7b3cf1b0ebda6dab85af
fmt: 4c2741a687cc09f0634a2e2c72a838b99f1ff120
Expand Down Expand Up @@ -1496,6 +1520,7 @@ SPEC CHECKSUMS:
React-runtimescheduler: cfbe85c3510c541ec6dc815c7729b41304b67961
React-utils: f242eb7e7889419d979ca0e1c02ccc0ea6e43b29
ReactCommon: f7da14a8827b72704169a48c929bcde802698361
RNReanimated: 58a768c2c17a5589ef732fa6bd8d7ed0eb6df1c1
SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d
Yoga: 2246eea72aaf1b816a68a35e6e4b74563653ae09

Expand Down
Loading

0 comments on commit 80e8e1e

Please sign in to comment.