Added elevenlabs commands /v11f - female /v11m - male

src/config.ts

@@ -162,5 +162,6 @@ export default {
     index: process.env.ES_INDEX
   },
   deepL: { apikey: process.env.DEEPL_API_KEY ?? '' },
-  gc: { credentials: process.env.GC_CREDENTIALS ?? '' }
+  gc: { credentials: process.env.GC_CREDENTIALS ?? '' },
+  elevenlabs: { apiKey: process.env.ELEVENLABS_API_KEY ?? '' }
 }

src/elevenlabs/elevenlabsClient.ts

@@ -1,5 +1,37 @@
 import axios, { type AxiosInstance } from 'axios'
 
+interface Voice {
+  'voice_id': string
+  'name': string
+  'samples': null
+  'category': 'premade'
+  'fine_tuning': {
+    'language': null
+    'is_allowed_to_fine_tune': boolean
+    'fine_tuning_requested': boolean
+    'finetuning_state': 'not_started'
+    'verification_attempts': null
+    'verification_failures': []
+    'verification_attempts_count': 0
+    'slice_ids': null
+    'manual_verification': null
+    'manual_verification_requested': false
+  }
+  'labels': {
+    'accent': 'american' | string
+    'description': 'strong' | string
+    'age': 'young' | string
+    'gender': 'female' | string
+    'use case': 'narration' | string
+  }
+  'description': null
+  'preview_url': string
+  'available_for_tiers': []
+  'settings': null
+  'sharing': null
+  'high_quality_base_model_ids': []
+}
+
 export class ElevenlabsClient {
   private readonly _token: string
   private readonly _httpClient: AxiosInstance
@@ -17,13 +49,20 @@ export class ElevenlabsClient {
   }
 
   public async textToSpeech ({ text, voiceId }: { text: string, voiceId: string }): Promise<string | Uint8Array | null | undefined> {
-    return await this._httpClient.post(`/v1/text-to-speech/${voiceId}`, {
-      text: 'string',
-      model_id: 'eleven_monolingual_v1',
+    const response = await this._httpClient.post(`/v1/text-to-speech/${voiceId}`, {
+      text,
+      model_id: 'eleven_multilingual_v2',
       voice_settings: {
         stability: 0.5,
         similarity_boost: 0.5
       }
-    })
+    }, { responseType: 'arraybuffer' })
+
+    return Buffer.from(response.data, 'binary')
+  }
+
+  public async voiceList (): Promise<Voice[]> {
+    const response = await this._httpClient.get<{ voices: Voice[] }>('/v1/voices')
+    return response.data.voices
   }
 }

src/elevenlabs/sandbox.ts

@@ -0,0 +1,19 @@
+import { ElevenlabsClient } from './elevenlabsClient'
+import config from '../config'
+
+function labelsToString (labels: Record<string, string>): string {
+  return Object.entries(labels).reduce((acc, item) => {
+    return acc + item.join(': ') + '; '
+  }, '')
+}
+
+async function main (): Promise<void> {
+  const client = new ElevenlabsClient(config.elevenlabs.apiKey)
+  const voiceList = await client.voiceList()
+
+  for (const voice of voiceList) {
+    console.log(voice.voice_id, voice.name, '\t', labelsToString(voice.labels))
+  }
+}
+
+main().then(() => { console.log('### finish') }).catch(console.log)

src/modules/text-to-speech/index.ts

@@ -5,15 +5,13 @@ import type { BotPayments } from '../payment'
 import type { OnMessageContext, PayableBot } from '../types'
 import { gcTextToSpeedClient, type TextToSpeechParams } from '../../google-cloud/gcTextToSpeechClient'
 import { getCommandList, getConfigByCommand } from './commandConfigList'
+import { ElevenlabsClient } from '../../elevenlabs/elevenlabsClient'
+import config from '../../config'
 
 enum SupportedCommands {
   VOICE = 'voice',
-  VOICEHK = 'voicehk',
-  VOICEHKF = 'voicehkf',
-  VOICERU = 'voiceru',
-  VOICECN = 'voicecn',
-  VOICEES = 'voicees'
-
+  V11M = 'v11m',
+  V11F = 'v11f',
 }
 
 export class TextToSpeechBot implements PayableBot {
@@ -56,6 +54,20 @@ export class TextToSpeechBot implements PayableBot {
       return
     }
 
+    if (ctx.hasCommand(SupportedCommands.V11M)) {
+      const text = this.getTextFromMessage(ctx)
+      // Liam accent: american; age: young; gender: male; use case: narration; description : neutral;
+      await this.onTextToSpeech11Labs(ctx, { text, voiceId: 'TX3LPaxmHKxFdv7VOQHJ' })
+      return
+    }
+
+    if (ctx.hasCommand(SupportedCommands.V11F)) {
+      const text = this.getTextFromMessage(ctx)
+      // Rachel accent: american; description: calm; age: young; gender: female; use case: narration;
+      await this.onTextToSpeech11Labs(ctx, { text, voiceId: '21m00Tcm4TlvDq8ikWAM' })
+      return
+    }
+
     // if (ctx.hasCommand(SupportedCommands.VOICEHK)) {
     //   const text = this.getTextFromMessage(ctx)
     //   await this.onTextToSpeech(ctx, { text, ssmlGender: 'MALE', languageCode: 'yue-Hant-HK' })
@@ -146,4 +158,33 @@ export class TextToSpeechBot implements PayableBot {
     await ctx.api.deleteMessage(ctx.chat.id, progressMessage.message_id)
     await ctx.replyWithVoice(inputFile)
   }
+
+  public async onTextToSpeech11Labs (ctx: OnMessageContext, params: { text: string, voiceId: string }): Promise<void> {
+    const { text, voiceId } = params
+
+    if (!params.text) {
+      await ctx.reply('/voice command should contain text.')
+      return
+    }
+
+    if (!ctx.chat?.id) {
+      throw new Error('Internal error')
+    }
+
+    const progressMessage = await ctx.reply('Generating...')
+
+    const elevenlabsClient = new ElevenlabsClient(config.elevenlabs.apiKey)
+
+    const voiceResult = await elevenlabsClient.textToSpeech({ text, voiceId })
+
+    if (!voiceResult) {
+      await ctx.api.editMessageText(ctx.chat.id, progressMessage.message_id, 'An error occurred during the process of generating the message.')
+      return
+    }
+
+    const inputFile = new InputFile(voiceResult)
+
+    await ctx.api.deleteMessage(ctx.chat.id, progressMessage.message_id)
+    await ctx.replyWithVoice(inputFile)
+  }
 }