diff --git a/app/widget/src/App.tsx b/app/widget/src/App.tsx index beddcf77..53281236 100644 --- a/app/widget/src/App.tsx +++ b/app/widget/src/App.tsx @@ -50,7 +50,12 @@ function App() { } else { setMessages([ ...messages, - { isBot: true, message: botStyle.data.first_message, sources: [] }, + { + isBot: true, + message: botStyle.data.first_message, + sources: [], + id: "first-message", + }, ]); } } diff --git a/app/widget/src/components/BotChatBubble.tsx b/app/widget/src/components/BotChatBubble.tsx index c9596f7a..619f1bdf 100644 --- a/app/widget/src/components/BotChatBubble.tsx +++ b/app/widget/src/components/BotChatBubble.tsx @@ -1,7 +1,9 @@ +import { PlayIcon, StopIcon } from "@heroicons/react/20/solid"; import { Message } from "../store"; import { BotStyle } from "../utils/types"; import BotSource from "./BotSource"; import Markdown from "./Markdown"; +import { useVoice } from "../hooks/useVoice"; export default function BotChatBubble({ message, @@ -10,6 +12,7 @@ export default function BotChatBubble({ message: Message; botStyle: BotStyle; }) { + const { cancel, isPlaying, loading, speak } = useVoice(); return (

+ {botStyle.data.tts && message.isBot && message.id !== "temp-id" && ( +
+ +
+ )}
{botStyle.data.show_reference && diff --git a/app/widget/src/components/BotHeader.tsx b/app/widget/src/components/BotHeader.tsx index 830812a5..0fab8461 100644 --- a/app/widget/src/components/BotHeader.tsx +++ b/app/widget/src/components/BotHeader.tsx @@ -26,6 +26,7 @@ export default function BotHeader({ setMessages([ { message: botStyle?.data?.first_message, + id: "first-message", isBot: true, sources: [], }, diff --git a/app/widget/src/hooks/useMessage.tsx b/app/widget/src/hooks/useMessage.tsx index 8e9e332e..72b58fff 100644 --- a/app/widget/src/hooks/useMessage.tsx +++ b/app/widget/src/hooks/useMessage.tsx @@ -7,7 +7,7 @@ import { notification } from "antd"; export type BotResponse = { bot: { - id: string; + chat_id: string; text: string; sourceDocuments: any[]; }; @@ -70,7 +70,7 @@ export const useMessage = () => { }); const data = response.data as BotResponse; newMessage[newMessage.length - 1].message = data.bot.text; - newMessage[newMessage.length - 1].id = data.bot.id; + newMessage[newMessage.length - 1].id = data.bot.chat_id; newMessage[newMessage.length - 1].sources = data.bot.sourceDocuments; localStorage.setItem("DS_MESSAGE", JSON.stringify(newMessage)); localStorage.setItem("DS_HISTORY", JSON.stringify(data.history)); @@ -160,10 +160,11 @@ export const useMessage = () => { count++; } else if (type === "result") { const responseData = JSON.parse(message) as BotResponse; + console.log(responseData); newMessage[appendingIndex].message = responseData.bot.text; newMessage[appendingIndex].sources = responseData.bot.sourceDocuments; - newMessage[appendingIndex].id = responseData.bot.id; + newMessage[appendingIndex].id = responseData.bot.chat_id; localStorage.setItem("DS_MESSAGE", JSON.stringify(newMessage)); localStorage.setItem( "DS_HISTORY", diff --git a/app/widget/src/hooks/useVoice.tsx b/app/widget/src/hooks/useVoice.tsx new file mode 100644 index 00000000..c28c47a1 --- /dev/null +++ b/app/widget/src/hooks/useVoice.tsx @@ -0,0 +1,70 @@ +import { useEffect, useRef, useState } from "react"; +import { notification } from "antd"; +import { useMutation } from "@tanstack/react-query"; +import axios from "axios"; +import { getUrl } from "../utils/getUrl"; + +type VoiceOptions = { + id: string; +}; + +export const useVoice = () => { + const audioRef = useRef(null); + const [isPlaying, setIsPlaying] = useState(false); + + const onSubmit = async (values: { id: string }) => { + const response = await axios.post(`${getUrl().split("?")[0]}/tts`, values, { + responseType: "arraybuffer", + }); + return response.data; + }; + + const { mutateAsync: generateAudio, isLoading } = useMutation(onSubmit); + + const speak = async (args: VoiceOptions) => { + try { + const data = await generateAudio({ + id: args.id, + }); + + const blob = new Blob([data], { type: "audio/mpeg" }); + const url = window.URL.createObjectURL(blob); + audioRef.current = new Audio(url); + + audioRef.current.onended = () => { + setIsPlaying(false); + }; + + audioRef.current.play(); + setIsPlaying(true); + } catch (error) { + console.log(error); + + notification.error({ + message: "Error", + description: "Something went wrong while trying to play the audio", + }); + } + }; + + const cancel = () => { + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current.currentTime = 0; + setIsPlaying(false); + } + }; + + useEffect(() => { + return () => { + cancel(); + }; + }, []); + + return { + speak, + cancel, + isPlaying, + loading: isLoading, + }; +}; diff --git a/app/widget/src/utils/types.ts b/app/widget/src/utils/types.ts index ee22e483..3b3f2b59 100644 --- a/app/widget/src/utils/types.ts +++ b/app/widget/src/utils/types.ts @@ -13,5 +13,6 @@ export type BotStyle = { text_color?: string; }; first_message: string; + tts: boolean; }; }; diff --git a/server/src/handlers/api/v1/bot/appearance/get.handler.ts b/server/src/handlers/api/v1/bot/appearance/get.handler.ts index 426bb5ce..d0192906 100644 --- a/server/src/handlers/api/v1/bot/appearance/get.handler.ts +++ b/server/src/handlers/api/v1/bot/appearance/get.handler.ts @@ -1,6 +1,6 @@ import { FastifyReply, FastifyRequest } from "fastify"; import { GetBotAppearanceById } from "./types"; -import { getElevenLabTTS, getOpenAITTS } from "../../../../../utils/elevenlabs"; +import { getElevenLabTTS, getOpenAITTS } from "../../../../../utils/voice"; export const getBotAppearanceByIdHandler = async ( request: FastifyRequest, diff --git a/server/src/handlers/api/v1/bot/playground/get.handler.ts b/server/src/handlers/api/v1/bot/playground/get.handler.ts index 7dbee9a4..c86df35a 100644 --- a/server/src/handlers/api/v1/bot/playground/get.handler.ts +++ b/server/src/handlers/api/v1/bot/playground/get.handler.ts @@ -5,7 +5,7 @@ import { } from "./types"; import { getElevenLab, -} from "../../../../../utils/elevenlabs"; +} from "../../../../../utils/voice"; export async function getPlaygroundHistoryByBotId( request: FastifyRequest, diff --git a/server/src/handlers/api/v1/voice/post.handler.ts b/server/src/handlers/api/v1/voice/post.handler.ts index 19439355..cc7998c2 100644 --- a/server/src/handlers/api/v1/voice/post.handler.ts +++ b/server/src/handlers/api/v1/voice/post.handler.ts @@ -4,7 +4,7 @@ import { isElevenLabAPIKeyPresent, isElevenLabAPIValid, textToSpeech, -} from "../../../../utils/elevenlabs"; +} from "../../../../utils/voice"; export const voiceTTSHandler = async ( request: FastifyRequest, diff --git a/server/src/handlers/bot/index.ts b/server/src/handlers/bot/index.ts index bd54ac7a..978c79c9 100644 --- a/server/src/handlers/bot/index.ts +++ b/server/src/handlers/bot/index.ts @@ -1,3 +1,4 @@ export * from "./post.handler" export * from "./get.handler" -export * from "./api.handler" \ No newline at end of file +export * from "./api.handler" +export * from "./voice.handler" \ No newline at end of file diff --git a/server/src/handlers/bot/types.ts b/server/src/handlers/bot/types.ts index c4d78fac..80611bec 100644 --- a/server/src/handlers/bot/types.ts +++ b/server/src/handlers/bot/types.ts @@ -47,3 +47,12 @@ export interface ChatAPIRequest { }; } +export interface ChatTTSRequest { + Params: { + id: string; + }; + + Body: { + id: string; + }; +} diff --git a/server/src/handlers/bot/voice.handler.ts b/server/src/handlers/bot/voice.handler.ts new file mode 100644 index 00000000..b4bbae32 --- /dev/null +++ b/server/src/handlers/bot/voice.handler.ts @@ -0,0 +1,100 @@ +import { FastifyReply, FastifyRequest } from "fastify"; +import { ChatTTSRequest } from "./types"; +import { + isElevenLabAPIKeyPresent, + isElevenLabAPIValid, + isOpenAIAPIKeyPresent, + textToSpeech, + textToSpeechOpenAI, +} from "../../utils/voice"; + +export const chatTTSHandler = async ( + request: FastifyRequest, + reply: FastifyReply +) => { + const bot_id = request.params.id; + const prisma = request.server.prisma; + const chat_id = request.body.id; + + const isBotExist = await prisma.bot.findFirst({ + where: { + publicId: bot_id, + }, + }); + + if (!isBotExist) { + return reply.status(404).send({ + message: "Bot not found", + }); + } + + let text = ""; + + const botAppearance = await prisma.botAppearance.findFirst({ + where: { + bot_id: isBotExist.id, + }, + }); + + if (!botAppearance) { + return reply.status(404).send({ + message: "Bot configuration not found", + }); + } + + if (chat_id === "first-message") { + text = botAppearance?.first_message! || ""; + } else { + const chat = await prisma.botWebHistory.findFirst({ + where: { + id: chat_id, + }, + }); + + if (!chat) { + return reply.status(404).send({ + message: "Chat not found", + }); + } + + text = chat.bot!; + } + + if (!botAppearance.tts) { + return reply.status(404).send({ + message: "TTS not enabled for this bot", + }); + } + if (botAppearance?.tts_provider === "eleven_labs") { + if (!isElevenLabAPIKeyPresent()) { + return reply.status(400).send({ + message: "Eleven Labs API key not present", + }); + } + const is11LabAPIVValid = await isElevenLabAPIValid(); + if (!is11LabAPIVValid) { + return reply.status(400).send({ + message: "Eleven Labs API key not valid", + }); + } + const buffer = await textToSpeech(text, botAppearance.tts_voice!); + return reply.status(200).send(buffer); + } else if (botAppearance.tts_provider === "openai") { + const isOpenAIKeyPresent = isOpenAIAPIKeyPresent(); + if (!isOpenAIKeyPresent) { + return reply.status(400).send({ + message: "OpenAI API key not present", + }); + } + const buffer = await textToSpeechOpenAI( + text, + botAppearance.tts_voice!, + botAppearance.tts_model! + ); + return reply.status(200).send(buffer); + } else { + return reply.status(404).send({ + message: "TTS provider not found", + }); + } +}; diff --git a/server/src/routes/bot/root.ts b/server/src/routes/bot/root.ts index 136ae952..b45f4125 100644 --- a/server/src/routes/bot/root.ts +++ b/server/src/routes/bot/root.ts @@ -4,12 +4,14 @@ import { chatRequestStreamHandler, getChatStyleByIdHandler, chatRequestAPIHandler, + chatTTSHandler, } from "../../handlers/bot"; import { chatRequestSchema, chatRequestStreamSchema, chatStyleSchema, chatAPIRequestSchema, + chatTTSSchema, } from "../../schema/bot"; const root: FastifyPluginAsync = async (fastify, _): Promise => { @@ -37,7 +39,13 @@ const root: FastifyPluginAsync = async (fastify, _): Promise => { }, getChatStyleByIdHandler ); - + fastify.post( + "/:id/tts", + { + schema: chatTTSSchema, + }, + chatTTSHandler + ); fastify.post( "/:id/api", { diff --git a/server/src/schema/bot/index.ts b/server/src/schema/bot/index.ts index 3d836861..dc0a923c 100644 --- a/server/src/schema/bot/index.ts +++ b/server/src/schema/bot/index.ts @@ -40,6 +40,29 @@ export const chatStyleSchema: FastifySchema = { }, }; +export const chatTTSSchema: FastifySchema = { + tags: ["Widget"], + summary: "API to get TTS of message", + params: { + type: "object", + required: ["id"], + properties: { + id: { + type: "string", + }, + }, + }, + body: { + type: "object", + required: ["id"], + properties: { + id: { + type: "string", + }, + }, + }, +}; + export const chatRequestStreamSchema: FastifySchema = { tags: ["Widget"], summary: "API to get stream of message from bot", diff --git a/server/src/utils/elevenlabs.ts b/server/src/utils/voice.ts similarity index 90% rename from server/src/utils/elevenlabs.ts rename to server/src/utils/voice.ts index b7329fb2..99b8a98c 100644 --- a/server/src/utils/elevenlabs.ts +++ b/server/src/utils/voice.ts @@ -242,3 +242,31 @@ export const textToSpeech = async (text: string, voiceId: string) => { return response.data; }; + +export const textToSpeechOpenAI = async ( + input: string, + voice: string, + model: string +) => { + const apiKey = process.env.OPENAI_API_KEY; + const response = await axios.post( + "https://api.openai.com/v1/audio/speech", + { + input: input, + voice: voice, + model: model, + }, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + responseType: "arraybuffer", + } + ); + + return response.data; +}; + +export const isOpenAIAPIKeyPresent = () => { + return !!process.env.OPENAI_API_KEY; +} \ No newline at end of file