feat: Play audio #2088 (#2200)

### What problem does this PR solve?
feat: Play audio #2088


### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/package-lock.json

@@ -28,6 +28,7 @@
         "jsencrypt": "^3.3.2",
         "lodash": "^4.17.21",
         "mammoth": "^1.7.2",
+        "openai-speech-stream-player": "^1.0.8",
         "rc-tween-one": "^3.0.6",
         "react-copy-to-clipboard": "^5.1.0",
         "react-force-graph": "^1.44.4",
@@ -20565,6 +20566,11 @@
         "node": ">=12"
       }
     },
+    "node_modules/openai-speech-stream-player": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmmirror.com/openai-speech-stream-player/-/openai-speech-stream-player-1.0.8.tgz",
+      "integrity": "sha512-0SUybbhStl65s66ezh2QaoZE5k1kNb2t5M8tDOqJFILdHpwHaBqnYy4uHl3Hk/8F5VFWxxHaLamjKOnfNDKgbw=="
+    },
     "node_modules/option": {
       "version": "0.2.4",
       "resolved": "https://registry.npmmirror.com/option/-/option-0.2.4.tgz",

web/package.json

@@ -39,6 +39,7 @@
     "jsencrypt": "^3.3.2",
     "lodash": "^4.17.21",
     "mammoth": "^1.7.2",
+    "openai-speech-stream-player": "^1.0.8",
     "rc-tween-one": "^3.0.6",
     "react-copy-to-clipboard": "^5.1.0",
     "react-force-graph": "^1.44.4",

web/src/components/message-item/group-button.tsx

@@ -5,6 +5,7 @@ import {
   DeleteOutlined,
   DislikeOutlined,
   LikeOutlined,
+  PauseCircleOutlined,
   SoundOutlined,
   SyncOutlined,
 } from '@ant-design/icons';
@@ -13,7 +14,7 @@ import { useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
 import SvgIcon from '../svg-icon';
 import FeedbackModal from './feedback-modal';
-import { useRemoveMessage, useSendFeedback } from './hooks';
+import { useRemoveMessage, useSendFeedback, useSpeech } from './hooks';
 import PromptModal from './prompt-modal';
 
 interface IProps {
@@ -37,6 +38,7 @@ export const AssistantGroupButton = ({
     showModal: showPromptModal,
   } = useSetModalState();
   const { t } = useTranslation();
+  const { handleRead, ref, isPlaying } = useSpeech(content);
 
   const handleLike = useCallback(() => {
     onFeedbackOk({ thumbup: true });
@@ -48,10 +50,11 @@ export const AssistantGroupButton = ({
         <Radio.Button value="a">
           <CopyToClipboard text={content}></CopyToClipboard>
         </Radio.Button>
-        <Radio.Button value="b">
+        <Radio.Button value="b" onClick={handleRead}>
           <Tooltip title={t('chat.read')}>
-            <SoundOutlined />
+            {isPlaying ? <PauseCircleOutlined /> : <SoundOutlined />}
           </Tooltip>
+          <audio src="" ref={ref}></audio>
         </Radio.Button>
         {showLikeButton && (
           <>

web/src/components/message-item/hooks.ts

@@ -1,9 +1,10 @@
 import { useDeleteMessage, useFeedback } from '@/hooks/chat-hooks';
 import { useSetModalState } from '@/hooks/common-hooks';
-import { IRemoveMessageById } from '@/hooks/logic-hooks';
+import { IRemoveMessageById, useSpeechWithSse } from '@/hooks/logic-hooks';
 import { IFeedbackRequestBody } from '@/interfaces/request/chat';
 import { getMessagePureId } from '@/utils/chat';
-import { useCallback } from 'react';
+import { SpeechPlayer } from 'openai-speech-stream-player';
+import { useCallback, useEffect, useRef, useState } from 'react';
 
 export const useSendFeedback = (messageId: string) => {
   const { visible, hideModal, showModal } = useSetModalState();
@@ -50,3 +51,52 @@ export const useRemoveMessage = (
 
   return { onRemoveMessage, loading };
 };
+
+export const useSpeech = (content: string) => {
+  const ref = useRef<HTMLAudioElement>(null);
+  const { read } = useSpeechWithSse();
+  const player = useRef<SpeechPlayer>();
+  const [isPlaying, setIsPlaying] = useState<boolean>(false);
+
+  const initialize = useCallback(async () => {
+    player.current = new SpeechPlayer({
+      audio: ref.current!,
+      onPlaying: () => {
+        setIsPlaying(true);
+      },
+      onPause: () => {
+        setIsPlaying(false);
+      },
+      onChunkEnd: () => {},
+      mimeType: 'audio/mpeg',
+    });
+    await player.current.init();
+  }, []);
+
+  const pause = useCallback(() => {
+    player.current?.pause();
+  }, []);
+
+  const speech = useCallback(async () => {
+    const response = await read({ text: content });
+    if (response) {
+      player?.current?.feedWithResponse(response);
+    }
+  }, [read, content]);
+
+  const handleRead = useCallback(async () => {
+    if (isPlaying) {
+      setIsPlaying(false);
+      pause();
+    } else {
+      setIsPlaying(true);
+      speech();
+    }
+  }, [setIsPlaying, speech, isPlaying, pause]);
+
+  useEffect(() => {
+    initialize();
+  }, [initialize]);
+
+  return { ref, handleRead, isPlaying };
+};

web/src/hooks/logic-hooks.ts

@@ -278,6 +278,88 @@ export const useSendMessageWithSse = (
   return { send, answer, done, setDone };
 };
 
+export const useSpeechWithSse = (url: string = api.tts) => {
+  const read = useCallback(
+    (body: any) => {
+      const response = fetch(url, {
+        method: 'POST',
+        headers: {
+          [Authorization]: getAuthorization(),
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(body),
+      });
+      return response;
+    },
+    [url],
+  );
+
+  return { read };
+};
+
+export const useFetchAudioWithSse = (url: string = api.tts) => {
+  // const [answer, setAnswer] = useState<IAnswer>({} as IAnswer);
+  const [done, setDone] = useState(true);
+
+  const read = useCallback(
+    async (
+      body: any,
+    ): Promise<{ response: Response; data: ResponseType } | undefined> => {
+      try {
+        setDone(false);
+        const response = await fetch(url, {
+          method: 'POST',
+          headers: {
+            [Authorization]: getAuthorization(),
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(body),
+        });
+
+        const res = response.clone().json();
+
+        const reader = response?.body?.getReader();
+
+        while (true) {
+          const x = await reader?.read();
+          if (x) {
+            const { done, value } = x;
+            try {
+              // const val = JSON.parse(value || '');
+              const val = value;
+              // const d = val?.data;
+              // if (typeof d !== 'boolean') {
+              // console.info('data:', d);
+              // setAnswer({
+              //   ...d,
+              //   conversationId: body?.conversation_id,
+              // });
+              // }
+            } catch (e) {
+              console.warn(e);
+            }
+            if (done) {
+              console.info('done');
+              break;
+            }
+          }
+        }
+        console.info('done?');
+        setDone(true);
+        // setAnswer({} as IAnswer);
+        return { data: await res, response };
+      } catch (e) {
+        setDone(true);
+        // setAnswer({} as IAnswer);
+        console.warn(e);
+      }
+    },
+    [url],
+  );
+
+  return { read, done, setDone };
+};
+
 //#region chat hooks
 
 export const useScrollToBottom = (messages?: unknown) => {