add speech-to-text switch

api/controllers/console/__init__.py

@@ -6,10 +6,10 @@ bp = Blueprint('console', __name__, url_prefix='/console/api')
 api = ExternalApi(bp)
 
 # Import other controllers
-from . import setup, version, apikey, admin, audio
+from . import setup, version, apikey, admin
 
 # Import app controllers
-from .app import app, site, completion, model_config, statistic, conversation, message, generator
+from .app import app, site, completion, model_config, statistic, conversation, message, generator, audio
 
 # Import auth controllers
 from .auth import login, oauth, data_source_oauth
@@ -21,4 +21,4 @@ from .datasets import datasets, datasets_document, datasets_segments, file, hit_
 from .workspace import workspace, members, providers, account
 
 # Import explore controllers
-from .explore import installed_app, recommended_app, completion, conversation, message, parameter, saved_message
+from .explore import installed_app, recommended_app, completion, conversation, message, parameter, saved_message, audio

api/controllers/console/app/app.py

@@ -22,6 +22,7 @@ model_config_fields = {
     'opening_statement': fields.String,
     'suggested_questions': fields.Raw(attribute='suggested_questions_list'),
     'suggested_questions_after_answer': fields.Raw(attribute='suggested_questions_after_answer_dict'),
+    'speech_to_text': fields.Raw(attribute='speech_to_text_dict'),
     'more_like_this': fields.Raw(attribute='more_like_this_dict'),
     'model': fields.Raw(attribute='model_dict'),
     'user_input_form': fields.Raw(attribute='user_input_form_list'),
@@ -144,6 +145,7 @@ class AppListApi(Resource):
                 opening_statement=model_configuration['opening_statement'],
                 suggested_questions=json.dumps(model_configuration['suggested_questions']),
                 suggested_questions_after_answer=json.dumps(model_configuration['suggested_questions_after_answer']),
+                speech_to_text=json.dumps(model_configuration['speech_to_text']),
                 more_like_this=json.dumps(model_configuration['more_like_this']),
                 model=json.dumps(model_configuration['model']),
                 user_input_form=json.dumps(model_configuration['user_input_form']),
@@ -434,6 +436,7 @@ class AppCopy(Resource):
             opening_statement=app_config.opening_statement,
             suggested_questions=app_config.suggested_questions,
             suggested_questions_after_answer=app_config.suggested_questions_after_answer,
+            speech_to_text=app_config.speech_to_text,
             more_like_this=app_config.more_like_this,
             model=app_config.model,
             user_input_form=app_config.user_input_form,

api/controllers/console/app/audio.py

+# -*- coding:utf-8 -*-
+import logging
+
+from flask import request
+from flask_login import login_required
+from werkzeug.exceptions import InternalServerError, NotFound
+
+import services
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.app.error import AppUnavailableError, \
+    ProviderNotInitializeError, CompletionRequestError, ProviderQuotaExceededError, \
+    ProviderModelCurrentlyNotSupportError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
+    LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+from flask_restful import Resource
+from models.model import AppModelConfig
+from services.audio_service import AudioService
+
+
+class ChatMessageAudioApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        app_id = str(app_id)
+
+        app_model = _get_app(app_id, 'chat')
+        app_model_config: AppModelConfig = app_model.app_model_config
+
+        if not app_model_config.speech_to_text_dict['enabled']:
+            raise AppUnavailableError()
+
+        file = request.files['file']
+
+        try:
+            response = AudioService.transcript(
+                tenant_id=app_model.tenant_id,
+                file=file,
+            )
+
+            return response
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+        
+
+api.add_resource(ChatMessageAudioApi, '/apps/<uuid:app_id>/audio-to-text')
\ No newline at end of file

api/controllers/console/app/model_config.py

@@ -41,6 +41,7 @@ class ModelConfigResource(Resource):
             opening_statement=model_configuration['opening_statement'],
             suggested_questions=json.dumps(model_configuration['suggested_questions']),
             suggested_questions_after_answer=json.dumps(model_configuration['suggested_questions_after_answer']),
+            speech_to_text=json.dumps(model_configuration['speech_to_text']),
             more_like_this=json.dumps(model_configuration['more_like_this']),
             model=json.dumps(model_configuration['model']),
             user_input_form=json.dumps(model_configuration['user_input_form']),

api/controllers/console/audio.py → api/controllers/console/explore/audio.py

@@ -2,25 +2,32 @@
 import logging
 
 from flask import request
-from flask_login import current_user
-from flask_restful import Resource
 from werkzeug.exceptions import InternalServerError
 
 import services
-from services.audio_service import AudioService
 from controllers.console import api
 from controllers.console.app.error import AppUnavailableError, ProviderNotInitializeError, \
     ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError, CompletionRequestError
+from controllers.console.explore.wraps import InstalledAppResource
 from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+from services.audio_service import AudioService
+from models.model import AppModelConfig
+
+
+class ChatAudioApi(InstalledAppResource):
+    def post(self, installed_app):
+        app_model = installed_app.app
+        app_model_config: AppModelConfig = app_model.app_model_config
+
+        if not app_model_config.speech_to_text_dict['enabled']:
+            raise AppUnavailableError()
 
-class AudioApi(Resource):
-    def post(self):
         file = request.files['file']
-        
+
         try:
             response = AudioService.transcript(
-                tenant_id=current_user.current_tenant_id,
+                tenant_id=app_model.tenant_id,
                 file=file,
             )
 
@@ -42,5 +49,6 @@ class AudioApi(Resource):
         except Exception as e:
             logging.exception("internal server error.")
             raise InternalServerError()
+        
 
-api.add_resource(AudioApi, '/audio-to-text')
\ No newline at end of file
+api.add_resource(ChatAudioApi, '/installed-apps/<uuid:installed_app_id>/audio-to-text', endpoint='installed_app_audio')
\ No newline at end of file

api/controllers/console/explore/parameter.py

@@ -21,6 +21,7 @@ class AppParameterApi(InstalledAppResource):
         'opening_statement': fields.String,
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
+        'speech_to_text': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
     }
@@ -35,6 +36,7 @@ class AppParameterApi(InstalledAppResource):
             'opening_statement': app_model_config.opening_statement,
             'suggested_questions': app_model_config.suggested_questions_list,
             'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
+            'speech_to_text': app_model_config.speech_to_text_dict,
             'more_like_this': app_model_config.more_like_this_dict,
             'user_input_form': app_model_config.user_input_form_list
         }

api/controllers/service_api/app/app.py

@@ -22,6 +22,7 @@ class AppParameterApi(AppApiResource):
         'opening_statement': fields.String,
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
+        'speech_to_text': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
     }
@@ -35,6 +36,7 @@ class AppParameterApi(AppApiResource):
             'opening_statement': app_model_config.opening_statement,
             'suggested_questions': app_model_config.suggested_questions_list,
             'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
+            'speech_to_text': app_model_config.speech_to_text_dict,
             'more_like_this': app_model_config.more_like_this_dict,
             'user_input_form': app_model_config.user_input_form_list
         }

api/controllers/service_api/app/audio.py

@@ -11,10 +11,15 @@ from controllers.service_api.app.error import AppUnavailableError, ProviderNotIn
 from controllers.service_api.wraps import AppApiResource
 from core.llm.error import LLMBadRequestError, LLMAuthorizationError, LLMAPIUnavailableError, LLMAPIConnectionError, \
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
-from models.model import App
+from models.model import App, AppModelConfig
 
 class AudioApi(AppApiResource):
     def post(self, app_model: App, end_user):
+        app_model_config: AppModelConfig = app_model.app_model_config
+
+        if not app_model_config.speech_to_text_dict['enabled']:
+            raise AppUnavailableError() 
+
         file = request.files['file']
 
         try:

api/controllers/web/app.py

@@ -21,6 +21,7 @@ class AppParameterApi(WebApiResource):
         'opening_statement': fields.String,
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
+        'speech_to_text': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
     }
@@ -34,6 +35,7 @@ class AppParameterApi(WebApiResource):
             'opening_statement': app_model_config.opening_statement,
             'suggested_questions': app_model_config.suggested_questions_list,
             'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
+            'speech_to_text': app_model_config.speech_to_text_dict,
             'more_like_this': app_model_config.more_like_this_dict,
             'user_input_form': app_model_config.user_input_form_list
         }

api/controllers/web/audio.py

@@ -12,11 +12,16 @@ from controllers.web.wraps import WebApiResource
 from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from services.audio_service import AudioService
-from models.model import App
+from models.model import App, AppModelConfig
 
 
 class AudioApi(WebApiResource):
     def post(self, app_model: App, end_user):
+        app_model_config: AppModelConfig = app_model.app_model_config
+
+        if not app_model_config.speech_to_text_dict['enabled']:
+            raise AppUnavailableError()
+
         file = request.files['file']
 
         try:

api/migrations/versions/a5b56fb053ef_app_config_add_speech_to_text.py

+"""app config add speech_to_text
+
+Revision ID: a5b56fb053ef
+Revises: d3d503a3471c
+Create Date: 2023-07-06 17:55:20.894149
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a5b56fb053ef'
+down_revision = 'd3d503a3471c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('speech_to_text', sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.drop_column('speech_to_text')
+
+    # ### end Alembic commands ###

api/models/model.py

@@ -81,6 +81,7 @@ class AppModelConfig(db.Model):
     opening_statement = db.Column(db.Text)
     suggested_questions = db.Column(db.Text)
     suggested_questions_after_answer = db.Column(db.Text)
+    speech_to_text = db.Column(db.Text)
     more_like_this = db.Column(db.Text)
     model = db.Column(db.Text)
     user_input_form = db.Column(db.Text)
@@ -104,6 +105,11 @@ class AppModelConfig(db.Model):
     def suggested_questions_after_answer_dict(self) -> dict:
         return json.loads(self.suggested_questions_after_answer) if self.suggested_questions_after_answer \
             else {"enabled": False}
+    
+    @property
+    def speech_to_text_dict(self) -> dict:
+        return json.loads(self.speech_to_text) if self.speech_to_text \
+            else {"enabled": False}
 
     @property
     def more_like_this_dict(self) -> dict:
@@ -223,6 +229,9 @@ class Conversation(db.Model):
                 model_config['suggested_questions_after_answer'] = override_model_configs[
                     'suggested_questions_after_answer'] \
                     if 'suggested_questions_after_answer' in override_model_configs else {"enabled": False}
+                model_config['speech_to_text'] = override_model_configs[
+                    'speech_to_text'] \
+                    if 'speech_to_text' in override_model_configs else {"enabled": False}
                 model_config['more_like_this'] = override_model_configs['more_like_this'] \
                     if 'more_like_this' in override_model_configs else {"enabled": False}
                 model_config['user_input_form'] = override_model_configs['user_input_form']
@@ -239,6 +248,7 @@ class Conversation(db.Model):
             model_config['opening_statement'] = app_model_config.opening_statement
             model_config['suggested_questions'] = app_model_config.suggested_questions_list
             model_config['suggested_questions_after_answer'] = app_model_config.suggested_questions_after_answer_dict
+            model_config['speech_to_text'] = app_model_config.speech_to_text_dict
             model_config['more_like_this'] = app_model_config.more_like_this_dict
             model_config['user_input_form'] = app_model_config.user_input_form_list
 

api/services/app_model_config_service.py

@@ -109,6 +109,21 @@ class AppModelConfigService:
         if not isinstance(config["suggested_questions_after_answer"]["enabled"], bool):
             raise ValueError("enabled in suggested_questions_after_answer must be of boolean type")
 
+        # speech_to_text
+        if 'speech_to_text' not in config or not config["speech_to_text"]:
+            config["speech_to_text"] = {
+                "enabled": False
+            }
+
+        if not isinstance(config["speech_to_text"], dict):
+            raise ValueError("speech_to_text must be of dict type")
+
+        if "enabled" not in config["speech_to_text"] or not config["speech_to_text"]["enabled"]:
+            config["speech_to_text"]["enabled"] = False
+
+        if not isinstance(config["speech_to_text"]["enabled"], bool):
+            raise ValueError("enabled in speech_to_text must be of boolean type")
+
         # more_like_this
         if 'more_like_this' not in config or not config["more_like_this"]:
             config["more_like_this"] = {
@@ -277,6 +292,7 @@ class AppModelConfigService:
             "opening_statement": config["opening_statement"],
             "suggested_questions": config["suggested_questions"],
             "suggested_questions_after_answer": config["suggested_questions_after_answer"],
+            "speech_to_text": config["speech_to_text"],
             "more_like_this": config["more_like_this"],
             "model": {
                 "provider": config["model"]["provider"],

web/app/components/app/chat/index.tsx

@@ -64,6 +64,7 @@ export type IChatProps = {
   controlFocus?: number
   isShowSuggestion?: boolean
   suggestionList?: string[]
+  isShowSpeechToText?: boolean
 }
 
 export type MessageMore = {
@@ -426,6 +427,7 @@ const Chat: FC<IChatProps> = ({
   controlFocus,
   isShowSuggestion,
   suggestionList,
+  isShowSpeechToText,
 }) => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
@@ -586,15 +588,17 @@ const Chat: FC<IChatProps> = ({
                         <XCircle className='w-4 h-4 text-[#98A2B3]' />
                       </div>
                     )
-                    : (
-                      <div
-                        className='group flex justify-center items-center w-8 h-8 hover:bg-primary-50 rounded-lg cursor-pointer'
-                        onClick={handleVoiceInputShow}
-                      >
-                        <Microphone01 className='block w-4 h-4 text-gray-500 group-hover:hidden' />
-                        <Microphone01Solid className='hidden w-4 h-4 text-primary-600 group-hover:block' />
-                      </div>
-                    )
+                    : isShowSpeechToText
+                      ? (
+                        <div
+                          className='group flex justify-center items-center w-8 h-8 hover:bg-primary-50 rounded-lg cursor-pointer'
+                          onClick={handleVoiceInputShow}
+                        >
+                          <Microphone01 className='block w-4 h-4 text-gray-500 group-hover:hidden' />
+                          <Microphone01Solid className='hidden w-4 h-4 text-primary-600 group-hover:block' />
+                        </div>
+                      )
+                      : null
                 }
                 <div className='mx-2 w-[1px] h-4 bg-black opacity-5' />
                 {isMobile

web/app/components/app/configuration/config/feature/choose-feature/feature-item/style.module.css

@@ -22,4 +22,8 @@
 
 .moreLikeThisPreview {
   background-image: url(./preview-imgs/more-like-this.svg);
+}
+
+.speechToTextPreview {
+  background-image: url(./preview-imgs/speech-to-text.svg);
 }
\ No newline at end of file

web/app/components/app/configuration/config/feature/choose-feature/index.tsx

@@ -7,10 +7,12 @@ import MoreLikeThisIcon from '../../../base/icons/more-like-this-icon'
 import FeatureItem from './feature-item'
 import Modal from '@/app/components/base/modal'
 import SuggestedQuestionsAfterAnswerIcon from '@/app/components/app/configuration/base/icons/suggested-questions-after-answer-icon'
+import { Microphone01 } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
 type IConfig = {
   openingStatement: boolean
   moreLikeThis: boolean
   suggestedQuestionsAfterAnswer: boolean
+  speechToText: boolean
 }
 
 export type IChooseFeatureProps = {
@@ -69,6 +71,14 @@ const ChooseFeature: FC<IChooseFeatureProps> = ({
                 value={config.suggestedQuestionsAfterAnswer}
                 onChange={value => onChange('suggestedQuestionsAfterAnswer', value)}
               />
+              <FeatureItem
+                icon={<Microphone01 className='w-4 h-4 text-[#7839EE]' />}
+                previewImgClassName='speechToTextPreview'
+                title={t('appDebug.feature.speechToText.title')}
+                description={t('appDebug.feature.speechToText.description')}
+                value={config.speechToText}
+                onChange={value => onChange('speechToText', value)}
+              />
             </>
           </FeatureGroup>
         )}

web/app/components/app/configuration/config/feature/use-feature.tsx

@@ -7,6 +7,8 @@ function useFeature({
   setMoreLikeThis,
   suggestedQuestionsAfterAnswer,
   setSuggestedQuestionsAfterAnswer,
+  speechToText,
+  setSpeechToText,
 }: {
   introduction: string
   setIntroduction: (introduction: string) => void
@@ -14,13 +16,14 @@ function useFeature({
   setMoreLikeThis: (moreLikeThis: boolean) => void
   suggestedQuestionsAfterAnswer: boolean
   setSuggestedQuestionsAfterAnswer: (suggestedQuestionsAfterAnswer: boolean) => void
+  speechToText: boolean
+  setSpeechToText: (speechToText: boolean) => void
 }) {
   const [tempshowOpeningStatement, setTempShowOpeningStatement] = React.useState(!!introduction)
   useEffect(() => {
     // wait to api data back
-    if (!!introduction) {
+    if (introduction)
       setTempShowOpeningStatement(true)
-    }
   }, [introduction])
 
   // const [tempMoreLikeThis, setTempMoreLikeThis] = React.useState(moreLikeThis)
@@ -30,15 +33,16 @@ function useFeature({
 
   const featureConfig = {
     openingStatement: tempshowOpeningStatement,
-    moreLikeThis: moreLikeThis,
-    suggestedQuestionsAfterAnswer: suggestedQuestionsAfterAnswer
+    moreLikeThis,
+    suggestedQuestionsAfterAnswer,
+    speechToText,
   }
   const handleFeatureChange = (key: string, value: boolean) => {
     switch (key) {
       case 'openingStatement':
-        if (!value) {
+        if (!value)
           setIntroduction('')
-        }
+
         setTempShowOpeningStatement(value)
         break
       case 'moreLikeThis':
@@ -47,12 +51,14 @@ function useFeature({
       case 'suggestedQuestionsAfterAnswer':
         setSuggestedQuestionsAfterAnswer(value)
         break
+      case 'speechToText':
+        setSpeechToText(value)
     }
   }
   return {
     featureConfig,
-    handleFeatureChange
+    handleFeatureChange,
   }
 }
 
-export default useFeature
\ No newline at end of file
+export default useFeature

web/app/components/app/configuration/config/index.tsx

@@ -33,6 +33,8 @@ const Config: FC = () => {
     setMoreLikeThisConfig,
     suggestedQuestionsAfterAnswerConfig,
     setSuggestedQuestionsAfterAnswerConfig,
+    speechToTextConfig,
+    setSpeechToTextConfig,
   } = useContext(ConfigContext)
   const isChatApp = mode === AppType.chat
 
@@ -78,9 +80,15 @@ const Config: FC = () => {
         draft.enabled = value
       }))
     },
+    speechToText: speechToTextConfig.enabled,
+    setSpeechToText: (value) => {
+      setSpeechToTextConfig(produce(speechToTextConfig, (draft) => {
+        draft.enabled = value
+      }))
+    },
   })
 
-  const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer)
+  const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer || featureConfig.speechToText)
   const hasToolbox = false
 
   const [showAutomatic, { setTrue: showAutomaticTrue, setFalse: showAutomaticFalse }] = useBoolean(false)
@@ -149,6 +157,7 @@ const Config: FC = () => {
                 }
               }
               isShowSuggestedQuestionsAfterAnswer={featureConfig.suggestedQuestionsAfterAnswer}
+              isShowSpeechText={featureConfig.speechToText}
             />
           )
         }

web/app/components/app/configuration/debug/index.tsx

@@ -38,6 +38,7 @@ const Debug: FC<IDebug> = ({
     mode,
     introduction,
     suggestedQuestionsAfterAnswerConfig,
+    speechToTextConfig,
     moreLikeThisConfig,
     inputs,
     // setInputs,
@@ -159,6 +160,7 @@ const Debug: FC<IDebug> = ({
         enabled: false,
       },
       suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
+      speech_to_text: speechToTextConfig,
       agent_mode: {
         enabled: true,
         tools: [...postDatasets],
@@ -308,6 +310,7 @@ const Debug: FC<IDebug> = ({
       user_input_form: promptVariablesToUserInputsForm(modelConfig.configs.prompt_variables),
       opening_statement: introduction,
       suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
+      speech_to_text: speechToTextConfig,
       more_like_this: moreLikeThisConfig,
       agent_mode: {
         enabled: true,
@@ -387,6 +390,7 @@ const Debug: FC<IDebug> = ({
                   isShowSuggestion={doShowSuggestion}
                   suggestionList={suggestQuestions}
                   displayScene='console'
+                  isShowSpeechToText={speechToTextConfig.enabled}
                 />
               </div>
             </div>

web/app/components/app/configuration/features/chat-group/index.tsx

 'use client'
-import React, { FC } from 'react'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
 import GroupName from '../../base/group-name'
-import OpeningStatement, { IOpeningStatementProps } from './opening-statement'
+import type { IOpeningStatementProps } from './opening-statement'
+import OpeningStatement from './opening-statement'
 import SuggestedQuestionsAfterAnswer from './suggested-questions-after-answer'
-import { useTranslation } from 'react-i18next'
+import SpeechToText from './speech-to-text'
 
 /*
-* Include 
+* Include
 * 1. Conversation Opener
 * 2. Opening Suggestion
 * 3. Next question suggestion
 */
-interface ChatGroupProps {
+type ChatGroupProps = {
   isShowOpeningStatement: boolean
   openingStatementConfig: IOpeningStatementProps
   isShowSuggestedQuestionsAfterAnswer: boolean
+  isShowSpeechText: boolean
 }
 const ChatGroup: FC<ChatGroupProps> = ({
   isShowOpeningStatement,
   openingStatementConfig,
-  isShowSuggestedQuestionsAfterAnswer
+  isShowSuggestedQuestionsAfterAnswer,
+  isShowSpeechText,
 }) => {
   const { t } = useTranslation()
 
@@ -33,6 +38,11 @@ const ChatGroup: FC<ChatGroupProps> = ({
         {isShowSuggestedQuestionsAfterAnswer && (
           <SuggestedQuestionsAfterAnswer />
         )}
+        {
+          isShowSpeechText && (
+            <SpeechToText />
+          )
+        }
       </div>
     </div>
   )

web/app/components/app/configuration/features/chat-group/speech-to-text/index.tsx

+'use client'
+import React, { type FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import Panel from '@/app/components/app/configuration/base/feature-panel'
+import { Microphone01 } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
+
+const SuggestedQuestionsAfterAnswer: FC = () => {
+  const { t } = useTranslation()
+
+  return (
+    <Panel
+      title={
+        <div className='flex items-center gap-2'>
+          <div>{t('appDebug.feature.speechToText.title')}</div>
+        </div>
+      }
+      headerIcon={<Microphone01 className='w-4 h-4 text-[#7839EE]' />}
+      headerRight={
+        <div className='text-xs text-gray-500'>{t('appDebug.feature.speechToText.resDes')}</div>
+      }
+      noBodySpacing
+    />
+  )
+}
+export default React.memo(SuggestedQuestionsAfterAnswer)

web/app/components/app/configuration/index.tsx

@@ -53,6 +53,9 @@ const Configuration: FC = () => {
   const [suggestedQuestionsAfterAnswerConfig, setSuggestedQuestionsAfterAnswerConfig] = useState<MoreLikeThisConfig>({
     enabled: false,
   })
+  const [speechToTextConfig, setSpeechToTextConfig] = useState<MoreLikeThisConfig>({
+    enabled: false,
+  })
   const [formattingChanged, setFormattingChanged] = useState(false)
   const [inputs, setInputs] = useState<Inputs>({})
   const [query, setQuery] = useState('')
@@ -73,6 +76,7 @@ const Configuration: FC = () => {
     opening_statement: '',
     more_like_this: null,
     suggested_questions_after_answer: null,
+    speech_to_text: null,
     dataSets: [],
   })
 
@@ -102,6 +106,9 @@ const Configuration: FC = () => {
     setSuggestedQuestionsAfterAnswerConfig(modelConfig.suggested_questions_after_answer || {
       enabled: false,
     })
+    setSpeechToTextConfig(modelConfig.speech_to_text || {
+      enabled: false,
+    })
   }
 
   const [hasSetCustomAPIKEY, setHasSetCustomerAPIKEY] = useState(true)
@@ -146,6 +153,9 @@ const Configuration: FC = () => {
       if (modelConfig.suggested_questions_after_answer)
         setSuggestedQuestionsAfterAnswerConfig(modelConfig.suggested_questions_after_answer)
 
+      if (modelConfig.speech_to_text)
+        setSpeechToTextConfig(modelConfig.speech_to_text)
+
       const config = {
         modelConfig: {
           provider: model.provider,
@@ -157,6 +167,7 @@ const Configuration: FC = () => {
           opening_statement: modelConfig.opening_statement,
           more_like_this: modelConfig.more_like_this,
           suggested_questions_after_answer: modelConfig.suggested_questions_after_answer,
+          speech_to_text: modelConfig.speech_to_text,
           dataSets: datasets || [],
         },
         completionParams: model.completion_params,
@@ -187,6 +198,7 @@ const Configuration: FC = () => {
       opening_statement: introduction || '',
       more_like_this: moreLikeThisConfig,
       suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
+      speech_to_text: speechToTextConfig,
       agent_mode: {
         enabled: true,
         tools: [...postDatasets],
@@ -203,6 +215,7 @@ const Configuration: FC = () => {
       draft.opening_statement = introduction
       draft.more_like_this = moreLikeThisConfig
       draft.suggested_questions_after_answer = suggestedQuestionsAfterAnswerConfig
+      draft.speech_to_text = speechToTextConfig
       draft.dataSets = dataSets
     })
     setPublishedConfig({
@@ -245,6 +258,8 @@ const Configuration: FC = () => {
       setMoreLikeThisConfig,
       suggestedQuestionsAfterAnswerConfig,
       setSuggestedQuestionsAfterAnswerConfig,
+      speechToTextConfig,
+      setSpeechToTextConfig,
       formattingChanged,
       setFormattingChanged,
       inputs,

web/app/components/base/voice-input/index.tsx

 import { useCallback, useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { useParams, usePathname } from 'next/navigation'
 import cn from 'classnames'
 import Recorder from 'js-audio-recorder'
 import { useRafInterval } from 'ahooks'
@@ -27,6 +28,8 @@ const VoiceInput = ({
   const [originDuration, setOriginDuration] = useState(0)
   const [startRecord, setStartRecord] = useState(false)
   const [startConvert, setStartConvert] = useState(false)
+  const pathname = usePathname()
+  const params = useParams()
   const clearInterval = useRafInterval(() => {
     setOriginDuration(originDuration + 1)
   }, 1000)
@@ -76,8 +79,20 @@ const VoiceInput = ({
     const formData = new FormData()
     formData.append('file', wavFile)
 
+    let url = ''
+
+    if (params.token) {
+      url = '/audio-to-text'
+    }
+    else if (params.appId) {
+      if (pathname.search('explore/installed') > -1)
+        url = `/installed-apps/${params.appId}/audio-to-text`
+      else
+        url = `/apps/${params.appId}/audio-to-text`
+    }
+
     try {
-      const audioResponse = await audioToText(isPublic, formData)
+      const audioResponse = await audioToText(url, isPublic, formData)
       onConverted(audioResponse.text)
       onCancel()
     }

web/app/components/share/chat/index.tsx

@@ -151,6 +151,7 @@ const Main: FC<IMainProps> = ({
   }
 
   const [suggestedQuestionsAfterAnswerConfig, setSuggestedQuestionsAfterAnswerConfig] = useState<SuggestedQuestionsAfterAnswerConfig | null>(null)
+  const [speechToTextConfig, setSpeechToTextConfig] = useState<SuggestedQuestionsAfterAnswerConfig | null>(null)
 
   const [conversationIdChangeBecauseOfNew, setConversationIdChangeBecauseOfNew, getConversationIdChangeBecauseOfNew] = useGetState(false)
   const [isChatStarted, { setTrue: setChatStarted, setFalse: setChatNotStarted }] = useBoolean(false)
@@ -328,7 +329,7 @@ const Main: FC<IMainProps> = ({
         const isNotNewConversation = allConversations.some(item => item.id === _conversationId)
         setAllConversationList(allConversations)
         // fetch new conversation info
-        const { user_input_form, opening_statement: introduction, suggested_questions_after_answer }: any = appParams
+        const { user_input_form, opening_statement: introduction, suggested_questions_after_answer, speech_to_text }: any = appParams
         const prompt_variables = userInputsFormToPromptVariables(user_input_form)
         if (siteInfo.default_language)
           changeLanguage(siteInfo.default_language)
@@ -343,6 +344,7 @@ const Main: FC<IMainProps> = ({
           prompt_variables,
         } as PromptConfig)
         setSuggestedQuestionsAfterAnswerConfig(suggested_questions_after_answer)
+        setSpeechToTextConfig(speech_to_text)
 
         // setConversationList(conversations as ConversationItem[])
 
@@ -623,6 +625,7 @@ const Main: FC<IMainProps> = ({
                     isShowSuggestion={doShowSuggestion}
                     suggestionList={suggestQuestions}
                     displayScene={displayScene}
+                    isShowSpeechToText={speechToTextConfig?.enabled}
                   />
                 </div>
               </div>)

web/context/debug-configuration.ts

 import { createContext } from 'use-context-selector'
-import type { CompletionParams, Inputs, ModelConfig, MoreLikeThisConfig, PromptConfig, SuggestedQuestionsAfterAnswerConfig } from '@/models/debug'
+import type { CompletionParams, Inputs, ModelConfig, MoreLikeThisConfig, PromptConfig, SpeechToTextConfig, SuggestedQuestionsAfterAnswerConfig } from '@/models/debug'
 import type { DataSet } from '@/models/datasets'
 
 type IDebugConfiguration = {
@@ -19,6 +19,8 @@ type IDebugConfiguration = {
   setMoreLikeThisConfig: (moreLikeThisConfig: MoreLikeThisConfig) => void
   suggestedQuestionsAfterAnswerConfig: SuggestedQuestionsAfterAnswerConfig
   setSuggestedQuestionsAfterAnswerConfig: (suggestedQuestionsAfterAnswerConfig: SuggestedQuestionsAfterAnswerConfig) => void
+  speechToTextConfig: SpeechToTextConfig
+  setSpeechToTextConfig: (speechToTextConfig: SpeechToTextConfig) => void
   formattingChanged: boolean
   setFormattingChanged: (formattingChanged: boolean) => void
   inputs: Inputs
@@ -59,6 +61,10 @@ const DebugConfigurationContext = createContext<IDebugConfiguration>({
     enabled: false,
   },
   setSuggestedQuestionsAfterAnswerConfig: () => { },
+  speechToTextConfig: {
+    enabled: false,
+  },
+  setSpeechToTextConfig: () => { },
   formattingChanged: false,
   setFormattingChanged: () => { },
   inputs: {},

web/i18n/lang/app-debug.en.ts

@@ -46,6 +46,11 @@ const translation = {
       generateNumTip: 'Number of each generated times',
       tip: 'Using this feature will incur additional tokens overhead',
     },
+    speechToText: {
+      title: 'Speech to Text',
+      description: 'Once enabled, you can use voice input.',
+      resDes: 'Voice input is enabled',
+    },
     dataSet: {
       title: 'Context',
       noData: 'You can import datasets as context',

web/i18n/lang/app-debug.zh.ts

@@ -46,6 +46,11 @@ const translation = {
       generateNumTip: '每次生成数',
       tip: '使用此功能将会额外消耗 tokens',
     },
+    speechToText: {
+      title: '语音转文字',
+      description: '启用后，您可以使用语音输入。',
+      resDes: '语音输入已启用',
+    },
     dataSet: {
       title: '上下文',
       noData: '您可以导入数据集作为上下文',

web/models/debug.ts

@@ -31,6 +31,8 @@ export type MoreLikeThisConfig = {
 
 export type SuggestedQuestionsAfterAnswerConfig = MoreLikeThisConfig
 
+export type SpeechToTextConfig = MoreLikeThisConfig
+
 // frontend use. Not the same as backend
 export type ModelConfig = {
   provider: string // LLM Provider: for example "OPENAI"
@@ -43,6 +45,9 @@ export type ModelConfig = {
   suggested_questions_after_answer: {
     enabled: boolean
   } | null
+  speech_to_text: {
+    enabled: boolean
+  } | null
   dataSets: any[]
 }
 

web/service/share.ts

@@ -115,6 +115,6 @@ export const fetchSuggestedQuestions = (messageId: string, isInstalledApp: boole
   return (getAction('get', isInstalledApp))(getUrl(`/messages/${messageId}/suggested-questions`, isInstalledApp, installedAppId))
 }
 
-export const audioToText = (isPublicAPI: boolean, body: FormData) => {
-  return (getAction('post', !isPublicAPI))('/audio-to-text', { body }, { bodyStringify: false, deleteContentType: true }) as Promise<{ text: string }>
+export const audioToText = (url: string, isPublicAPI: boolean, body: FormData) => {
+  return (getAction('post', !isPublicAPI))(url, { body }, { bodyStringify: false, deleteContentType: true }) as Promise<{ text: string }>
 }

web/types/app.ts

@@ -85,6 +85,9 @@ export type ModelConfig = {
   suggested_questions_after_answer: {
     enabled: boolean
   }
+  speech_to_text: {
+    enabled: boolean
+  }
   agent_mode: {
     enabled: boolean
     tools: ToolItem[]