fix: audio

api/controllers/console/__init__.py

@@ -6,7 +6,7 @@ bp = Blueprint('console', __name__, url_prefix='/console/api')
 api = ExternalApi(bp)
 
 # Import other controllers
-from . import setup, version, apikey, admin
+from . import setup, version, apikey, admin, audio
 
 # Import app controllers
 from .app import app, site, completion, model_config, statistic, conversation, message, generator

api/controllers/console/audio.py

+# -*- coding:utf-8 -*-
+import logging
+
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource
+from werkzeug.exceptions import InternalServerError
+
+import services
+from services.audio_service import AudioService
+from controllers.console import api
+from controllers.console.app.error import AppUnavailableError, ProviderNotInitializeError, \
+    ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError, CompletionRequestError
+from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
+    LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+
+class AudioApi(Resource):
+    def post(self):
+        file = request.files['file']
+        
+        try:
+            response = AudioService.transcript(
+                tenant_id=current_user.current_tenant_id,
+                file=file,
+            )
+
+            return response
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+api.add_resource(AudioApi, '/audio-to-text')
\ No newline at end of file

api/controllers/console/explore/completion.py

@@ -3,7 +3,7 @@ import json
 import logging
 from typing import Generator, Union
 
-from flask import Response, stream_with_context, request
+from flask import Response, stream_with_context
 from flask_login import current_user
 from flask_restful import reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
@@ -20,7 +20,6 @@ from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthor
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
 from services.completion_service import CompletionService
-from controllers.console.datasets.error import NoFileUploadedError, TooManyFilesError
 
 
 # define completion api for user
@@ -140,52 +139,6 @@ class ChatStopApi(InstalledAppResource):
         PubHandler.stop(current_user, task_id)
 
         return {'result': 'success'}, 200
-    
-class AudioApi(InstalledAppResource):
-    def post(self, installed_app):
-        app_model = installed_app.app
-        if app_model.mode != 'chat':
-            raise NotChatAppError()
-        
-        file = request.files['file']
-
-        # check file
-        if 'file' not in request.files:
-            raise NoFileUploadedError()
-        
-        if len(request.files) > 1:
-            raise TooManyFilesError()
-        
-        from services.audio_service import AudioService
-
-        try:
-            response = AudioService.transcript(
-                app_model=app_model,
-                file=file,
-            )
-
-            return response
-        except services.errors.conversation.ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-        except services.errors.conversation.ConversationCompletedError:
-            raise ConversationCompletedError()
-        except services.errors.app_model_config.AppModelConfigBrokenError:
-            logging.exception("App model config broken.")
-            raise AppUnavailableError()
-        except ProviderTokenNotInitError:
-            raise ProviderNotInitializeError()
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
-                LLMRateLimitError, LLMAuthorizationError) as e:
-            raise CompletionRequestError(str(e))
-        except ValueError as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
 
 
 def compact_response(response: Union[dict | Generator]) -> Response:
@@ -226,4 +179,3 @@ api.add_resource(CompletionApi, '/installed-apps/<uuid:installed_app_id>/complet
 api.add_resource(CompletionStopApi, '/installed-apps/<uuid:installed_app_id>/completion-messages/<string:task_id>/stop', endpoint='installed_app_stop_completion')
 api.add_resource(ChatApi, '/installed-apps/<uuid:installed_app_id>/chat-messages', endpoint='installed_app_chat_completion')
 api.add_resource(ChatStopApi, '/installed-apps/<uuid:installed_app_id>/chat-messages/<string:task_id>/stop', endpoint='installed_app_stop_chat_completion')
-api.add_resource(AudioApi, '/installed-apps/<uuid:installed_app_id>/audio-to-text')

api/controllers/service_api/__init__.py

@@ -7,6 +7,6 @@ bp = Blueprint('service_api', __name__, url_prefix='/v1')
 api = ExternalApi(bp)
 
 
-from .app import completion, app, conversation, message
+from .app import completion, app, conversation, message, audio
 
 from .dataset import document

api/controllers/service_api/app/audio.py

+import logging
+
+from flask import request
+from werkzeug.exceptions import InternalServerError
+
+import services
+from services.audio_service import AudioService
+from controllers.service_api import api
+from controllers.service_api.app.error import AppUnavailableError, ProviderNotInitializeError, CompletionRequestError, ProviderQuotaExceededError, \
+    ProviderModelCurrentlyNotSupportError
+from controllers.service_api.wraps import AppApiResource
+from core.llm.error import LLMBadRequestError, LLMAuthorizationError, LLMAPIUnavailableError, LLMAPIConnectionError, \
+    LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+from models.model import App
+
+class AudioApi(AppApiResource):
+    def post(self, app_model: App, end_user):
+        file = request.files['file']
+
+        try:
+            response = AudioService.transcript(
+                tenant_id=app_model.tenant_id,
+                file=file,
+            )
+
+            return response
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+        
+api.add_resource(AudioApi, '/audio-to-text')
\ No newline at end of file

api/controllers/web/__init__.py

@@ -7,4 +7,4 @@ bp = Blueprint('web', __name__, url_prefix='/api')
 api = ExternalApi(bp)
 
 
-from . import completion, app, conversation, message, site, saved_message
+from . import completion, app, conversation, message, site, saved_message, audio

api/controllers/web/audio.py

+# -*- coding:utf-8 -*-
+import logging
+
+from flask import request
+from werkzeug.exceptions import InternalServerError
+
+import services
+from controllers.web import api
+from controllers.web.error import AppUnavailableError, ProviderNotInitializeError, CompletionRequestError, \
+    ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError
+from controllers.web.wraps import WebApiResource
+from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
+    LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+from services.audio_service import AudioService
+from models.model import App
+
+
+class AudioApi(WebApiResource):
+    def post(self, app_model: App, end_user):
+        file = request.files['file']
+
+        try:
+            response = AudioService.transcript(
+                tenant_id=app_model.tenant_id,
+                file=file,
+            )
+
+            return response
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+api.add_resource(AudioApi, '/audio-to-text')
\ No newline at end of file

api/controllers/web/completion.py

@@ -3,7 +3,7 @@ import json
 import logging
 from typing import Generator, Union
 
-from flask import Response, stream_with_context, request
+from flask import Response, stream_with_context
 from flask_restful import reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
 
@@ -18,8 +18,6 @@ from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthor
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
-from services.audio_service import AudioService
-from controllers.console.datasets.error import NoFileUploadedError, TooManyFilesError
 
 # define completion api for user
 class CompletionApi(WebApiResource):
@@ -134,49 +132,6 @@ class ChatStopApi(WebApiResource):
         PubHandler.stop(end_user, task_id)
 
         return {'result': 'success'}, 200
-    
-class AudioApi(WebApiResource):
-    def post(self, app_model, end_user):
-        if app_model.mode != 'chat':
-            raise NotChatAppError()
-        
-        file = request.files['file']
-
-        # check file
-        if 'file' not in request.files:
-            raise NoFileUploadedError()
-        
-        if len(request.files) > 1:
-            raise TooManyFilesError()
-
-        try:
-            response = AudioService.transcript(
-                app_model=app_model,
-                file=file,
-            )
-
-            return response
-        except services.errors.conversation.ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-        except services.errors.conversation.ConversationCompletedError:
-            raise ConversationCompletedError()
-        except services.errors.app_model_config.AppModelConfigBrokenError:
-            logging.exception("App model config broken.")
-            raise AppUnavailableError()
-        except ProviderTokenNotInitError:
-            raise ProviderNotInitializeError()
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
-                LLMRateLimitError, LLMAuthorizationError) as e:
-            raise CompletionRequestError(str(e))
-        except ValueError as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
 
 
 def compact_response(response: Union[dict | Generator]) -> Response:
@@ -217,4 +172,3 @@ api.add_resource(CompletionApi, '/completion-messages')
 api.add_resource(CompletionStopApi, '/completion-messages/<string:task_id>/stop')
 api.add_resource(ChatApi, '/chat-messages')
 api.add_resource(ChatStopApi, '/chat-messages/<string:task_id>/stop')
-api.add_resource(AudioApi, '/audio-to-text')

api/services/audio_service.py

-import openai
 import io
+import openai
 from werkzeug.datastructures import FileStorage
 from core.llm.llm_builder import LLMBuilder
 from core.llm.provider.llm_provider_service import LLMProviderService
-from models.model import App
-from controllers.console.datasets.error import FileTooLargeError, UnsupportedFileTypeError
+from services.errors.audio import NoAudioUploadedError, AudioTooLargeError, UnsupportedAudioTypeError
 
-FILE_SIZE_LIMIT = 25 * 1024 * 1024  # 25MB
+FILE_SIZE_LIMIT = 25 * 1024 * 1024
 ALLOWED_EXTENSIONS = ['mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav', 'webm']
 
 class AudioService:
     @classmethod
-    def transcript(cls, app_model: App, file: FileStorage, **params):
+    def transcript(cls, tenant_id: str, file: FileStorage, **params):
+        if file is None:
+            raise NoAudioUploadedError()
+        
+        extension = file.mimetype
+        if extension not in [f'audio/{ext}' for ext in ALLOWED_EXTENSIONS]:
+            raise AudioTooLargeError()
+
         file_content = file.read()
         file_size = len(file_content)
 
         if file_size > FILE_SIZE_LIMIT:
             message = f"({file_size} > {FILE_SIZE_LIMIT})"
-            raise FileTooLargeError(message)
+            raise UnsupportedAudioTypeError(message)
         
-        extension = file.mimetype
-        if extension not in [f'audio/{ext}' for ext in ALLOWED_EXTENSIONS]:
-            raise UnsupportedFileTypeError()
-
-        provider_name = LLMBuilder.get_default_provider(app_model.tenant_id)
-        provider = LLMProviderService(app_model.tenant_id, provider_name)
+        provider_name = LLMBuilder.get_default_provider(tenant_id)
+        provider = LLMProviderService(tenant_id, provider_name)
         credentials = provider.get_credentials(provider_name)
 
         buffer = io.BytesIO(file_content)

api/services/errors/__init__.py

 # -*- coding:utf-8 -*-
 __all__ = [
     'base', 'conversation', 'message', 'index', 'app_model_config', 'account', 'document', 'dataset',
-    'app', 'completion'
+    'app', 'completion', 'audio'
 ]
 
 from . import *

api/services/errors/audio.py

+from libs.exception import BaseHTTPException
+
+class NoAudioUploadedError(BaseHTTPException):
+    error_code = 'no_audio_uploaded'
+    description = "Please upload your audio."
+    code = 400
+
+
+class AudioTooLargeError(BaseHTTPException):
+    error_code = 'audio_too_large'
+    description = "Audio size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedAudioTypeError(BaseHTTPException):
+    error_code = 'unsupported_audio_type'
+    description = "Audio type not allowed."
+    code = 415
\ No newline at end of file

web/app/components/app/chat/index.tsx

@@ -63,8 +63,6 @@ export type IChatProps = {
   controlFocus?: number
   isShowSuggestion?: boolean
   suggestionList?: string[]
-  isInstalledApp: boolean
-  installedAppId: string
 }
 
 export type MessageMore = {
@@ -427,8 +425,6 @@ const Chat: FC<IChatProps> = ({
   controlFocus,
   isShowSuggestion,
   suggestionList,
-  isInstalledApp,
-  installedAppId,
 }) => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
@@ -612,8 +608,7 @@ const Chat: FC<IChatProps> = ({
               {
                 voiceInputShow && (
                   <VoiceInput
-                    isInstalledApp={isInstalledApp}
-                    installedAppId={installedAppId}
+                    isPublic={displayScene === 'web'}
                     onCancel={() => setVoiceInputShow(false)}
                     onConverted={text => setQuery(text)}
                   />

web/app/components/app/configuration/debug/index.tsx

@@ -386,6 +386,7 @@ const Debug: FC<IDebug> = ({
                   }}
                   isShowSuggestion={doShowSuggestion}
                   suggestionList={suggestQuestions}
+                  displayScene='console'
                 />
               </div>
             </div>

web/app/components/base/voice-input/index.tsx

@@ -8,15 +8,13 @@ import { Loading02, XClose } from '@/app/components/base/icons/src/vender/line/g
 import { audioToText } from '@/service/share'
 
 type VoiceInputTypes = {
-  isInstalledApp: boolean
-  installedAppId: string
+  isPublic: boolean
   onConverted: (text: string) => void
   onCancel: () => void
 }
 
 const VoiceInput = ({
-  isInstalledApp,
-  installedAppId,
+  isPublic,
   onCancel,
   onConverted,
 }: VoiceInputTypes) => {
@@ -74,7 +72,7 @@ const VoiceInput = ({
     formData.append('file', wavFile)
 
     try {
-      const audioResponse = await audioToText(isInstalledApp, installedAppId, formData)
+      const audioResponse = await audioToText(isPublic, formData)
       onConverted(audioResponse.text)
       onCancel()
     }

web/app/components/explore/installed-app/index.tsx

@@ -29,7 +29,7 @@ const InstalledApp: FC<IInstalledAppProps> = ({
     <div className='h-full p-2'>
       {installedApp?.app.mode === 'chat'
         ? (
-          <ChatApp isInstalledApp installedAppInfo={installedApp}/>
+          <ChatApp isInstalledApp installedAppInfo={installedApp} displayScene='console' />
         )
         : (
           <TextGenerationApp isInstalledApp installedAppInfo={installedApp}/>

web/app/components/share/chat/index.tsx

@@ -17,7 +17,7 @@ import Header from '@/app/components/share/header'
 import { delConversation, fetchAppInfo, fetchAppParams, fetchChatList, fetchConversations, fetchSuggestedQuestions, pinConversation, sendChatMessage, stopChatMessageResponding, unpinConversation, updateFeedback } from '@/service/share'
 import type { ConversationItem, SiteInfo } from '@/models/share'
 import type { PromptConfig, SuggestedQuestionsAfterAnswerConfig } from '@/models/debug'
-import type { Feedbacktype, IChatItem } from '@/app/components/app/chat'
+import type { DisplayScene, Feedbacktype, IChatItem } from '@/app/components/app/chat'
 import Chat from '@/app/components/app/chat'
 import { changeLanguage } from '@/i18n/i18next-config'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
@@ -28,11 +28,13 @@ import type { InstalledApp } from '@/models/explore'
 import Confirm from '@/app/components/base/confirm'
 
 export type IMainProps = {
+  displayScene?: DisplayScene
   isInstalledApp?: boolean
   installedAppInfo?: InstalledApp
 }
 
 const Main: FC<IMainProps> = ({
+  displayScene = 'web',
   isInstalledApp = false,
   installedAppInfo,
 }) => {
@@ -620,8 +622,7 @@ const Main: FC<IMainProps> = ({
                     controlFocus={controlFocus}
                     isShowSuggestion={doShowSuggestion}
                     suggestionList={suggestQuestions}
-                    isInstalledApp={isInstalledApp}
-                    installedAppId={installedAppInfo?.id || ''}
+                    displayScene={displayScene}
                   />
                 </div>
               </div>)

web/service/share.ts

@@ -115,6 +115,6 @@ export const fetchSuggestedQuestions = (messageId: string, isInstalledApp: boole
   return (getAction('get', isInstalledApp))(getUrl(`/messages/${messageId}/suggested-questions`, isInstalledApp, installedAppId))
 }
 
-export const audioToText = (isInstalledApp: boolean, installedAppId: string, body: FormData) => {
-  return (getAction('post', isInstalledApp))(getUrl('/audio-to-text', isInstalledApp, installedAppId), { body }, { bodyStringify: false, deleteContentType: true }) as Promise<{ text: string }>
+export const audioToText = (isPublicAPI: boolean, body: FormData) => {
+  return (getAction('post', !isPublicAPI))('/audio-to-text', { body }, { bodyStringify: false, deleteContentType: true }) as Promise<{ text: string }>
 }