Merge branch 'feat/chat-support-voice-input' into deploy/dev

api/controllers/console/app/audio.py

@@ -10,13 +10,16 @@ from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.app.error import AppUnavailableError, \
     ProviderNotInitializeError, CompletionRequestError, ProviderQuotaExceededError, \
-    ProviderModelCurrentlyNotSupportError
+    ProviderModelCurrentlyNotSupportError, NoAudioUploadedError, AudioTooLargeError, \
+    UnsupportedAudioTypeError, ProviderNotSupportSpeechToTextError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from flask_restful import Resource
 from services.audio_service import AudioService
+from services.errors.audio import NoAudioUploadedServiceError, AudioTooLargeServiceError, \
+    UnsupportedAudioTypeServiceError, ProviderNotSupportSpeechToTextServiceError
 
 
 class ChatMessageAudioApi(Resource):
@@ -39,6 +42,14 @@ class ChatMessageAudioApi(Resource):
         except services.errors.app_model_config.AppModelConfigBrokenError:
             logging.exception("App model config broken.")
             raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
         except ProviderTokenNotInitError:
             raise ProviderNotInitializeError()
         except QuotaExceededError:

api/controllers/console/app/error.py

@@ -49,3 +49,27 @@ class AppMoreLikeThisDisabledError(BaseHTTPException):
     error_code = 'app_more_like_this_disabled'
     description = "The 'More like this' feature is disabled. Please refresh your page."
     code = 403
+
+
+class NoAudioUploadedError(BaseHTTPException):
+    error_code = 'no_audio_uploaded'
+    description = "Please upload your audio."
+    code = 400
+
+
+class AudioTooLargeError(BaseHTTPException):
+    error_code = 'audio_too_large'
+    description = "Audio size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedAudioTypeError(BaseHTTPException):
+    error_code = 'unsupported_audio_type'
+    description = "Audio type not allowed."
+    code = 415
+
+
+class ProviderNotSupportSpeechToTextError(BaseHTTPException):
+    error_code = 'provider_not_support_speech_to_text'
+    description = "Provider not support speech to text."
+    code = 400
\ No newline at end of file

api/controllers/console/explore/audio.py

@@ -7,11 +7,15 @@ from werkzeug.exceptions import InternalServerError
 import services
 from controllers.console import api
 from controllers.console.app.error import AppUnavailableError, ProviderNotInitializeError, \
-    ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError, CompletionRequestError
+    ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError, CompletionRequestError, \
+    NoAudioUploadedError, AudioTooLargeError, \
+    UnsupportedAudioTypeError, ProviderNotSupportSpeechToTextError
 from controllers.console.explore.wraps import InstalledAppResource
 from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from services.audio_service import AudioService
+from services.errors.audio import NoAudioUploadedServiceError, AudioTooLargeServiceError, \
+    UnsupportedAudioTypeServiceError, ProviderNotSupportSpeechToTextServiceError
 from models.model import AppModelConfig
 
 
@@ -35,6 +39,14 @@ class ChatAudioApi(InstalledAppResource):
         except services.errors.app_model_config.AppModelConfigBrokenError:
             logging.exception("App model config broken.")
             raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
         except ProviderTokenNotInitError:
             raise ProviderNotInitializeError()
         except QuotaExceededError:

api/controllers/service_api/app/audio.py

@@ -4,14 +4,17 @@ from flask import request
 from werkzeug.exceptions import InternalServerError
 
 import services
-from services.audio_service import AudioService
 from controllers.service_api import api
 from controllers.service_api.app.error import AppUnavailableError, ProviderNotInitializeError, CompletionRequestError, ProviderQuotaExceededError, \
-    ProviderModelCurrentlyNotSupportError
+    ProviderModelCurrentlyNotSupportError, NoAudioUploadedError, AudioTooLargeError, UnsupportedAudioTypeError, \
+    ProviderNotSupportSpeechToTextError
 from controllers.service_api.wraps import AppApiResource
 from core.llm.error import LLMBadRequestError, LLMAuthorizationError, LLMAPIUnavailableError, LLMAPIConnectionError, \
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from models.model import App, AppModelConfig
+from services.audio_service import AudioService
+from services.errors.audio import NoAudioUploadedServiceError, AudioTooLargeServiceError, \
+    UnsupportedAudioTypeServiceError, ProviderNotSupportSpeechToTextServiceError
 
 class AudioApi(AppApiResource):
     def post(self, app_model: App, end_user):
@@ -32,6 +35,14 @@ class AudioApi(AppApiResource):
         except services.errors.app_model_config.AppModelConfigBrokenError:
             logging.exception("App model config broken.")
             raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
         except ProviderTokenNotInitError:
             raise ProviderNotInitializeError()
         except QuotaExceededError:

api/controllers/service_api/app/error.py

@@ -51,3 +51,27 @@ class CompletionRequestError(BaseHTTPException):
     description = "Completion request failed."
     code = 400
 
+
+class NoAudioUploadedError(BaseHTTPException):
+    error_code = 'no_audio_uploaded'
+    description = "Please upload your audio."
+    code = 400
+
+
+class AudioTooLargeError(BaseHTTPException):
+    error_code = 'audio_too_large'
+    description = "Audio size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedAudioTypeError(BaseHTTPException):
+    error_code = 'unsupported_audio_type'
+    description = "Audio type not allowed."
+    code = 415
+
+
+class ProviderNotSupportSpeechToTextError(BaseHTTPException):
+    error_code = 'provider_not_support_speech_to_text'
+    description = "Provider not support speech to text."
+    code = 400
+

api/controllers/web/audio.py

@@ -7,11 +7,14 @@ from werkzeug.exceptions import InternalServerError
 import services
 from controllers.web import api
 from controllers.web.error import AppUnavailableError, ProviderNotInitializeError, CompletionRequestError, \
-    ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError
+    ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError, NoAudioUploadedError, AudioTooLargeError, \
+    UnsupportedAudioTypeError, ProviderNotSupportSpeechToTextError
 from controllers.web.wraps import WebApiResource
 from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from services.audio_service import AudioService
+from services.errors.audio import NoAudioUploadedServiceError, AudioTooLargeServiceError, \
+    UnsupportedAudioTypeServiceError, ProviderNotSupportSpeechToTextServiceError
 from models.model import App, AppModelConfig
 
 
@@ -34,6 +37,14 @@ class AudioApi(WebApiResource):
         except services.errors.app_model_config.AppModelConfigBrokenError:
             logging.exception("App model config broken.")
             raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
         except ProviderTokenNotInitError:
             raise ProviderNotInitializeError()
         except QuotaExceededError:

api/controllers/web/error.py

@@ -62,3 +62,27 @@ class AppSuggestedQuestionsAfterAnswerDisabledError(BaseHTTPException):
     error_code = 'app_suggested_questions_after_answer_disabled'
     description = "The 'Suggested Questions After Answer' feature is disabled. Please refresh your page."
     code = 403
+
+
+class NoAudioUploadedError(BaseHTTPException):
+    error_code = 'no_audio_uploaded'
+    description = "Please upload your audio."
+    code = 400
+
+
+class AudioTooLargeError(BaseHTTPException):
+    error_code = 'audio_too_large'
+    description = "Audio size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedAudioTypeError(BaseHTTPException):
+    error_code = 'unsupported_audio_type'
+    description = "Audio type not allowed."
+    code = 415
+
+
+class ProviderNotSupportSpeechToTextError(BaseHTTPException):
+    error_code = 'provider_not_support_speech_to_text'
+    description = "Provider not support speech to text."
+    code = 400
\ No newline at end of file

api/core/llm/provider/azure_provider.py

@@ -44,6 +44,7 @@ class AzureProvider(BaseProvider):
         config['openai_api_type'] = 'azure'
         if model_id == 'text-embedding-ada-002':
             config['deployment'] = model_id.replace('.', '') if model_id else None
+            config['chunk_size'] = 1
         else:
             config['deployment_name'] = model_id.replace('.', '') if model_id else None
         return config

api/core/llm/whisper.py

+import openai
+from models.provider import ProviderName
+from core.llm.error_handle_wraps import handle_llm_exceptions
+from core.llm.provider.base import BaseProvider
+
+
+class Whisper:
+
+    def __init__(self, provider: BaseProvider):
+        self.provider = provider
+
+        if self.provider.get_provider_name() == ProviderName.OPENAI:
+            self.client = openai.Audio
+            self.credentials = provider.get_credentials()
+
+    @handle_llm_exceptions
+    def transcribe(self, file):
+        return self.client.transcribe(
+            model='whisper-1', 
+            file=file,
+            api_key=self.credentials.get('openai_api_key'),
+            api_base=self.credentials.get('openai_api_base'),
+            api_type=self.credentials.get('openai_api_type'),
+            api_version=self.credentials.get('openai_api_version'),
+        )

api/services/audio_service.py

 import io
-import openai
 from werkzeug.datastructures import FileStorage
 from core.llm.llm_builder import LLMBuilder
 from core.llm.provider.llm_provider_service import LLMProviderService
-from services.errors.audio import NoAudioUploadedError, AudioTooLargeError, UnsupportedAudioTypeError
-from openai.error import InvalidRequestError
+from services.errors.audio import NoAudioUploadedServiceError, AudioTooLargeServiceError, UnsupportedAudioTypeServiceError, ProviderNotSupportSpeechToTextServiceError
+from core.llm.whisper import Whisper
+from models.provider import ProviderName
 
 FILE_SIZE_LIMIT = 1 * 1024 * 1024
 ALLOWED_EXTENSIONS = ['mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav', 'webm']
 
 class AudioService:
     @classmethod
-    def transcript(cls, tenant_id: str, file: FileStorage, **params):
+    def transcript(cls, tenant_id: str, file: FileStorage):
         if file is None:
-            raise NoAudioUploadedError()
+            raise NoAudioUploadedServiceError()
         
         extension = file.mimetype
         if extension not in [f'audio/{ext}' for ext in ALLOWED_EXTENSIONS]:
-            raise AudioTooLargeError()
+            raise UnsupportedAudioTypeServiceError()
 
         file_content = file.read()
         file_size = len(file_content)
 
         if file_size > FILE_SIZE_LIMIT:
             message = f"({file_size} > {FILE_SIZE_LIMIT})"
-            raise UnsupportedAudioTypeError(message)
+            raise AudioTooLargeServiceError(message)
         
         provider_name = LLMBuilder.get_default_provider(tenant_id)
-        provider = LLMProviderService(tenant_id, provider_name)
-        credentials = provider.get_credentials(provider_name)
+        if provider_name != ProviderName.OPENAI.value:
+            raise ProviderNotSupportSpeechToTextServiceError('haha')
+
+        provider_service = LLMProviderService(tenant_id, provider_name)
 
         buffer = io.BytesIO(file_content)
         buffer.name = 'temp.wav'
 
-        try:
-            transcript = openai.Audio.transcribe(
-                        model='whisper-1', 
-                        file=buffer,
-                        api_key=credentials.get('openai_api_key'),
-                        api_base=credentials.get('openai_api_base'),
-                        api_type=credentials.get('openai_api_type'),
-                        api_version=credentials.get('openai_api_version'),
-                        params=params
-                    )
-        
-            return transcript
-        except InvalidRequestError as e:
-            return {'message': str(e)}, 400
-    
+        return Whisper(provider_service.provider).transcribe(buffer)
+
 
 
         

api/services/errors/audio.py

-from libs.exception import BaseHTTPException
+from services.errors.base import BaseServiceError
 
-class NoAudioUploadedError(BaseHTTPException):
+class NoAudioUploadedServiceError(BaseServiceError):
     error_code = 'no_audio_uploaded'
     description = "Please upload your audio."
     code = 400
 
 
-class AudioTooLargeError(BaseHTTPException):
+class AudioTooLargeServiceError(BaseServiceError):
     error_code = 'audio_too_large'
     description = "Audio size exceeded. {message}"
     code = 413
 
 
-class UnsupportedAudioTypeError(BaseHTTPException):
+class UnsupportedAudioTypeServiceError(BaseServiceError):
     error_code = 'unsupported_audio_type'
     description = "Audio type not allowed."
-    code = 415
\ No newline at end of file
+    code = 415
+
+class ProviderNotSupportSpeechToTextServiceError(BaseServiceError):
+    error_code = 'provider_not_support_speech_to_text'
+    description = "Provider not support speech to text. {message}"
+    code = 400
\ No newline at end of file

web/app/components/base/icons/src/vender/solid/general/index.ts

+export { default as Download02 } from './Download02'
 export { default as XCircle } from './XCircle'
 export { default as Download02 } from './Download02'