Merge branch 'feat/universal-chat' into deploy/dev

.github/workflows/check_no_chinese_comments.py

@@ -19,7 +19,7 @@ def check_file_for_chinese_comments(file_path):
 
 def main():
     has_chinese = False
-    excluded_files = ["model_template.py", 'stopwords.py', 'commands.py', 'indexing_runner.py']
+    excluded_files = ["model_template.py", 'stopwords.py', 'commands.py', 'indexing_runner.py', 'web_reader_tool.py']
 
     for root, _, files in os.walk("."):
         for file in files:

api/core/completion.py

 import logging
+import re
 from typing import Optional, List, Union, Tuple
 
 from langchain.base_language import BaseLanguageModel
@@ -270,6 +271,9 @@ And answer according to the language of the user's question.
 
             messages.append(human_message)
 
+            for message in messages:
+                message.content = re.sub(r'<\|.*?\|>', '', message.content)
+
             return messages, ['\nHuman:', '</histories>']
 
     @classmethod

api/core/orchestrator_rule_parser.py

@@ -31,6 +31,7 @@ class OrchestratorRuleParser:
     def __init__(self, tenant_id: str, app_model_config: AppModelConfig):
         self.tenant_id = tenant_id
         self.app_model_config = app_model_config
+        self.agent_summary_model_name = "gpt-3.5-turbo-16k"
 
     def to_agent_executor(self, conversation_message_task: ConversationMessageTask, memory: Optional[BaseChatMemory],
                        rest_tokens: int, chain_callback: MainChainGatherCallbackHandler) \
@@ -71,7 +72,7 @@ class OrchestratorRuleParser:
 
             summary_llm = LLMBuilder.to_llm(
                 tenant_id=self.tenant_id,
-                model_name=agent_model_name,
+                model_name=self.agent_summary_model_name,
                 temperature=0,
                 max_tokens=500,
                 callbacks=[DifyStdOutCallbackHandler()]
@@ -80,7 +81,7 @@ class OrchestratorRuleParser:
             tools = self.to_tools(
                 tool_configs=tool_configs,
                 conversation_message_task=conversation_message_task,
-                model_name=agent_model_name,
+                model_name=self.agent_summary_model_name,
                 rest_tokens=rest_tokens,
                 callbacks=[agent_callback, DifyStdOutCallbackHandler()]
             )