Feat/assistant app (#2086)

Co-authored-by: chenhe <guchenhe@gmail.com>
Co-authored-by: Pascal M <11357019+perzeuss@users.noreply.github.com>

.github/workflows/tool-tests.yaml

+name: Run Tool Pytest
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: ./api/requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r ./api/requirements.txt
+
+      - name: Run pytest
+        run: pytest ./api/tests/integration_tests/tools/test_all_provider.py

CONTRIBUTING.md

@@ -91,6 +91,8 @@ To validate your set up, head over to [http://localhost:3000](http://localhost:3
 
 If you are adding a model provider, [this guide](https://github.com/langgenius/dify/blob/main/api/core/model_runtime/README.md) is for you.
 
+If you are adding a tool provider to Agent or Workflow, [this guide](./api/core/tools/README.md) is for you.
+
 To help you quickly navigate where your contribution fits, a brief, annotated outline of Dify's backend & frontend is as follows:
 
 ### Backend

api/app.py

@@ -30,7 +30,7 @@ from flask import Flask, Response, request
 from flask_cors import CORS
 from libs.passport import PassportService
 # DO NOT REMOVE BELOW
-from models import account, dataset, model, source, task, tool, web
+from models import account, dataset, model, source, task, tool, web, tools
 from services.account_service import AccountService
 
 # DO NOT REMOVE ABOVE

api/commands.py

@@ -22,7 +22,7 @@ from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
 from models.account import InvitationCode, Tenant, TenantAccountJoin
 from models.dataset import Dataset, DatasetCollectionBinding, DatasetQuery, Document
-from models.model import Account, App, AppModelConfig, Message, MessageAnnotation
+from models.model import Account, App, AppModelConfig, Message, MessageAnnotation, InstalledApp
 from models.provider import Provider, ProviderModel, ProviderQuotaType, ProviderType
 from qdrant_client.http.models import TextIndexParams, TextIndexType, TokenizerType
 from tqdm import tqdm
@@ -775,6 +775,66 @@ def add_annotation_question_field_value():
             click.echo(
                 click.style(f'Congratulations! add annotation question value successful. Deal count {message_annotation_deal_count}', fg='green'))
 
+@click.command('migrate-universal-chat-to-installed-app', help='Migrate universal chat to installed app.')
+@click.option("--batch-size", default=500, help="Number of records to migrate in each batch.")
+def migrate_universal_chat_to_installed_app(batch_size):
+    total_records = db.session.query(App).filter(
+        App.is_universal == True
+    ).count()
+    if total_records == 0:
+        click.secho("No data to migrate.", fg='green')
+        return
+
+    num_batches = (total_records + batch_size - 1) // batch_size
+
+    with tqdm(total=total_records, desc="Migrating Data") as pbar:
+        for i in range(num_batches):
+            offset = i * batch_size
+            limit = min(batch_size, total_records - offset)
+
+            click.secho(f"Fetching batch {i + 1}/{num_batches} from source database...", fg='green')
+
+            data_batch: list[App] = db.session.query(App) \
+                .filter(App.is_universal == True) \
+                .order_by(App.created_at) \
+                .offset(offset).limit(limit).all()
+
+            if not data_batch:
+                click.secho("No more data to migrate.", fg='green')
+                break
+
+            try:
+                click.secho(f"Migrating {len(data_batch)} records...", fg='green')
+                for data in data_batch:
+                    # check if the app is already installed
+                    installed_app = db.session.query(InstalledApp).filter(
+                        InstalledApp.app_id == data.id
+                    ).first()
+
+                    if installed_app:
+                        continue
+
+                    # insert installed app
+                    installed_app = InstalledApp(
+                        app_id=data.id,
+                        tenant_id=data.tenant_id,
+                        position=0,
+                        app_owner_tenant_id=data.tenant_id,
+                        is_pinned=True,
+                        last_used_at=datetime.datetime.utcnow(),
+                    )
+
+                    db.session.add(installed_app)
+
+                db.session.commit()
+
+            except Exception as e:
+                click.secho(f"Error while migrating data: {e}, app_id: {data.id}", fg='red')
+                continue
+
+            click.secho(f"Successfully migrated batch {i + 1}/{num_batches}.", fg='green')
+
+            pbar.update(len(data_batch))
 
 def register_commands(app):
     app.cli.add_command(reset_password)
@@ -791,3 +851,4 @@ def register_commands(app):
     app.cli.add_command(migrate_default_input_to_dataset_query_variable)
     app.cli.add_command(add_qdrant_full_text_index)
     app.cli.add_command(add_annotation_question_field_value)
+    app.cli.add_command(migrate_universal_chat_to_installed_app)

api/controllers/console/__init__.py

@@ -16,7 +16,5 @@ from .billing import billing
 from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing
 # Import explore controllers
 from .explore import audio, completion, conversation, installed_app, message, parameter, recommended_app, saved_message
-# Import universal chat controllers
-from .universal_chat import audio, chat, conversation, message, parameter
 # Import workspace controllers
 from .workspace import account, members, model_providers, models, tool_providers, workspace

api/controllers/console/app/app.py

@@ -16,14 +16,15 @@ from events.app_event import app_was_created, app_was_deleted
 from extensions.ext_database import db
 from fields.app_fields import (app_detail_fields, app_detail_fields_with_site, app_pagination_fields,
                                template_list_fields)
+from flask import current_app
 from flask_login import current_user
 from flask_restful import Resource, abort, inputs, marshal_with, reqparse
 from libs.login import login_required
 from models.model import App, AppModelConfig, Site
+from models.tools import ApiToolProvider
 from services.app_model_config_service import AppModelConfigService
 from werkzeug.exceptions import Forbidden
 
-
 def _get_app(app_id, tenant_id):
     app = db.session.query(App).filter(App.id == app_id, App.tenant_id == tenant_id).first()
     if not app:
@@ -42,14 +43,30 @@ class AppListApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument('page', type=inputs.int_range(1, 99999), required=False, default=1, location='args')
         parser.add_argument('limit', type=inputs.int_range(1, 100), required=False, default=20, location='args')
+        parser.add_argument('mode', type=str, choices=['chat', 'completion', 'all'], default='all', location='args', required=False)
+        parser.add_argument('name', type=str, location='args', required=False)
         args = parser.parse_args()
 
+        filters = [
+            App.tenant_id == current_user.current_tenant_id,
+        ]
+
+        if args['mode'] == 'completion':
+            filters.append(App.mode == 'completion')
+        elif args['mode'] == 'chat':
+            filters.append(App.mode == 'chat')
+        else:
+            pass
+
+        if 'name' in args and args['name']:
+            filters.append(App.name.ilike(f'%{args["name"]}%'))
+
         app_models = db.paginate(
-            db.select(App).where(App.tenant_id == current_user.current_tenant_id,
-                                 App.is_universal == False).order_by(App.created_at.desc()),
+            db.select(App).where(*filters).order_by(App.created_at.desc()),
             page=args['page'],
             per_page=args['limit'],
-            error_out=False)
+            error_out=False
+        )
 
         return app_models
 
@@ -62,7 +79,7 @@ class AppListApi(Resource):
         """Create app"""
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, location='json')
-        parser.add_argument('mode', type=str, choices=['completion', 'chat'], location='json')
+        parser.add_argument('mode', type=str, choices=['completion', 'chat', 'assistant'], location='json')
         parser.add_argument('icon', type=str, location='json')
         parser.add_argument('icon_background', type=str, location='json')
         parser.add_argument('model_config', type=dict, location='json')
@@ -178,7 +195,7 @@ class AppListApi(Resource):
         app_was_created.send(app)
 
         return app, 201
-
+    
 
 class AppTemplateApi(Resource):
 

api/controllers/console/explore/installed_app.py

@@ -33,8 +33,9 @@ class InstalledAppsListApi(Resource):
                 'app_owner_tenant_id': installed_app.app_owner_tenant_id,
                 'is_pinned': installed_app.is_pinned,
                 'last_used_at': installed_app.last_used_at,
-                "editable": current_user.role in ["owner", "admin"],
-                "uninstallable": current_tenant_id == installed_app.app_owner_tenant_id
+                'editable': current_user.role in ["owner", "admin"],
+                'uninstallable': current_tenant_id == installed_app.app_owner_tenant_id,
+                'is_agent': installed_app.is_agent
             }
             for installed_app in installed_apps
         ]

api/controllers/console/explore/message.py

@@ -17,9 +17,9 @@ from core.model_runtime.errors.invoke import InvokeError
 from fields.message_fields import message_infinite_scroll_pagination_fields
 from flask import Response, stream_with_context
 from flask_login import current_user
-from flask_restful import marshal_with, reqparse
+from flask_restful import marshal_with, reqparse, fields
 from flask_restful.inputs import int_range
-from libs.helper import uuid_value
+from libs.helper import uuid_value, TimestampField
 from services.completion_service import CompletionService
 from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
@@ -29,7 +29,6 @@ from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class MessageListApi(InstalledAppResource):
-
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, installed_app):
         app_model = installed_app.app
@@ -51,7 +50,6 @@ class MessageListApi(InstalledAppResource):
         except services.errors.message.FirstMessageNotExistsError:
             raise NotFound("First Message Not Exists.")
 
-
 class MessageFeedbackApi(InstalledAppResource):
     def post(self, installed_app, message_id):
         app_model = installed_app.app

api/controllers/console/explore/parameter.py

 # -*- coding:utf-8 -*-
+import json
+
 from controllers.console import api
 from controllers.console.explore.wraps import InstalledAppResource
 from flask import current_app
 from flask_restful import fields, marshal_with
-from models.model import InstalledApp
+from models.model import InstalledApp, AppModelConfig
+from models.tools import ApiToolProvider
 
+from extensions.ext_database import db
 
 class AppParameterApi(InstalledAppResource):
     """Resource for app variables."""
@@ -58,5 +62,42 @@ class AppParameterApi(InstalledAppResource):
             }
         }
 
+class ExploreAppMetaApi(InstalledAppResource):
+    def get(self, installed_app: InstalledApp):
+        """Get app meta"""
+        app_model_config: AppModelConfig = installed_app.app.app_model_config
+
+        agent_config = app_model_config.agent_mode_dict or {}
+        meta = {
+            'tool_icons': {}
+        }
+
+        # get all tools
+        tools = agent_config.get('tools', [])
+        url_prefix = (current_app.config.get("CONSOLE_API_URL")
+                  + f"/console/api/workspaces/current/tool-provider/builtin/")
+        for tool in tools:
+            keys = list(tool.keys())
+            if len(keys) >= 4:
+                # current tool standard
+                provider_type = tool.get('provider_type')
+                provider_id = tool.get('provider_id')
+                tool_name = tool.get('tool_name')
+                if provider_type == 'builtin':
+                    meta['tool_icons'][tool_name] = url_prefix + provider_id + '/icon'
+                elif provider_type == 'api':
+                    try:
+                        provider: ApiToolProvider = db.session.query(ApiToolProvider).filter(
+                            ApiToolProvider.id == provider_id
+                        )
+                        meta['tool_icons'][tool_name] = json.loads(provider.icon)
+                    except:
+                        meta['tool_icons'][tool_name] =  {
+                            "background": "#252525",
+                            "content": "\ud83d\ude01"
+                        }
+
+        return meta
 
 api.add_resource(AppParameterApi, '/installed-apps/<uuid:installed_app_id>/parameters', endpoint='installed_app_parameters')
+api.add_resource(ExploreAppMetaApi, '/installed-apps/<uuid:installed_app_id>/meta', endpoint='installed_app_meta')

api/controllers/console/explore/recommended_app.py

@@ -29,7 +29,8 @@ recommended_app_fields = {
     'is_listed': fields.Boolean,
     'install_count': fields.Integer,
     'installed': fields.Boolean,
-    'editable': fields.Boolean
+    'editable': fields.Boolean,
+    'is_agent': fields.Boolean
 }
 
 recommended_app_list_fields = {
@@ -82,6 +83,7 @@ class RecommendedAppListApi(Resource):
                 'install_count': recommended_app.install_count,
                 'installed': installed,
                 'editable': current_user.role in ['owner', 'admin'],
+                "is_agent": app.is_agent
             }
             recommended_apps_result.append(recommended_app_result)
 

api/controllers/console/universal_chat/audio.py

@@ -60,5 +60,3 @@ class UniversalChatAudioApi(UniversalChatResource):
             logging.exception("internal server error.")
             raise InternalServerError()
         
-
-api.add_resource(UniversalChatAudioApi, '/universal-chat/audio-to-text')
\ No newline at end of file

api/controllers/console/universal_chat/chat.py

-import json
-import logging
-from typing import Generator, Union
-
-import services
-from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from core.application_queue_manager import ApplicationQueueManager
-from core.entities.application_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import reqparse
-from libs.helper import uuid_value
-from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
-
-
-class UniversalChatApi(UniversalChatResource):
-    def post(self, universal_app):
-        app_model = universal_app
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('query', type=str, required=True, location='json')
-        parser.add_argument('files', type=list, required=False, location='json')
-        parser.add_argument('conversation_id', type=uuid_value, location='json')
-        parser.add_argument('provider', type=str, required=True, location='json')
-        parser.add_argument('model', type=str, required=True, location='json')
-        parser.add_argument('tools', type=list, required=True, location='json')
-        parser.add_argument('retriever_from', type=str, required=False, default='universal_app', location='json')
-        args = parser.parse_args()
-
-        app_model_config = app_model.app_model_config
-
-        # update app model config
-        args['model_config'] = app_model_config.to_dict()
-        args['model_config']['model']['name'] = args['model']
-        args['model_config']['model']['provider'] = args['provider']
-        args['model_config']['agent_mode']['tools'] = args['tools']
-
-        if not args['model_config']['agent_mode']['tools']:
-            args['model_config']['agent_mode']['tools'] = [
-                {
-                    "current_datetime": {
-                        "enabled": True
-                    }
-                }
-            ]
-        else:
-            args['model_config']['agent_mode']['tools'].append({
-                    "current_datetime": {
-                        "enabled": True
-                    }
-                })
-
-        args['inputs'] = {}
-
-        del args['model']
-        del args['tools']
-
-        args['auto_generate_name'] = False
-
-        try:
-            response = CompletionService.completion(
-                app_model=app_model,
-                user=current_user,
-                args=args,
-                invoke_from=InvokeFrom.EXPLORE,
-                streaming=True,
-                is_model_config_override=True,
-            )
-
-            return compact_response(response)
-        except services.errors.conversation.ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-        except services.errors.conversation.ConversationCompletedError:
-            raise ConversationCompletedError()
-        except services.errors.app_model_config.AppModelConfigBrokenError:
-            logging.exception("App model config broken.")
-            raise AppUnavailableError()
-        except ProviderTokenNotInitError:
-            raise ProviderNotInitializeError()
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-        except ValueError as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
-
-
-class UniversalChatStopApi(UniversalChatResource):
-    def post(self, universal_app, task_id):
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)
-
-        return {'result': 'success'}, 200
-
-
-def compact_response(response: Union[dict, Generator]) -> Response:
-    if isinstance(response, dict):
-        return Response(response=json.dumps(response), status=200, mimetype='application/json')
-    else:
-        def generate() -> Generator:
-            for chunk in response:
-                yield chunk
-
-        return Response(stream_with_context(generate()), status=200,
-                        mimetype='text/event-stream')
-
-
-api.add_resource(UniversalChatApi, '/universal-chat/messages')
-api.add_resource(UniversalChatStopApi, '/universal-chat/messages/<string:task_id>/stop')

api/controllers/console/universal_chat/conversation.py

-# -*- coding:utf-8 -*-
-from controllers.console import api
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from fields.conversation_fields import (conversation_with_model_config_fields,
-                                        conversation_with_model_config_infinite_scroll_pagination_fields)
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
-from services.conversation_service import ConversationService
-from services.errors.conversation import ConversationNotExistsError, LastConversationNotExistsError
-from services.web_conversation_service import WebConversationService
-from werkzeug.exceptions import NotFound
-
-
-class UniversalChatConversationListApi(UniversalChatResource):
-
-    @marshal_with(conversation_with_model_config_infinite_scroll_pagination_fields)
-    def get(self, universal_app):
-        app_model = universal_app
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('last_id', type=uuid_value, location='args')
-        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
-        parser.add_argument('pinned', type=str, choices=['true', 'false', None], location='args')
-        args = parser.parse_args()
-
-        pinned = None
-        if 'pinned' in args and args['pinned'] is not None:
-            pinned = True if args['pinned'] == 'true' else False
-
-        try:
-            return WebConversationService.pagination_by_last_id(
-                app_model=app_model,
-                user=current_user,
-                last_id=args['last_id'],
-                limit=args['limit'],
-                pinned=pinned
-            )
-        except LastConversationNotExistsError:
-            raise NotFound("Last Conversation Not Exists.")
-
-
-class UniversalChatConversationApi(UniversalChatResource):
-    def delete(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-
-        try:
-            ConversationService.delete(app_model, conversation_id, current_user)
-        except ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-
-        WebConversationService.unpin(app_model, conversation_id, current_user)
-
-        return {"result": "success"}, 204
-
-
-class UniversalChatConversationRenameApi(UniversalChatResource):
-
-    @marshal_with(conversation_with_model_config_fields)
-    def post(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('name', type=str, required=False, location='json')
-        parser.add_argument('auto_generate', type=bool, required=False, default=False, location='json')
-        args = parser.parse_args()
-
-        try:
-            return ConversationService.rename(
-                app_model,
-                conversation_id,
-                current_user,
-                args['name'],
-                args['auto_generate']
-            )
-        except ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-
-
-class UniversalChatConversationPinApi(UniversalChatResource):
-
-    def patch(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-
-        try:
-            WebConversationService.pin(app_model, conversation_id, current_user)
-        except ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-
-        return {"result": "success"}
-
-
-class UniversalChatConversationUnPinApi(UniversalChatResource):
-    def patch(self, universal_app, c_id):
-        app_model = universal_app
-        conversation_id = str(c_id)
-        WebConversationService.unpin(app_model, conversation_id, current_user)
-
-        return {"result": "success"}
-
-
-api.add_resource(UniversalChatConversationRenameApi, '/universal-chat/conversations/<uuid:c_id>/name')
-api.add_resource(UniversalChatConversationListApi, '/universal-chat/conversations')
-api.add_resource(UniversalChatConversationApi, '/universal-chat/conversations/<uuid:c_id>')
-api.add_resource(UniversalChatConversationPinApi, '/universal-chat/conversations/<uuid:c_id>/pin')
-api.add_resource(UniversalChatConversationUnPinApi, '/universal-chat/conversations/<uuid:c_id>/unpin')

api/controllers/console/universal_chat/message.py

-# -*- coding:utf-8 -*-
-import logging
-
-import services
-from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
-from controllers.console.explore.error import AppSuggestedQuestionsAfterAnswerDisabledError
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
-from core.model_runtime.errors.invoke import InvokeError
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
-from services.errors.conversation import ConversationNotExistsError
-from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
-from services.message_service import MessageService
-from werkzeug.exceptions import InternalServerError, NotFound
-
-
-class UniversalChatMessageListApi(UniversalChatResource):
-    feedback_fields = {
-        'rating': fields.String
-    }
-
-    agent_thought_fields = {
-        'id': fields.String,
-        'chain_id': fields.String,
-        'message_id': fields.String,
-        'position': fields.Integer,
-        'thought': fields.String,
-        'tool': fields.String,
-        'tool_input': fields.String,
-        'created_at': TimestampField
-    }
-
-    retriever_resource_fields = {
-        'id': fields.String,
-        'message_id': fields.String,
-        'position': fields.Integer,
-        'dataset_id': fields.String,
-        'dataset_name': fields.String,
-        'document_id': fields.String,
-        'document_name': fields.String,
-        'data_source_type': fields.String,
-        'segment_id': fields.String,
-        'score': fields.Float,
-        'hit_count': fields.Integer,
-        'word_count': fields.Integer,
-        'segment_position': fields.Integer,
-        'index_node_hash': fields.String,
-        'content': fields.String,
-        'created_at': TimestampField
-    }
-
-    message_fields = {
-        'id': fields.String,
-        'conversation_id': fields.String,
-        'inputs': fields.Raw,
-        'query': fields.String,
-        'answer': fields.String,
-        'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
-        'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
-        'created_at': TimestampField,
-        'agent_thoughts': fields.List(fields.Nested(agent_thought_fields))
-    }
-
-    message_infinite_scroll_pagination_fields = {
-        'limit': fields.Integer,
-        'has_more': fields.Boolean,
-        'data': fields.List(fields.Nested(message_fields))
-    }
-
-    @marshal_with(message_infinite_scroll_pagination_fields)
-    def get(self, universal_app):
-        app_model = universal_app
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('conversation_id', required=True, type=uuid_value, location='args')
-        parser.add_argument('first_id', type=uuid_value, location='args')
-        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
-        args = parser.parse_args()
-
-        try:
-            return MessageService.pagination_by_first_id(app_model, current_user,
-                                                     args['conversation_id'], args['first_id'], args['limit'])
-        except services.errors.conversation.ConversationNotExistsError:
-            raise NotFound("Conversation Not Exists.")
-        except services.errors.message.FirstMessageNotExistsError:
-            raise NotFound("First Message Not Exists.")
-
-
-class UniversalChatMessageFeedbackApi(UniversalChatResource):
-    def post(self, universal_app, message_id):
-        app_model = universal_app
-        message_id = str(message_id)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('rating', type=str, choices=['like', 'dislike', None], location='json')
-        args = parser.parse_args()
-
-        try:
-            MessageService.create_feedback(app_model, message_id, current_user, args['rating'])
-        except services.errors.message.MessageNotExistsError:
-            raise NotFound("Message Not Exists.")
-
-        return {'result': 'success'}
-
-
-class UniversalChatMessageSuggestedQuestionApi(UniversalChatResource):
-    def get(self, universal_app, message_id):
-        app_model = universal_app
-        message_id = str(message_id)
-
-        try:
-            questions = MessageService.get_suggested_questions_after_answer(
-                app_model=app_model,
-                user=current_user,
-                message_id=message_id
-            )
-        except MessageNotExistsError:
-            raise NotFound("Message not found")
-        except ConversationNotExistsError:
-            raise NotFound("Conversation not found")
-        except SuggestedQuestionsAfterAnswerDisabledError:
-            raise AppSuggestedQuestionsAfterAnswerDisabledError()
-        except ProviderTokenNotInitError:
-            raise ProviderNotInitializeError()
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-        except Exception:
-            logging.exception("internal server error.")
-            raise InternalServerError()
-
-        return {'data': questions}
-
-
-api.add_resource(UniversalChatMessageListApi, '/universal-chat/messages')
-api.add_resource(UniversalChatMessageFeedbackApi, '/universal-chat/messages/<uuid:message_id>/feedbacks')
-api.add_resource(UniversalChatMessageSuggestedQuestionApi, '/universal-chat/messages/<uuid:message_id>/suggested-questions')

api/controllers/console/universal_chat/parameter.py

-# -*- coding:utf-8 -*-
-import json
-
-from controllers.console import api
-from controllers.console.universal_chat.wraps import UniversalChatResource
-from flask_restful import fields, marshal_with
-from models.model import App
-
-
-class UniversalChatParameterApi(UniversalChatResource):
-    """Resource for app variables."""
-    parameters_fields = {
-        'opening_statement': fields.String,
-        'suggested_questions': fields.Raw,
-        'suggested_questions_after_answer': fields.Raw,
-        'speech_to_text': fields.Raw,
-        'retriever_resource': fields.Raw,
-        'annotation_reply': fields.Raw
-    }
-
-    @marshal_with(parameters_fields)
-    def get(self, universal_app: App):
-        """Retrieve app parameters."""
-        app_model = universal_app
-        app_model_config = app_model.app_model_config
-        app_model_config.retriever_resource = json.dumps({'enabled': True})
-
-        return {
-            'opening_statement': app_model_config.opening_statement,
-            'suggested_questions': app_model_config.suggested_questions_list,
-            'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
-            'speech_to_text': app_model_config.speech_to_text_dict,
-            'retriever_resource': app_model_config.retriever_resource_dict,
-            'annotation_reply': app_model_config.annotation_reply_dict,
-        }
-
-
-api.add_resource(UniversalChatParameterApi, '/universal-chat/parameters')

api/controllers/console/universal_chat/wraps.py

-import json
-from functools import wraps
-
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource
-from libs.login import login_required
-from models.model import App, AppModelConfig
-
-
-def universal_chat_app_required(view=None):
-    def decorator(view):
-        @wraps(view)
-        def decorated(*args, **kwargs):
-            # get universal chat app
-            universal_app = db.session.query(App).filter(
-                App.tenant_id == current_user.current_tenant_id,
-                App.is_universal == True
-            ).first()
-
-            if universal_app is None:
-                # create universal app if not exists
-                universal_app = App(
-                    tenant_id=current_user.current_tenant_id,
-                    name='Universal Chat',
-                    mode='chat',
-                    is_universal=True,
-                    icon='',
-                    icon_background='',
-                    api_rpm=0,
-                    api_rph=0,
-                    enable_site=False,
-                    enable_api=False,
-                    status='normal'
-                )
-
-                db.session.add(universal_app)
-                db.session.flush()
-
-                app_model_config = AppModelConfig(
-                    provider="",
-                    model_id="",
-                    configs={},
-                    opening_statement='',
-                    suggested_questions=json.dumps([]),
-                    suggested_questions_after_answer=json.dumps({'enabled': True}),
-                    speech_to_text=json.dumps({'enabled': True}),
-                    retriever_resource=json.dumps({'enabled': True}),
-                    more_like_this=None,
-                    sensitive_word_avoidance=None,
-                    model=json.dumps({
-                        "provider": "openai",
-                        "name": "gpt-3.5-turbo-16k",
-                        "completion_params": {
-                            "max_tokens": 800,
-                            "temperature": 0.8,
-                            "top_p": 1,
-                            "presence_penalty": 0,
-                            "frequency_penalty": 0
-                        }
-                    }),
-                    user_input_form=json.dumps([]),
-                    pre_prompt='',
-                    agent_mode=json.dumps({"enabled": True, "strategy": "function_call", "tools": []}),
-                )
-
-                app_model_config.app_id = universal_app.id
-                db.session.add(app_model_config)
-                db.session.flush()
-
-                universal_app.app_model_config_id = app_model_config.id
-                db.session.commit()
-
-            return view(universal_app, *args, **kwargs)
-        return decorated
-
-    if view:
-        return decorator(view)
-    return decorator
-
-
-class UniversalChatResource(Resource):
-    # must be reversed if there are multiple decorators
-    method_decorators = [universal_chat_app_required, account_initialization_required, login_required, setup_required]

api/controllers/files/__init__.py

@@ -7,3 +7,4 @@ api = ExternalApi(bp)
 
 
 from . import image_preview
+from . import tool_files
\ No newline at end of file

api/controllers/files/tool_files.py

+from controllers.files import api
+from flask import Response
+from flask_restful import Resource, reqparse
+from libs.exception import BaseHTTPException
+from werkzeug.exceptions import NotFound, Forbidden
+
+from core.tools.tool_file_manager import ToolFileManager
+
+class ToolFilePreviewApi(Resource):
+    def get(self, file_id, extension):
+        file_id = str(file_id)
+
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('timestamp', type=str, required=True, location='args')
+        parser.add_argument('nonce', type=str, required=True, location='args')
+        parser.add_argument('sign', type=str, required=True, location='args')
+
+        args = parser.parse_args()
+
+        if not ToolFileManager.verify_file(file_id=file_id,
+                                            timestamp=args['timestamp'],
+                                            nonce=args['nonce'],
+                                            sign=args['sign'],
+        ):
+            raise Forbidden('Invalid request.')
+        
+        try:
+            result = ToolFileManager.get_file_generator_by_message_file_id(
+                file_id,
+            )
+
+            if not result:
+                raise NotFound(f'file is not found')
+            
+            generator, mimetype = result
+        except Exception:
+            raise UnsupportedFileTypeError()
+
+        return Response(generator, mimetype=mimetype)
+
+api.add_resource(ToolFilePreviewApi, '/files/tools/<uuid:file_id>.<string:extension>')
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = 'unsupported_file_type'
+    description = "File type not allowed."
+    code = 415

api/controllers/service_api/app/app.py

@@ -3,7 +3,12 @@ from controllers.service_api import api
 from controllers.service_api.wraps import AppApiResource
 from flask import current_app
 from flask_restful import fields, marshal_with
-from models.model import App
+from models.model import App, AppModelConfig
+from models.tools import ApiToolProvider
+
+import json
+
+from extensions.ext_database import db
 
 
 class AppParameterApi(AppApiResource):
@@ -58,5 +63,42 @@ class AppParameterApi(AppApiResource):
             }
         }
 
+class AppMetaApi(AppApiResource):
+    def get(self, app_model: App, end_user):
+        """Get app meta"""
+        app_model_config: AppModelConfig = app_model.app_model_config
+
+        agent_config = app_model_config.agent_mode_dict or {}
+        meta = {
+            'tool_icons': {}
+        }
+
+        # get all tools
+        tools = agent_config.get('tools', [])
+        url_prefix = (current_app.config.get("CONSOLE_API_URL")
+                  + f"/console/api/workspaces/current/tool-provider/builtin/")
+        for tool in tools:
+            keys = list(tool.keys())
+            if len(keys) >= 4:
+                # current tool standard
+                provider_type = tool.get('provider_type')
+                provider_id = tool.get('provider_id')
+                tool_name = tool.get('tool_name')
+                if provider_type == 'builtin':
+                    meta['tool_icons'][tool_name] = url_prefix + provider_id + '/icon'
+                elif provider_type == 'api':
+                    try:
+                        provider: ApiToolProvider = db.session.query(ApiToolProvider).filter(
+                            ApiToolProvider.id == provider_id
+                        )
+                        meta['tool_icons'][tool_name] = json.loads(provider.icon)
+                    except:
+                        meta['tool_icons'][tool_name] =  {
+                            "background": "#252525",
+                            "content": "\ud83d\ude01"
+                        }
+
+        return meta
 
 api.add_resource(AppParameterApi, '/parameters')
+api.add_resource(AppMetaApi, '/meta')

api/controllers/service_api/app/message.py

@@ -37,6 +37,19 @@ class MessageListApi(AppApiResource):
         'created_at': TimestampField
     }
 
+    agent_thought_fields = {
+        'id': fields.String,
+        'chain_id': fields.String,
+        'message_id': fields.String,
+        'position': fields.Integer,
+        'thought': fields.String,
+        'tool': fields.String,
+        'tool_input': fields.String,
+        'created_at': TimestampField,
+        'observation': fields.String,
+        'message_files': fields.List(fields.String, attribute='files')
+    }
+
     message_fields = {
         'id': fields.String,
         'conversation_id': fields.String,
@@ -46,7 +59,8 @@ class MessageListApi(AppApiResource):
         'message_files': fields.List(fields.Nested(message_file_fields), attribute='files'),
         'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
         'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
-        'created_at': TimestampField
+        'created_at': TimestampField,
+        'agent_thoughts': fields.List(fields.Nested(agent_thought_fields))
     }
 
     message_infinite_scroll_pagination_fields = {

api/controllers/web/app.py

@@ -3,7 +3,12 @@ from controllers.web import api
 from controllers.web.wraps import WebApiResource
 from flask import current_app
 from flask_restful import fields, marshal_with
-from models.model import App
+from models.model import App, AppModelConfig
+from models.tools import ApiToolProvider
+
+from extensions.ext_database import db
+
+import json
 
 
 class AppParameterApi(WebApiResource):
@@ -57,5 +62,42 @@ class AppParameterApi(WebApiResource):
             }
         }
 
+class AppMeta(WebApiResource):
+    def get(self, app_model: App, end_user):
+        """Get app meta"""
+        app_model_config: AppModelConfig = app_model.app_model_config
+
+        agent_config = app_model_config.agent_mode_dict or {}
+        meta = {
+            'tool_icons': {}
+        }
+
+        # get all tools
+        tools = agent_config.get('tools', [])
+        url_prefix = (current_app.config.get("CONSOLE_API_URL")
+                  + f"/console/api/workspaces/current/tool-provider/builtin/")
+        for tool in tools:
+            keys = list(tool.keys())
+            if len(keys) >= 4:
+                # current tool standard
+                provider_type = tool.get('provider_type')
+                provider_id = tool.get('provider_id')
+                tool_name = tool.get('tool_name')
+                if provider_type == 'builtin':
+                    meta['tool_icons'][tool_name] = url_prefix + provider_id + '/icon'
+                elif provider_type == 'api':
+                    try:
+                        provider: ApiToolProvider = db.session.query(ApiToolProvider).filter(
+                            ApiToolProvider.id == provider_id
+                        )
+                        meta['tool_icons'][tool_name] = json.loads(provider.icon)
+                    except:
+                        meta['tool_icons'][tool_name] =  {
+                            "background": "#252525",
+                            "content": "\ud83d\ude01"
+                        }
+
+        return meta
 
 api.add_resource(AppParameterApi, '/parameters')
+api.add_resource(AppMeta, '/meta')
\ No newline at end of file

api/controllers/web/message.py

@@ -14,6 +14,7 @@ from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from fields.conversation_fields import message_file_fields
+from fields.message_fields import agent_thought_fields
 from flask import Response, stream_with_context
 from flask_restful import fields, marshal_with, reqparse
 from flask_restful.inputs import int_range
@@ -59,7 +60,8 @@ class MessageListApi(WebApiResource):
         'message_files': fields.List(fields.Nested(message_file_fields), attribute='files'),
         'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
         'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
-        'created_at': TimestampField
+        'created_at': TimestampField,
+        'agent_thoughts': fields.List(fields.Nested(agent_thought_fields))
     }
 
     message_infinite_scroll_pagination_fields = {

api/core/agent/agent_executor.py

@@ -13,8 +13,8 @@ from core.entities.message_entities import prompt_messages_to_lc_messages
 from core.helper import moderation
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.errors.invoke import InvokeError
-from core.tool.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
-from core.tool.dataset_retriever_tool import DatasetRetrieverTool
+from core.tools.tool.dataset_retriever.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
+from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool
 from langchain.agents import AgentExecutor as LCAgentExecutor
 from langchain.agents import BaseMultiActionAgent, BaseSingleActionAgent
 from langchain.callbacks.manager import Callbacks

api/core/app_runner/app_runner.py

@@ -2,7 +2,8 @@ import time
 from typing import Generator, List, Optional, Tuple, Union, cast
 
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
-from core.entities.application_entities import AppOrchestrationConfigEntity, ModelConfigEntity, PromptTemplateEntity
+from core.entities.application_entities import AppOrchestrationConfigEntity, ModelConfigEntity, \
+    PromptTemplateEntity, ExternalDataVariableEntity, ApplicationGenerateEntity, InvokeFrom
 from core.file.file_obj import FileObj
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
@@ -10,9 +11,12 @@ from core.model_runtime.entities.message_entities import AssistantPromptMessage,
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.errors.invoke import InvokeBadRequestError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.features.hosting_moderation import HostingModerationFeature
+from core.features.moderation import ModerationFeature
+from core.features.external_data_fetch import ExternalDataFetchFeature
+from core.features.annotation_reply import AnnotationReplyFeature
 from core.prompt.prompt_transform import PromptTransform
-from models.model import App
-
+from models.model import App, MessageAnnotation, Message
 
 class AppRunner:
     def get_pre_calculate_rest_tokens(self, app_record: App,
@@ -199,7 +203,8 @@ class AppRunner:
 
     def _handle_invoke_result(self, invoke_result: Union[LLMResult, Generator],
                               queue_manager: ApplicationQueueManager,
-                              stream: bool) -> None:
+                              stream: bool,
+                              agent: bool = False) -> None:
         """
         Handle invoke result
         :param invoke_result: invoke result
@@ -210,16 +215,19 @@ class AppRunner:
         if not stream:
             self._handle_invoke_result_direct(
                 invoke_result=invoke_result,
-                queue_manager=queue_manager
+                queue_manager=queue_manager,
+                agent=agent
             )
         else:
             self._handle_invoke_result_stream(
                 invoke_result=invoke_result,
-                queue_manager=queue_manager
+                queue_manager=queue_manager,
+                agent=agent
             )
 
     def _handle_invoke_result_direct(self, invoke_result: LLMResult,
-                                     queue_manager: ApplicationQueueManager) -> None:
+                                     queue_manager: ApplicationQueueManager,
+                                     agent: bool) -> None:
         """
         Handle invoke result direct
         :param invoke_result: invoke result
@@ -232,7 +240,8 @@ class AppRunner:
         )
 
     def _handle_invoke_result_stream(self, invoke_result: Generator,
-                                     queue_manager: ApplicationQueueManager) -> None:
+                                     queue_manager: ApplicationQueueManager,
+                                     agent: bool) -> None:
         """
         Handle invoke result
         :param invoke_result: invoke result
@@ -244,7 +253,10 @@ class AppRunner:
         text = ''
         usage = None
         for result in invoke_result:
-            queue_manager.publish_chunk_message(result, PublishFrom.APPLICATION_MANAGER)
+            if not agent:
+                queue_manager.publish_chunk_message(result, PublishFrom.APPLICATION_MANAGER)
+            else:
+                queue_manager.publish_agent_chunk_message(result, PublishFrom.APPLICATION_MANAGER)
 
             text += result.delta.message.content
 
@@ -271,3 +283,101 @@ class AppRunner:
             llm_result=llm_result,
             pub_from=PublishFrom.APPLICATION_MANAGER
         )
+
+    def moderation_for_inputs(self, app_id: str,
+                              tenant_id: str,
+                              app_orchestration_config_entity: AppOrchestrationConfigEntity,
+                              inputs: dict,
+                              query: str) -> Tuple[bool, dict, str]:
+        """
+        Process sensitive_word_avoidance.
+        :param app_id: app id
+        :param tenant_id: tenant id
+        :param app_orchestration_config_entity: app orchestration config entity
+        :param inputs: inputs
+        :param query: query
+        :return:
+        """
+        moderation_feature = ModerationFeature()
+        return moderation_feature.check(
+            app_id=app_id,
+            tenant_id=tenant_id,
+            app_orchestration_config_entity=app_orchestration_config_entity,
+            inputs=inputs,
+            query=query,
+        )
+    
+    def check_hosting_moderation(self, application_generate_entity: ApplicationGenerateEntity,
+                                 queue_manager: ApplicationQueueManager,
+                                 prompt_messages: list[PromptMessage]) -> bool:
+        """
+        Check hosting moderation
+        :param application_generate_entity: application generate entity
+        :param queue_manager: queue manager
+        :param prompt_messages: prompt messages
+        :return:
+        """
+        hosting_moderation_feature = HostingModerationFeature()
+        moderation_result = hosting_moderation_feature.check(
+            application_generate_entity=application_generate_entity,
+            prompt_messages=prompt_messages
+        )
+
+        if moderation_result:
+            self.direct_output(
+                queue_manager=queue_manager,
+                app_orchestration_config=application_generate_entity.app_orchestration_config_entity,
+                prompt_messages=prompt_messages,
+                text="I apologize for any confusion, " \
+                     "but I'm an AI assistant to be helpful, harmless, and honest.",
+                stream=application_generate_entity.stream
+            )
+
+        return moderation_result
+
+    def fill_in_inputs_from_external_data_tools(self, tenant_id: str,
+                                                app_id: str,
+                                                external_data_tools: list[ExternalDataVariableEntity],
+                                                inputs: dict,
+                                                query: str) -> dict:
+        """
+        Fill in variable inputs from external data tools if exists.
+
+        :param tenant_id: workspace id
+        :param app_id: app id
+        :param external_data_tools: external data tools configs
+        :param inputs: the inputs
+        :param query: the query
+        :return: the filled inputs
+        """
+        external_data_fetch_feature = ExternalDataFetchFeature()
+        return external_data_fetch_feature.fetch(
+            tenant_id=tenant_id,
+            app_id=app_id,
+            external_data_tools=external_data_tools,
+            inputs=inputs,
+            query=query
+        )
+    
+    def query_app_annotations_to_reply(self, app_record: App,
+                                       message: Message,
+                                       query: str,
+                                       user_id: str,
+                                       invoke_from: InvokeFrom) -> Optional[MessageAnnotation]:
+        """
+        Query app annotations to reply
+        :param app_record: app record
+        :param message: message
+        :param query: query
+        :param user_id: user id
+        :param invoke_from: invoke from
+        :return:
+        """
+        annotation_reply_feature = AnnotationReplyFeature()
+        return annotation_reply_feature.query(
+            app_record=app_record,
+            message=message,
+            query=query,
+            user_id=user_id,
+            invoke_from=invoke_from
+        )
\ No newline at end of file

api/core/app_runner/basic_app_runner.py

 import logging
-from typing import Optional, Tuple
+from typing import Optional
 
 from core.app_runner.app_runner import AppRunner
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
-from core.entities.application_entities import (ApplicationGenerateEntity, AppOrchestrationConfigEntity, DatasetEntity,
-                                                ExternalDataVariableEntity, InvokeFrom, ModelConfigEntity)
-from core.features.annotation_reply import AnnotationReplyFeature
+from core.entities.application_entities import (ApplicationGenerateEntity, DatasetEntity,
+                                                 InvokeFrom, ModelConfigEntity)
 from core.features.dataset_retrieval import DatasetRetrievalFeature
-from core.features.external_data_fetch import ExternalDataFetchFeature
-from core.features.hosting_moderation import HostingModerationFeature
-from core.features.moderation import ModerationFeature
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance
-from core.model_runtime.entities.message_entities import PromptMessage
 from core.moderation.base import ModerationException
 from core.prompt.prompt_transform import AppMode
 from extensions.ext_database import db
-from models.model import App, Conversation, Message, MessageAnnotation
+from models.model import App, Conversation, Message
 
 logger = logging.getLogger(__name__)
 
@@ -213,76 +208,6 @@ class BasicApplicationRunner(AppRunner):
             stream=application_generate_entity.stream
         )
 
-    def moderation_for_inputs(self, app_id: str,
-                              tenant_id: str,
-                              app_orchestration_config_entity: AppOrchestrationConfigEntity,
-                              inputs: dict,
-                              query: str) -> Tuple[bool, dict, str]:
-        """
-        Process sensitive_word_avoidance.
-        :param app_id: app id
-        :param tenant_id: tenant id
-        :param app_orchestration_config_entity: app orchestration config entity
-        :param inputs: inputs
-        :param query: query
-        :return:
-        """
-        moderation_feature = ModerationFeature()
-        return moderation_feature.check(
-            app_id=app_id,
-            tenant_id=tenant_id,
-            app_orchestration_config_entity=app_orchestration_config_entity,
-            inputs=inputs,
-            query=query,
-        )
-
-    def query_app_annotations_to_reply(self, app_record: App,
-                                       message: Message,
-                                       query: str,
-                                       user_id: str,
-                                       invoke_from: InvokeFrom) -> Optional[MessageAnnotation]:
-        """
-        Query app annotations to reply
-        :param app_record: app record
-        :param message: message
-        :param query: query
-        :param user_id: user id
-        :param invoke_from: invoke from
-        :return:
-        """
-        annotation_reply_feature = AnnotationReplyFeature()
-        return annotation_reply_feature.query(
-            app_record=app_record,
-            message=message,
-            query=query,
-            user_id=user_id,
-            invoke_from=invoke_from
-        )
-
-    def fill_in_inputs_from_external_data_tools(self, tenant_id: str,
-                                                app_id: str,
-                                                external_data_tools: list[ExternalDataVariableEntity],
-                                                inputs: dict,
-                                                query: str) -> dict:
-        """
-        Fill in variable inputs from external data tools if exists.
-
-        :param tenant_id: workspace id
-        :param app_id: app id
-        :param external_data_tools: external data tools configs
-        :param inputs: the inputs
-        :param query: the query
-        :return: the filled inputs
-        """
-        external_data_fetch_feature = ExternalDataFetchFeature()
-        return external_data_fetch_feature.fetch(
-            tenant_id=tenant_id,
-            app_id=app_id,
-            external_data_tools=external_data_tools,
-            inputs=inputs,
-            query=query
-        )
-
     def retrieve_dataset_context(self, tenant_id: str,
                                  app_record: App,
                                  queue_manager: ApplicationQueueManager,
@@ -334,31 +259,4 @@ class BasicApplicationRunner(AppRunner):
             hit_callback=hit_callback,
             memory=memory
         )
-
-    def check_hosting_moderation(self, application_generate_entity: ApplicationGenerateEntity,
-                                 queue_manager: ApplicationQueueManager,
-                                 prompt_messages: list[PromptMessage]) -> bool:
-        """
-        Check hosting moderation
-        :param application_generate_entity: application generate entity
-        :param queue_manager: queue manager
-        :param prompt_messages: prompt messages
-        :return:
-        """
-        hosting_moderation_feature = HostingModerationFeature()
-        moderation_result = hosting_moderation_feature.check(
-            application_generate_entity=application_generate_entity,
-            prompt_messages=prompt_messages
-        )
-
-        if moderation_result:
-            self.direct_output(
-                queue_manager=queue_manager,
-                app_orchestration_config=application_generate_entity.app_orchestration_config_entity,
-                prompt_messages=prompt_messages,
-                text="I apologize for any confusion, " \
-                     "but I'm an AI assistant to be helpful, harmless, and honest.",
-                stream=application_generate_entity.stream
-            )
-
-        return moderation_result
+    
\ No newline at end of file

api/core/app_runner/generate_task_pipeline.py

@@ -8,7 +8,8 @@ from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.entities.application_entities import ApplicationGenerateEntity, InvokeFrom
 from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentThoughtEvent, QueueErrorEvent,
                                           QueueMessageEndEvent, QueueMessageEvent, QueueMessageReplaceEvent,
-                                          QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent)
+                                          QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent,
+                                          QueueMessageFileEvent, QueueAgentMessageEvent)
 from core.errors.error import ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent,
@@ -16,11 +17,12 @@ from core.model_runtime.entities.message_entities import (AssistantPromptMessage
                                                           TextPromptMessageContent)
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.tools.tool_file_manager import ToolFileManager
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.prompt.prompt_template import PromptTemplateParser
 from events.message_event import message_was_created
 from extensions.ext_database import db
-from models.model import Conversation, Message, MessageAgentThought
+from models.model import Conversation, Message, MessageAgentThought, MessageFile
 from pydantic import BaseModel
 from services.annotation_service import AppAnnotationService
 
@@ -284,6 +286,7 @@ class GenerateTaskPipeline:
                     .filter(MessageAgentThought.id == event.agent_thought_id)
                     .first()
                 )
+                db.session.refresh(agent_thought)
 
                 if agent_thought:
                     response = {
@@ -293,16 +296,48 @@ class GenerateTaskPipeline:
                         'message_id': self._message.id,
                         'position': agent_thought.position,
                         'thought': agent_thought.thought,
+                        'observation': agent_thought.observation,
                         'tool': agent_thought.tool,
                         'tool_input': agent_thought.tool_input,
-                        'created_at': int(self._message.created_at.timestamp())
+                        'created_at': int(self._message.created_at.timestamp()),
+                        'message_files': agent_thought.files
                     }
 
                     if self._conversation.mode == 'chat':
                         response['conversation_id'] = self._conversation.id
 
                     yield self._yield_response(response)
-            elif isinstance(event, QueueMessageEvent):
+            elif isinstance(event, QueueMessageFileEvent):
+                message_file: MessageFile = (
+                    db.session.query(MessageFile)
+                    .filter(MessageFile.id == event.message_file_id)
+                    .first()
+                )
+                # get extension
+                if '.' in message_file.url:
+                    extension = f'.{message_file.url.split(".")[-1]}'
+                    if len(extension) > 10:
+                        extension = '.bin'
+                else:
+                    extension = '.bin'
+                # add sign url
+                url = ToolFileManager.sign_file(file_id=message_file.id, extension=extension)
+
+                if message_file:
+                    response = {
+                        'event': 'message_file',
+                        'id': message_file.id,
+                        'type': message_file.type,
+                        'belongs_to': message_file.belongs_to or 'user',
+                        'url': url
+                    }
+
+                    if self._conversation.mode == 'chat':
+                        response['conversation_id'] = self._conversation.id
+
+                    yield self._yield_response(response)
+
+            elif isinstance(event, (QueueMessageEvent, QueueAgentMessageEvent)):
                 chunk = event.chunk
                 delta_text = chunk.delta.message.content
                 if delta_text is None:
@@ -332,7 +367,7 @@ class GenerateTaskPipeline:
                         self._output_moderation_handler.append_new_token(delta_text)
 
                 self._task_state.llm_result.message.content += delta_text
-                response = self._handle_chunk(delta_text)
+                response = self._handle_chunk(delta_text, agent=isinstance(event, QueueAgentMessageEvent))
                 yield self._yield_response(response)
             elif isinstance(event, QueueMessageReplaceEvent):
                 response = {
@@ -384,14 +419,14 @@ class GenerateTaskPipeline:
             extras=self._application_generate_entity.extras
         )
 
-    def _handle_chunk(self, text: str) -> dict:
+    def _handle_chunk(self, text: str, agent: bool = False) -> dict:
         """
         Handle completed event.
         :param text: text
         :return:
         """
         response = {
-            'event': 'message',
+            'event': 'message' if not agent else 'agent_message',
             'id': self._message.id,
             'task_id': self._application_generate_entity.task_id,
             'message_id': self._message.id,

api/core/application_queue_manager.py

@@ -7,10 +7,10 @@ from core.entities.application_entities import InvokeFrom
 from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentThoughtEvent, QueueErrorEvent,
                                           QueueMessage, QueueMessageEndEvent, QueueMessageEvent,
                                           QueueMessageReplaceEvent, QueuePingEvent, QueueRetrieverResourcesEvent,
-                                          QueueStopEvent)
+                                          QueueStopEvent, QueueMessageFileEvent, QueueAgentMessageEvent)
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
 from extensions.ext_redis import redis_client
-from models.model import MessageAgentThought
+from models.model import MessageAgentThought, MessageFile
 from sqlalchemy.orm import DeclarativeMeta
 
 
@@ -96,6 +96,18 @@ class ApplicationQueueManager:
             chunk=chunk
         ), pub_from)
 
+    def publish_agent_chunk_message(self, chunk: LLMResultChunk, pub_from: PublishFrom) -> None:
+        """
+        Publish agent chunk message to channel
+
+        :param chunk: chunk
+        :param pub_from: publish from
+        :return:
+        """
+        self.publish(QueueAgentMessageEvent(
+            chunk=chunk
+        ), pub_from)
+
     def publish_message_replace(self, text: str, pub_from: PublishFrom) -> None:
         """
         Publish message replace
@@ -144,6 +156,17 @@ class ApplicationQueueManager:
             agent_thought_id=message_agent_thought.id
         ), pub_from)
 
+    def publish_message_file(self, message_file: MessageFile, pub_from: PublishFrom) -> None:
+        """
+        Publish agent thought
+        :param message_file: message file
+        :param pub_from: publish from
+        :return:
+        """
+        self.publish(QueueMessageFileEvent(
+            message_file_id=message_file.id
+        ), pub_from)
+
     def publish_error(self, e, pub_from: PublishFrom) -> None:
         """
         Publish error

api/core/callback_handler/agent_tool_callback_handler.py

+import os
+from typing import Any, Dict, Optional, Union
+from pydantic import BaseModel
+
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.input import print_text
+
+class DifyAgentCallbackHandler(BaseCallbackHandler, BaseModel):
+    """Callback Handler that prints to std out."""
+    color: Optional[str] = ''
+    current_loop = 1
+
+    def __init__(self, color: Optional[str] = None) -> None:
+        super().__init__()
+        """Initialize callback handler."""
+        # use a specific color is not specified
+        self.color = color or 'green'
+        self.current_loop = 1
+
+    def on_tool_start(
+        self,
+        tool_name: str,
+        tool_inputs: Dict[str, Any],
+    ) -> None:
+        """Do nothing."""
+        print_text("\n[on_tool_start] ToolCall:" + tool_name + "\n" + str(tool_inputs) + "\n", color=self.color)
+
+    def on_tool_end(
+        self,
+        tool_name: str,
+        tool_inputs: Dict[str, Any],
+        tool_outputs: str,
+    ) -> None:
+        """If not the final action, print out observation."""
+        print_text("\n[on_tool_end]\n", color=self.color)
+        print_text("Tool: " + tool_name + "\n", color=self.color)
+        print_text("Inputs: " + str(tool_inputs) + "\n", color=self.color)
+        print_text("Outputs: " + str(tool_outputs) + "\n", color=self.color)
+        print_text("\n")
+
+    def on_tool_error(
+        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
+    ) -> None:
+        """Do nothing."""
+        print_text("\n[on_tool_error] Error: " + str(error) + "\n", color='red')
+
+    def on_agent_start(
+        self, thought: str
+    ) -> None:
+        """Run on agent start."""
+        if thought:
+            print_text("\n[on_agent_start] \nCurrent Loop: " + \
+                        str(self.current_loop) + \
+                        "\nThought: " + thought + "\n", color=self.color)
+        else:
+            print_text("\n[on_agent_start] \nCurrent Loop: " + str(self.current_loop) + "\n", color=self.color)
+
+    def on_agent_finish(
+        self, color: Optional[str] = None, **kwargs: Any
+    ) -> None:
+        """Run on agent end."""
+        print_text("\n[on_agent_finish]\n Loop: " + str(self.current_loop) + "\n", color=self.color)
+
+        self.current_loop += 1
+
+    @property
+    def ignore_agent(self) -> bool:
+        """Whether to ignore agent callbacks."""
+        return not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true'
+
+    @property
+    def ignore_chat_model(self) -> bool:
+        """Whether to ignore chat model callbacks."""
+        return not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true'

api/core/entities/application_entities.py

 from enum import Enum
-from typing import Any, Optional, cast
+from typing import Optional, Any, cast, Literal, Union
+
+from pydantic import BaseModel
 
 from core.entities.provider_configuration import ProviderModelBundle
 from core.file.file_obj import FileObj
 from core.model_runtime.entities.message_entities import PromptMessageRole
 from core.model_runtime.entities.model_entities import AIModelEntity
-from pydantic import BaseModel
 
 
 class ModelConfigEntity(BaseModel):
@@ -153,9 +154,35 @@ class AgentToolEntity(BaseModel):
     """
     Agent Tool Entity.
     """
-    tool_id: str
-    config: dict[str, Any] = {}
+    provider_type: Literal["builtin", "api"]
+    provider_id: str
+    tool_name: str
+    tool_parameters: dict[str, Any] = {}
+
+class AgentPromptEntity(BaseModel):
+    """
+    Agent Prompt Entity.
+    """
+    first_prompt: str
+    next_iteration: str
 
+class AgentScratchpadUnit(BaseModel):
+    """
+    Agent First Prompt Entity.
+    """
+
+    class Action(BaseModel):
+        """
+        Action Entity.
+        """
+        action_name: str
+        action_input: Union[dict, str]
+
+    agent_response: Optional[str] = None
+    thought: Optional[str] = None
+    action_str: Optional[str] = None
+    observation: Optional[str] = None
+    action: Optional[Action] = None    
 
 class AgentEntity(BaseModel):
     """
@@ -171,8 +198,9 @@ class AgentEntity(BaseModel):
     provider: str
     model: str
     strategy: Strategy
-    tools: list[AgentToolEntity] = []
-
+    prompt: Optional[AgentPromptEntity] = None
+    tools: list[AgentToolEntity] = None
+    max_iteration: int = 5
 
 class AppOrchestrationConfigEntity(BaseModel):
     """

api/core/entities/queue_entities.py

@@ -10,11 +10,13 @@ class QueueEvent(Enum):
     QueueEvent enum
     """
     MESSAGE = "message"
+    AGENT_MESSAGE = "agent_message"
     MESSAGE_REPLACE = "message-replace"
     MESSAGE_END = "message-end"
     RETRIEVER_RESOURCES = "retriever-resources"
     ANNOTATION_REPLY = "annotation-reply"
     AGENT_THOUGHT = "agent-thought"
+    MESSAGE_FILE = "message-file"
     ERROR = "error"
     PING = "ping"
     STOP = "stop"
@@ -33,7 +35,14 @@ class QueueMessageEvent(AppQueueEvent):
     """
     event = QueueEvent.MESSAGE
     chunk: LLMResultChunk
-    
+
+class QueueAgentMessageEvent(AppQueueEvent):
+    """
+    QueueMessageEvent entity
+    """
+    event = QueueEvent.AGENT_MESSAGE
+    chunk: LLMResultChunk
+
     
 class QueueMessageReplaceEvent(AppQueueEvent):
     """
@@ -73,7 +82,13 @@ class QueueAgentThoughtEvent(AppQueueEvent):
     """
     event = QueueEvent.AGENT_THOUGHT
     agent_thought_id: str
-    
+
+class QueueMessageFileEvent(AppQueueEvent):
+    """
+    QueueAgentThoughtEvent entity
+    """
+    event = QueueEvent.MESSAGE_FILE
+    message_file_id: str
     
 class QueueErrorEvent(AppQueueEvent):
     """

api/core/features/agent_runner.py

 import logging
-from typing import List, Optional, cast
+from typing import cast, Optional, List
+
+from langchain import WikipediaAPIWrapper
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.tools import BaseTool, WikipediaQueryRun, Tool
+from pydantic import BaseModel, Field
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
-from core.agent.agent_executor import AgentConfiguration, AgentExecutor, PlanningStrategy
+from core.agent.agent_executor import PlanningStrategy, AgentConfiguration, AgentExecutor
 from core.application_queue_manager import ApplicationQueueManager
 from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
 from core.callback_handler.std_out_callback_handler import DifyStdOutCallbackHandler
-from core.entities.application_entities import (AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity, InvokeFrom,
-                                                ModelConfigEntity)
+from core.entities.application_entities import ModelConfigEntity, InvokeFrom, \
+    AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.model_entities import ModelFeature, ModelType
 from core.model_runtime.model_providers import model_provider_factory
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.tool.current_datetime_tool import DatetimeTool
-from core.tool.dataset_retriever_tool import DatasetRetrieverTool
-from core.tool.provider.serpapi_provider import SerpAPIToolProvider
-from core.tool.serpapi_wrapper import OptimizedSerpAPIInput, OptimizedSerpAPIWrapper
-from core.tool.web_reader_tool import WebReaderTool
+from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool
 from extensions.ext_database import db
-from langchain import WikipediaAPIWrapper
-from langchain.callbacks.base import BaseCallbackHandler
-from langchain.tools import BaseTool, Tool, WikipediaQueryRun
 from models.dataset import Dataset
 from models.model import Message
-from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
@@ -132,55 +129,6 @@ class AgentRunnerFeature:
             logger.exception("agent_executor run failed")
             return None
 
-    def to_tools(self, tool_configs: list[AgentToolEntity],
-                 invoke_from: InvokeFrom,
-                 callbacks: list[BaseCallbackHandler]) \
-            -> Optional[List[BaseTool]]:
-        """
-        Convert tool configs to tools
-        :param tool_configs: tool configs
-        :param invoke_from: invoke from
-        :param callbacks: callbacks
-        """
-        tools = []
-        for tool_config in tool_configs:
-            tool = None
-            if tool_config.tool_id == "dataset":
-                tool = self.to_dataset_retriever_tool(
-                    tool_config=tool_config.config,
-                    invoke_from=invoke_from
-                )
-            elif tool_config.tool_id == "web_reader":
-                tool = self.to_web_reader_tool(
-                    tool_config=tool_config.config,
-                    invoke_from=invoke_from
-                )
-            elif tool_config.tool_id == "google_search":
-                tool = self.to_google_search_tool(
-                    tool_config=tool_config.config,
-                    invoke_from=invoke_from
-                )
-            elif tool_config.tool_id == "wikipedia":
-                tool = self.to_wikipedia_tool(
-                    tool_config=tool_config.config,
-                    invoke_from=invoke_from
-                )
-            elif tool_config.tool_id == "current_datetime":
-                tool = self.to_current_datetime_tool(
-                    tool_config=tool_config.config,
-                    invoke_from=invoke_from
-                )
-
-            if tool:
-                if tool.callbacks is not None:
-                    tool.callbacks.extend(callbacks)
-                else:
-                    tool.callbacks = callbacks
-
-                tools.append(tool)
-
-        return tools
-
     def to_dataset_retriever_tool(self, tool_config: dict,
                                   invoke_from: InvokeFrom) \
             -> Optional[BaseTool]:
@@ -247,78 +195,4 @@ class AgentRunnerFeature:
             retriever_from=invoke_from.to_source()
         )
 
-        return tool
-
-    def to_web_reader_tool(self, tool_config: dict,
-                           invoke_from: InvokeFrom) -> Optional[BaseTool]:
-        """
-        A tool for reading web pages
-        :param tool_config: tool config
-        :param invoke_from: invoke from
-        :return:
-        """
-        model_parameters = {
-            "temperature": 0,
-            "max_tokens": 500
-        }
-
-        tool = WebReaderTool(
-            model_config=self.model_config,
-            model_parameters=model_parameters,
-            max_chunk_length=4000,
-            continue_reading=True
-        )
-
-        return tool
-
-    def to_google_search_tool(self, tool_config: dict,
-                              invoke_from: InvokeFrom) -> Optional[BaseTool]:
-        """
-        A tool for performing a Google search and extracting snippets and webpages
-        :param tool_config: tool config
-        :param invoke_from: invoke from
-        :return:
-        """
-        tool_provider = SerpAPIToolProvider(tenant_id=self.tenant_id)
-        func_kwargs = tool_provider.credentials_to_func_kwargs()
-        if not func_kwargs:
-            return None
-
-        tool = Tool(
-            name="google_search",
-            description="A tool for performing a Google search and extracting snippets and webpages "
-                        "when you need to search for something you don't know or when your information "
-                        "is not up to date. "
-                        "Input should be a search query.",
-            func=OptimizedSerpAPIWrapper(**func_kwargs).run,
-            args_schema=OptimizedSerpAPIInput
-        )
-
-        return tool
-
-    def to_current_datetime_tool(self, tool_config: dict,
-                                 invoke_from: InvokeFrom) -> Optional[BaseTool]:
-        """
-        A tool for getting the current date and time
-        :param tool_config: tool config
-        :param invoke_from: invoke from
-        :return:
-        """
-        return DatetimeTool()
-
-    def to_wikipedia_tool(self, tool_config: dict,
-                          invoke_from: InvokeFrom) -> Optional[BaseTool]:
-        """
-        A tool for searching Wikipedia
-        :param tool_config: tool config
-        :param invoke_from: invoke from
-        :return:
-        """
-        class WikipediaInput(BaseModel):
-            query: str = Field(..., description="search query.")
-
-        return WikipediaQueryRun(
-            name="wikipedia",
-            api_wrapper=WikipediaAPIWrapper(doc_content_chars_max=4000),
-            args_schema=WikipediaInput
-        )
+        return tool
\ No newline at end of file

api/core/features/dataset_retrieval.py

@@ -6,8 +6,8 @@ from core.entities.application_entities import DatasetEntity, DatasetRetrieveCon
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.tool.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
-from core.tool.dataset_retriever_tool import DatasetRetrieverTool
+from core.tools.tool.dataset_retriever.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
+from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool
 from extensions.ext_database import db
 from langchain.tools import BaseTool
 from models.dataset import Dataset

api/core/file/file_obj.py

@@ -22,6 +22,7 @@ class FileType(enum.Enum):
 class FileTransferMethod(enum.Enum):
     REMOTE_URL = 'remote_url'
     LOCAL_FILE = 'local_file'
+    TOOL_FILE = 'tool_file'
 
     @staticmethod
     def value_of(value):
@@ -30,6 +31,16 @@ class FileTransferMethod(enum.Enum):
                 return member
         raise ValueError(f"No matching enum found for value '{value}'")
 
+class FileBelongsTo(enum.Enum):
+    USER = 'user'
+    ASSISTANT = 'assistant'
+
+    @staticmethod
+    def value_of(value):
+        for member in FileBelongsTo:
+            if member.value == value:
+                return member
+        raise ValueError(f"No matching enum found for value '{value}'")
 
 class FileObj(BaseModel):
     id: Optional[str]

api/core/file/message_file_parser.py

 from typing import Dict, List, Optional, Union
 
 import requests
-from core.file.file_obj import FileObj, FileTransferMethod, FileType
+from core.file.file_obj import FileObj, FileTransferMethod, FileType, FileBelongsTo
 from services.file_service import IMAGE_EXTENSIONS
 from extensions.ext_database import db
 from models.account import Account
@@ -128,6 +128,9 @@ class MessageFileParser:
 
         # group by file type and convert file args or message files to FileObj
         for file in files:
+            if file.belongs_to == FileBelongsTo.ASSISTANT.value:
+                continue
+
             file_obj = self._to_file_obj(file, file_upload_config)
             if file_obj.type not in type_file_objs:
                 continue

api/core/file/tool_file_parser.py

+tool_file_manager = {
+    'manager': None
+}
+
+class ToolFileParser:
+    @staticmethod
+    def get_tool_file_manager() -> 'ToolFileManager':
+        return tool_file_manager['manager']
\ No newline at end of file

api/core/model_runtime/model_providers/openai/llm/llm.py

@@ -485,19 +485,37 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel):
         :return: llm response chunk generator
         """
         full_assistant_content = ''
+        delta_assistant_message_function_call_storage: ChoiceDeltaFunctionCall = None
         for chunk in response:
             if len(chunk.choices) == 0:
                 continue
 
             delta = chunk.choices[0]
 
-            if delta.finish_reason is None and (delta.delta.content is None or delta.delta.content == ''):
+            if delta.finish_reason is None and (delta.delta.content is None or delta.delta.content == '') and \
+                delta.delta.function_call is None:
                 continue
 
             # assistant_message_tool_calls = delta.delta.tool_calls
             assistant_message_function_call = delta.delta.function_call
 
             # extract tool calls from response
+            if delta_assistant_message_function_call_storage is not None:
+                # handle process of stream function call
+                if assistant_message_function_call:
+                    # message has not ended ever
+                    delta_assistant_message_function_call_storage.arguments += assistant_message_function_call.arguments
+                    continue
+                else:
+                    # message has ended
+                    assistant_message_function_call = delta_assistant_message_function_call_storage
+                    delta_assistant_message_function_call_storage = None
+            else:
+                if assistant_message_function_call:
+                    # start of stream function call
+                    delta_assistant_message_function_call_storage = assistant_message_function_call
+                    continue
+
             # tool_calls = self._extract_response_tool_calls(assistant_message_tool_calls)
             function_call = self._extract_response_function_call(assistant_message_function_call)
             tool_calls = [function_call] if function_call else []

api/core/tools/README.md

+# Tools
+
+This module implements built-in tools used in Agent Assistants and Workflows within Dify. You could define and display your own tools in this module, without modifying the frontend logic. This decoupling allows for easier horizontal scaling of Dify's capabilities.
+
+## Feature Introduction
+
+The tools provided for Agents and Workflows are currently divided into two categories:
+- `Built-in Tools` are internally implemented within our product and are hardcoded for use in Agents and Workflows. 
+- `Api-Based Tools`  leverage third-party APIs for implementation. You don't need to code to integrate these -- simply provide interface definitions in formats like `OpenAPI` , `Swagger`, or the `OpenAI-plugin` on the front-end.
+
+### Built-in Tool Providers
+![Alt text](docs/zh_Hans/images/index/image.png)
+
+### API Tool Providers
+![Alt text](docs/zh_Hans/images/index/image-1.png)
+
+## Tool Integration
+
+To enable developers to build flexible and powerful tools, we provide two guides:
+
+### [Quick Integration 👈🏻](./docs/en_US/tool_scale_out.md)
+Quick integration aims at quickly getting you up to speed with tool integration by walking over an example Google Search tool.
+
+### [Advanced Integration 👈🏻](./docs/en_US/advanced_scale_out.md)
+Advanced integration will offer a deeper dive into the module interfaces, and explain how to implement more complex capabilities, such as generating images, combining multiple tools, and managing the flow of parameters, images, and files between different tools.
\ No newline at end of file

api/core/tools/README_CN.md

+# Tools
+
+该模块提供了各Agent和Workflow中会使用的内置工具的调用、鉴权接口，并为 Dify 提供了统一的工具供应商的信息和凭据表单规则。
+
+- 一方面将工具和业务代码解耦，方便开发者对模型横向扩展，
+- 另一方面提供了只需在后端定义供应商和工具，即可在前端页面直接展示，无需修改前端逻辑。
+
+## 功能介绍
+
+对于给Agent和Workflow提供的工具，我们当前将其分为两类：
+- `Built-in Tools` 内置工具，即Dify内部实现的工具，通过硬编码的方式提供给Agent和Workflow使用。
+- `Api-Based Tools` 基于API的工具，即通过调用第三方API实现的工具，`Api-Based Tool`不需要再额外定义，只需提供`OpenAPI` `Swagger` `OpenAI plugin`等接口文档即可。
+
+### 内置工具供应商
+![Alt text](docs/zh_Hans/images/index/image.png)
+
+### API工具供应商
+![Alt text](docs/zh_Hans/images/index/image-1.png)
+
+## 工具接入
+为了实现更灵活更强大的功能，Tools提供了一系列的接口，帮助开发者快速构建想要的工具，本文作为开发者的入门指南，将会以[快速接入](./docs/zh_Hans/tool_scale_out.md)和[高级接入](./docs/zh_Hans/advanced_scale_out.md)两部分介绍如何接入工具。
+
+### [快速接入 👈🏻](./docs/zh_Hans/tool_scale_out.md)
+快速接入可以帮助你在10~20分钟内完成工具的接入，但是这种接入方式只能实现简单的功能，如果你想要实现更复杂的功能，可以参考下面的高级接入。
+
+### [高级接入 👈🏻](./docs/zh_Hans/advanced_scale_out.md)
+高级接入将介绍如何实现更复杂的功能配置，包括实现图生图、实现多个工具的组合、实现参数、图片、文件在多个工具之间的流转。
\ No newline at end of file

api/core/tools/entities/common_entities.py

+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class I18nObject(BaseModel):
+    """
+    Model class for i18n object.
+    """
+    zh_Hans: Optional[str] = None
+    en_US: str
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        if not self.zh_Hans:
+            self.zh_Hans = self.en_US
+
+    def to_dict(self) -> dict:
+        return {
+            'zh_Hans': self.zh_Hans,
+            'en_US': self.en_US,
+        }
\ No newline at end of file

api/core/tools/entities/constant.py

+class DEFAULT_PROVIDERS:
+    API_BASED = '__api_based'
+    APP_BASED = '__app_based'
\ No newline at end of file

api/core/tools/entities/tool_bundle.py

+from pydantic import BaseModel
+from typing import Dict, Optional, Any, List
+
+from core.tools.entities.tool_entities import ToolProviderType, ToolParamter
+
+class ApiBasedToolBundle(BaseModel):
+    """
+    This class is used to store the schema information of an api based tool. such as the url, the method, the parameters, etc.
+    """
+    # server_url
+    server_url: str
+    # method
+    method: str
+    # summary
+    summary: Optional[str] = None
+    # operation_id
+    operation_id: str = None
+    # parameters
+    parameters: Optional[List[ToolParamter]] = None
+    # author
+    author: str
+    # icon
+    icon: Optional[str] = None
+    # openapi operation
+    openapi: dict
+
+class AppToolBundle(BaseModel):
+    """
+    This class is used to store the schema information of an tool for an app.
+    """
+    type: ToolProviderType
+    credential: Optional[Dict[str, Any]] = None
+    provider_id: str
+    tool_name: str
\ No newline at end of file

api/core/tools/model/errors.py

+class InvokeModelError(Exception):
+    pass
\ No newline at end of file

api/core/tools/provider/builtin/chart/chart.yaml

+identity:
+  author: Dify
+  name: chart
+  label:
+    en_US: ChartGenerator
+    zh_Hans: 图表生成
+  description:
+    en_US: Chart Generator is a tool for generating statistical charts like bar chart, line chart, pie chart, etc.
+    zh_Hans: 图表生成是一个用于生成可视化图表的工具，你可以通过它来生成柱状图、折线图、饼图等各类图表
+  icon: icon.png
+credentails_for_provider: