Merge branch 'main' into tp

README.md

@@ -7,9 +7,6 @@
 
 [Website](https://dify.ai) • [Docs](https://docs.dify.ai) • [Twitter](https://twitter.com/dify_ai) • [Discord](https://discord.gg/FngNHpbcY7)
 
-Vote for us on Product Hunt ↓  
-<a href="https://www.producthunt.com/posts/dify-ai"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?sanitize=true&post_id=dify-ai&theme=light" alt="Product Hunt Badge" width="250" height="54"></a>
-
 **Dify** is an easy-to-use LLMOps platform designed to empower more people to create sustainable, AI-native applications. With visual orchestration for various application types, Dify offers out-of-the-box, ready-to-use applications that can also serve as Backend-as-a-Service APIs. Unify your development process with one API for plugins and datasets integration, and streamline your operations using a single interface for prompt engineering, visual analytics, and continuous improvement.
 
 Applications created with Dify include:

README_CN.md

@@ -8,9 +8,6 @@
 
 [官方网站](https://dify.ai) • [文档](https://docs.dify.ai/v/zh-hans) • [Twitter](https://twitter.com/dify_ai) •  [Discord](https://discord.gg/FngNHpbcY7)
 
-在 Product Hunt 上投我们一票吧 ↓  
-<a href="https://www.producthunt.com/posts/dify-ai"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?sanitize=true&post_id=dify-ai&theme=light" alt="Product Hunt Badge" width="250" height="54"></a>
-
 **Dify** 是一个易用的 LLMOps 平台，旨在让更多人可以创建可持续运营的原生 AI 应用。Dify 提供多种类型应用的可视化编排，应用可开箱即用，也能以“后端即服务”的 API 提供服务。
 
 通过 Dify 创建的应用包含了：

README_JA.md

@@ -7,9 +7,6 @@
 
 [Web サイト](https://dify.ai) • [ドキュメント](https://docs.dify.ai) • [Twitter](https://twitter.com/dify_ai) • [Discord](https://discord.gg/FngNHpbcY7)
 
-Product Huntで私たちに投票してください ↓  
-<a href="https://www.producthunt.com/posts/dify-ai"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?sanitize=true&post_id=dify-ai&theme=light" alt="Product Hunt Badge" width="250" height="54"></a>
-
 
 **Dify** は、より多くの人々が持続可能な AI ネイティブアプリケーションを作成できるように設計された、使いやすい LLMOps プラットフォームです。様々なアプリケーションタイプに対応したビジュアルオーケストレーションにより Dify は Backend-as-a-Service API としても機能する、すぐに使えるアプリケーションを提供します。プラグインやデータセットを統合するための1つの API で開発プロセスを統一し、プロンプトエンジニアリング、ビジュアル分析、継続的な改善のための1つのインターフェイスを使って業務を合理化します。
 

api/commands.py

@@ -3,15 +3,19 @@ import random
 import string
 
 import click
+from flask import current_app
 
 from libs.password import password_pattern, valid_password, hash_password
 from libs.helper import email as email_validate
 from extensions.ext_database import db
-from models.account import InvitationCode
+from libs.rsa import generate_key_pair
+from models.account import InvitationCode, Tenant
 from models.model import Account
 import secrets
 import base64
 
+from models.provider import Provider
+
 
 @click.command('reset-password', help='Reset the account password.')
 @click.option('--email', prompt=True, help='The email address of the account whose password you need to reset')
@@ -73,6 +77,31 @@ def reset_email(email, new_email, email_confirm):
     click.echo(click.style('Congratulations!, email has been reset.', fg='green'))
 
 
+@click.command('reset-encrypt-key-pair', help='Reset the asymmetric key pair of workspace for encrypt LLM credentials. '
+                                              'After the reset, all LLM credentials will become invalid, '
+                                              'requiring re-entry.'
+                                              'Only support SELF_HOSTED mode.')
+@click.confirmation_option(prompt=click.style('Are you sure you want to reset encrypt key pair?'
+                                              ' this operation cannot be rolled back!', fg='red'))
+def reset_encrypt_key_pair():
+    if current_app.config['EDITION'] != 'SELF_HOSTED':
+        click.echo(click.style('Sorry, only support SELF_HOSTED mode.', fg='red'))
+        return
+
+    tenant = db.session.query(Tenant).first()
+    if not tenant:
+        click.echo(click.style('Sorry, no workspace found. Please enter /install to initialize.', fg='red'))
+        return
+
+    tenant.encrypt_public_key = generate_key_pair(tenant.id)
+
+    db.session.query(Provider).filter(Provider.provider_type == 'custom').delete()
+    db.session.commit()
+
+    click.echo(click.style('Congratulations! '
+                           'the asymmetric key pair of workspace {} has been reset.'.format(tenant.id), fg='green'))
+
+
 @click.command('generate-invitation-codes', help='Generate invitation codes.')
 @click.option('--batch', help='The batch of invitation codes.')
 @click.option('--count', prompt=True, help='Invitation codes count.')
@@ -134,3 +163,4 @@ def register_commands(app):
     app.cli.add_command(reset_password)
     app.cli.add_command(reset_email)
     app.cli.add_command(generate_invitation_codes)
+    app.cli.add_command(reset_encrypt_key_pair)

api/controllers/console/datasets/file.py

@@ -18,6 +18,7 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.index.readers.html_parser import HTMLParser
 from core.index.readers.pdf_parser import PDFParser
+from core.index.readers.xlsx_parser import XLSXParser
 from extensions.ext_storage import storage
 from libs.helper import TimestampField
 from extensions.ext_database import db
@@ -26,7 +27,7 @@ from models.model import UploadFile
 cache = TTLCache(maxsize=None, ttl=30)
 
 FILE_SIZE_LIMIT = 15 * 1024 * 1024  # 15MB
-ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm']
+ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx']
 PREVIEW_WORDS_LIMIT = 3000
 
 
@@ -133,6 +134,9 @@ class FilePreviewApi(Resource):
                 # Use BeautifulSoup to extract text
                 parser = HTMLParser()
                 text = parser.parse_file(Path(filepath))
+            elif extension == 'xlsx':
+                parser = XLSXParser()
+                text = parser.parse_file(filepath)
             else:
                 # ['txt', 'markdown', 'md']
                 with open(filepath, "rb") as fp:

api/core/index/readers/xlsx_parser.py

+from pathlib import Path
+import json
+from typing import Dict
+from openpyxl import load_workbook
+
+from llama_index.readers.file.base_parser import BaseParser
+from flask import current_app
+
+
+class XLSXParser(BaseParser):
+    """XLSX parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser"""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        data = []
+        keys = []
+        with open(file, "r") as fp:
+            wb = load_workbook(filename=file, read_only=True)
+            # loop over all sheets
+            for sheet in wb:
+                for row in sheet.iter_rows(values_only=True):
+                    if all(v is None for v in row):
+                        continue
+                    if keys == []:
+                        keys = row
+                    else:
+                        data.append(json.dumps(dict(zip(keys, row)), ensure_ascii=False))
+        return data

api/core/indexing_runner.py

@@ -17,6 +17,7 @@ from llama_index.readers.file.base import DEFAULT_FILE_EXTRACTOR
 from llama_index.readers.file.markdown_parser import MarkdownParser
 
 from core.data_source.notion import NotionPageReader
+from core.index.readers.xlsx_parser import XLSXParser
 from core.docstore.dataset_docstore import DatesetDocumentStore
 from core.index.keyword_table_index import KeywordTableIndex
 from core.index.readers.html_parser import HTMLParser
@@ -333,6 +334,7 @@ class IndexingRunner:
             file_extractor[".html"] = HTMLParser()
             file_extractor[".htm"] = HTMLParser()
             file_extractor[".pdf"] = PDFParser({'upload_file': upload_file})
+            file_extractor[".xlsx"] = XLSXParser()
 
             loader = SimpleDirectoryReader(input_files=[filepath], file_extractor=file_extractor)
             text_docs = loader.load_data()

api/requirements.txt

@@ -30,3 +30,4 @@ jieba==0.42.1
 celery==5.2.7
 redis~=4.5.4
 pypdf==3.8.1
+openpyxl==3.1.2
\ No newline at end of file

api/services/account_service.py

@@ -268,6 +268,7 @@ class TenantService:
         if action not in ['add', 'remove', 'update']:
             raise InvalidActionError("Invalid action.")
         
+        if member:
             if operator.id == member.id:
                 raise CannotOperateSelfError("Cannot operate self.")
 
@@ -365,6 +366,7 @@ class RegisterService:
         account = Account.query.filter_by(email=email).first()
 
         if not account:
+            TenantService.check_member_permission(tenant, inviter, None, 'add')
             name = email.split('@')[0]
             account = AccountService.create_account(email, name)
             account.status = AccountStatus.PENDING.value

sdks/nodejs-client/README.md

@@ -12,8 +12,12 @@ After installing the SDK, you can use it in your project like this:
 ```js
 import { DifyClient, ChatClient, CompletionClient } from 'dify-client'
 
-const API_KEY = 'your-api-key-here';
-const user = `random-user-id`;
+const API_KEY = 'your-api-key-here'
+const user = `random-user-id`
+const inputs = {
+  name: 'test name a'
+}
+const query = "Please tell me a short story in 10 words or less."
 
 // Create a completion client
 const completionClient = new CompletionClient(API_KEY)
@@ -22,8 +26,15 @@ completionClient.createCompletionMessage(inputs, query, responseMode, user)
 
 // Create a chat client
 const chatClient = new ChatClient(API_KEY)
-// Create a chat message
-chatClient.createChatMessage(inputs, query, responseMode, user, conversationId)
+// Create a chat message in stream mode
+const response = await chatClient.createChatMessage(inputs, query, user, true, null)
+const stream = response.data;
+stream.on('data', data => {
+    console.log(data);
+});
+stream.on('end', () => {
+    console.log("stream done");
+});
 // Fetch conversations
 chatClient.getConversations(user)
 // Fetch conversation messages

sdks/nodejs-client/index.js

-import axios from 'axios'
-
-export const BASE_URL = 'https://api.dify.ai/v1'
+import axios from "axios";
+export const BASE_URL = "https://api.dify.ai/v1";
 
 export const routes = {
   application: {
-    method: 'GET',
-    url: () => `/parameters`
+    method: "GET",
+    url: () => `/parameters`,
   },
   feedback: {
-    method: 'POST',
-    url: (messageId) => `/messages/${messageId}/feedbacks`,
+    method: "POST",
+    url: (message_id) => `/messages/${message_id}/feedbacks`,
   },
   createCompletionMessage: {
-    method: 'POST',
+    method: "POST",
     url: () => `/completion-messages`,
   },
   createChatMessage: {
-    method: 'POST',
+    method: "POST",
     url: () => `/chat-messages`,
   },
   getConversationMessages: {
-    method: 'GET',
-    url: () => '/messages',
+    method: "GET",
+    url: () => "/messages",
   },
   getConversations: {
-    method: 'GET',
-    url: () => '/conversations',
+    method: "GET",
+    url: () => "/conversations",
   },
   renameConversation: {
-    method: 'PATCH',
-    url: (conversationId) => `/conversations/${conversationId}`,
-  }
-
-}
+    method: "PATCH",
+    url: (conversation_id) => `/conversations/${conversation_id}`,
+  },
+};
 
 export class DifyClient {
   constructor(apiKey, baseUrl = BASE_URL) {
-    this.apiKey = apiKey
-    this.baseUrl = baseUrl
+    this.apiKey = apiKey;
+    this.baseUrl = baseUrl;
   }
 
   updateApiKey(apiKey) {
-    this.apiKey = apiKey
+    this.apiKey = apiKey;
   }
 
-  async sendRequest(method, endpoint, data = null, params = null, stream = false) {
+  async sendRequest(
+    method,
+    endpoint,
+    data = null,
+    params = null,
+    stream = false
+  ) {
     const headers = {
-      'Authorization': `Bearer ${this.apiKey}`,
-      'Content-Type': 'application/json',
-    }
+      Authorization: `Bearer ${this.apiKey}`,
+      "Content-Type": "application/json",
+    };
 
-    const url = `${this.baseUrl}${endpoint}`
-    let response
-    if (!stream) {
+    const url = `${this.baseUrl}${endpoint}`;
+    let response;
+    if (stream) {
       response = await axios({
         method,
         url,
         data,
         params,
         headers,
-        responseType: stream ? 'stream' : 'json',
-      })
+        responseType: "stream",
+      });
     } else {
-      response = await fetch(url, {
-        headers,
+      response = await axios({
         method,
-        body: JSON.stringify(data),
-      })
+        url,
+        data,
+        params,
+        headers,
+        responseType: "json",
+      });
     }
 
-    return response
+    return response;
   }
 
-  messageFeedback(messageId, rating, user) {
+  messageFeedback(message_id, rating, user) {
     const data = {
       rating,
       user,
-    }
-    return this.sendRequest(routes.feedback.method, routes.feedback.url(messageId), data)
+    };
+    return this.sendRequest(
+      routes.feedback.method,
+      routes.feedback.url(message_id),
+      data
+    );
   }
 
   getApplicationParameters(user) {
-    const params = { user }
-    return this.sendRequest(routes.application.method, routes.application.url(), null, params)
+    const params = { user };
+    return this.sendRequest(
+      routes.application.method,
+      routes.application.url(),
+      null,
+      params
+    );
   }
 }
 
 export class CompletionClient extends DifyClient {
-  createCompletionMessage(inputs, query, user, responseMode) {
+  createCompletionMessage(inputs, query, user, stream = false) {
     const data = {
       inputs,
       query,
-      responseMode,
       user,
-    }
-    return this.sendRequest(routes.createCompletionMessage.method, routes.createCompletionMessage.url(), data, null, responseMode === 'streaming')
+      response_mode: stream ? "streaming" : "blocking",
+    };
+    return this.sendRequest(
+      routes.createCompletionMessage.method,
+      routes.createCompletionMessage.url(),
+      data,
+      null,
+      stream
+    );
   }
 }
 
 export class ChatClient extends DifyClient {
-  createChatMessage(inputs, query, user, responseMode = 'blocking', conversationId = null) {
+  createChatMessage(
+    inputs,
+    query,
+    user,
+    stream = false,
+    conversation_id = null
+  ) {
     const data = {
       inputs,
       query,
       user,
-      responseMode,
-    }
-    if (conversationId)
-      data.conversation_id = conversationId
+      response_mode: stream ? "streaming" : "blocking",
+    };
+    if (conversation_id) data.conversation_id = conversation_id;
 
-    return this.sendRequest(routes.createChatMessage.method, routes.createChatMessage.url(), data, null, responseMode === 'streaming')
+    return this.sendRequest(
+      routes.createChatMessage.method,
+      routes.createChatMessage.url(),
+      data,
+      null,
+      stream
+    );
   }
 
-  getConversationMessages(user, conversationId = '', firstId = null, limit = null) {
-    const params = { user }
+  getConversationMessages(
+    user,
+    conversation_id = "",
+    first_id = null,
+    limit = null
+  ) {
+    const params = { user };
 
-    if (conversationId)
-      params.conversation_id = conversationId
+    if (conversation_id) params.conversation_id = conversation_id;
 
-    if (firstId)
-      params.first_id = firstId
+    if (first_id) params.first_id = first_id;
 
-    if (limit)
-      params.limit = limit
+    if (limit) params.limit = limit;
 
-    return this.sendRequest(routes.getConversationMessages.method, routes.getConversationMessages.url(), null, params)
+    return this.sendRequest(
+      routes.getConversationMessages.method,
+      routes.getConversationMessages.url(),
+      null,
+      params
+    );
   }
 
-  getConversations(user, firstId = null, limit = null, pinned = null) {
-    const params = { user, first_id: firstId, limit, pinned }
-    return this.sendRequest(routes.getConversations.method, routes.getConversations.url(), null, params)
+  getConversations(user, first_id = null, limit = null, pinned = null) {
+    const params = { user, first_id: first_id, limit, pinned };
+    return this.sendRequest(
+      routes.getConversations.method,
+      routes.getConversations.url(),
+      null,
+      params
+    );
   }
 
-  renameConversation(conversationId, name, user) {
-    const data = { name, user }
-    return this.sendRequest(routes.renameConversation.method, routes.renameConversation.url(conversationId), data)
+  renameConversation(conversation_id, name, user) {
+    const data = { name, user };
+    return this.sendRequest(
+      routes.renameConversation.method,
+      routes.renameConversation.url(conversation_id),
+      data
+    );
   }
 }
\ No newline at end of file
-

sdks/nodejs-client/package.json

 {
   "name": "dify-client",
-  "version": "1.0.3",
+  "version": "2.0.0",
   "description": "This is the Node.js SDK for the Dify.AI API, which allows you to easily integrate Dify.AI into your Node.js applications.",
   "main": "index.js",
   "type": "module",

web/app/components/datasets/create/file-uploader/index.tsx

@@ -22,7 +22,7 @@ const ACCEPTS = [
   '.md',
   '.markdown',
   '.txt',
-  '.xls',
+  // '.xls',
   '.xlsx',
   '.csv',
 ]

web/i18n/lang/dataset-creation.en.ts

@@ -22,7 +22,7 @@ const translation = {
       title: 'Upload text file',
       button: 'Drag and drop file, or',
       browse: 'Browse',
-      tip: 'Supports txt, html, markdown, xlsx, xls, and pdf.',
+      tip: 'Supports txt, html, markdown, xlsx, and pdf.',
       validation: {
         typeError: 'File type not supported',
         size: 'File too large. Maximum is 15MB',

web/i18n/lang/dataset-creation.zh.ts

@@ -22,7 +22,7 @@ const translation = {
       title: '上传文本文件',
       button: '拖拽文件至此，或者',
       browse: '选择文件',
-      tip: '已支持 TXT, HTML, Markdown, PDF, XLSX, XLS',
+      tip: '已支持 TXT, HTML, Markdown, PDF, XLSX',
       validation: {
         typeError: '文件类型不支持',
         size: '文件太大了，不能超过 15MB',