Commit 6b884ab2 authored by JzoNg's avatar JzoNg

Merge branch 'main' into tp

parents 7104fba0 b0554701
......@@ -7,9 +7,6 @@
[Website](https://dify.ai)[Docs](https://docs.dify.ai)[Twitter](https://twitter.com/dify_ai)[Discord](https://discord.gg/FngNHpbcY7)
Vote for us on Product Hunt ↓
<a href="https://www.producthunt.com/posts/dify-ai"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?sanitize=true&post_id=dify-ai&theme=light" alt="Product Hunt Badge" width="250" height="54"></a>
**Dify** is an easy-to-use LLMOps platform designed to empower more people to create sustainable, AI-native applications. With visual orchestration for various application types, Dify offers out-of-the-box, ready-to-use applications that can also serve as Backend-as-a-Service APIs. Unify your development process with one API for plugins and datasets integration, and streamline your operations using a single interface for prompt engineering, visual analytics, and continuous improvement.
Applications created with Dify include:
......
......@@ -8,9 +8,6 @@
[官方网站](https://dify.ai)[文档](https://docs.dify.ai/v/zh-hans)[Twitter](https://twitter.com/dify_ai)[Discord](https://discord.gg/FngNHpbcY7)
在 Product Hunt 上投我们一票吧 ↓
<a href="https://www.producthunt.com/posts/dify-ai"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?sanitize=true&post_id=dify-ai&theme=light" alt="Product Hunt Badge" width="250" height="54"></a>
**Dify** 是一个易用的 LLMOps 平台,旨在让更多人可以创建可持续运营的原生 AI 应用。Dify 提供多种类型应用的可视化编排,应用可开箱即用,也能以“后端即服务”的 API 提供服务。
通过 Dify 创建的应用包含了:
......
......@@ -7,9 +7,6 @@
[Web サイト](https://dify.ai)[ドキュメント](https://docs.dify.ai)[Twitter](https://twitter.com/dify_ai)[Discord](https://discord.gg/FngNHpbcY7)
Product Huntで私たちに投票してください ↓
<a href="https://www.producthunt.com/posts/dify-ai"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?sanitize=true&post_id=dify-ai&theme=light" alt="Product Hunt Badge" width="250" height="54"></a>
**Dify** は、より多くの人々が持続可能な AI ネイティブアプリケーションを作成できるように設計された、使いやすい LLMOps プラットフォームです。様々なアプリケーションタイプに対応したビジュアルオーケストレーションにより Dify は Backend-as-a-Service API としても機能する、すぐに使えるアプリケーションを提供します。プラグインやデータセットを統合するための1つの API で開発プロセスを統一し、プロンプトエンジニアリング、ビジュアル分析、継続的な改善のための1つのインターフェイスを使って業務を合理化します。
......
......@@ -3,15 +3,19 @@ import random
import string
import click
from flask import current_app
from libs.password import password_pattern, valid_password, hash_password
from libs.helper import email as email_validate
from extensions.ext_database import db
from models.account import InvitationCode
from libs.rsa import generate_key_pair
from models.account import InvitationCode, Tenant
from models.model import Account
import secrets
import base64
from models.provider import Provider
@click.command('reset-password', help='Reset the account password.')
@click.option('--email', prompt=True, help='The email address of the account whose password you need to reset')
......@@ -73,6 +77,31 @@ def reset_email(email, new_email, email_confirm):
click.echo(click.style('Congratulations!, email has been reset.', fg='green'))
@click.command('reset-encrypt-key-pair', help='Reset the asymmetric key pair of workspace for encrypt LLM credentials. '
'After the reset, all LLM credentials will become invalid, '
'requiring re-entry.'
'Only support SELF_HOSTED mode.')
@click.confirmation_option(prompt=click.style('Are you sure you want to reset encrypt key pair?'
' this operation cannot be rolled back!', fg='red'))
def reset_encrypt_key_pair():
if current_app.config['EDITION'] != 'SELF_HOSTED':
click.echo(click.style('Sorry, only support SELF_HOSTED mode.', fg='red'))
return
tenant = db.session.query(Tenant).first()
if not tenant:
click.echo(click.style('Sorry, no workspace found. Please enter /install to initialize.', fg='red'))
return
tenant.encrypt_public_key = generate_key_pair(tenant.id)
db.session.query(Provider).filter(Provider.provider_type == 'custom').delete()
db.session.commit()
click.echo(click.style('Congratulations! '
'the asymmetric key pair of workspace {} has been reset.'.format(tenant.id), fg='green'))
@click.command('generate-invitation-codes', help='Generate invitation codes.')
@click.option('--batch', help='The batch of invitation codes.')
@click.option('--count', prompt=True, help='Invitation codes count.')
......@@ -134,3 +163,4 @@ def register_commands(app):
app.cli.add_command(reset_password)
app.cli.add_command(reset_email)
app.cli.add_command(generate_invitation_codes)
app.cli.add_command(reset_encrypt_key_pair)
......@@ -18,6 +18,7 @@ from controllers.console.setup import setup_required
from controllers.console.wraps import account_initialization_required
from core.index.readers.html_parser import HTMLParser
from core.index.readers.pdf_parser import PDFParser
from core.index.readers.xlsx_parser import XLSXParser
from extensions.ext_storage import storage
from libs.helper import TimestampField
from extensions.ext_database import db
......@@ -26,7 +27,7 @@ from models.model import UploadFile
cache = TTLCache(maxsize=None, ttl=30)
FILE_SIZE_LIMIT = 15 * 1024 * 1024 # 15MB
ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm']
ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx']
PREVIEW_WORDS_LIMIT = 3000
......@@ -133,6 +134,9 @@ class FilePreviewApi(Resource):
# Use BeautifulSoup to extract text
parser = HTMLParser()
text = parser.parse_file(Path(filepath))
elif extension == 'xlsx':
parser = XLSXParser()
text = parser.parse_file(filepath)
else:
# ['txt', 'markdown', 'md']
with open(filepath, "rb") as fp:
......
from pathlib import Path
import json
from typing import Dict
from openpyxl import load_workbook
from llama_index.readers.file.base_parser import BaseParser
from flask import current_app
class XLSXParser(BaseParser):
"""XLSX parser."""
def _init_parser(self) -> Dict:
"""Init parser"""
return {}
def parse_file(self, file: Path, errors: str = "ignore") -> str:
data = []
keys = []
with open(file, "r") as fp:
wb = load_workbook(filename=file, read_only=True)
# loop over all sheets
for sheet in wb:
for row in sheet.iter_rows(values_only=True):
if all(v is None for v in row):
continue
if keys == []:
keys = row
else:
data.append(json.dumps(dict(zip(keys, row)), ensure_ascii=False))
return data
......@@ -17,6 +17,7 @@ from llama_index.readers.file.base import DEFAULT_FILE_EXTRACTOR
from llama_index.readers.file.markdown_parser import MarkdownParser
from core.data_source.notion import NotionPageReader
from core.index.readers.xlsx_parser import XLSXParser
from core.docstore.dataset_docstore import DatesetDocumentStore
from core.index.keyword_table_index import KeywordTableIndex
from core.index.readers.html_parser import HTMLParser
......@@ -333,6 +334,7 @@ class IndexingRunner:
file_extractor[".html"] = HTMLParser()
file_extractor[".htm"] = HTMLParser()
file_extractor[".pdf"] = PDFParser({'upload_file': upload_file})
file_extractor[".xlsx"] = XLSXParser()
loader = SimpleDirectoryReader(input_files=[filepath], file_extractor=file_extractor)
text_docs = loader.load_data()
......
......@@ -30,3 +30,4 @@ jieba==0.42.1
celery==5.2.7
redis~=4.5.4
pypdf==3.8.1
openpyxl==3.1.2
\ No newline at end of file
......@@ -268,6 +268,7 @@ class TenantService:
if action not in ['add', 'remove', 'update']:
raise InvalidActionError("Invalid action.")
if member:
if operator.id == member.id:
raise CannotOperateSelfError("Cannot operate self.")
......@@ -365,6 +366,7 @@ class RegisterService:
account = Account.query.filter_by(email=email).first()
if not account:
TenantService.check_member_permission(tenant, inviter, None, 'add')
name = email.split('@')[0]
account = AccountService.create_account(email, name)
account.status = AccountStatus.PENDING.value
......
......@@ -12,8 +12,12 @@ After installing the SDK, you can use it in your project like this:
```js
import { DifyClient, ChatClient, CompletionClient } from 'dify-client'
const API_KEY = 'your-api-key-here';
const user = `random-user-id`;
const API_KEY = 'your-api-key-here'
const user = `random-user-id`
const inputs = {
name: 'test name a'
}
const query = "Please tell me a short story in 10 words or less."
// Create a completion client
const completionClient = new CompletionClient(API_KEY)
......@@ -22,8 +26,15 @@ completionClient.createCompletionMessage(inputs, query, responseMode, user)
// Create a chat client
const chatClient = new ChatClient(API_KEY)
// Create a chat message
chatClient.createChatMessage(inputs, query, responseMode, user, conversationId)
// Create a chat message in stream mode
const response = await chatClient.createChatMessage(inputs, query, user, true, null)
const stream = response.data;
stream.on('data', data => {
console.log(data);
});
stream.on('end', () => {
console.log("stream done");
});
// Fetch conversations
chatClient.getConversations(user)
// Fetch conversation messages
......
import axios from 'axios'
export const BASE_URL = 'https://api.dify.ai/v1'
import axios from "axios";
export const BASE_URL = "https://api.dify.ai/v1";
export const routes = {
application: {
method: 'GET',
url: () => `/parameters`
method: "GET",
url: () => `/parameters`,
},
feedback: {
method: 'POST',
url: (messageId) => `/messages/${messageId}/feedbacks`,
method: "POST",
url: (message_id) => `/messages/${message_id}/feedbacks`,
},
createCompletionMessage: {
method: 'POST',
method: "POST",
url: () => `/completion-messages`,
},
createChatMessage: {
method: 'POST',
method: "POST",
url: () => `/chat-messages`,
},
getConversationMessages: {
method: 'GET',
url: () => '/messages',
method: "GET",
url: () => "/messages",
},
getConversations: {
method: 'GET',
url: () => '/conversations',
method: "GET",
url: () => "/conversations",
},
renameConversation: {
method: 'PATCH',
url: (conversationId) => `/conversations/${conversationId}`,
}
}
method: "PATCH",
url: (conversation_id) => `/conversations/${conversation_id}`,
},
};
export class DifyClient {
constructor(apiKey, baseUrl = BASE_URL) {
this.apiKey = apiKey
this.baseUrl = baseUrl
this.apiKey = apiKey;
this.baseUrl = baseUrl;
}
updateApiKey(apiKey) {
this.apiKey = apiKey
this.apiKey = apiKey;
}
async sendRequest(method, endpoint, data = null, params = null, stream = false) {
async sendRequest(
method,
endpoint,
data = null,
params = null,
stream = false
) {
const headers = {
'Authorization': `Bearer ${this.apiKey}`,
'Content-Type': 'application/json',
}
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
};
const url = `${this.baseUrl}${endpoint}`
let response
if (!stream) {
const url = `${this.baseUrl}${endpoint}`;
let response;
if (stream) {
response = await axios({
method,
url,
data,
params,
headers,
responseType: stream ? 'stream' : 'json',
})
responseType: "stream",
});
} else {
response = await fetch(url, {
headers,
response = await axios({
method,
body: JSON.stringify(data),
})
url,
data,
params,
headers,
responseType: "json",
});
}
return response
return response;
}
messageFeedback(messageId, rating, user) {
messageFeedback(message_id, rating, user) {
const data = {
rating,
user,
}
return this.sendRequest(routes.feedback.method, routes.feedback.url(messageId), data)
};
return this.sendRequest(
routes.feedback.method,
routes.feedback.url(message_id),
data
);
}
getApplicationParameters(user) {
const params = { user }
return this.sendRequest(routes.application.method, routes.application.url(), null, params)
const params = { user };
return this.sendRequest(
routes.application.method,
routes.application.url(),
null,
params
);
}
}
export class CompletionClient extends DifyClient {
createCompletionMessage(inputs, query, user, responseMode) {
createCompletionMessage(inputs, query, user, stream = false) {
const data = {
inputs,
query,
responseMode,
user,
}
return this.sendRequest(routes.createCompletionMessage.method, routes.createCompletionMessage.url(), data, null, responseMode === 'streaming')
response_mode: stream ? "streaming" : "blocking",
};
return this.sendRequest(
routes.createCompletionMessage.method,
routes.createCompletionMessage.url(),
data,
null,
stream
);
}
}
export class ChatClient extends DifyClient {
createChatMessage(inputs, query, user, responseMode = 'blocking', conversationId = null) {
createChatMessage(
inputs,
query,
user,
stream = false,
conversation_id = null
) {
const data = {
inputs,
query,
user,
responseMode,
}
if (conversationId)
data.conversation_id = conversationId
response_mode: stream ? "streaming" : "blocking",
};
if (conversation_id) data.conversation_id = conversation_id;
return this.sendRequest(routes.createChatMessage.method, routes.createChatMessage.url(), data, null, responseMode === 'streaming')
return this.sendRequest(
routes.createChatMessage.method,
routes.createChatMessage.url(),
data,
null,
stream
);
}
getConversationMessages(user, conversationId = '', firstId = null, limit = null) {
const params = { user }
getConversationMessages(
user,
conversation_id = "",
first_id = null,
limit = null
) {
const params = { user };
if (conversationId)
params.conversation_id = conversationId
if (conversation_id) params.conversation_id = conversation_id;
if (firstId)
params.first_id = firstId
if (first_id) params.first_id = first_id;
if (limit)
params.limit = limit
if (limit) params.limit = limit;
return this.sendRequest(routes.getConversationMessages.method, routes.getConversationMessages.url(), null, params)
return this.sendRequest(
routes.getConversationMessages.method,
routes.getConversationMessages.url(),
null,
params
);
}
getConversations(user, firstId = null, limit = null, pinned = null) {
const params = { user, first_id: firstId, limit, pinned }
return this.sendRequest(routes.getConversations.method, routes.getConversations.url(), null, params)
getConversations(user, first_id = null, limit = null, pinned = null) {
const params = { user, first_id: first_id, limit, pinned };
return this.sendRequest(
routes.getConversations.method,
routes.getConversations.url(),
null,
params
);
}
renameConversation(conversationId, name, user) {
const data = { name, user }
return this.sendRequest(routes.renameConversation.method, routes.renameConversation.url(conversationId), data)
renameConversation(conversation_id, name, user) {
const data = { name, user };
return this.sendRequest(
routes.renameConversation.method,
routes.renameConversation.url(conversation_id),
data
);
}
}
\ No newline at end of file
{
"name": "dify-client",
"version": "1.0.3",
"version": "2.0.0",
"description": "This is the Node.js SDK for the Dify.AI API, which allows you to easily integrate Dify.AI into your Node.js applications.",
"main": "index.js",
"type": "module",
......
......@@ -22,7 +22,7 @@ const ACCEPTS = [
'.md',
'.markdown',
'.txt',
'.xls',
// '.xls',
'.xlsx',
'.csv',
]
......
......@@ -22,7 +22,7 @@ const translation = {
title: 'Upload text file',
button: 'Drag and drop file, or',
browse: 'Browse',
tip: 'Supports txt, html, markdown, xlsx, xls, and pdf.',
tip: 'Supports txt, html, markdown, xlsx, and pdf.',
validation: {
typeError: 'File type not supported',
size: 'File too large. Maximum is 15MB',
......
......@@ -22,7 +22,7 @@ const translation = {
title: '上传文本文件',
button: '拖拽文件至此,或者',
browse: '选择文件',
tip: '已支持 TXT, HTML, Markdown, PDF, XLSX, XLS',
tip: '已支持 TXT, HTML, Markdown, PDF, XLSX',
validation: {
typeError: '文件类型不支持',
size: '文件太大了,不能超过 15MB',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment