Initial commit

.github/workflows/build-api-image.sh

+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SHA=$(git rev-parse HEAD)
+REPO_NAME=langgenius/dify
+API_REPO_NAME="${REPO_NAME}-api"
+
+if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
+  LATEST_TAG="pr-${PR_NUM}"
+  CACHE_FROM_TAG="latest"
+elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="latest"
+else
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="${REFSPEC}"
+fi
+
+if [[ "${REFSPEC}" == "main" ]]; then
+  LATEST_TAG="latest"
+  CACHE_FROM_TAG="latest"
+fi
+
+echo "Pulling cache image ${API_REPO_NAME}:${CACHE_FROM_TAG}"
+if docker pull "${API_REPO_NAME}:${CACHE_FROM_TAG}"; then
+  API_CACHE_FROM_SCRIPT="--cache-from ${API_REPO_NAME}:${CACHE_FROM_TAG}"
+else
+  echo "WARNING: Failed to pull ${API_REPO_NAME}:${CACHE_FROM_TAG}, disable build image cache."
+  API_CACHE_FROM_SCRIPT=""
+fi
+
+
+cat<<EOF
+  Rolling with tags:
+  - ${API_REPO_NAME}:${SHA}
+  - ${API_REPO_NAME}:${REFSPEC}
+  - ${API_REPO_NAME}:${LATEST_TAG}
+EOF
+
+#
+# Build image
+#
+cd api
+docker build \
+  ${API_CACHE_FROM_SCRIPT} \
+  --build-arg COMMIT_SHA=${SHA} \
+  -t "${API_REPO_NAME}:${SHA}" \
+  -t "${API_REPO_NAME}:${REFSPEC}" \
+  -t "${API_REPO_NAME}:${LATEST_TAG}" \
+  --label "sha=${SHA}" \
+  --label "built_at=$(date)" \
+  --label "build_actor=${GITHUB_ACTOR}" \
+  .
+
+# push
+docker push --all-tags "${API_REPO_NAME}"

.github/workflows/build-api-image.yml

+name: Build and Push API Image
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'deploy/dev'
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    steps:
+    - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_USER }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push Docker image
+      shell: bash
+      env:
+        DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+        DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      run: |
+        /bin/bash .github/workflows/build-api-image.sh
+
+    - name: Deploy to server
+      if: github.ref == 'refs/heads/deploy/dev'
+      uses: appleboy/ssh-action@v0.1.8
+      with:
+        host: ${{ secrets.SSH_HOST }}
+        username: ${{ secrets.SSH_USER }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script: |
+          ${{ secrets.SSH_SCRIPT }}

.github/workflows/build-web-image.sh

+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SHA=$(git rev-parse HEAD)
+REPO_NAME=langgenius/dify
+WEB_REPO_NAME="${REPO_NAME}-web"
+
+if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
+  LATEST_TAG="pr-${PR_NUM}"
+  CACHE_FROM_TAG="latest"
+elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="latest"
+else
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="${REFSPEC}"
+fi
+
+if [[ "${REFSPEC}" == "main" ]]; then
+  LATEST_TAG="latest"
+  CACHE_FROM_TAG="latest"
+fi
+
+echo "Pulling cache image ${WEB_REPO_NAME}:${CACHE_FROM_TAG}"
+if docker pull "${WEB_REPO_NAME}:${CACHE_FROM_TAG}"; then
+  WEB_CACHE_FROM_SCRIPT="--cache-from ${WEB_REPO_NAME}:${CACHE_FROM_TAG}"
+else
+  echo "WARNING: Failed to pull ${WEB_REPO_NAME}:${CACHE_FROM_TAG}, disable build image cache."
+  WEB_CACHE_FROM_SCRIPT=""
+fi
+
+
+cat<<EOF
+  Rolling with tags:
+  - ${WEB_REPO_NAME}:${SHA}
+  - ${WEB_REPO_NAME}:${REFSPEC}
+  - ${WEB_REPO_NAME}:${LATEST_TAG}
+EOF
+
+#
+# Build image
+#
+cd web
+docker build \
+  ${WEB_CACHE_FROM_SCRIPT} \
+  --build-arg COMMIT_SHA=${SHA} \
+  -t "${WEB_REPO_NAME}:${SHA}" \
+  -t "${WEB_REPO_NAME}:${REFSPEC}" \
+  -t "${WEB_REPO_NAME}:${LATEST_TAG}" \
+  --label "sha=${SHA}" \
+  --label "built_at=$(date)" \
+  --label "build_actor=${GITHUB_ACTOR}" \
+  .
+
+docker push --all-tags "${WEB_REPO_NAME}"
\ No newline at end of file

.github/workflows/build-web-image.yml

+name: Build and Push WEB Image
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'deploy/dev'
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    steps:
+    - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_USER }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push Docker image
+      shell: bash
+      env:
+        DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+        DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      run: |
+        /bin/bash .github/workflows/build-web-image.sh
+
+    - name: Deploy to server
+      if: github.ref == 'refs/heads/deploy/dev'
+      uses: appleboy/ssh-action@v0.1.8
+      with:
+        host: ${{ secrets.SSH_HOST }}
+        username: ${{ secrets.SSH_USER }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script: |
+          ${{ secrets.SSH_SCRIPT }}

.github/workflows/check_no_chinese_comments.py

+import os
+import re
+from zhon.hanzi import punctuation
+
+def has_chinese_characters(text):
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff' or char in punctuation:
+            return True
+    return False
+
+def check_file_for_chinese_comments(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        for line_number, line in enumerate(file, start=1):
+            if has_chinese_characters(line):
+                print(f"Found Chinese characters in {file_path} on line {line_number}:")
+                print(line.strip())
+                return True
+    return False
+
+def main():
+    has_chinese = False
+    excluded_files = ["model_template.py", 'stopwords.py', 'commands.py', 'indexing_runner.py']
+
+    for root, _, files in os.walk("."):
+        for file in files:
+            if file.endswith(".py") and file not in excluded_files:
+                file_path = os.path.join(root, file)
+                if check_file_for_chinese_comments(file_path):
+                    has_chinese = True
+
+    if has_chinese:
+        raise Exception("Found Chinese characters in Python files. Please remove them.")
+
+if __name__ == "__main__":
+    main()

.github/workflows/check_no_chinese_comments.yml

+name: Check for Chinese comments
+
+on:
+  push:
+    branches:
+      - 'main'
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  check-chinese-comments:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install zhon
+
+    - name: Run script to check for Chinese comments
+      run: |
+        python .github/workflows/check_no_chinese_comments.py

.github/workflows/flake8.yml

+name: PEP8 Check
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+jobs:
+  pep8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install flake8
+        run: pip install flake8
+      - name: Run flake8
+        run: flake8 --ignore=E501 .

.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+.idea/'
+
+.DS_Store
+.vscode
+
+# Intellij IDEA Files
+.idea/
+.ideaDataSources/
+
+api/.env
+api/storage/*
+
+docker/volumes/app/storage/privkeys/*
+docker/volumes/db/data/*
+docker/volumes/redis/data/*
+docker/volumes/weaviate/*
+
+sdks/python-client/build
+sdks/python-client/dist
+sdks/python-client/dify_client.egg-info

AUTHORS

+nite-knite
+goocarlos
+crazywoola
+iamjoel
+idsong
+takatost

CONTRIBUTING.md

+# Contributing
+
+Thanks for your interest in [Dify](https://dify.ai) and for wanting to contribute! Before you begin, read the
+[code of conduct](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md) and check out the
+[existing issues](https://github.com/langgenius/langgenius-gateway/issues).
+This document describes how to set up your development environment to build and test [Dify](https://dify.ai).
+
+### Install dependencies
+
+You need to install and configure the following dependencies on your machine to build [Dify](https://dify.ai):
+
+- [Git](http://git-scm.com/)
+- [Docker](https://www.docker.com/)
+- [Docker Compose](https://docs.docker.com/compose/install/)
+- [Node.js v18.x (LTS)](http://nodejs.org)
+- [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
+- [Python](https://www.python.org/) version 3.10.x
+
+## Local development
+
+To set up a working development environment, just fork the project git repository and install the backend and frontend dependencies using the proper package manager and create run the docker-compose stack.
+
+### Fork the repository
+
+you need to fork the [repository](https://github.com/langgenius/langgenius-gateway).
+
+### Clone the repo
+
+Clone your GitHub forked repository:
+
+```
+git clone git@github.com:<github_username>/langgenius-gateway.git
+```
+
+### Install backend
+
+To learn how to install the backend application, please refer to the [Backend README](api/README.md).
+
+### Install frontend
+
+To learn how to install the frontend application, please refer to the [Frontend README](web/README.md).
+
+### Visit dify in your browser
+
+Finally, you can now visit [http://localhost:3000](http://localhost:3000) to view the [Dify](https://dify.ai) in local environment.
+
+
+## Create a pull request
+
+After making your changes, open a pull request (PR). Once you submit your pull request, others from the Dify team/community will review it with you.
+
+Did you have an issue, like a merge conflict, or don't know how to open a pull request? Check out [GitHub's pull request tutorial](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests) on how to resolve merge conflicts and other issues. Once your PR has been merged, you will be proudly listed as a contributor in the [contributor chart](https://github.com/langgenius/langgenius-gateway/graphs/contributors).
+
+## Community channels
+
+Stuck somewhere? Have any questions? Join the [Discord Community Server](https://discord.gg/AhzKf7dNgk). We are here to help!

CONTRIBUTING_CN.md

+# 贡献
+
+感谢您对 [Dify](https://dify.ai) 的兴趣，并希望您能够做出贡献！在开始之前，请先阅读[行为准则](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)并查看[现有问题](https://github.com/langgenius/dify/issues)。
+本文档介绍了如何设置开发环境以构建和测试 [Dify](https://dify.ai)。
+
+### 安装依赖项
+
+您需要在计算机上安装和配置以下依赖项才能构建 [Dify](https://dify.ai)：
+
+- [Git](http://git-scm.com/)
+- [Docker](https://www.docker.com/)
+- [Docker Compose](https://docs.docker.com/compose/install/)
+- [Node.js v18.x (LTS)](http://nodejs.org)
+- [npm](https://www.npmjs.com/) 版本 8.x.x 或 [Yarn](https://yarnpkg.com/)
+- [Python](https://www.python.org/) 版本 3.10.x
+
+## 本地开发
+
+要设置一个可工作的开发环境，只需 fork 项目的 git 存储库，并使用适当的软件包管理器安装后端和前端依赖项，然后创建并运行 docker-compose 堆栈。
+
+### Fork存储库
+
+您需要 fork [存储库](https://github.com/langgenius/dify)。
+
+### 克隆存储库
+
+克隆您在 GitHub 上 fork 的存储库：
+
+```
+git clone git@github.com:<github_username>/dify.git
+```
+
+### 安装后端
+
+要了解如何安装后端应用程序，请参阅[后端 README](api/README.md)。
+
+### 安装前端
+
+要了解如何安装前端应用程序，请参阅[前端 README](web/README.md)。
+
+### 在浏览器中访问 Dify
+
+最后，您现在可以访问 [http://localhost:3000](http://localhost:3000) 在本地环境中查看 [Dify](https://dify.ai)。
+
+## 创建拉取请求
+
+在进行更改后，打开一个拉取请求（PR）。提交拉取请求后，Dify 团队/社区的其他人将与您一起审查它。
+
+如果遇到问题，比如合并冲突或不知道如何打开拉取请求，请查看 GitHub 的[拉取请求教程](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests)，了解如何解决合并冲突和其他问题。一旦您的 PR 被合并，您将自豪地被列为[贡献者表](https://github.com/langgenius/dify/graphs/contributors)中的一员。
+
+## 社区渠道
+
+遇到困难了吗？有任何问题吗? 加入 [Discord Community Server](https://discord.gg/AhzKf7dNgk)，我们将为您提供帮助。

LICENSE

+# Dify Open Source License
+
+The Dify project uses a combination of the Apache License 2.0, MIT License, and an additional agreement to protect against direct competition with Dify Cloud services.
+
+As a contributor, you should agree that your contributed code:
+a. Might be subject to a more permissive open source license in the future.
+b. Can be used for commercial purposes, such as Dify's cloud business.
+
+The following components are open source under the MIT license, allowing you to build and develop applications based on them:
+- WebApp elements, e.g., web/app/components/share
+- Derived WebApp Template projects
+
+The remaining parts of the project are open source under the Apache License 2.0.
+
+With the Apache License 2.0, MIT License, and this supplementary agreement, anyone can freely use, modify, and distribute Dify, provided that:
+
+- If you use Dify solely as a backend service for other applications, no authorization is needed for commercial or closed source purposes.
+- If you wish to use Dify for commercial and closed source SaaS services similar to Dify Cloud, please contact us for authorization.
+
+The interactive design of this product is protected by appearance patent.
+
+© 2023 LangGenius, Inc.
+
+----------
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+----------
+The MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

+![](./images/describe-en.png)
+<p align="center">
+  <a href="./README.md">English</a> |
+  <a href="./README_CN.md">简体中文</a>
+</p>
+
+[Website](http://dify.ai) • [Docs](https://docs.dify.ai) • [Twitter](https://twitter.com/dify_ai)
+
+**Dify** is an easy-to-use LLMOps platform designed to empower more people to create sustainable, AI-native applications. With visual orchestration for various application types, Dify offers out-of-the-box, ready-to-use applications that can also serve as Backend-as-a-Service APIs. Unify your development process with one API for plugins and datasets integration, and streamline your operations using a single interface for prompt engineering, visual analytics, and continuous improvement.
+
+Applications created with Dify include:
+
+Out-of-the-box web sites supporting form mode and chat conversation mode
+A single API encompassing plugin capabilities, context enhancement, and more, saving you backend coding effort
+Visual data analysis, log review, and annotation for applications
+Dify is compatible with Langchain, meaning we'll gradually support multiple LLMs, currently supported:
+
+- GPT 3 (text-davinci-003)
+- GPT 3.5 Turbo(ChatGPT)
+- GPT-4
+
+## Use Cloud Services
+
+Visit [Dify.ai](http://dify.ai)
+
+## Install the Community Edition
+
+### System Requirements
+
+Before installing Dify, make sure your machine meets the following minimum system requirements:
+
+- CPU >= 1 Core
+- RAM >= 4GB
+
+### Quick Start
+
+The easiest way to start the Dify server is to run our [docker-compose.yml](docker/docker-compose.yaml) file. Before running the installation command, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine:
+
+```bash
+cd docker
+docker-compose up -d
+```
+
+After running, you can access the Dify console in your browser at [http://localhost](http://localhost) and start the initialization operation.
+
+### Configuration
+
+If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run 'docker-compose up -d' again.
+
+## Roadmap
+
+Features under development:
+
+- **Datasets**, supporting more datasets, e.g. syncing content from Notion or webpages
+We will support more datasets, including text, webpages, and even Notion content. Users can build AI applications based on their own data sources.
+- **Plugins**, introducing ChatGPT Plugin-standard plugins for applications, or using Dify-produced plugins
+We will release plugins complying with ChatGPT standard, or Dify's own plugins to enable more capabilities in applications. 
+- **Open-source models**, e.g. adopting Llama as a model provider or for further fine-tuning
+We will work with excellent open-source models like Llama, by providing them as model options in our platform, or using them for further fine-tuning.
+
+
+## Q&A
+
+**Q: What can I do with Dify?**
+
+A: Dify is a simple yet powerful LLM development and operations tool. You can use it to build commercial-grade applications, personal assistants. If you want to develop your own applications, LangDifyGenius can save you backend work in integrating with OpenAI and offer visual operations capabilities, allowing you to continuously improve and train your GPT model.
+
+**Q: How do I use Dify to "train" my own model?**
+
+A: A valuable application consists of Prompt Engineering, context enhancement, and Fine-tuning. We've created a hybrid programming approach combining Prompts with programming languages (similar to a template engine), making it easy to accomplish long-text embedding or capturing subtitles from a user-input Youtube video - all of which will be submitted as context for LLMs to process. We place great emphasis on application operability, with data generated by users during App usage available for analysis, annotation, and continuous training. Without the right tools, these steps can be time-consuming.
+
+**Q: What do I need to prepare if I want to create my own application?**
+
+A: We assume you already have an OpenAI API Key; if not, please register for one. If you already have some content that can serve as training context, that's great!
+
+**Q: What interface languages are available?**
+
+A: English and Chinese are currently supported, and you can contribute language packs to us.
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
+
+## Contact Us
+
+If you have any questions, suggestions, or partnership inquiries, feel free to contact us through the following channels:
+
+- Submit an Issue or PR on our GitHub Repo
+- Join the discussion in our [Discord](https://discord.gg/AhzKf7dNgk) Community
+- Send an email to hello@dify.ai
+
+We're eager to assist you and together create more fun and useful AI applications!
+
+## Contributing
+
+To ensure proper review, all code contributions - including those from contributors with direct commit access - must be submitted via pull requests and approved by the core development team prior to being merged.
+
+We welcome all pull requests! If you'd like to help, check out the [Contribution Guide](CONTRIBUTING.md) for more information on how to get started.
+
+## Security
+
+To protect your privacy, please avoid posting security issues on GitHub. Instead, send your questions to security@dify.ai and we will provide you with a more detailed answer.
+
+## Citation
+
+This software uses the following open-source software:
+
+- Chase, H. (2022). LangChain [Computer software]. https://github.com/hwchase17/langchain
+- Liu, J. (2022). LlamaIndex [Computer software]. doi: 10.5281/zenodo.1234.
+
+For more information, please refer to the official website or license text of the respective software.
+
+## License
+
+This repository is available under the [Dify Open Source License](LICENSE).

README_CN.md

+![](./images/describe-cn.jpg)
+<p align="center">
+  <a href="./README.md">English</a> |
+  <a href="./README_CN.md">简体中文</a>
+</p>
+
+
+[官方网站](http://dify.ai) • [文档](https://docs.dify.ai/v/zh-hans) • [Twitter](https://twitter.com/dify_ai)
+
+**Dify** 是一个易用的 LLMOps 平台，旨在让更多人可以创建可持续运营的原生 AI 应用。Dify 提供多种类型应用的可视化编排，应用可开箱即用，也能以“后端即服务”的 API 提供服务。
+
+通过 Dify 创建的应用包含了：
+
+- 开箱即用的的 Web 站点，支持表单模式和聊天对话模式
+- 一套 API 即可包含插件、上下文增强等能力，替你省下了后端代码的编写工作
+- 可视化的对应用进行数据分析，查阅日志或进行标注
+
+Dify 兼容 Langchain，这意味着我们将逐步支持多种 LLMs ，目前已支持：
+
+- GPT 3 (text-davinci-003)
+- GPT 3.5 Turbo(ChatGPT)
+- GPT-4
+
+## 使用云服务
+
+访问 [Dify.ai](http://cloud.dify.ai)
+
+## 安装社区版
+
+### 系统要求
+
+在安装 Dify 之前，请确保您的机器满足以下最低系统要求：
+
+- CPU >= 1 Core
+- RAM >= 4GB
+
+### 快速启动
+
+启动 Dify 服务器的最简单方法是运行我们的 [docker-compose.yml](docker/docker-compose.yaml) 文件。在运行安装命令之前，请确保您的机器上安装了 [Docker](https://docs.docker.com/get-docker/) 和 [Docker Compose](https://docs.docker.com/compose/install/)：
+
+```bash
+cd docker
+docker-compose up -d
+```
+
+运行后，可以在浏览器上访问 [http://localhost](http://localhost) 进入 Dify 控制台，并开始初始化操作。
+
+### 配置
+
+需要自定义配置，请参考我们的 [docker-compose.yml](docker/docker-compose.yaml) 文件中的注释，并手动设置环境配置，修改完毕后，请再次执行 `docker-compose up -d`。
+
+## Roadmap
+
+我们正在开发中的功能：
+
+- **数据集**，支持更多的数据集，例如同步 Notion 或网页的内容
+我们将支持更多的数据集，包括文本、网页，甚至 Notion 内容。用户可以根据自己的数据源构建 AI 应用程序。
+- **插件**，推出符合 ChatGPT 标准的插件，或使用 Dify 产生的插件
+我们将发布符合 ChatGPT 标准的插件，或者 Dify 自己的插件，以在应用程序中启用更多功能。
+- **开源模型**，例如采用 Llama 作为模型提供者，或进行进一步的微调
+我们将与优秀的开源模型如 Llama 合作，通过在我们的平台中提供它们作为模型选项，或使用它们进行进一步的微调。
+
+## Q&A
+
+**Q: 我能用 Dify 做什么？**
+
+A: Dify 是一个简单且能力丰富的 LLM 开发和运营工具。你可以用它搭建商用级应用，个人助理。如果你想自己开发应用，Dify 也能为你省下接入 OpenAI 的后端工作，使用我们逐步提供的可视化运营能力，你可以持续的改进和训练你的 GPT 模型。
+
+**Q: 如何使用 Dify “训练”自己的模型？**
+
+A: 一个有价值的应用由 Prompt Engineering、上下文增强和 Fine-tune 三个环节组成。我们创造了一种 Prompt 结合编程语言的 Hybrid 编程方式（类似一个模版引擎），你可以轻松的完成长文本嵌入，或抓取用户输入的一个 Youtube 视频的字幕——这些都将作为上下文提交给 LLMs 进行计算。我们十分注重应用的可运营性，你的用户在使用 App 期间产生的数据，可进行分析、标记和持续训练。以上环节如果没有好的工具支持，可能会消耗你大量的时间。
+
+**Q: 如果要创建一个自己的应用，我需要准备什么？**
+
+A: 我们假定你已经有了 OpenAI API Key，如果没有请去注册一个。如果你已经有了一些内容可以作为训练上下文，就太好了。
+
+**Q: 提供哪些界面语言？**
+
+A: 现已支持英文与中文，你可以为我们贡献语言包。
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
+
+## 联系我们
+
+如果您有任何问题、建议或合作意向，欢迎通过以下方式联系我们：
+
+- 在我们的 [GitHub Repo](https://github.com/langgenius/dify) 上提交 Issue 或 PR
+- 在我们的 [Discord 社区](https://discord.gg/AhzKf7dNgk) 上加入讨论
+- 发送邮件至 hello@dify.ai
+
+## 贡献代码
+
+为了确保正确审查，所有代码贡献 - 包括来自具有直接提交更改权限的贡献者 - 都必须提交 PR 请求并在合并分支之前得到核心开发人员的批准。
+
+我们欢迎所有人提交 PR！如果您愿意提供帮助，可以在 [贡献指南](CONTRIBUTING_CN.md) 中了解有关如何为项目做出贡献的更多信息。
+
+## 安全
+
+为了保护您的隐私，请避免在 GitHub 上发布安全问题。发送问题至 security@dify.ai，我们将为您做更细致的解答。
+
+## Citation
+
+本软件使用了以下开源软件：
+
+- Chase, H. (2022). LangChain [Computer software]. https://github.com/hwchase17/langchain
+- Liu, J. (2022). LlamaIndex [Computer software]. doi: 10.5281/zenodo.1234.
+
+更多信息，请参考相应软件的官方网站或许可证文本。
+
+## License
+
+本仓库遵循 [Dify Open Source License](LICENSE) 开源协议。

api/.dockerignore

+.env
+storage/privkeys/*
\ No newline at end of file

api/.env.example

+# Server Edition
+EDITION=SELF_HOSTED
+
+# Your App secret key will be used for securely signing the session cookie
+# Make sure you are changing this key for your deployment with a strong key.
+# You can generate a strong key using `openssl rand -base64 42`.
+# Alternatively you can set it with `SECRET_KEY` environment variable.
+SECRET_KEY=
+
+# Console API base URL
+CONSOLE_URL=http://127.0.0.1:5001
+
+# Service API base URL
+API_URL=http://127.0.0.1:5001
+
+# Web APP base URL
+APP_URL=http://127.0.0.1:5001
+
+# celery configuration
+CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1
+
+# redis configuration
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_PASSWORD=difyai123456
+REDIS_DB=0
+
+# PostgreSQL database configuration
+DB_USERNAME=postgres
+DB_PASSWORD=difyai123456
+DB_HOST=localhost
+DB_PORT=5432
+DB_DATABASE=dify
+
+# Storage configuration
+# use for store upload files, private keys...
+# storage type: local, s3
+STORAGE_TYPE=local
+STORAGE_LOCAL_PATH=storage
+S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com
+S3_BUCKET_NAME=your-bucket-name
+S3_ACCESS_KEY=your-access-key
+S3_SECRET_KEY=your-secret-key
+S3_REGION=your-region
+
+# CORS configuration
+WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+
+# Cookie configuration
+COOKIE_HTTPONLY=true
+COOKIE_SAMESITE=None
+COOKIE_SECURE=true
+
+# Session configuration
+SESSION_PERMANENT=true
+SESSION_USE_SIGNER=true
+
+## support redis, sqlalchemy
+SESSION_TYPE=redis
+
+# session redis configuration
+SESSION_REDIS_HOST=localhost
+SESSION_REDIS_PORT=6379
+SESSION_REDIS_PASSWORD=difyai123456
+SESSION_REDIS_DB=2
+
+# Vector database configuration, support: weaviate, qdrant
+VECTOR_STORE=weaviate
+
+# Weaviate configuration
+WEAVIATE_ENDPOINT=http://localhost:8080
+WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
+WEAVIATE_GRPC_ENABLED=false
+
+# Qdrant configuration, use `path:` prefix for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode
+QDRANT_URL=path:storage/qdrant
+QDRANT_API_KEY=your-qdrant-api-key
+
+# Sentry configuration
+SENTRY_DSN=
+
+# DEBUG
+DEBUG=false
+SQLALCHEMY_ECHO=false

api/Dockerfile

+FROM langgenius/base:1.0.0-bullseye-slim as langgenius-api
+
+LABEL maintainer="takatost@gmail.com"
+
+ENV FLASK_APP app.py
+ENV EDITION SELF_HOSTED
+ENV DEPLOY_ENV PRODUCTION
+ENV CONSOLE_URL http://127.0.0.1:5001
+ENV API_URL http://127.0.0.1:5001
+ENV APP_URL http://127.0.0.1:5001
+
+EXPOSE 5001
+
+WORKDIR /app/api
+
+COPY requirements.txt /app/api/requirements.txt
+
+RUN pip install -r requirements.txt
+
+COPY . /app/api/
+
+COPY docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ARG COMMIT_SHA
+ENV COMMIT_SHA ${COMMIT_SHA}
+
+ENTRYPOINT ["/entrypoint.sh"]
\ No newline at end of file

api/README.md

+# Dify Backend API
+
+## Usage
+
+1. Start the docker-compose stack
+
+   The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
+   
+   ```bash
+   cd ../docker
+   docker-compose -f docker-compose.middleware.yaml up -d
+   cd ../api
+   ```
+2. Copy `.env.example` to `.env`
+3. Generate a `SECRET_KEY` in the `.env` file.
+
+   ```bash
+   openssl rand -base64 42
+   ```
+4. Install dependencies
+   ```bash
+   pip install -r requirements.txt
+   ```
+5. Run migrate
+
+   Before the first launch, migrate the database to the latest version.
+
+   ```bash
+   flask db upgrade
+   ```
+6. Start backend:
+   ```bash
+   flask run --host 0.0.0.0 --port=5001 --debug
+   ```
+7. Setup your application by visiting http://localhost:5001/console/api/setup or other apis...

api/app.py

+# -*- coding:utf-8 -*-
+import os
+if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true':
+    from gevent import monkey
+    monkey.patch_all()
+
+import logging
+import json
+import threading
+
+from flask import Flask, request, Response, session
+import flask_login
+from flask_cors import CORS
+
+from extensions import ext_session, ext_celery, ext_sentry, ext_redis, ext_login, ext_vector_store, ext_migrate, \
+    ext_database, ext_storage
+from extensions.ext_database import db
+from extensions.ext_login import login_manager
+
+# DO NOT REMOVE BELOW
+from models import model, account, dataset, web, task
+from events import event_handlers
+# DO NOT REMOVE ABOVE
+
+import core
+from config import Config, CloudEditionConfig
+from commands import register_commands
+from models.account import TenantAccountJoin
+from models.model import Account, EndUser, App
+
+import warnings
+warnings.simplefilter("ignore", ResourceWarning)
+
+
+class DifyApp(Flask):
+    pass
+
+# -------------
+# Configuration
+# -------------
+
+
+config_type = os.getenv('EDITION', default='SELF_HOSTED')  # ce edition first
+
+# ----------------------------
+# Application Factory Function
+# ----------------------------
+
+
+def create_app(test_config=None) -> Flask:
+    app = DifyApp(__name__)
+
+    if test_config:
+        app.config.from_object(test_config)
+    else:
+        if config_type == "CLOUD":
+            app.config.from_object(CloudEditionConfig())
+        else:
+            app.config.from_object(Config())
+
+    app.secret_key = app.config['SECRET_KEY']
+
+    logging.basicConfig(level=app.config.get('LOG_LEVEL', 'INFO'))
+
+    initialize_extensions(app)
+    register_blueprints(app)
+    register_commands(app)
+
+    core.init_app(app)
+
+    return app
+
+
+def initialize_extensions(app):
+    # Since the application instance is now created, pass it to each Flask
+    # extension instance to bind it to the Flask application instance (app)
+    ext_database.init_app(app)
+    ext_migrate.init(app, db)
+    ext_redis.init_app(app)
+    ext_vector_store.init_app(app)
+    ext_storage.init_app(app)
+    ext_celery.init_app(app)
+    ext_session.init_app(app)
+    ext_login.init_app(app)
+    ext_sentry.init_app(app)
+
+
+# Flask-Login configuration
+@login_manager.user_loader
+def load_user(user_id):
+    """Load user based on the user_id."""
+    if request.blueprint == 'console':
+        # Check if the user_id contains a dot, indicating the old format
+        if '.' in user_id:
+            tenant_id, account_id = user_id.split('.')
+        else:
+            account_id = user_id
+
+        account = db.session.query(Account).filter(Account.id == account_id).first()
+
+        if account:
+            workspace_id = session.get('workspace_id')
+            if workspace_id:
+                tenant_account_join = db.session.query(TenantAccountJoin).filter(
+                    TenantAccountJoin.account_id == account.id,
+                    TenantAccountJoin.tenant_id == workspace_id
+                ).first()
+
+                if not tenant_account_join:
+                    tenant_account_join = db.session.query(TenantAccountJoin).filter(
+                        TenantAccountJoin.account_id == account.id).first()
+
+                    if tenant_account_join:
+                        account.current_tenant_id = tenant_account_join.tenant_id
+                        session['workspace_id'] = account.current_tenant_id
+                else:
+                    account.current_tenant_id = workspace_id
+            else:
+                tenant_account_join = db.session.query(TenantAccountJoin).filter(
+                    TenantAccountJoin.account_id == account.id).first()
+                if tenant_account_join:
+                    account.current_tenant_id = tenant_account_join.tenant_id
+                    session['workspace_id'] = account.current_tenant_id
+
+            # Log in the user with the updated user_id
+            flask_login.login_user(account, remember=True)
+
+        return account
+    else:
+        return None
+
+
+@login_manager.unauthorized_handler
+def unauthorized_handler():
+    """Handle unauthorized requests."""
+    return Response(json.dumps({
+        'code': 'unauthorized',
+        'message': "Unauthorized."
+    }), status=401, content_type="application/json")
+
+
+# register blueprint routers
+def register_blueprints(app):
+    from controllers.service_api import bp as service_api_bp
+    from controllers.web import bp as web_bp
+    from controllers.console import bp as console_app_bp
+
+    app.register_blueprint(service_api_bp)
+
+    CORS(web_bp,
+         resources={
+             r"/*": {"origins": app.config['WEB_API_CORS_ALLOW_ORIGINS']}},
+         supports_credentials=True,
+         allow_headers=['Content-Type', 'Authorization'],
+         methods=['GET', 'PUT', 'POST', 'DELETE', 'OPTIONS', 'PATCH'],
+         expose_headers=['X-Version', 'X-Env']
+         )
+
+    app.register_blueprint(web_bp)
+
+    CORS(console_app_bp,
+         resources={
+             r"/*": {"origins": app.config['CONSOLE_CORS_ALLOW_ORIGINS']}},
+         supports_credentials=True,
+         allow_headers=['Content-Type', 'Authorization'],
+         methods=['GET', 'PUT', 'POST', 'DELETE', 'OPTIONS', 'PATCH'],
+         expose_headers=['X-Version', 'X-Env']
+         )
+
+    app.register_blueprint(console_app_bp)
+
+
+# create app
+app = create_app()
+celery = app.extensions["celery"]
+
+
+if app.config['TESTING']:
+    print("App is running in TESTING mode")
+
+
+@app.after_request
+def after_request(response):
+    """Add Version headers to the response."""
+    response.headers.add('X-Version', app.config['CURRENT_VERSION'])
+    response.headers.add('X-Env', app.config['DEPLOY_ENV'])
+    return response
+
+
+@app.route('/health')
+def health():
+    return Response(json.dumps({
+        'status': 'ok',
+        'version': app.config['CURRENT_VERSION']
+    }), status=200, content_type="application/json")
+
+
+@app.route('/threads')
+def threads():
+    num_threads = threading.active_count()
+    threads = threading.enumerate()
+
+    thread_list = []
+    for thread in threads:
+        thread_name = thread.name
+        thread_id = thread.ident
+        is_alive = thread.is_alive()
+
+        thread_list.append({
+            'name': thread_name,
+            'id': thread_id,
+            'is_alive': is_alive
+        })
+
+    return {
+        'thread_num': num_threads,
+        'threads': thread_list
+    }
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5001)

api/commands.py

+import datetime
+import json
+import random
+import string
+
+import click
+
+from libs.password import password_pattern, valid_password, hash_password
+from libs.helper import email as email_validate
+from extensions.ext_database import db
+from models.account import InvitationCode
+from models.model import Account, AppModelConfig, ApiToken, Site, App, RecommendedApp
+import secrets
+import base64
+
+
+@click.command('reset-password', help='Reset the account password.')
+@click.option('--email', prompt=True, help='The email address of the account whose password you need to reset')
+@click.option('--new-password', prompt=True, help='the new password.')
+@click.option('--password-confirm', prompt=True, help='the new password confirm.')
+def reset_password(email, new_password, password_confirm):
+    if str(new_password).strip() != str(password_confirm).strip():
+        click.echo(click.style('sorry. The two passwords do not match.', fg='red'))
+        return
+    account = db.session.query(Account). \
+        filter(Account.email == email). \
+        one_or_none()
+    if not account:
+        click.echo(click.style('sorry. the account: [{}] not exist .'.format(email), fg='red'))
+        return
+    try:
+        valid_password(new_password)
+    except:
+        click.echo(
+            click.style('sorry. The passwords must match {} '.format(password_pattern), fg='red'))
+        return
+
+    # generate password salt
+    salt = secrets.token_bytes(16)
+    base64_salt = base64.b64encode(salt).decode()
+
+    # encrypt password with salt
+    password_hashed = hash_password(new_password, salt)
+    base64_password_hashed = base64.b64encode(password_hashed).decode()
+    account.password = base64_password_hashed
+    account.password_salt = base64_salt
+    db.session.commit()
+    click.echo(click.style('Congratulations!, password has been reset.', fg='green'))
+
+
+@click.command('reset-email', help='Reset the account email.')
+@click.option('--email', prompt=True, help='The old email address of the account whose email you need to reset')
+@click.option('--new-email', prompt=True, help='the new email.')
+@click.option('--email-confirm', prompt=True, help='the new email confirm.')
+def reset_email(email, new_email, email_confirm):
+    if str(new_email).strip() != str(email_confirm).strip():
+        click.echo(click.style('Sorry, new email and confirm email do not match.', fg='red'))
+        return
+    account = db.session.query(Account). \
+        filter(Account.email == email). \
+        one_or_none()
+    if not account:
+        click.echo(click.style('sorry. the account: [{}] not exist .'.format(email), fg='red'))
+        return
+    try:
+        email_validate(new_email)
+    except:
+        click.echo(
+            click.style('sorry. {} is not a valid email. '.format(email), fg='red'))
+        return
+
+    account.email = new_email
+    db.session.commit()
+    click.echo(click.style('Congratulations!, email has been reset.', fg='green'))
+
+
+@click.command('generate-invitation-codes', help='Generate invitation codes.')
+@click.option('--batch', help='The batch of invitation codes.')
+@click.option('--count', prompt=True, help='Invitation codes count.')
+def generate_invitation_codes(batch, count):
+    if not batch:
+        now = datetime.datetime.now()
+        batch = now.strftime('%Y%m%d%H%M%S')
+
+    if not count or int(count) <= 0:
+        click.echo(click.style('sorry. the count must be greater than 0.', fg='red'))
+        return
+
+    count = int(count)
+
+    click.echo('Start generate {} invitation codes for batch {}.'.format(count, batch))
+
+    codes = ''
+    for i in range(count):
+        code = generate_invitation_code()
+        invitation_code = InvitationCode(
+            code=code,
+            batch=batch
+        )
+        db.session.add(invitation_code)
+        click.echo(code)
+
+        codes += code + "\n"
+    db.session.commit()
+
+    filename = 'storage/invitation-codes-{}.txt'.format(batch)
+
+    with open(filename, 'w') as f:
+        f.write(codes)
+
+    click.echo(click.style(
+        'Congratulations! Generated {} invitation codes for batch {} and saved to the file \'{}\''.format(count, batch,
+                                                                                                          filename),
+        fg='green'))
+
+
+def generate_invitation_code():
+    code = generate_upper_string()
+    while db.session.query(InvitationCode).filter(InvitationCode.code == code).count() > 0:
+        code = generate_upper_string()
+
+    return code
+
+
+def generate_upper_string():
+    letters_digits = string.ascii_uppercase + string.digits
+    result = ""
+    for i in range(8):
+        result += random.choice(letters_digits)
+
+    return result
+
+
+@click.command('gen-recommended-apps', help='Number of records to generate')
+def generate_recommended_apps():
+    print('Generating recommended app data...')
+    apps = App.query.all()
+    for app in apps:
+        recommended_app = RecommendedApp(
+            app_id=app.id,
+            description={
+                'en': 'Description for ' + app.name,
+                'zh': '描述 ' + app.name
+            },
+            copyright='Copyright ' + str(random.randint(1990, 2020)),
+            privacy_policy='https://privacypolicy.example.com',
+            category=random.choice(['Games', 'News', 'Music', 'Sports']),
+            position=random.randint(1, 100),
+            install_count=random.randint(100, 100000)
+        )
+        db.session.add(recommended_app)
+    db.session.commit()
+    print('Done!')
+
+
+def register_commands(app):
+    app.cli.add_command(reset_password)
+    app.cli.add_command(reset_email)
+    app.cli.add_command(generate_invitation_codes)
+    app.cli.add_command(generate_recommended_apps)

api/config.py

+# -*- coding:utf-8 -*-
+import os
+from datetime import timedelta
+
+import dotenv
+
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+
+dotenv.load_dotenv()
+
+DEFAULTS = {
+    'COOKIE_HTTPONLY': 'True',
+    'COOKIE_SECURE': 'True',
+    'COOKIE_SAMESITE': 'None',
+    'DB_USERNAME': 'postgres',
+    'DB_PASSWORD': '',
+    'DB_HOST': 'localhost',
+    'DB_PORT': '5432',
+    'DB_DATABASE': 'dify',
+    'REDIS_HOST': 'localhost',
+    'REDIS_PORT': '6379',
+    'REDIS_DB': '0',
+    'SESSION_REDIS_HOST': 'localhost',
+    'SESSION_REDIS_PORT': '6379',
+    'SESSION_REDIS_DB': '2',
+    'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize',
+    'OAUTH_REDIRECT_INDEX_PATH': '/',
+    'CONSOLE_URL': 'https://cloud.dify.ai',
+    'API_URL': 'https://api.dify.ai',
+    'APP_URL': 'https://udify.app',
+    'STORAGE_TYPE': 'local',
+    'STORAGE_LOCAL_PATH': 'storage',
+    'CHECK_UPDATE_URL': 'https://updates.dify.ai',
+    'SESSION_TYPE': 'sqlalchemy',
+    'SESSION_PERMANENT': 'True',
+    'SESSION_USE_SIGNER': 'True',
+    'DEPLOY_ENV': 'PRODUCTION',
+    'SQLALCHEMY_POOL_SIZE': 30,
+    'SQLALCHEMY_ECHO': 'False',
+    'SENTRY_TRACES_SAMPLE_RATE': 1.0,
+    'SENTRY_PROFILES_SAMPLE_RATE': 1.0,
+    'WEAVIATE_GRPC_ENABLED': 'True',
+    'CELERY_BACKEND': 'database',
+    'PDF_PREVIEW': 'True',
+    'LOG_LEVEL': 'INFO',
+}
+
+
+def get_env(key):
+    return os.environ.get(key, DEFAULTS.get(key))
+
+
+def get_bool_env(key):
+    return get_env(key).lower() == 'true'
+
+
+def get_cors_allow_origins(env, default):
+    cors_allow_origins = []
+    if get_env(env):
+        for origin in get_env(env).split(','):
+            cors_allow_origins.append(origin)
+    else:
+        cors_allow_origins = [default]
+
+    return cors_allow_origins
+
+
+class Config:
+    """Application configuration class."""
+
+    def __init__(self):
+        # app settings
+        self.CONSOLE_URL = get_env('CONSOLE_URL')
+        self.API_URL = get_env('API_URL')
+        self.APP_URL = get_env('APP_URL')
+        self.CURRENT_VERSION = "0.2.0"
+        self.COMMIT_SHA = get_env('COMMIT_SHA')
+        self.EDITION = "SELF_HOSTED"
+        self.DEPLOY_ENV = get_env('DEPLOY_ENV')
+        self.TESTING = False
+        self.LOG_LEVEL = get_env('LOG_LEVEL')
+        self.PDF_PREVIEW = get_bool_env('PDF_PREVIEW')
+
+        # Your App secret key will be used for securely signing the session cookie
+        # Make sure you are changing this key for your deployment with a strong key.
+        # You can generate a strong key using `openssl rand -base64 42`.
+        # Alternatively you can set it with `SECRET_KEY` environment variable.
+        self.SECRET_KEY = get_env('SECRET_KEY')
+
+        # cookie settings
+        self.REMEMBER_COOKIE_HTTPONLY = get_bool_env('COOKIE_HTTPONLY')
+        self.SESSION_COOKIE_HTTPONLY = get_bool_env('COOKIE_HTTPONLY')
+        self.REMEMBER_COOKIE_SAMESITE = get_env('COOKIE_SAMESITE')
+        self.SESSION_COOKIE_SAMESITE = get_env('COOKIE_SAMESITE')
+        self.REMEMBER_COOKIE_SECURE = get_bool_env('COOKIE_SECURE')
+        self.SESSION_COOKIE_SECURE = get_bool_env('COOKIE_SECURE')
+        self.PERMANENT_SESSION_LIFETIME = timedelta(days=7)
+
+        # session settings, only support sqlalchemy, redis
+        self.SESSION_TYPE = get_env('SESSION_TYPE')
+        self.SESSION_PERMANENT = get_bool_env('SESSION_PERMANENT')
+        self.SESSION_USE_SIGNER = get_bool_env('SESSION_USE_SIGNER')
+
+        # redis settings
+        self.REDIS_HOST = get_env('REDIS_HOST')
+        self.REDIS_PORT = get_env('REDIS_PORT')
+        self.REDIS_PASSWORD = get_env('REDIS_PASSWORD')
+        self.REDIS_DB = get_env('REDIS_DB')
+
+        # session redis settings
+        self.SESSION_REDIS_HOST = get_env('SESSION_REDIS_HOST')
+        self.SESSION_REDIS_PORT = get_env('SESSION_REDIS_PORT')
+        self.SESSION_REDIS_PASSWORD = get_env('SESSION_REDIS_PASSWORD')
+        self.SESSION_REDIS_DB = get_env('SESSION_REDIS_DB')
+
+        # storage settings
+        self.STORAGE_TYPE = get_env('STORAGE_TYPE')
+        self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH')
+        self.S3_ENDPOINT = get_env('S3_ENDPOINT')
+        self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME')
+        self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
+        self.S3_SECRET_KEY = get_env('S3_SECRET_KEY')
+        self.S3_REGION = get_env('S3_REGION')
+
+        # vector store settings, only support weaviate, qdrant
+        self.VECTOR_STORE = get_env('VECTOR_STORE')
+
+        # weaviate settings
+        self.WEAVIATE_ENDPOINT = get_env('WEAVIATE_ENDPOINT')
+        self.WEAVIATE_API_KEY = get_env('WEAVIATE_API_KEY')
+        self.WEAVIATE_GRPC_ENABLED = get_bool_env('WEAVIATE_GRPC_ENABLED')
+
+        # qdrant settings
+        self.QDRANT_URL = get_env('QDRANT_URL')
+        self.QDRANT_API_KEY = get_env('QDRANT_API_KEY')
+
+        # cors settings
+        self.CONSOLE_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
+            'CONSOLE_CORS_ALLOW_ORIGINS', self.CONSOLE_URL)
+        self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
+            'WEB_API_CORS_ALLOW_ORIGINS', '*')
+
+        # sentry settings
+        self.SENTRY_DSN = get_env('SENTRY_DSN')
+        self.SENTRY_TRACES_SAMPLE_RATE = float(get_env('SENTRY_TRACES_SAMPLE_RATE'))
+        self.SENTRY_PROFILES_SAMPLE_RATE = float(get_env('SENTRY_PROFILES_SAMPLE_RATE'))
+
+        # check update url
+        self.CHECK_UPDATE_URL = get_env('CHECK_UPDATE_URL')
+
+        # database settings
+        db_credentials = {
+            key: get_env(key) for key in
+            ['DB_USERNAME', 'DB_PASSWORD', 'DB_HOST', 'DB_PORT', 'DB_DATABASE']
+        }
+
+        self.SQLALCHEMY_DATABASE_URI = f"postgresql://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/{db_credentials['DB_DATABASE']}"
+        self.SQLALCHEMY_ENGINE_OPTIONS = {'pool_size': int(get_env('SQLALCHEMY_POOL_SIZE'))}
+
+        self.SQLALCHEMY_ECHO = get_bool_env('SQLALCHEMY_ECHO')
+
+        # celery settings
+        self.CELERY_BROKER_URL = get_env('CELERY_BROKER_URL')
+        self.CELERY_BACKEND = get_env('CELERY_BACKEND')
+        self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
+            if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
+
+        # hosted provider credentials
+        self.OPENAI_API_KEY = get_env('OPENAI_API_KEY')
+
+
+class CloudEditionConfig(Config):
+
+    def __init__(self):
+        super().__init__()
+
+        self.EDITION = "CLOUD"
+
+        self.GITHUB_CLIENT_ID = get_env('GITHUB_CLIENT_ID')
+        self.GITHUB_CLIENT_SECRET = get_env('GITHUB_CLIENT_SECRET')
+        self.GOOGLE_CLIENT_ID = get_env('GOOGLE_CLIENT_ID')
+        self.GOOGLE_CLIENT_SECRET = get_env('GOOGLE_CLIENT_SECRET')
+        self.OAUTH_REDIRECT_PATH = get_env('OAUTH_REDIRECT_PATH')
+
+
+class TestConfig(Config):
+
+    def __init__(self):
+        super().__init__()
+
+        self.EDITION = "SELF_HOSTED"
+        self.TESTING = True
+
+        db_credentials = {
+            key: get_env(key) for key in ['DB_USERNAME', 'DB_PASSWORD', 'DB_HOST', 'DB_PORT']
+        }
+
+        # use a different database for testing: dify_test
+        self.SQLALCHEMY_DATABASE_URI = f"postgresql://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/dify_test"

api/controllers/__init__.py

+# -*- coding:utf-8 -*-
+
+
+

api/controllers/console/__init__.py

+from flask import Blueprint
+
+from libs.external_api import ExternalApi
+
+bp = Blueprint('console', __name__, url_prefix='/console/api')
+api = ExternalApi(bp)
+
+# Import app controllers
+from .app import app, site, explore, completion, model_config, statistic, conversation, message
+
+# Import auth controllers
+from .auth import login, oauth
+
+# Import datasets controllers
+from .datasets import datasets, datasets_document, datasets_segments, file, hit_testing
+
+# Import other controllers
+from . import setup, version, apikey
+
+from .workspace import workspace, members, providers, account

api/controllers/console/apikey.py

+from flask_login import login_required, current_user
+import flask_restful
+from flask_restful import Resource, fields, marshal_with
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
+from models.model import App, ApiToken
+from models.dataset import Dataset
+
+from . import api
+from .setup import setup_required
+from .wraps import account_initialization_required
+from libs.helper import TimestampField
+
+api_key_fields = {
+    'id': fields.String,
+    'type': fields.String,
+    'token': fields.String,
+    'last_used_at': TimestampField,
+    'created_at': TimestampField
+}
+
+api_key_list = {
+    'data': fields.List(fields.Nested(api_key_fields), attribute="items")
+}
+
+
+def _get_resource(resource_id, tenant_id, resource_model):
+    resource = resource_model.query.filter_by(
+        id=resource_id, tenant_id=tenant_id
+    ).first()
+
+    if resource is None:
+        flask_restful.abort(
+            404, message=f"{resource_model.__name__} not found.")
+
+    return resource
+
+
+class BaseApiKeyListResource(Resource):
+    method_decorators = [account_initialization_required, login_required, setup_required]
+
+    resource_type = None
+    resource_model = None
+    resource_id_field = None
+    token_prefix = None
+    max_keys = 10
+
+    @marshal_with(api_key_list)
+    def get(self, resource_id):
+        resource_id = str(resource_id)
+        _get_resource(resource_id, current_user.current_tenant_id,
+                      self.resource_model)
+        keys = db.session.query(ApiToken). \
+            filter(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id). \
+            all()
+        return {"items": keys}
+
+    @marshal_with(api_key_fields)
+    def post(self, resource_id):
+        resource_id = str(resource_id)
+        _get_resource(resource_id, current_user.current_tenant_id,
+                      self.resource_model)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        current_key_count = db.session.query(ApiToken). \
+            filter(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id). \
+            count()
+
+        if current_key_count >= self.max_keys:
+            flask_restful.abort(
+                400,
+                message=f"Cannot create more than {self.max_keys} API keys for this resource type.",
+                code='max_keys_exceeded'
+            )
+
+        key = ApiToken.generate_api_key(self.token_prefix, 24)
+        api_token = ApiToken()
+        setattr(api_token, self.resource_id_field, resource_id)
+        api_token.token = key
+        api_token.type = self.resource_type
+        db.session.add(api_token)
+        db.session.commit()
+        return api_token, 201
+
+
+class BaseApiKeyResource(Resource):
+    method_decorators = [account_initialization_required, login_required, setup_required]
+
+    resource_type = None
+    resource_model = None
+    resource_id_field = None
+
+    def delete(self, resource_id, api_key_id):
+        resource_id = str(resource_id)
+        api_key_id = str(api_key_id)
+        _get_resource(resource_id, current_user.current_tenant_id,
+                      self.resource_model)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        key = db.session.query(ApiToken). \
+            filter(getattr(ApiToken, self.resource_id_field) == resource_id, ApiToken.type == self.resource_type, ApiToken.id == api_key_id). \
+            first()
+
+        if key is None:
+            flask_restful.abort(404, message='API key not found')
+
+        db.session.query(ApiToken).filter(ApiToken.id == api_key_id).delete()
+        db.session.commit()
+
+        return {'result': 'success'}, 204
+
+
+class AppApiKeyListResource(BaseApiKeyListResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+
+    resource_type = 'app'
+    resource_model = App
+    resource_id_field = 'app_id'
+    token_prefix = 'app-'
+
+
+class AppApiKeyResource(BaseApiKeyResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+
+    resource_type = 'app'
+    resource_model = App
+    resource_id_field = 'app_id'
+
+
+class DatasetApiKeyListResource(BaseApiKeyListResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+
+    resource_type = 'dataset'
+    resource_model = Dataset
+    resource_id_field = 'dataset_id'
+    token_prefix = 'ds-'
+
+
+class DatasetApiKeyResource(BaseApiKeyResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+    resource_type = 'dataset'
+    resource_model = Dataset
+    resource_id_field = 'dataset_id'
+
+
+api.add_resource(AppApiKeyListResource, '/apps/<uuid:resource_id>/api-keys')
+api.add_resource(AppApiKeyResource,
+                 '/apps/<uuid:resource_id>/api-keys/<uuid:api_key_id>')
+api.add_resource(DatasetApiKeyListResource,
+                 '/datasets/<uuid:resource_id>/api-keys')
+api.add_resource(DatasetApiKeyResource,
+                 '/datasets/<uuid:resource_id>/api-keys/<uuid:api_key_id>')

api/controllers/console/app/__init__.py

+from flask_login import current_user
+from werkzeug.exceptions import NotFound
+
+from controllers.console.app.error import AppUnavailableError
+from extensions.ext_database import db
+from models.model import App
+
+
+def _get_app(app_id, mode=None):
+    app = db.session.query(App).filter(
+        App.id == app_id,
+        App.tenant_id == current_user.current_tenant_id,
+        App.status == 'normal'
+    ).first()
+
+    if not app:
+        raise NotFound("App not found")
+
+    if mode and app.mode != mode:
+        raise AppUnavailableError()
+
+    return app

api/controllers/console/app/completion.py

+# -*- coding:utf-8 -*-
+import json
+import logging
+from typing import Generator, Union
+
+import flask_login
+from flask import Response, stream_with_context
+from flask_login import login_required
+from werkzeug.exceptions import InternalServerError, NotFound
+
+import services
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.app.error import ConversationCompletedError, AppUnavailableError, \
+    ProviderNotInitializeError, CompletionRequestError, ProviderQuotaExceededError, \
+    ProviderModelCurrentlyNotSupportError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.conversation_message_task import PubHandler
+from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
+    LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+from libs.helper import uuid_value
+from flask_restful import Resource, reqparse
+
+from services.completion_service import CompletionService
+
+
+# define completion message api for user
+class CompletionMessageApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app_model = _get_app(app_id, 'completion')
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, required=True, location='json')
+        parser.add_argument('query', type=str, location='json')
+        parser.add_argument('model_config', type=dict, required=True, location='json')
+        args = parser.parse_args()
+
+        account = flask_login.current_user
+
+        try:
+            response = CompletionService.completion(
+                app_model=app_model,
+                user=account,
+                args=args,
+                from_source='console',
+                streaming=True,
+                is_model_config_override=True
+            )
+
+            return compact_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+class CompletionMessageStopApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id, task_id):
+        app_id = str(app_id)
+
+        # get app info
+        _get_app(app_id, 'completion')
+
+        account = flask_login.current_user
+
+        PubHandler.stop(account, task_id)
+
+        return {'result': 'success'}, 200
+
+
+class ChatMessageApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app_model = _get_app(app_id, 'chat')
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, required=True, location='json')
+        parser.add_argument('query', type=str, required=True, location='json')
+        parser.add_argument('model_config', type=dict, required=True, location='json')
+        parser.add_argument('conversation_id', type=uuid_value, location='json')
+        args = parser.parse_args()
+
+        account = flask_login.current_user
+
+        try:
+            response = CompletionService.completion(
+                app_model=app_model,
+                user=account,
+                args=args,
+                from_source='console',
+                streaming=True,
+                is_model_config_override=True
+            )
+
+            return compact_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+def compact_response(response: Union[dict | Generator]) -> Response:
+    if isinstance(response, dict):
+        return Response(response=json.dumps(response), status=200, mimetype='application/json')
+    else:
+        def generate() -> Generator:
+            try:
+                for chunk in response:
+                    yield chunk
+            except services.errors.conversation.ConversationNotExistsError:
+                yield "data: " + json.dumps(api.handle_error(NotFound("Conversation Not Exists.")).get_json()) + "\n\n"
+            except services.errors.conversation.ConversationCompletedError:
+                yield "data: " + json.dumps(api.handle_error(ConversationCompletedError()).get_json()) + "\n\n"
+            except services.errors.app_model_config.AppModelConfigBrokenError:
+                logging.exception("App model config broken.")
+                yield "data: " + json.dumps(api.handle_error(AppUnavailableError()).get_json()) + "\n\n"
+            except ProviderTokenNotInitError:
+                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError()).get_json()) + "\n\n"
+            except QuotaExceededError:
+                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
+            except ModelCurrentlyNotSupportError:
+                yield "data: " + json.dumps(api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
+            except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                    LLMRateLimitError, LLMAuthorizationError) as e:
+                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(str(e))).get_json()) + "\n\n"
+            except ValueError as e:
+                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
+            except Exception:
+                logging.exception("internal server error.")
+                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
+
+        return Response(stream_with_context(generate()), status=200,
+                        mimetype='text/event-stream')
+
+
+class ChatMessageStopApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id, task_id):
+        app_id = str(app_id)
+
+        # get app info
+        _get_app(app_id, 'chat')
+
+        account = flask_login.current_user
+
+        PubHandler.stop(account, task_id)
+
+        return {'result': 'success'}, 200
+
+
+api.add_resource(CompletionMessageApi, '/apps/<uuid:app_id>/completion-messages')
+api.add_resource(CompletionMessageStopApi, '/apps/<uuid:app_id>/completion-messages/<string:task_id>/stop')
+api.add_resource(ChatMessageApi, '/apps/<uuid:app_id>/chat-messages')
+api.add_resource(ChatMessageStopApi, '/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop')

api/controllers/console/app/error.py

+from libs.exception import BaseHTTPException
+
+
+class AppNotFoundError(BaseHTTPException):
+    error_code = 'app_not_found'
+    description = "App not found."
+    code = 404
+
+
+class ProviderNotInitializeError(BaseHTTPException):
+    error_code = 'provider_not_initialize'
+    description = "Provider Token not initialize."
+    code = 400
+
+
+class ProviderQuotaExceededError(BaseHTTPException):
+    error_code = 'provider_quota_exceeded'
+    description = "Provider quota exceeded."
+    code = 400
+
+
+class ProviderModelCurrentlyNotSupportError(BaseHTTPException):
+    error_code = 'model_currently_not_support'
+    description = "GPT-4 currently not support."
+    code = 400
+
+
+class ConversationCompletedError(BaseHTTPException):
+    error_code = 'conversation_completed'
+    description = "Conversation was completed."
+    code = 400
+
+
+class AppUnavailableError(BaseHTTPException):
+    error_code = 'app_unavailable'
+    description = "App unavailable."
+    code = 400
+
+
+class CompletionRequestError(BaseHTTPException):
+    error_code = 'completion_request_error'
+    description = "Completion request failed."
+    code = 400
+
+
+class AppMoreLikeThisDisabledError(BaseHTTPException):
+    error_code = 'app_more_like_this_disabled'
+    description = "More like this disabled."
+    code = 403

api/controllers/console/app/explore.py

+# -*- coding:utf-8 -*-
+from datetime import datetime
+
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal_with, abort, inputs
+from sqlalchemy import and_
+
+from controllers.console import api
+from extensions.ext_database import db
+from models.model import Tenant, App, InstalledApp, RecommendedApp
+from services.account_service import TenantService
+
+app_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String
+}
+
+installed_app_fields = {
+    'id': fields.String,
+    'app': fields.Nested(app_fields, attribute='app'),
+    'app_owner_tenant_id': fields.String,
+    'is_pinned': fields.Boolean,
+    'last_used_at': fields.DateTime,
+    'editable': fields.Boolean
+}
+
+installed_app_list_fields = {
+    'installed_apps': fields.List(fields.Nested(installed_app_fields))
+}
+
+recommended_app_fields = {
+    'app': fields.Nested(app_fields, attribute='app'),
+    'app_id': fields.String,
+    'description': fields.String(attribute='description'),
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'category': fields.String,
+    'position': fields.Integer,
+    'is_listed': fields.Boolean,
+    'install_count': fields.Integer,
+    'installed': fields.Boolean,
+    'editable': fields.Boolean
+}
+
+recommended_app_list_fields = {
+    'recommended_apps': fields.List(fields.Nested(recommended_app_fields)),
+    'categories': fields.List(fields.String)
+}
+
+
+class InstalledAppsListResource(Resource):
+    @login_required
+    @marshal_with(installed_app_list_fields)
+    def get(self):
+        current_tenant_id = Tenant.query.first().id
+        installed_apps = db.session.query(InstalledApp).filter(
+            InstalledApp.tenant_id == current_tenant_id
+        ).all()
+
+        current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
+        installed_apps = [
+            {
+                **installed_app,
+                "editable": current_user.role in ["owner", "admin"],
+            }
+            for installed_app in installed_apps
+        ]
+        installed_apps.sort(key=lambda app: (-app.is_pinned, app.last_used_at))
+
+        return {'installed_apps': installed_apps}
+
+    @login_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('app_id', type=str, required=True, help='Invalid app_id')
+        args = parser.parse_args()
+
+        current_tenant_id = Tenant.query.first().id
+        app = App.query.get(args['app_id'])
+        if app is None:
+            abort(404, message='App not found')
+        recommended_app = RecommendedApp.query.filter(RecommendedApp.app_id == args['app_id']).first()
+        if recommended_app is None:
+            abort(404, message='App not found')
+        if not app.is_public:
+            abort(403, message="You can't install a non-public app")
+
+        installed_app = InstalledApp.query.filter(and_(
+            InstalledApp.app_id == args['app_id'],
+            InstalledApp.tenant_id == current_tenant_id
+        )).first()
+
+        if installed_app is None:
+            # todo: position
+            recommended_app.install_count += 1
+
+            new_installed_app = InstalledApp(
+                app_id=args['app_id'],
+                tenant_id=current_tenant_id,
+                is_pinned=False,
+                last_used_at=datetime.utcnow()
+            )
+            db.session.add(new_installed_app)
+            db.session.commit()
+
+        return {'message': 'App installed successfully'}
+
+
+class InstalledAppResource(Resource):
+
+    @login_required
+    def delete(self, installed_app_id):
+
+        installed_app = InstalledApp.query.filter(and_(
+            InstalledApp.id == str(installed_app_id),
+            InstalledApp.tenant_id == current_user.current_tenant_id
+        )).first()
+
+        if installed_app is None:
+            abort(404, message='App not found')
+
+        if installed_app.app_owner_tenant_id == current_user.current_tenant_id:
+            abort(400, message="You can't uninstall an app owned by the current tenant")
+
+        db.session.delete(installed_app)
+        db.session.commit()
+
+        return {'result': 'success', 'message': 'App uninstalled successfully'}
+
+    @login_required
+    def patch(self, installed_app_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('is_pinned', type=inputs.boolean)
+        args = parser.parse_args()
+
+        current_tenant_id = Tenant.query.first().id
+        installed_app = InstalledApp.query.filter(and_(
+            InstalledApp.id == str(installed_app_id),
+            InstalledApp.tenant_id == current_tenant_id
+        )).first()
+
+        if installed_app is None:
+            abort(404, message='Installed app not found')
+
+        commit_args = False
+        if 'is_pinned' in args:
+            installed_app.is_pinned = args['is_pinned']
+            commit_args = True
+
+        if commit_args:
+            db.session.commit()
+
+        return {'result': 'success', 'message': 'App info updated successfully'}
+
+
+class RecommendedAppsResource(Resource):
+    @login_required
+    @marshal_with(recommended_app_list_fields)
+    def get(self):
+        recommended_apps = db.session.query(RecommendedApp).filter(
+            RecommendedApp.is_listed == True
+        ).all()
+
+        categories = set()
+        current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
+        recommended_apps_result = []
+        for recommended_app in recommended_apps:
+            installed = db.session.query(InstalledApp).filter(
+                and_(
+                    InstalledApp.app_id == recommended_app.app_id,
+                    InstalledApp.tenant_id == current_user.current_tenant_id
+                )
+            ).first() is not None
+
+            language_prefix = current_user.interface_language.split('-')[0]
+            desc = None
+            if recommended_app.description:
+                if language_prefix in recommended_app.description:
+                    desc = recommended_app.description[language_prefix]
+                elif 'en' in recommended_app.description:
+                    desc = recommended_app.description['en']
+
+            recommended_app_result = {
+                'id': recommended_app.id,
+                'app': recommended_app.app,
+                'app_id': recommended_app.app_id,
+                'description': desc,
+                'copyright': recommended_app.copyright,
+                'privacy_policy': recommended_app.privacy_policy,
+                'category': recommended_app.category,
+                'position': recommended_app.position,
+                'is_listed': recommended_app.is_listed,
+                'install_count': recommended_app.install_count,
+                'installed': installed,
+                'editable': current_user.role in ['owner', 'admin'],
+            }
+            recommended_apps_result.append(recommended_app_result)
+
+            categories.add(recommended_app.category)  # add category to categories
+
+        return {'recommended_apps': recommended_apps_result, 'categories': list(categories)}
+
+
+api.add_resource(InstalledAppsListResource, '/installed-apps')
+api.add_resource(InstalledAppResource, '/installed-apps/<uuid:installed_app_id>')
+api.add_resource(RecommendedAppsResource, '/explore/apps')

api/controllers/console/app/model_config.py

+# -*- coding:utf-8 -*-
+import json
+
+from flask import request
+from flask_restful import Resource
+from flask_login import login_required, current_user
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from events.app_event import app_model_config_was_updated
+from extensions.ext_database import db
+from models.model import AppModelConfig
+from services.app_model_config_service import AppModelConfigService
+
+
+class ModelConfigResource(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        """Modify app model config"""
+        app_id = str(app_id)
+
+        app_model = _get_app(app_id)
+
+        # validate config
+        model_configuration = AppModelConfigService.validate_configuration(
+            account=current_user,
+            config=request.json,
+            mode=app_model.mode
+        )
+
+        new_app_model_config = AppModelConfig(
+            app_id=app_model.id,
+            provider="",
+            model_id="",
+            configs={},
+            opening_statement=model_configuration['opening_statement'],
+            suggested_questions=json.dumps(model_configuration['suggested_questions']),
+            suggested_questions_after_answer=json.dumps(model_configuration['suggested_questions_after_answer']),
+            more_like_this=json.dumps(model_configuration['more_like_this']),
+            model=json.dumps(model_configuration['model']),
+            user_input_form=json.dumps(model_configuration['user_input_form']),
+            pre_prompt=model_configuration['pre_prompt'],
+            agent_mode=json.dumps(model_configuration['agent_mode']),
+        )
+
+        db.session.add(new_app_model_config)
+        db.session.flush()
+
+        app_model.app_model_config_id = new_app_model_config.id
+        db.session.commit()
+
+        app_model_config_was_updated.send(
+            app_model,
+            app_model_config=new_app_model_config
+        )
+
+        return {'result': 'success'}
+
+
+api.add_resource(ModelConfigResource, '/apps/<uuid:app_id>/model-config')

api/controllers/console/app/site.py

+# -*- coding:utf-8 -*-
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal_with
+from werkzeug.exceptions import NotFound, Forbidden
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import supported_language
+from extensions.ext_database import db
+from models.model import Site
+
+app_site_fields = {
+    'app_id': fields.String,
+    'access_token': fields.String(attribute='code'),
+    'code': fields.String,
+    'title': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'default_language': fields.String,
+    'customize_domain': fields.String,
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'customize_token_strategy': fields.String,
+    'prompt_public': fields.Boolean
+}
+
+
+def parse_app_site_args():
+    parser = reqparse.RequestParser()
+    parser.add_argument('title', type=str, required=False, location='json')
+    parser.add_argument('icon', type=str, required=False, location='json')
+    parser.add_argument('icon_background', type=str, required=False, location='json')
+    parser.add_argument('description', type=str, required=False, location='json')
+    parser.add_argument('default_language', type=supported_language, required=False, location='json')
+    parser.add_argument('customize_domain', type=str, required=False, location='json')
+    parser.add_argument('copyright', type=str, required=False, location='json')
+    parser.add_argument('privacy_policy', type=str, required=False, location='json')
+    parser.add_argument('customize_token_strategy', type=str, choices=['must', 'allow', 'not_allow'],
+                        required=False,
+                        location='json')
+    parser.add_argument('prompt_public', type=bool, required=False, location='json')
+    return parser.parse_args()
+
+
+class AppSite(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_site_fields)
+    def post(self, app_id):
+        args = parse_app_site_args()
+
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        site = db.session.query(Site). \
+            filter(Site.app_id == app_model.id). \
+            one_or_404()
+
+        for attr_name in [
+            'title',
+            'icon',
+            'icon_background',
+            'description',
+            'default_language',
+            'customize_domain',
+            'copyright',
+            'privacy_policy',
+            'customize_token_strategy',
+            'prompt_public'
+        ]:
+            value = args.get(attr_name)
+            if value is not None:
+                setattr(site, attr_name, value)
+
+        db.session.commit()
+
+        return site
+
+
+class AppSiteAccessTokenReset(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_site_fields)
+    def post(self, app_id):
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        site = db.session.query(Site).filter(Site.app_id == app_model.id).first()
+
+        if not site:
+            raise NotFound
+
+        site.code = Site.generate_code(16)
+        db.session.commit()
+
+        return site
+
+
+api.add_resource(AppSite, '/apps/<uuid:app_id>/site')
+api.add_resource(AppSiteAccessTokenReset, '/apps/<uuid:app_id>/site/access-token-reset')

api/controllers/console/app/statistic.py

+# -*- coding:utf-8 -*-
+from datetime import datetime
+
+import pytz
+from flask import jsonify
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import datetime_string
+from extensions.ext_database import db
+
+
+class DailyConversationStatistic(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        account = current_user
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+        SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.conversation_id) AS conversation_count
+            FROM messages where app_id = :app_id 
+        '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+        rs = db.session.execute(sql_query, arg_dict)
+
+        response_date = []
+
+        for i in rs:
+            response_date.append({
+                'date': str(i.date),
+                'conversation_count': i.conversation_count
+            })
+
+        return jsonify({
+            'data': response_date
+        })
+
+
+class DailyTerminalsStatistic(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        account = current_user
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.from_end_user_id) AS terminal_count
+                    FROM messages where app_id = :app_id 
+                '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+        rs = db.session.execute(sql_query, arg_dict)
+
+        response_date = []
+
+        for i in rs:
+            response_date.append({
+                'date': str(i.date),
+                'terminal_count': i.terminal_count
+            })
+
+        return jsonify({
+            'data': response_date
+        })
+
+
+class DailyTokenCostStatistic(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        account = current_user
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
+                    (sum(messages.message_tokens) + sum(messages.answer_tokens)) as token_count,
+                    sum(total_price) as total_price
+                    FROM messages where app_id = :app_id 
+                '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+        rs = db.session.execute(sql_query, arg_dict)
+
+        response_date = []
+
+        for i in rs:
+            response_date.append({
+                'date': str(i.date),
+                'token_count': i.token_count,
+                'total_price': i.total_price,
+                'currency': 'USD'
+            })
+
+        return jsonify({
+            'data': response_date
+        })
+
+
+api.add_resource(DailyConversationStatistic, '/apps/<uuid:app_id>/statistics/daily-conversations')
+api.add_resource(DailyTerminalsStatistic, '/apps/<uuid:app_id>/statistics/daily-end-users')
+api.add_resource(DailyTokenCostStatistic, '/apps/<uuid:app_id>/statistics/token-costs')

api/controllers/console/auth/login.py

+# -*- coding:utf-8 -*-
+import flask
+import flask_login
+from flask import request, current_app
+from flask_restful import Resource, reqparse
+
+import services
+from controllers.console import api
+from controllers.console.error import AccountNotLinkTenantError
+from controllers.console.setup import setup_required
+from libs.helper import email
+from libs.password import valid_password
+from services.account_service import AccountService, TenantService
+
+
+class LoginApi(Resource):
+    """Resource for user login."""
+
+    @setup_required
+    def post(self):
+        """Authenticate user and login."""
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=email, required=True, location='json')
+        parser.add_argument('password', type=valid_password, required=True, location='json')
+        parser.add_argument('remember_me', type=bool, required=False, default=False, location='json')
+        args = parser.parse_args()
+
+        # todo: Verify the recaptcha
+
+        try:
+            account = AccountService.authenticate(args['email'], args['password'])
+        except services.errors.account.AccountLoginError:
+            return {'code': 'unauthorized', 'message': 'Invalid email or password'}, 401
+
+        try:
+            TenantService.switch_tenant(account)
+        except Exception:
+            raise AccountNotLinkTenantError("Account not link tenant")
+
+        flask_login.login_user(account, remember=args['remember_me'])
+        AccountService.update_last_login(account, request)
+
+        # todo: return the user info
+
+        return {'result': 'success'}
+
+
+class LogoutApi(Resource):
+
+    @setup_required
+    def get(self):
+        flask.session.pop('workspace_id', None)
+        flask_login.logout_user()
+        return {'result': 'success'}
+
+
+class ResetPasswordApi(Resource):
+    @setup_required
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=email, required=True, location='json')
+        args = parser.parse_args()
+
+        # import mailchimp_transactional as MailchimpTransactional
+        # from mailchimp_transactional.api_client import ApiClientError
+
+        account = {'email': args['email']}
+        # account = AccountService.get_by_email(args['email'])
+        # if account is None:
+        #     raise ValueError('Email not found')
+        # new_password = AccountService.generate_password()
+        # AccountService.update_password(account, new_password)
+
+        # todo: Send email
+        MAILCHIMP_API_KEY = current_app.config['MAILCHIMP_TRANSACTIONAL_API_KEY']
+        # mailchimp = MailchimpTransactional(MAILCHIMP_API_KEY)
+
+        message = {
+            'from_email': 'noreply@example.com',
+            'to': [{'email': account.email}],
+            'subject': 'Reset your Dify password',
+            'html': """
+                <p>Dear User,</p>
+                <p>The Dify team has generated a new password for you, details as follows:</p> 
+                <p><strong>{new_password}</strong></p>
+                <p>Please change your password to log in as soon as possible.</p>
+                <p>Regards,</p>
+                <p>The Dify Team</p> 
+            """
+        }
+
+        # response = mailchimp.messages.send({
+        #     'message': message,
+        #     # required for transactional email
+        #     ' settings': {
+        #         'sandbox_mode': current_app.config['MAILCHIMP_SANDBOX_MODE'],
+        #     },
+        # })
+
+        # Check if MSG was sent
+        # if response.status_code != 200:
+        #     # handle error
+        #     pass
+
+        return {'result': 'success'}
+
+
+api.add_resource(LoginApi, '/login')
+api.add_resource(LogoutApi, '/logout')

api/controllers/console/auth/oauth.py

+import logging
+from datetime import datetime
+from typing import Optional
+
+import flask_login
+import requests
+from flask import request, redirect, current_app, session
+from flask_restful import Resource
+
+from libs.oauth import OAuthUserInfo, GitHubOAuth, GoogleOAuth
+from extensions.ext_database import db
+from models.account import Account, AccountStatus
+from services.account_service import AccountService, RegisterService
+from .. import api
+
+
+def get_oauth_providers():
+    with current_app.app_context():
+        github_oauth = GitHubOAuth(client_id=current_app.config.get('GITHUB_CLIENT_ID'),
+                                   client_secret=current_app.config.get(
+                                       'GITHUB_CLIENT_SECRET'),
+                                   redirect_uri=current_app.config.get(
+                                       'CONSOLE_URL') + '/console/api/oauth/authorize/github')
+
+        google_oauth = GoogleOAuth(client_id=current_app.config.get('GOOGLE_CLIENT_ID'),
+                                   client_secret=current_app.config.get(
+                                       'GOOGLE_CLIENT_SECRET'),
+                                   redirect_uri=current_app.config.get(
+                                       'CONSOLE_URL') + '/console/api/oauth/authorize/google')
+
+        OAUTH_PROVIDERS = {
+            'github': github_oauth,
+            'google': google_oauth
+        }
+        return OAUTH_PROVIDERS
+
+
+class OAuthLogin(Resource):
+    def get(self, provider: str):
+        OAUTH_PROVIDERS = get_oauth_providers()
+        with current_app.app_context():
+            oauth_provider = OAUTH_PROVIDERS.get(provider)
+            print(vars(oauth_provider))
+        if not oauth_provider:
+            return {'error': 'Invalid provider'}, 400
+
+        auth_url = oauth_provider.get_authorization_url()
+        return redirect(auth_url)
+
+
+class OAuthCallback(Resource):
+    def get(self, provider: str):
+        OAUTH_PROVIDERS = get_oauth_providers()
+        with current_app.app_context():
+            oauth_provider = OAUTH_PROVIDERS.get(provider)
+        if not oauth_provider:
+            return {'error': 'Invalid provider'}, 400
+
+        code = request.args.get('code')
+        try:
+            token = oauth_provider.get_access_token(code)
+            user_info = oauth_provider.get_user_info(token)
+        except requests.exceptions.HTTPError as e:
+            logging.exception(
+                f"An error occurred during the OAuth process with {provider}: {e.response.text}")
+            return {'error': 'OAuth process failed'}, 400
+
+        account = _generate_account(provider, user_info)
+        # Check account status
+        if account.status == AccountStatus.BANNED.value or account.status == AccountStatus.CLOSED.value:
+            return {'error': 'Account is banned or closed.'}, 403
+
+        if account.status == AccountStatus.PENDING.value:
+            account.status = AccountStatus.ACTIVE.value
+            account.initialized_at = datetime.utcnow()
+            db.session.commit()
+
+        # login user
+        session.clear()
+        flask_login.login_user(account, remember=True)
+        AccountService.update_last_login(account, request)
+
+        return redirect(f'{current_app.config.get("CONSOLE_URL")}?oauth_login=success')
+
+
+def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Optional[Account]:
+    account = Account.get_by_openid(provider, user_info.id)
+
+    if not account:
+        account = Account.query.filter_by(email=user_info.email).first()
+
+    return account
+
+
+def _generate_account(provider: str, user_info: OAuthUserInfo):
+    # Get account by openid or email.
+    account = _get_account_by_openid_or_email(provider, user_info)
+
+    if not account:
+        # Create account
+        account_name = user_info.name if user_info.name else 'Dify'
+        account = RegisterService.register(
+            email=user_info.email,
+            name=account_name,
+            password=None,
+            open_id=user_info.id,
+            provider=provider
+        )
+
+        # Set interface language
+        preferred_lang = request.accept_languages.best_match(['zh', 'en'])
+        if preferred_lang == 'zh':
+            interface_language = 'zh-Hans'
+        else:
+            interface_language = 'en-US'
+        account.interface_language = interface_language
+        db.session.commit()
+
+    # Link account
+    AccountService.link_account_integrate(provider, user_info.id, account)
+
+    return account
+
+
+api.add_resource(OAuthLogin, '/oauth/login/<provider>')
+api.add_resource(OAuthCallback, '/oauth/authorize/<provider>')

api/controllers/console/datasets/datasets.py

+# -*- coding:utf-8 -*-
+from flask import request
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal, marshal_with
+from werkzeug.exceptions import NotFound, Forbidden
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import DatasetNameDuplicateError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.indexing_runner import IndexingRunner
+from libs.helper import TimestampField
+from extensions.ext_database import db
+from models.model import UploadFile
+from services.dataset_service import DatasetService
+
+dataset_detail_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'description': fields.String,
+    'provider': fields.String,
+    'permission': fields.String,
+    'data_source_type': fields.String,
+    'indexing_technique': fields.String,
+    'app_count': fields.Integer,
+    'document_count': fields.Integer,
+    'word_count': fields.Integer,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'updated_by': fields.String,
+    'updated_at': TimestampField,
+}
+
+dataset_query_detail_fields = {
+    "id": fields.String,
+    "content": fields.String,
+    "source": fields.String,
+    "source_app_id": fields.String,
+    "created_by_role": fields.String,
+    "created_by": fields.String,
+    "created_at": TimestampField
+}
+
+
+def _validate_name(name):
+    if not name or len(name) < 1 or len(name) > 40:
+        raise ValueError('Name must be between 1 to 40 characters.')
+    return name
+
+
+def _validate_description_length(description):
+    if len(description) > 200:
+        raise ValueError('Description cannot exceed 200 characters.')
+    return description
+
+
+class DatasetListApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        page = request.args.get('page', default=1, type=int)
+        limit = request.args.get('limit', default=20, type=int)
+        ids = request.args.getlist('ids')
+        provider = request.args.get('provider', default="vendor")
+        if ids:
+            datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id)
+        else:
+            datasets, total = DatasetService.get_datasets(page, limit, provider,
+                                                          current_user.current_tenant_id, current_user)
+
+        response = {
+            'data': marshal(datasets, dataset_detail_fields),
+            'has_more': len(datasets) == limit,
+            'limit': limit,
+            'total': total,
+            'page': page
+        }
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', nullable=False, required=True,
+                            help='type is required. Name must be between 1 to 40 characters.',
+                            type=_validate_name)
+        parser.add_argument('indexing_technique', type=str, location='json',
+                            choices=('high_quality', 'economy'),
+                            help='Invalid indexing technique.')
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        try:
+            dataset = DatasetService.create_empty_dataset(
+                tenant_id=current_user.current_tenant_id,
+                name=args['name'],
+                indexing_technique=args['indexing_technique'],
+                account=current_user
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return marshal(dataset, dataset_detail_fields), 201
+
+
+class DatasetApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(
+                dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        return marshal(dataset, dataset_detail_fields), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', nullable=False,
+                            help='type is required. Name must be between 1 to 40 characters.',
+                            type=_validate_name)
+        parser.add_argument('description',
+                            location='json', store_missing=False,
+                            type=_validate_description_length)
+        parser.add_argument('indexing_technique', type=str, location='json',
+                            choices=('high_quality', 'economy'),
+                            help='Invalid indexing technique.')
+        parser.add_argument('permission', type=str, location='json', choices=(
+            'only_me', 'all_team_members'), help='Invalid permission.')
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        dataset = DatasetService.update_dataset(
+            dataset_id_str, args, current_user)
+
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        return marshal(dataset, dataset_detail_fields), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        if DatasetService.delete_dataset(dataset_id_str, current_user):
+            return {'result': 'success'}, 204
+        else:
+            raise NotFound("Dataset not found.")
+
+
+class DatasetQueryApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        page = request.args.get('page', default=1, type=int)
+        limit = request.args.get('limit', default=20, type=int)
+
+        dataset_queries, total = DatasetService.get_dataset_queries(
+            dataset_id=dataset.id,
+            page=page,
+            per_page=limit
+        )
+
+        response = {
+            'data': marshal(dataset_queries, dataset_query_detail_fields),
+            'has_more': len(dataset_queries) == limit,
+            'limit': limit,
+            'total': total,
+            'page': page
+        }
+        return response, 200
+
+
+class DatasetIndexingEstimateApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        segment_rule = request.get_json()
+        file_detail = db.session.query(UploadFile).filter(
+            UploadFile.tenant_id == current_user.current_tenant_id,
+            UploadFile.id == segment_rule["file_id"]
+        ).first()
+
+        if file_detail is None:
+            raise NotFound("File not found.")
+
+        indexing_runner = IndexingRunner()
+        response = indexing_runner.indexing_estimate(file_detail, segment_rule['process_rule'])
+        return response, 200
+
+
+class DatasetRelatedAppListApi(Resource):
+    app_detail_kernel_fields = {
+        'id': fields.String,
+        'name': fields.String,
+        'mode': fields.String,
+        'icon': fields.String,
+        'icon_background': fields.String,
+    }
+
+    related_app_list = {
+        'data': fields.List(fields.Nested(app_detail_kernel_fields)),
+        'total': fields.Integer,
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(related_app_list)
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        app_dataset_joins = DatasetService.get_related_apps(dataset.id)
+
+        related_apps = []
+        for app_dataset_join in app_dataset_joins:
+            app_model = app_dataset_join.app
+            if app_model:
+                related_apps.append(app_model)
+
+        return {
+            'data': related_apps,
+            'total': len(related_apps)
+        }, 200
+
+
+api.add_resource(DatasetListApi, '/datasets')
+api.add_resource(DatasetApi, '/datasets/<uuid:dataset_id>')
+api.add_resource(DatasetQueryApi, '/datasets/<uuid:dataset_id>/queries')
+api.add_resource(DatasetIndexingEstimateApi, '/datasets/file-indexing-estimate')
+api.add_resource(DatasetRelatedAppListApi, '/datasets/<uuid:dataset_id>/related-apps')

api/controllers/console/datasets/datasets_segments.py

+# -*- coding:utf-8 -*-
+from datetime import datetime
+
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal
+from werkzeug.exceptions import NotFound, Forbidden
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import InvalidActionError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from models.dataset import DocumentSegment
+
+from libs.helper import TimestampField
+from services.dataset_service import DatasetService, DocumentService
+from tasks.add_segment_to_index_task import add_segment_to_index_task
+from tasks.remove_segment_from_index_task import remove_segment_from_index_task
+
+segment_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'document_id': fields.String,
+    'content': fields.String,
+    'word_count': fields.Integer,
+    'tokens': fields.Integer,
+    'keywords': fields.List(fields.String),
+    'index_node_id': fields.String,
+    'index_node_hash': fields.String,
+    'hit_count': fields.Integer,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'status': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'indexing_at': TimestampField,
+    'completed_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField
+}
+
+segment_list_response = {
+    'data': fields.List(fields.Nested(segment_fields)),
+    'has_more': fields.Boolean,
+    'limit': fields.Integer
+}
+
+
+class DatasetDocumentSegmentListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        document = DocumentService.get_document(dataset_id, document_id)
+
+        if not document:
+            raise NotFound('Document not found.')
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('last_id', type=str, default=None, location='args')
+        parser.add_argument('limit', type=int, default=20, location='args')
+        parser.add_argument('status', type=str,
+                            action='append', default=[], location='args')
+        parser.add_argument('hit_count_gte', type=int,
+                            default=None, location='args')
+        parser.add_argument('enabled', type=str, default='all', location='args')
+        args = parser.parse_args()
+
+        last_id = args['last_id']
+        limit = min(args['limit'], 100)
+        status_list = args['status']
+        hit_count_gte = args['hit_count_gte']
+
+        query = DocumentSegment.query.filter(
+            DocumentSegment.document_id == str(document_id),
+            DocumentSegment.tenant_id == current_user.current_tenant_id
+        )
+
+        if last_id is not None:
+            last_segment = DocumentSegment.query.get(str(last_id))
+            if last_segment:
+                query = query.filter(
+                    DocumentSegment.position > last_segment.position)
+            else:
+                return {'data': [], 'has_more': False, 'limit': limit}, 200
+
+        if status_list:
+            query = query.filter(DocumentSegment.status.in_(status_list))
+
+        if hit_count_gte is not None:
+            query = query.filter(DocumentSegment.hit_count >= hit_count_gte)
+
+        if args['enabled'].lower() != 'all':
+            if args['enabled'].lower() == 'true':
+                query = query.filter(DocumentSegment.enabled == True)
+            elif args['enabled'].lower() == 'false':
+                query = query.filter(DocumentSegment.enabled == False)
+
+        total = query.count()
+        segments = query.order_by(DocumentSegment.position).limit(limit + 1).all()
+
+        has_more = False
+        if len(segments) > limit:
+            has_more = True
+            segments = segments[:-1]
+
+        return {
+            'data': marshal(segments, segment_fields),
+            'has_more': has_more,
+            'limit': limit,
+            'total': total
+        }, 200
+
+
+class DatasetDocumentSegmentApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, dataset_id, segment_id, action):
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id),
+            DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+
+        if not segment:
+            raise NotFound('Segment not found.')
+
+        document_indexing_cache_key = 'document_{}_indexing'.format(segment.document_id)
+        cache_result = redis_client.get(document_indexing_cache_key)
+        if cache_result is not None:
+            raise InvalidActionError("Document is being indexed, please try again later")
+
+        indexing_cache_key = 'segment_{}_indexing'.format(segment.id)
+        cache_result = redis_client.get(indexing_cache_key)
+        if cache_result is not None:
+            raise InvalidActionError("Segment is being indexed, please try again later")
+
+        if action == "enable":
+            if segment.enabled:
+                raise InvalidActionError("Segment is already enabled.")
+
+            segment.enabled = True
+            segment.disabled_at = None
+            segment.disabled_by = None
+            db.session.commit()
+
+            # Set cache to prevent indexing the same segment multiple times
+            redis_client.setex(indexing_cache_key, 600, 1)
+
+            add_segment_to_index_task.delay(segment.id)
+
+            return {'result': 'success'}, 200
+        elif action == "disable":
+            if not segment.enabled:
+                raise InvalidActionError("Segment is already disabled.")
+
+            segment.enabled = False
+            segment.disabled_at = datetime.utcnow()
+            segment.disabled_by = current_user.id
+            db.session.commit()
+
+            # Set cache to prevent indexing the same segment multiple times
+            redis_client.setex(indexing_cache_key, 600, 1)
+
+            remove_segment_from_index_task.delay(segment.id)
+
+            return {'result': 'success'}, 200
+        else:
+            raise InvalidActionError()
+
+
+api.add_resource(DatasetDocumentSegmentListApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments')
+api.add_resource(DatasetDocumentSegmentApi,
+                 '/datasets/<uuid:dataset_id>/segments/<uuid:segment_id>/<string:action>')

api/controllers/console/error.py

+from libs.exception import BaseHTTPException
+
+
+class AlreadySetupError(BaseHTTPException):
+    error_code = 'already_setup'
+    description = "Application already setup."
+    code = 403
+
+
+class NotSetupError(BaseHTTPException):
+    error_code = 'not_setup'
+    description = "Application not setup."
+    code = 401
+
+
+class AccountNotLinkTenantError(BaseHTTPException):
+    error_code = 'account_not_link_tenant'
+    description = "Account not link tenant."
+    code = 403

api/controllers/console/version.py

+# -*- coding:utf-8 -*-
+
+import json
+import logging
+
+import requests
+from flask import current_app
+from flask_restful import reqparse, Resource
+from werkzeug.exceptions import InternalServerError
+
+from . import api
+
+
+class VersionApi(Resource):
+
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('current_version', type=str, required=True, location='args')
+        args = parser.parse_args()
+        check_update_url = current_app.config['CHECK_UPDATE_URL']
+
+        try:
+            response = requests.get(check_update_url, {
+                'current_version': args.get('current_version')
+            })
+        except Exception as error:
+            logging.exception("Check update error.")
+            raise InternalServerError()
+
+        content = json.loads(response.content)
+        return {
+            'version': content['version'],
+            'release_date': content['releaseDate'],
+            'release_notes': content['releaseNotes'],
+            'can_auto_update': content['canAutoUpdate']
+        }
+
+
+api.add_resource(VersionApi, '/version')

api/core/callback_handler/entity/dataset_query.py

+from pydantic import BaseModel
+
+
+class DatasetQueryObj(BaseModel):
+    dataset_id: str = None
+    query: str = None