feat: ollama support (#2003)

api/core/app_runner/generate_task_pipeline.py

@@ -459,10 +459,33 @@ class GenerateTaskPipeline:
                     "files": files
                 })
         else:
-            prompts.append({
+            prompt_message = prompt_messages[0]
+            text = ''
+            files = []
+            if isinstance(prompt_message.content, list):
+                for content in prompt_message.content:
+                    if content.type == PromptMessageContentType.TEXT:
+                        content = cast(TextPromptMessageContent, content)
+                        text += content.data
+                    else:
+                        content = cast(ImagePromptMessageContent, content)
+                        files.append({
+                            "type": 'image',
+                            "data": content.data[:10] + '...[TRUNCATED]...' + content.data[-10:],
+                            "detail": content.detail.value
+                        })
+            else:
+                text = prompt_message.content
+
+            params = {
                 "role": 'user',
-                "text": prompt_messages[0].content
-            })
+                "text": text,
+            }
+
+            if files:
+                params['files'] = files
+
+            prompts.append(params)
 
         return prompts
 

api/core/model_runtime/model_providers/_position.yaml

@@ -6,6 +6,7 @@
 - huggingface_hub
 - cohere
 - togetherai
+- ollama
 - zhipuai
 - baichuan
 - spark

api/core/model_runtime/model_providers/localai/localai.yaml

@@ -54,5 +54,5 @@ model_credential_schema:
       type: text-input
       required: true
       placeholder:
-        zh_Hans: 在此输入LocalAI的服务器地址，如 https://example.com/xxx
-        en_US: Enter the url of your LocalAI, for example https://example.com/xxx
+        zh_Hans: 在此输入LocalAI的服务器地址，如 http://192.168.1.100:8080
+        en_US: Enter the url of your LocalAI, e.g. http://192.168.1.100:8080

api/core/model_runtime/model_providers/ollama/_assets/icon_s_en.svg

+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g clip-path="url(#clip0_16325_59237)">
+<rect width="24" height="24" rx="5" fill="white"/>
+<rect x="3.5" width="17" height="24" fill="url(#pattern0)"/>
+</g>
+<defs>
+<pattern id="pattern0" patternContentUnits="objectBoundingBox" width="1" height="1">
+<use xlink:href="#image0_16325_59237" transform="matrix(0.00552486 0 0 0.00391344 0 -0.00092081)"/>
+</pattern>
+<clipPath id="clip0_16325_59237">
+<rect width="24" height="24" fill="white"/>
+</clipPath>
+<image id="image0_16325_59237" width="181" height="256" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALUAAAEACAMAAADC/cfpAAAC8VBMVEUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADF5N8AAAA+nRSTlMAQP4BYIAC/AP4/foEJCYH+59j9ggKCUSkE/Dy2g8G+W71Gn+nCyGX8SI4IEHTV1YNEO/R0u0V5XYvGLbWsw7BrEt+UgUZPvdw1Tl3xoSP8+dF3BbdH+bQib/fJ2U61ze+cgwbdeP0YYF6TFqyR2Lkq7DhxbjYnVSpjdRp6RFZ6DIqPcoSraBvPy3HSOvASc7LbCx7WJ4zFx0wfe5t4KJGFEJNT19KpY5Vaxy9aimMi5qRpl2bNZajUbnqu9vDxHk8hXzM7HivnFxTMTTik04eySsuiiXeqtlnkrzIrjuht2SYXpRQtIi1mSious+HkHSGg5VxNiOxzVtoemhgZAAAEO5JREFUeF7t3WVwHNeWB/C/uqWZEcsRWbLAki2ZQWaGmGNmZmbH9BJzzOyAHVOcOGjHkDhML8zwwkybDT7mpf+nLclS1bk93T13ZvSs3i39PqpKM0ddd85cOkf4f6tWrVq1atWqVatWrVq1zOTFJ+LiFvbph3+Vfn0WxsWdWJxsorqUzRnEy4y932Wg+mV8t9fgZYPmlKE6BOq3p3CqpYnqZbY8RaF9/QCidq4nLVaUoTqVraBFz3OI0vGODDKtFNWndBqDdDyOqNx6kjaWp6C6pCynjZO3IgrJG2jrk2JUj+JPaGtDMiJW0o4OvkpFdUj9ig7alSBSl+joIVSHh+joEiK01KCjhGxELzuBjoyliEjGKLoYlYVoZbm/QQYi0Ymurka0rqarTohAWXu6qncU0Tlaj67alyF8LzGE6xGd6xnCSwhbxmYqipYNTqNiXgdEo8M8KtIGLyuiYnMGwnUPFf7ewCMGFScQjRNUGI8Avf1U3INwraJiKAA8QEXHvohc345UPAAAQ6lYhTClGJTWpAJAajcqnkTknqSi2+U3WEPJSEF44qjogwpLqbgWkbuWiqWo0IeKOISnBaV2Plw2g1JhDiKVU0hpBi7ztaPUAmFJpmIZKmVbfx6pZVRkO/w8GeGoTym3BJXMFpS6I1LdKbUwUakkl1L9KF70dVTBbZTSMxGZzHRKt6EKXo/8sSQ1pjRSfDcUUtqHyOyjVCi+r0ZSapwEfbdSOinzcldKDyIyD1LqKvP4SUq3Qp86Dq6DMIXSjYjMjZSmQLjOMnb03e78osUGhbQeiESPNApGsfNjuR36BlFKgdSOUnNEojmldpBSKA2Ctow0CvtNSMModaqGBccwSOZ+CmkZ0HWcUlco3qT0KSLxKaU3oehK6Th0taI0FIp+lLrBwnzvyVu+nTHtVDppbJi0rftD66fOglU3Sv2gGEqpFXTdQuksVCspJKZCaLJ+xQgGSTt05t5MCKmJFFZCdZbSLdC1ltKHrt+bN6DKxQUt6Kjo68WzUOUG12nBh5TWQtczFIwsqF6kdDcqtP58G0MYfuQaExXupvQiVFkGhWciS3wjYLGY0j8AYOyZXOq4cXFfAPgHpcWwGBFZ6mtDYS8s3qZ0BmiytpC6xlzVFzhD6W1Y7KXQBpoClH4PizJKvTp8UchwfPam2YvSaVj8nlIAes5R+g2scilsOMlwTVxDIRdWv6F0DnqahFi7dWN16hZizdoEeqZSugCr86xO52F1gdJU6ImhFOs6I4ze7bCKpRQTUdR9YPUFq9MXsOpTDVHHuI676MVpvX/0UXdiKMby5566feeKGW3nMaROVyjq+XTT+NqZ37RGJXPsr64/ZNDN/CsUdSwdtf9tdgGsZl/omUBHsTUd9XObMmDvPya8cmWjvgtWLWnH6H4NXJTsWk5bLWF1VzVEHeu8VpWebYoQsjq1oY3m1ZSvt1Ka4jSCpHGLTYQ2uo5BwSmqKZS2Qk9yiBO6mbR6Yjv0jHyHVjNDnOglQ88xSsNg0aMNVfUWQ9vG12jRpgcshlE6Bj1JlOrC4nmqfn0DwmAuTKDqeVjUpZQETfUo9ESVnJhNc+IWNNhAxd4OCE/LeCo2NFgQN2dTTA6q9KRQD7ryKEwCZr0QG3dfi+G0c6A1wrU0nXaGt7gvLvaFWcAkCnnQNZGCf+D5Ijp6vzXCtyiNjuo9ushPYSJ01aWuU0cRifuprS50/Y2aEqcjIuYT1PU36IqlphOIUNlKaoqFrm+o50ASIrU1kXq+ga6SAdQxoBki9wC1DCiBtobUcT2iUNaGOhpCW/4Batifg2jMpI6us6Ap52XqmIOopC6njvdzoGV0W+oY0AHRWUAtbUdDw/YW1HIfonQukVrWbdd40jdqvtZYROumQmq5MeTTHtuN9ozPGt6yKeZis9/FtFz4ep1Lt7VG9N7+uU6dYRcONi0tfWzf52e+zKWDbmPhKmUu7RR9+kgX/MsFHmvQlrbmprgG3ZE22n2ejyvEPN5oHm10dAm73yQGe3mRD1fS6ReHM9ikfnCQP55BOr7hw5XWb6fBIOPzYcv3E62MR8tQE5qPYZCffLDzJ1qd/BE1pEddBvkTbIw0aNGiCWqMeVsaLYyRCNJhOS2eyUdN2jecFss7wGoHLa5NRc2a2oYWO0Lu3S0pQE1rWi/EnmDBNKqezULNG5lI1bQCSBeoGnMaXnAVLS5AKNlARVFneEMjqjbIZeSrVE2BR2SMp+pVsYu/n4olJrxiTzwV4xyHT+5seEcDKgakopL5X1QMhIdk/JqK/7a/S8vPCuAlfaiY6FD51hKeYn5FKaEUFZJGUBrkg7f0ts0iMVQsg7dYqwJutKvH2p8Fr/kfSsY5m2vLw+A5PYZTugoAZhuU3oL31KX0MAAcppRnwnvepTTXDMp7D8CDMosovQfgOUrZ8KJ/sxbNmfUoxGfCiwZbK4aLLcnQk16wVsLspvTv8KTWCRQOAd9TugneNIbCK8DCoIov738cA5hAKRnRMpNbLni456iV8QlMiF85qufDC1omm9W8fjyGOpSy4Cz/jth7svPh5tj6HfsZbP+O9cfgJj/7ntg78rVXNE3UqNPg6Ph18SQZf93NcLD9piEGnRhDbtoOBzdXvfRxOJlCaaoa9Ttw4BvqZ6X4gSZsfPB6Ot2lv/4BbJgD41nJP9SndR8hRo16Ehw8HqKTwA29EhhaQq8bQnRGeDySqFfC3mH31h45jfzU42+U496k5HAEUcfDVmAUFd18kDY9TX1Pb4Lk60bFqIDGuN6qlUNi3W5p5jdkeBrmu90ejdXIIcm4PnS+9uXR4kWRAPIYrjyRhl6kRZ4vdL4eq/PdOJ1WdVBlaWNa+cfv7HR2a5NmY5s12Xq2087xflo1XooqdWg1PfRUtcAyzu+HjZnOUbcspGpEo7t6QNXjrkYjqCps6Rz1TNgYZZmH3Bl6zhdHSa54WqVR8r92MAA7gYOv+SmltXK8SBSnM+ebTellvah/hQrZhUokR5LhLPlIGoXCbFT4lVbUu61Vt2YuhcTWOs2Icmeh3OQ2FL78AO4++JJCm8koNytXp/XQS5QaAHiW0l1OKwmpEcpltpAfsGUmQjGXyY9ui0yRHYQXEOyAdd2IMyEHtm8NFemlKPeDLCn/GDo+lsXsP6BcaToVa3wIsj2R0uygkbU5CcHupmTMR7n+hij+bg09rbuK1+mPcvMNSncj2HwGTZY6JFDqDRt/lkHvQrkk8dh6FUBXgajIbJeEcrtk2H+GjfOUdqLcy5T+EzZ8f01gJf/AoFOR1QXQV7DasmMHDPSzUsJffQiWkkBpsU1i8yfDztTVfpZbF4MKSWNYZVo+wpE/jVXGJKFCzDqW86+eCjs/UErMQbnOVDwKezmLFsa9utUHa56N74zwdI635n34tr4at3BRDmx1KaL0B1QwB1FKTIGGp1wqU3rMrHNpUwGAZnF1Hm/u1ivuKWgQZbxiWGFw+F2cuvhZaWUGLPrlkeRn9/YYVkjSWACrjJWs5O+CkC4mUpqXXxVCIRXvhnPAvd5pt9l4mhWMt2G1Xjy4UHwTqTiCKvdR0TEHruRv5GbCqk2oehhk5upfG72JdGiUc6tBxbU+7R2sIwjSmKqFCHKElcYghM7pVMxQGo2ounaAqxJDZAGrh6kwHoOkZiCjBK5GryEd99f/aVC1biOcyVYaXRDkvc22vQOkLqzinjZPd6PqgAnhOlrcrlcB2R42Loqjb2NYADbaa1Uvmr9QZTSF1KyIKn+yVtRtYafHQ2m8bFRv2GqrFfXNtHgCqltosUsr6gMQgotkO2bB3gGtqHdRdbIfVAXjqXpcK+o82LtTLOjs5GlF/TgF2+zeuZCKv8NFf1YaEHDtlJcegK3AAFbqDxd/p+KQCavJiVTM1OvVsQe2drqniD16fTZmUpE4OWSWvQgXgXiXezuybdgm2BrISvEBuLhoUPEwLH7nD2sKtY2VhsDWJPfasyGstC2slpr+37m2eGLjZrrdcrfAzhLXFLFFtyirWWO6NX/qEs+w+j1ms8oS2Ol8+e16mbCzRPsAvD4V8V3cmjq3TYK7pM3uy2NMbvSHn7rf1jfElaDNId+nrUv76sA7VNyBUD5ilXFdEJ4u41jlI4RyBxXvyFR6LxXnTYQyNpFVVmUiHJmrWCVxLEIxz1Nxr3M9Y3+E9qjYWQgn7MzVcmEdWn/HesdZ8ygNMRFacT3xC0eh6+gQUcRdjNDMIZTmzRJdBCNo5DdQrtmyoSe7o+7N0tDB1aU0rgCC3jLUWKvzuI8eMWR9vA86CsbZ16cntac0AXqKlZcb/nMx3BX/PJzCuGLomUCpfVWybErFzdA0vYhSYq8+mXCS2adXIqWi6ZD01wZNbVcE60zoOhhPVfozcYtKTajM0kVxz6RTFX8Qusx1to0on6I0FPr2FTFYwtw1E5/9ZNL4ieMnffLsxDVzExisaB/0DaX0FCr46lG6BmGYPo6RGDcdYbiGUj2fTcfNNgGEY3ZPhq/nbIQj0Mamq+gblJYgPIHnBzA8A54PIDxLKL1hk1kGI1wpvzAcv6QgXINtMnP3qJss36k/THreGXUL5+7BXXCMjYhE07rpDC29blNEYqMR1CnHTKQwFxEqm79kHt3MWzK/DBGaSyHRDOqxPAORS909eMe0RAZLnLZj8O5URG5GUA/ntyhdQpTM4j0xqj3FJqJ0Keg6/khKV8OLrg66evO9+1aaB+scvw+6inU3vOhuSrFBf0cMvCgmaDycoHQNvOgaSieCjtFL4UWllOKCok6BF6UERT2H0m540W5Kc4IuujSHFzWnND8o8x2GFx0Oynx9vF/EgZso9QnaSJsAL5oQ9NlLprQCXrSC0mQAJZTawovaUioBgM0U4vvCe/rGU9iMcs95vpb0LUrPodxvKa2H96yn9Fubc6aG8J6GlOrbTKie9sFrfE/bTEuz4tUfenyeGp+FCkMoPQiveZDSENud1rkBeEtgLqWh9odjvb3dHaK//blMV3hLV/tzGeyklLAHXrIngdJOVDlIxQoPz5x4EFX6jqCUMB3eMT2B0oi+TvNXbgvAKwLb6Dj/b+Kn4i/wir9Q4W8C4TUq0rfAG7akU/EapKZUjdkOL9g+hqqmUKymqqEHJ3vkaqgeS6DC+BE170eDioTHQnX7z0PNywv5XwBmN6ZqMmraZKoazw79nw/eRE17kwxZnuubQUUsalosFTN8sLGWilaoaa2oWKsxjeVY1LRjVPUO3b3vj6h5f6Ti1xmh7rHyXc/11CLrh/rvAavgBauoWJ4K1SNUTYUXTKXqEfe/age8YYfrCJhsWCszvSElkZIxGVIDerS14DAqGkAaT2lEa3hF6xGUxkM4ZlCaA++YQ8l4z/Emw4AceEfOAMeiqico9YKX9HKsujtF6bCHT0hPiTpTSmkb4SUb0yiNdijheB/e8r5DEcdgT2+7P+hwI/VbSmfhLWcpfSsq6KRSD1/EkVV6r1huZXuLeqv9FVTKonQIXnPIttXax5QaevyQ9GPb06SP4DUf2Z4mLaJUH15Tn9Ii292SN+A1b1CKtY36TnjNnbZR7/L4TdUY21YKcZQ+hNd8SCnONupm8JpmtlE3oLQFXrPFdsF7//+pcX2/bQ65C16zzzaH3EPpO49XFdxjW6m+AF6zgNIdtqO9Lrymrm22GO3xO59tbZe7aptP/0Z4y0Y/hVwTlfa6HSN57FBpr0Pv4K/hLV9TauRwsDc8FV6SOtzhGLQZFZ3hJZ3pNE1aR6kpvKQppUFORwX+0/CS0/7gudP/AtRWD4XPH31GAAAAAElFTkSuQmCC"/>
+</defs>
+</svg>

api/core/model_runtime/model_providers/ollama/ollama.py

+import logging
+
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAIProvider(ModelProvider):
+
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        pass

api/core/model_runtime/model_providers/ollama/ollama.yaml

+provider: ollama
+label:
+  en_US: Ollama
+icon_large:
+  en_US: icon_l_en.svg
+icon_small:
+  en_US: icon_s_en.svg
+background: "#F9FAFB"
+help:
+  title:
+    en_US: How to integrate with Ollama
+    zh_Hans: 如何集成 Ollama
+  url:
+    en_US: https://docs.dify.ai/advanced/model-configuration/ollama
+supported_model_types:
+  - llm
+  - text-embedding
+configurate_methods:
+  - customizable-model
+model_credential_schema:
+  model:
+    label:
+      en_US: Model Name
+      zh_Hans: 模型名称
+    placeholder:
+      en_US: Enter your model name
+      zh_Hans: 输入模型名称
+  credential_form_schemas:
+    - variable: base_url
+      label:
+        zh_Hans: 基础 URL
+        en_US: Base URL
+      type: text-input
+      required: true
+      placeholder:
+        zh_Hans: Ollama server 的基础 URL，例如 http://192.168.1.100:11434
+        en_US: Base url of Ollama server, e.g. http://192.168.1.100:11434
+    - variable: mode
+      show_on:
+        - variable: __model_type
+          value: llm
+      label:
+        zh_Hans: 模型类型
+        en_US: Completion mode
+      type: select
+      required: true
+      default: chat
+      placeholder:
+        zh_Hans: 选择对话类型
+        en_US: Select completion mode
+      options:
+        - value: completion
+          label:
+            en_US: Completion
+            zh_Hans: 补全
+        - value: chat
+          label:
+            en_US: Chat
+            zh_Hans: 对话
+    - variable: context_size
+      label:
+        zh_Hans: 模型上下文长度
+        en_US: Model context size
+      required: true
+      type: text-input
+      default: '4096'
+      placeholder:
+        zh_Hans: 在此输入您的模型上下文长度
+        en_US: Enter your Model context size
+    - variable: max_tokens
+      label:
+        zh_Hans: 最大 token 上限
+        en_US: Upper bound for max tokens
+      show_on:
+        - variable: __model_type
+          value: llm
+      default: '4096'
+      type: text-input
+      required: true
+    - variable: vision_support
+      label:
+        zh_Hans: 是否支持 Vision
+        en_US: Vision support
+      show_on:
+        - variable: __model_type
+          value: llm
+      default: 'false'
+      type: radio
+      required: false
+      options:
+        - value: 'true'
+          label:
+            en_US: Yes
+            zh_Hans: 是
+        - value: 'false'
+          label:
+            en_US: No
+            zh_Hans: 否

api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py

+import logging
+import time
+from decimal import Decimal
+from typing import Optional
+from urllib.parse import urljoin
+import requests
+import json
+
+import numpy as np
+
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.entities.model_entities import PriceType, ModelPropertyKey, ModelType, AIModelEntity, FetchFrom, \
+    PriceConfig
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError, InvokeBadRequestError, \
+    InvokeRateLimitError, InvokeServerUnavailableError, InvokeConnectionError
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
+
+logger = logging.getLogger(__name__)
+
+
+class OllamaEmbeddingModel(TextEmbeddingModel):
+    """
+    Model class for an Ollama text embedding model.
+    """
+
+    def _invoke(self, model: str, credentials: dict,
+                texts: list[str], user: Optional[str] = None) \
+            -> TextEmbeddingResult:
+        """
+        Invoke text embedding model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :param user: unique user id
+        :return: embeddings result
+        """
+
+        # Prepare headers and payload for the request
+        headers = {
+            'Content-Type': 'application/json'
+        }
+
+        endpoint_url = credentials.get('base_url')
+        if not endpoint_url.endswith('/'):
+            endpoint_url += '/'
+
+        endpoint_url = urljoin(endpoint_url, 'api/embeddings')
+
+        # get model properties
+        context_size = self._get_context_size(model, credentials)
+
+        inputs = []
+        used_tokens = 0
+
+        for i, text in enumerate(texts):
+            # Here token count is only an approximation based on the GPT2 tokenizer
+            num_tokens = self._get_num_tokens_by_gpt2(text)
+
+            if num_tokens >= context_size:
+                cutoff = int(len(text) * (np.floor(context_size / num_tokens)))
+                # if num tokens is larger than context length, only use the start
+                inputs.append(text[0: cutoff])
+            else:
+                inputs.append(text)
+
+        batched_embeddings = []
+
+        for text in inputs:
+            # Prepare the payload for the request
+            payload = {
+                'prompt': text,
+                'model': model,
+            }
+
+            # Make the request to the OpenAI API
+            response = requests.post(
+                endpoint_url,
+                headers=headers,
+                data=json.dumps(payload),
+                timeout=(10, 300)
+            )
+
+            response.raise_for_status()  # Raise an exception for HTTP errors
+            response_data = response.json()
+
+            # Extract embeddings and used tokens from the response
+            embeddings = response_data['embedding']
+            embedding_used_tokens = self.get_num_tokens(model, credentials, [text])
+
+            used_tokens += embedding_used_tokens
+            batched_embeddings.append(embeddings)
+
+        # calc usage
+        usage = self._calc_response_usage(
+            model=model,
+            credentials=credentials,
+            tokens=used_tokens
+        )
+
+        return TextEmbeddingResult(
+            embeddings=batched_embeddings,
+            usage=usage,
+            model=model
+        )
+
+    def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int:
+        """
+        Approximate number of tokens for given messages using GPT2 tokenizer
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :return:
+        """
+        return sum(self._get_num_tokens_by_gpt2(text) for text in texts)
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        try:
+            self._invoke(
+                model=model,
+                credentials=credentials,
+                texts=['ping']
+            )
+        except InvokeError as ex:
+            raise CredentialsValidateFailedError(f'An error occurred during credentials validation: {ex.description}')
+        except Exception as ex:
+            raise CredentialsValidateFailedError(f'An error occurred during credentials validation: {str(ex)}')
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity:
+        """
+            generate custom model entities from credentials
+        """
+        entity = AIModelEntity(
+            model=model,
+            label=I18nObject(en_US=model),
+            model_type=ModelType.TEXT_EMBEDDING,
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get('context_size')),
+                ModelPropertyKey.MAX_CHUNKS: 1,
+            },
+            parameter_rules=[],
+            pricing=PriceConfig(
+                input=Decimal(credentials.get('input_price', 0)),
+                unit=Decimal(credentials.get('unit', 0)),
+                currency=credentials.get('currency', "USD")
+            )
+        )
+
+        return entity
+
+    def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage:
+        """
+        Calculate response usage
+
+        :param model: model name
+        :param credentials: model credentials
+        :param tokens: input tokens
+        :return: usage
+        """
+        # get input price info
+        input_price_info = self.get_price(
+            model=model,
+            credentials=credentials,
+            price_type=PriceType.INPUT,
+            tokens=tokens
+        )
+
+        # transform usage
+        usage = EmbeddingUsage(
+            tokens=tokens,
+            total_tokens=tokens,
+            unit_price=input_price_info.unit_price,
+            price_unit=input_price_info.unit,
+            total_price=input_price_info.total_amount,
+            currency=input_price_info.currency,
+            latency=time.perf_counter() - self.started_at
+        )
+
+        return usage
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        The key is the error type thrown to the caller
+        The value is the error type thrown by the model,
+        which needs to be converted into a unified error type for the caller.
+
+        :return: Invoke error mapping
+        """
+        return {
+            InvokeAuthorizationError: [
+                requests.exceptions.InvalidHeader,  # Missing or Invalid API Key
+            ],
+            InvokeBadRequestError: [
+                requests.exceptions.HTTPError,  # Invalid Endpoint URL or model name
+                requests.exceptions.InvalidURL,  # Misconfigured request or other API error
+            ],
+            InvokeRateLimitError: [
+                requests.exceptions.RetryError  # Too many requests sent in a short period of time
+            ],
+            InvokeServerUnavailableError: [
+                requests.exceptions.ConnectionError,  # Engine Overloaded
+                requests.exceptions.HTTPError  # Server Error
+            ],
+            InvokeConnectionError: [
+                requests.exceptions.ConnectTimeout,  # Timeout
+                requests.exceptions.ReadTimeout  # Timeout
+            ]
+        }

api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py

@@ -360,6 +360,7 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel):
                         message=AssistantPromptMessage(content=""),
                         finish_reason="Non-JSON encountered."
                     )
+                    break
 
                 if not chunk_json or len(chunk_json['choices']) == 0:
                     continue

api/core/model_runtime/model_providers/openai_api_compatible/openai_api_compatible.yaml

@@ -33,8 +33,8 @@ model_credential_schema:
       type: text-input
       required: true
       placeholder:
-        zh_Hans: Base URL, eg. https://api.openai.com/v1
-        en_US: Base URL, eg. https://api.openai.com/v1
+        zh_Hans: Base URL, e.g. https://api.openai.com/v1
+        en_US: Base URL, e.g. https://api.openai.com/v1
     - variable: mode
       show_on:
         - variable: __model_type

api/core/model_runtime/model_providers/openllm/openllm.yaml

@@ -33,5 +33,5 @@ model_credential_schema:
       type: text-input
       required: true
       placeholder:
-        zh_Hans: 在此输入OpenLLM的服务器地址，如 https://example.com/xxx
-        en_US: Enter the url of your OpenLLM, for example https://example.com/xxx
+        zh_Hans: 在此输入OpenLLM的服务器地址，如 http://192.168.1.100:3000
+        en_US: Enter the url of your OpenLLM, e.g. http://192.168.1.100:3000

api/core/model_runtime/model_providers/xinference/xinference.yaml

@@ -34,8 +34,8 @@ model_credential_schema:
       type: secret-input
       required: true
       placeholder:
-        zh_Hans: 在此输入Xinference的服务器地址，如 https://example.com/xxx
-        en_US: Enter the url of your Xinference, for example https://example.com/xxx
+        zh_Hans: 在此输入Xinference的服务器地址，如 http://192.168.1.100:9997
+        en_US: Enter the url of your Xinference, e.g. http://192.168.1.100:9997
     - variable: model_uid
       label:
         zh_Hans: 模型UID

api/core/prompt/prompt_transform.py

@@ -121,6 +121,7 @@ class PromptTransform:
                     prompt_template_entity=prompt_template_entity,
                     inputs=inputs,
                     query=query,
+                    files=files,
                     context=context,
                     memory=memory,
                     model_config=model_config
@@ -343,7 +344,14 @@ class PromptTransform:
 
             prompt_message = UserPromptMessage(content=prompt_message_contents)
         else:
-            prompt_message = UserPromptMessage(content=prompt)
+            if files:
+                prompt_message_contents = [TextPromptMessageContent(data=prompt)]
+                for file in files:
+                    prompt_message_contents.append(file.prompt_message_content)
+
+                prompt_message = UserPromptMessage(content=prompt_message_contents)
+            else:
+                prompt_message = UserPromptMessage(content=prompt)
 
         return [prompt_message]
 
@@ -434,6 +442,7 @@ class PromptTransform:
                                                        prompt_template_entity: PromptTemplateEntity,
                                                        inputs: dict,
                                                        query: str,
+                                                       files: List[FileObj],
                                                        context: Optional[str],
                                                        memory: Optional[TokenBufferMemory],
                                                        model_config: ModelConfigEntity) -> List[PromptMessage]:
@@ -461,7 +470,14 @@ class PromptTransform:
 
         prompt = self._format_prompt(prompt_template, prompt_inputs)
 
-        prompt_messages.append(UserPromptMessage(content=prompt))
+        if files:
+            prompt_message_contents = [TextPromptMessageContent(data=prompt)]
+            for file in files:
+                prompt_message_contents.append(file.prompt_message_content)
+
+            prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
+        else:
+            prompt_messages.append(UserPromptMessage(content=prompt))
 
         return prompt_messages
 

api/tests/integration_tests/.env.example

@@ -62,5 +62,8 @@ COHERE_API_KEY=
 # Jina Credentials
 JINA_API_KEY=
 
+# Ollama Credentials
+OLLAMA_BASE_URL=
+
 # Mock Switch
 MOCK_SWITCH=false
\ No newline at end of file

api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py

+import os
+
+import pytest
+
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.ollama.text_embedding.text_embedding import OllamaEmbeddingModel
+
+
+def test_validate_credentials():
+    model = OllamaEmbeddingModel()
+
+    with pytest.raises(CredentialsValidateFailedError):
+        model.validate_credentials(
+            model='mistral:text',
+            credentials={
+                'base_url': 'http://localhost:21434',
+                'mode': 'chat',
+                'context_size': 4096,
+            }
+        )
+
+    model.validate_credentials(
+        model='mistral:text',
+        credentials={
+            'base_url': os.environ.get('OLLAMA_BASE_URL'),
+            'mode': 'chat',
+            'context_size': 4096,
+        }
+    )
+
+
+def test_invoke_model():
+    model = OllamaEmbeddingModel()
+
+    result = model.invoke(
+        model='mistral:text',
+        credentials={
+            'base_url': os.environ.get('OLLAMA_BASE_URL'),
+            'mode': 'chat',
+            'context_size': 4096,
+        },
+        texts=[
+            "hello",
+            "world"
+        ],
+        user="abc-123"
+    )
+
+    assert isinstance(result, TextEmbeddingResult)
+    assert len(result.embeddings) == 2
+    assert result.usage.total_tokens == 2
+
+
+def test_get_num_tokens():
+    model = OllamaEmbeddingModel()
+
+    num_tokens = model.get_num_tokens(
+        model='mistral:text',
+        credentials={
+            'base_url': os.environ.get('OLLAMA_BASE_URL'),
+            'mode': 'chat',
+            'context_size': 4096,
+        },
+        texts=[
+            "hello",
+            "world"
+        ]
+    )
+
+    assert num_tokens == 2