Merge branch 'main' into feat/enterprise

.github/workflows/style.yml

@@ -41,6 +41,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Setup NodeJS
         uses: actions/setup-node@v4
@@ -60,11 +62,10 @@ jobs:
           yarn run lint
 
       - name: Super-linter
-        uses: super-linter/super-linter/slim@v5
+        uses: super-linter/super-linter/slim@v6
         env:
           BASH_SEVERITY: warning
           DEFAULT_BRANCH: main
-          ERROR_ON_MISSING_EXEC_BIT: true
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           IGNORE_GENERATED_FILES: true
           IGNORE_GITIGNORED_FILES: true

api/commands.py

@@ -150,7 +150,7 @@ def vdb_migrate():
                         continue
                 if vector_type == "weaviate":
                     dataset_id = dataset.id
-                    collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node'
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
                         "type": 'weaviate',
                         "vector_store": {"class_prefix": collection_name}
@@ -167,7 +167,7 @@ def vdb_migrate():
                             raise ValueError('Dataset Collection Bindings is not exist!')
                     else:
                         dataset_id = dataset.id
-                        collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node'
+                        collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
                         "type": 'qdrant',
                         "vector_store": {"class_prefix": collection_name}
@@ -176,7 +176,7 @@ def vdb_migrate():
 
                 elif vector_type == "milvus":
                     dataset_id = dataset.id
-                    collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node'
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
                         "type": 'milvus',
                         "vector_store": {"class_prefix": collection_name}

api/constants/model_template.py

@@ -13,30 +13,14 @@ model_templates = {
             'status': 'normal'
         },
         'model_config': {
-            'provider': 'openai',
-            'model_id': 'gpt-3.5-turbo-instruct',
-            'configs': {
-                'prompt_template': '',
-                'prompt_variables': [],
-                'completion_params': {
-                    'max_token': 512,
-                    'temperature': 1,
-                    'top_p': 1,
-                    'presence_penalty': 0,
-                    'frequency_penalty': 0,
-                }
-            },
+            'provider': '',
+            'model_id': '',
+            'configs': {},
             'model': json.dumps({
                 "provider": "openai",
                 "name": "gpt-3.5-turbo-instruct",
                 "mode": "completion",
-                "completion_params": {
-                    "max_tokens": 512,
-                    "temperature": 1,
-                    "top_p": 1,
-                    "presence_penalty": 0,
-                    "frequency_penalty": 0
-                }
+                "completion_params": {}
             }),
             'user_input_form': json.dumps([
                 {
@@ -64,30 +48,14 @@ model_templates = {
             'status': 'normal'
         },
         'model_config': {
-            'provider': 'openai',
-            'model_id': 'gpt-3.5-turbo',
-            'configs': {
-                'prompt_template': '',
-                'prompt_variables': [],
-                'completion_params': {
-                    'max_token': 512,
-                    'temperature': 1,
-                    'top_p': 1,
-                    'presence_penalty': 0,
-                    'frequency_penalty': 0,
-                }
-            },
+            'provider': '',
+            'model_id': '',
+            'configs': {},
             'model': json.dumps({
                 "provider": "openai",
                 "name": "gpt-3.5-turbo",
                 "mode": "chat",
-                "completion_params": {
-                    "max_tokens": 512,
-                    "temperature": 1,
-                    "top_p": 1,
-                    "presence_penalty": 0,
-                    "frequency_penalty": 0
-                }
+                "completion_params": {}
             })
         }
     },

api/core/rag/datasource/vdb/vector_factory.py

@@ -39,7 +39,7 @@ class Vector:
                 collection_name = class_prefix
             else:
                 dataset_id = self._dataset.id
-                collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node'
+                collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                 index_struct_dict = {
                     "type": 'weaviate',
                     "vector_store": {"class_prefix": collection_name}
@@ -70,7 +70,7 @@ class Vector:
                     collection_name = class_prefix
                 else:
                     dataset_id = self._dataset.id
-                    collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node'
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
 
             if not self._dataset.index_struct_dict:
                 index_struct_dict = {
@@ -96,7 +96,7 @@ class Vector:
                 collection_name = class_prefix
             else:
                 dataset_id = self._dataset.id
-                collection_name = "Vector_index_" + dataset_id.replace("-", "_") + '_Node'
+                collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                 index_struct_dict = {
                     "type": 'milvus',
                     "vector_store": {"class_prefix": collection_name}

api/core/rag/datasource/vdb/weaviate/weaviate_vector.py

@@ -70,7 +70,7 @@ class WeaviateVector(BaseVector):
             return class_prefix
 
         dataset_id = dataset.id
-        return "Vector_index_" + dataset_id.replace("-", "_") + '_Node'
+        return Dataset.gen_collection_name_by_id(dataset_id)
 
     def to_index_struct(self) -> dict:
         return {

api/core/tools/tool/api_tool.py

@@ -67,9 +67,9 @@ class ApiTool(Tool):
             
             if 'api_key_header_prefix' in credentials:
                 api_key_header_prefix = credentials['api_key_header_prefix']
-                if api_key_header_prefix == 'basic':
+                if api_key_header_prefix == 'basic' and credentials['api_key_value']:
                     credentials['api_key_value'] = f'Basic {credentials["api_key_value"]}'
-                elif api_key_header_prefix == 'bearer':
+                elif api_key_header_prefix == 'bearer' and credentials['api_key_value']:
                     credentials['api_key_value'] = f'Bearer {credentials["api_key_value"]}'
                 elif api_key_header_prefix == 'custom':
                     pass
@@ -184,21 +184,7 @@ class ApiTool(Tool):
                     for name, property in properties.items():
                         if name in parameters:
                             # convert type
-                            try:
-                                value = parameters[name]
-                                if property['type'] == 'integer':
-                                    value = int(value)
-                                elif property['type'] == 'number':
-                                    # check if it is a float
-                                    if '.' in value:
-                                        value = float(value)
-                                    else:
-                                        value = int(value)
-                                elif property['type'] == 'boolean':
-                                    value = bool(value)
-                                body[name] = value
-                            except ValueError as e:
-                                body[name] = parameters[name]
+                            body[name] = self._convert_body_property_type(property, parameters[name])
                         elif name in required:
                             raise ToolProviderCredentialValidationError(
                                 f"Missing required parameter {name} in operation {self.api_bundle.operation_id}"
@@ -228,10 +214,6 @@ class ApiTool(Tool):
         elif method == 'put':
             response = ssrf_proxy.put(url, params=params, headers=headers, cookies=cookies, data=body, timeout=10, follow_redirects=True)
         elif method == 'delete':
-            """
-            request body data is unsupported for DELETE method in standard http protocol
-            however, OpenAPI 3.0 supports request body data for DELETE method, so we support it here by using requests
-            """
             response = ssrf_proxy.delete(url, params=params, headers=headers, cookies=cookies, data=body, timeout=10, allow_redirects=True)
         elif method == 'patch':
             response = ssrf_proxy.patch(url, params=params, headers=headers, cookies=cookies, data=body, timeout=10, follow_redirects=True)
@@ -243,6 +225,66 @@ class ApiTool(Tool):
             raise ValueError(f'Invalid http method {method}')
         
         return response
+    
+    def _convert_body_property_any_of(self, property: dict[str, Any], value: Any, any_of: list[dict[str, Any]], max_recursive=10) -> Any:
+        if max_recursive <= 0:
+            raise Exception("Max recursion depth reached")
+        for option in any_of or []:
+            try:
+                if 'type' in option:
+                    # Attempt to convert the value based on the type.
+                    if option['type'] == 'integer' or option['type'] == 'int':
+                        return int(value)
+                    elif option['type'] == 'number':
+                        if '.' in str(value):
+                            return float(value)
+                        else:
+                            return int(value)
+                    elif option['type'] == 'string':
+                        return str(value)
+                    elif option['type'] == 'boolean':
+                        if str(value).lower() in ['true', '1']:
+                            return True
+                        elif str(value).lower() in ['false', '0']:
+                            return False
+                        else:
+                            continue  # Not a boolean, try next option
+                    elif option['type'] == 'null' and not value:
+                        return None
+                    else:
+                        continue  # Unsupported type, try next option
+                elif 'anyOf' in option and isinstance(option['anyOf'], list):
+                    # Recursive call to handle nested anyOf
+                    return self._convert_body_property_any_of(property, value, option['anyOf'], max_recursive - 1)
+            except ValueError:
+                continue  # Conversion failed, try next option
+        # If no option succeeded, you might want to return the value as is or raise an error
+        return value  # or raise ValueError(f"Cannot convert value '{value}' to any specified type in anyOf")
+
+    def _convert_body_property_type(self, property: dict[str, Any], value: Any) -> Any:
+        try:
+            if 'type' in property:
+                if property['type'] == 'integer' or property['type'] == 'int':
+                    return int(value)
+                elif property['type'] == 'number':
+                    # check if it is a float
+                    if '.' in value:
+                        return float(value)
+                    else:
+                        return int(value)
+                elif property['type'] == 'string':
+                    return str(value)
+                elif property['type'] == 'boolean':
+                    return bool(value)
+                elif property['type'] == 'null':
+                    if value is None:
+                        return None
+                else:
+                    raise ValueError(f"Invalid type {property['type']} for property {property}")
+            elif 'anyOf' in property and isinstance(property['anyOf'], list):
+                return self._convert_body_property_any_of(property, value, property['anyOf'])
+        except ValueError as e:
+            return value
 
     def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage | list[ToolInvokeMessage]:
         """

api/models/dataset.py

@@ -116,6 +116,10 @@ class Dataset(db.Model):
         }
         return self.retrieval_model if self.retrieval_model else default_retrieval_model
 
+    @staticmethod
+    def gen_collection_name_by_id(dataset_id: str) -> str:
+        normalized_dataset_id = dataset_id.replace("-", "_")
+        return f'Vector_index_{normalized_dataset_id}_Node'
 
 class DatasetProcessRule(db.Model):
     __tablename__ = 'dataset_process_rules'

api/services/dataset_service.py

@@ -1244,7 +1244,7 @@ class DatasetCollectionBindingService:
             dataset_collection_binding = DatasetCollectionBinding(
                 provider_name=provider_name,
                 model_name=model_name,
-                collection_name="Vector_index_" + str(uuid.uuid4()).replace("-", "_") + '_Node',
+                collection_name=Dataset.gen_collection_name_by_id(str(uuid.uuid4())),
                 type=collection_type
             )
             db.session.add(dataset_collection_binding)

web/app/components/header/account-setting/model-provider-page/model-parameter-modal/parameter-item.tsx

@@ -115,6 +115,11 @@ const ParameterItem: FC<ParameterItemProps> = ({
     }
   }
 
+  useEffect(() => {
+    if ((parameterRule.type === 'int' || parameterRule.type === 'float') && numberInputRef.current)
+      numberInputRef.current.value = `${renderValue}`
+  }, [value])
+
   const renderInput = () => {
     const numberInputWithSlide = (parameterRule.type === 'int' || parameterRule.type === 'float')
     && !isNullOrUndefined(parameterRule.min)
@@ -207,11 +212,6 @@ const ParameterItem: FC<ParameterItemProps> = ({
     return null
   }
 
-  useEffect(() => {
-    if (numberInputRef.current)
-      numberInputRef.current.value = `${renderValue}`
-  }, [])
-
   return (
     <div className={`flex items-center justify-between ${className}`}>
       <div>

web/app/components/tools/edit-custom-collection-modal/index.tsx

@@ -118,6 +118,13 @@ const EditCustomCollectionModal: FC<Props> = ({
   const handleSave = () => {
     const postData = clone(customCollection)
     delete postData.tools
+
+    if (postData.credentials.auth_type === AuthType.none) {
+      delete postData.credentials.api_key_header
+      delete postData.credentials.api_key_header_prefix
+      delete postData.credentials.api_key_value
+    }
+
     if (isAdd) {
       onAdd?.(postData)
       return

web/app/components/tools/edit-custom-collection-modal/test-api.tsx

@@ -5,7 +5,7 @@ import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
 import { Settings01 } from '../../base/icons/src/vender/line/general'
 import ConfigCredentials from './config-credentials'
-import type { Credential, CustomCollectionBackend, CustomParamSchema } from '@/app/components/tools/types'
+import { AuthType, type Credential, type CustomCollectionBackend, type CustomParamSchema } from '@/app/components/tools/types'
 import Button from '@/app/components/base/button'
 import Drawer from '@/app/components/base/drawer-plus'
 import I18n from '@/context/i18n'
@@ -34,9 +34,16 @@ const TestApi: FC<Props> = ({
   const { operation_id: toolName, parameters } = tool
   const [parametersValue, setParametersValue] = useState<Record<string, string>>({})
   const handleTest = async () => {
+    // clone test schema
+    const credentials = JSON.parse(JSON.stringify(tempCredential)) as Credential
+    if (credentials.auth_type === AuthType.none) {
+      delete credentials.api_key_header_prefix
+      delete credentials.api_key_header
+      delete credentials.api_key_value
+    }
     const data = {
       tool_name: toolName,
-      credentials: tempCredential,
+      credentials,
       schema_type: customCollection.schema_type,
       schema: customCollection.schema,
       parameters: parametersValue,