fix: dataset switch indexing_technique invalid

api/core/index/index.py

@@ -10,9 +10,9 @@ from models.dataset import Dataset
 
 class IndexBuilder:
     @classmethod
-    def get_index(cls, dataset: Dataset, indexing_technique: str):
+    def get_index(cls, dataset: Dataset, indexing_technique: str, ignore_high_quality_check: bool = False):
         if indexing_technique == "high_quality":
-            if dataset.indexing_technique != 'high_quality':
+            if not ignore_high_quality_check and dataset.indexing_technique != 'high_quality':
                 return None
 
             model_credentials = LLMBuilder.get_model_credentials(

api/tasks/deal_dataset_vector_index_task.py

@@ -26,42 +26,47 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):
         dataset = Dataset.query.filter_by(
             id=dataset_id
         ).first()
+
         if not dataset:
             raise Exception('Dataset not found')
-        dataset_documents = DatasetDocument.query.filter_by(dataset_id=dataset_id).all()
-        if dataset_documents:
-            # save vector index
-            index = IndexBuilder.get_index(dataset, 'high_quality')
-            if index:
+
+        if action == "remove":
+            index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=True)
+            index.delete()
+        elif action == "add":
+            dataset_documents = db.session.query(DatasetDocument).filter(
+                DatasetDocument.dataset_id == dataset_id,
+                DatasetDocument.indexing_status == 'completed',
+                DatasetDocument.enabled == True,
+                DatasetDocument.archived == False,
+            ).all()
+
+            if dataset_documents:
+                # save vector index
+                index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=True)
                 for dataset_document in dataset_documents:
                     # delete from vector index
-                    if action == "remove":
-                        index.delete_by_document_id(dataset_document.id)
-                    elif action == "add":
-                        segments = db.session.query(DocumentSegment).filter(
-                            DocumentSegment.document_id == dataset_document.id,
-                            DocumentSegment.enabled == True
-                        ) .order_by(DocumentSegment.position.asc()).all()
+                    segments = db.session.query(DocumentSegment).filter(
+                        DocumentSegment.document_id == dataset_document.id,
+                        DocumentSegment.enabled == True
+                    ) .order_by(DocumentSegment.position.asc()).all()
 
-                        documents = []
-                        for segment in segments:
-                            document = Document(
-                                page_content=segment.content,
-                                metadata={
-                                    "doc_id": segment.index_node_id,
-                                    "doc_hash": segment.index_node_hash,
-                                    "document_id": segment.document_id,
-                                    "dataset_id": segment.dataset_id,
-                                }
-                            )
+                    documents = []
+                    for segment in segments:
+                        document = Document(
+                            page_content=segment.content,
+                            metadata={
+                                "doc_id": segment.index_node_id,
+                                "doc_hash": segment.index_node_hash,
+                                "document_id": segment.document_id,
+                                "dataset_id": segment.dataset_id,
+                            }
+                        )
 
-                            documents.append(document)
+                        documents.append(document)
 
-                        # save vector index
-                        index.add_texts(
-                            documents,
-                            duplicate_check=True
-                        )
+                    # save vector index
+                    index.add_texts(documents)
 
         end_at = time.perf_counter()
         logging.info(