Merge branch 'feat/milvus-support' into deploy/dev

api/core/indexing_runner.py

@@ -269,7 +269,7 @@ class IndexingRunner:
                 return {
                     "total_segments": total_segments,
                     "tokens": total_segments * 2000,
-                    "total_price": '{:f}'.format(TokenCalculator.get_token_price('gpt-3.5-turbo', tokens, 'completion')),
+                    "total_price": '{:f}'.format(TokenCalculator.get_token_price('gpt-3.5-turbo', total_segments * 2000, 'completion')),
                     "currency": TokenCalculator.get_currency(self.embedding_model_name),
                     "qa_preview": document_qa_list,
                     "preview": preview_texts
@@ -282,7 +282,7 @@ class IndexingRunner:
             "preview": preview_texts
         }
 
-    def notion_indexing_estimate(self, notion_info_list: list, tmp_processing_rule: dict, doc_from: str = None) -> dict:
+    def notion_indexing_estimate(self, notion_info_list: list, tmp_processing_rule: dict, doc_form: str = None) -> dict:
         """
         Estimate the indexing for the document.
         """
@@ -332,7 +332,19 @@ class IndexingRunner:
                         preview_texts.append(document.page_content)
 
                     tokens += TokenCalculator.get_num_tokens(self.embedding_model_name, document.page_content)
-
+        if doc_form and doc_form == 'qa_model':
+            if len(preview_texts) > 0:
+                # qa model document
+                response = LLMGenerator.generate_qa_document(current_user.current_tenant_id, preview_texts[0])
+                document_qa_list = self.format_split_text(response)
+                return {
+                    "total_segments": total_segments,
+                    "tokens": total_segments * 2000,
+                    "total_price": '{:f}'.format(TokenCalculator.get_token_price('gpt-3.5-turbo', total_segments * 2000, 'completion')),
+                    "currency": TokenCalculator.get_currency(self.embedding_model_name),
+                    "qa_preview": document_qa_list,
+                    "preview": preview_texts
+                }
         return {
             "total_segments": total_segments,
             "tokens": tokens,

web/app/components/datasets/documents/detail/completed/index.tsx

@@ -101,7 +101,7 @@ export const SegmentDetail: FC<ISegmentDetailProps> = memo(({
       <AutoHeightTextarea
         className='leading-6 text-md text-gray-800'
         value={question}
-        placeholder={t('datasetDocuments.segment.questionPlaceholder') || ''}
+        placeholder={t('datasetDocuments.segment.contentPlaceholder') || ''}
         onChange={e => setQuestion(e.target.value)}
         disabled={!isEditing}
         autoFocus
@@ -262,9 +262,22 @@ const Completed: FC<ICompletedProps> = () => {
   }
 
   const handleUpdateSegment = async (segmentId: string, question: string, answer: string) => {
-    const params: SegmentUpdator = { content: question }
-    if (docForm === 'qa_model')
+    const params: SegmentUpdator = { content: '' }
+    if (docForm === 'qa_model') {
+      if (!question.trim())
+        return notify({ type: 'error', message: t('datasetDocuments.segment.questionEmpty') })
+      if (!answer.trim())
+        return notify({ type: 'error', message: t('datasetDocuments.segment.answerEmpty') })
+
+      params.content = question
       params.answer = answer
+    }
+    else {
+      if (!question.trim())
+        return notify({ type: 'error', message: t('datasetDocuments.segment.contentEmpty') })
+
+      params.content = question
+    }
 
     const res = await updateSegment({ datasetId, documentId, segmentId, body: params })
     notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })

web/i18n/lang/dataset-documents.en.ts

@@ -311,7 +311,11 @@ const translation = {
     hitCount: 'hit count',
     vectorHash: 'Vector hash: ',
     questionPlaceholder: 'add question here',
+    questionEmpty: 'Question can not be empty',
     answerPlaceholder: 'add answer here',
+    answerEmpty: 'Answer can not be empty',
+    contentPlaceholder: 'add content here',
+    contentEmpty: 'Content can not be empty',
   },
 }
 

web/i18n/lang/dataset-documents.zh.ts

@@ -310,7 +310,11 @@ const translation = {
     hitCount: '命中次数',
     vectorHash: '向量哈希：',
     questionPlaceholder: '在这里添加问题',
+    questionEmpty: '问题不能为空',
     answerPlaceholder: '在这里添加答案',
+    answerEmpty: '答案不能为空',
+    contentPlaceholder: '在这里添加内容',
+    contentEmpty: '内容不能为空',
   },
 }