feat: language selection supported in segmentation

web/app/components/base/popover/index.tsx

@@ -9,6 +9,7 @@ type IPopover = {
   position?: 'bottom' | 'br'
   btnElement?: string | React.ReactNode
   btnClassName?: string | ((open: boolean) => string)
+  manualClose?: boolean
 }
 
 const timeoutDuration = 100
@@ -20,6 +21,7 @@ export default function CustomPopover({
   btnElement,
   className,
   btnClassName,
+  manualClose,
 }: IPopover) {
   const buttonRef = useRef<HTMLButtonElement>(null)
   const timeOutRef = useRef<NodeJS.Timeout | null>(null)
@@ -65,14 +67,16 @@ export default function CustomPopover({
                 leaveTo="opacity-0 translate-y-1"
               >
                 <Popover.Panel
-                  className={`${s.popupPanel} ${position === 'br' ? 'right-0' : 'transform -translate-x-1/2 left-1/2'} ${className}`}
+                  className={`${s.popupPanel} ${position === 'br' ? 'right-0' : 'translate-x-1/2 left-1/2'} ${className}`}
                   {...(trigger !== 'hover'
                     ? {}
                     : {
                       onMouseLeave: () => onMouseLeave(open),
                       onMouseEnter: () => onMouseEnter(open),
                     })
-                  }>
+                  }
+                >
+                  {({ close }) => (
                     <div
                       className={s.panelContainer}
                       {...(trigger !== 'hover'
@@ -85,8 +89,14 @@ export default function CustomPopover({
                     >
                       {cloneElement(htmlContent as React.ReactElement, {
                         onClose: () => onMouseLeave(open),
+                        ...(manualClose
+                          ? {
+                            onClick: close,
+                          }
+                          : {}),
                       })}
                     </div>
+                  )}
                 </Popover.Panel>
               </Transition>
             </div>

web/app/components/datasets/create/step-two/index.tsx

@@ -2,12 +2,14 @@
 'use client'
 import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
 import { useBoolean } from 'ahooks'
 import { XMarkIcon } from '@heroicons/react/20/solid'
 import cn from 'classnames'
 import Link from 'next/link'
 import { groupBy } from 'lodash-es'
 import PreviewItem, { PreviewType } from './preview-item'
+import LanguageSelect from './language-select'
 import s from './index.module.css'
 import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
 import {
@@ -26,7 +28,10 @@ import { DataSourceType } from '@/models/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import Switch from '@/app/components/base/switch'
 import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
+import { XClose } from '@/app/components/base/icons/src/vender/line/general'
 import { useDatasetDetailContext } from '@/context/dataset-detail'
+import I18n from '@/context/i18n'
+import { IS_CE_EDITION } from '@/config'
 
 type Page = DataSourceNotionPage & { workspace_id: string }
 
@@ -77,6 +82,8 @@ const StepTwo = ({
   onCancel,
 }: StepTwoProps) => {
   const { t } = useTranslation()
+  const { locale } = useContext(I18n)
+
   const { mutateDatasetRes } = useDatasetDetailContext()
   const scrollRef = useRef<HTMLDivElement>(null)
   const [scrolled, setScrolled] = useState(false)
@@ -97,6 +104,8 @@ const StepTwo = ({
   const [docForm, setDocForm] = useState<DocForm | string>(
     datasetId && documentDetail ? documentDetail.doc_form : DocForm.TEXT,
   )
+  const [docLanguage, setDocLanguage] = useState<string>(locale === 'en' ? 'English' : 'Chinese')
+  const [QATipHide, setQATipHide] = useState(false)
   const [previewSwitched, setPreviewSwitched] = useState(false)
   const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
   const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
@@ -229,6 +238,7 @@ const StepTwo = ({
         indexing_technique: getIndexing_technique(),
         process_rule: getProcessRule(),
         doc_form: docForm,
+        doc_language: docLanguage,
       }
     }
     if (dataSourceType === DataSourceType.NOTION) {
@@ -240,6 +250,7 @@ const StepTwo = ({
         indexing_technique: getIndexing_technique(),
         process_rule: getProcessRule(),
         doc_form: docForm,
+        doc_language: docLanguage,
       }
     }
     return params
@@ -251,6 +262,7 @@ const StepTwo = ({
       params = {
         original_document_id: documentDetail?.id,
         doc_form: docForm,
+        doc_language: docLanguage,
         process_rule: getProcessRule(),
       } as CreateDocumentReq
     }
@@ -265,6 +277,7 @@ const StepTwo = ({
         indexing_technique: getIndexing_technique(),
         process_rule: getProcessRule(),
         doc_form: docForm,
+        doc_language: docLanguage,
       } as CreateDocumentReq
       if (dataSourceType === DataSourceType.FILE) {
         params.data_source.info_list.file_info_list = {
@@ -347,6 +360,10 @@ const StepTwo = ({
       setDocForm(DocForm.TEXT)
   }
 
+  const handleSelect = (language: string) => {
+    setDocLanguage(language)
+  }
+
   const changeToEconomicalType = () => {
     if (!hasSetIndexType) {
       setIndexType(IndexingType.ECONOMICAL)
@@ -572,14 +589,18 @@ const StepTwo = ({
                 <Link className='text-[#155EEF]' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
               </div>
             )}
-            {indexType === IndexingType.QUALIFIED && (
-              <div className='flex justify-between items-center mt-3 px-5 py-4 rounded-xl bg-gray-50 border border-gray-100'>
+            {!IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
+              <div className='mt-3 rounded-xl bg-gray-50 border border-gray-100'>
+                <div className='flex justify-between items-center px-5 py-4'>
                   <div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
                     <MessageChatSquare className='w-4 h-4' />
                   </div>
                   <div className='grow mx-3'>
                     <div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
-                  <div className='text-[13px] leading-[18px] text-gray-500'>{t('datasetCreation.stepTwo.QATip')}</div>
+                    <div className='inline-flex items-center text-[13px] leading-[18px] text-gray-500'>
+                      <span className='pr-1'>{t('datasetCreation.stepTwo.QALanguage')}</span>
+                      <LanguageSelect currentLanguage={docLanguage} onSelect={handleSelect} />
+                    </div>
                   </div>
                   <div className='shrink-0'>
                     <Switch
@@ -589,6 +610,13 @@ const StepTwo = ({
                     />
                   </div>
                 </div>
+                {docForm === DocForm.QA && !QATipHide && (
+                  <div className='flex justify-between items-center px-5 py-2 bg-orange-50 border-t border-amber-100 rounded-b-xl text-[13px] leading-[18px] text-medium text-amber-500'>
+                    {t('datasetCreation.stepTwo.QATip')}
+                    <XClose className='w-4 h-4 text-gray-500 cursor-pointer' onClick={() => setQATipHide(true)} />
+                  </div>
+                )}
+              </div>
             )}
             <div className={s.source}>
               <div className={s.sourceContent}>

web/app/components/datasets/create/step-two/language-select/index.tsx

+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from 'classnames'
+import { ChevronDown } from '@/app/components/base/icons/src/vender/line/arrows'
+import Popover from '@/app/components/base/popover'
+
+export type ILanguageSelectProps = {
+  currentLanguage: string
+  onSelect: (language: string) => void
+}
+
+const LanguageSelect: FC<ILanguageSelectProps> = ({
+  currentLanguage,
+  onSelect,
+}) => {
+  return (
+    <Popover
+      manualClose
+      trigger='click'
+      htmlContent={
+        <div className='w-full py-1'>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('English')}>English</div>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('Chinese')}>简体中文</div>
+        </div>
+      }
+      btnElement={
+        <div className='inline-flex items-center'>
+          <span className='pr-[2px] text-xs leading-[18px] font-medium'>{currentLanguage === 'English' ? 'English' : '简体中文'}</span>
+          <ChevronDown className='w-3 h-3 opacity-60' />
+        </div>
+      }
+      btnClassName={open => cn('!border-0 !px-0 !py-0 !bg-inherit !hover:bg-inherit', open ? 'text-blue-600' : 'text-gray-500')}
+      className='!w-[120px] h-fit !z-20 !translate-x-0 !left-[-16px]'
+    />
+  )
+}
+export default React.memo(LanguageSelect)

web/i18n/lang/dataset-creation.en.ts

@@ -75,6 +75,7 @@ const translation = {
     economicalTip: 'Use offline vector engines, keyword indexes, etc. to reduce accuracy without spending tokens',
     QATitle: 'Segmenting in Question & Answer format',
     QATip: 'Enable this option will consume more tokens',
+    QALanguage: 'Segment using',
     emstimateCost: 'Estimation',
     emstimateSegment: 'Estimated segments',
     segmentCount: 'segments',

web/i18n/lang/dataset-creation.zh.ts

@@ -75,6 +75,7 @@ const translation = {
     economicalTip: '使用离线的向量引擎、关键词索引等方式，降低了准确度但无需花费 Token',
     QATitle: '采用 Q&A 分段模式',
     QATip: '开启后将会消耗额外的 token',
+    QALanguage: '分段使用',
     emstimateCost: '执行嵌入预估消耗',
     emstimateSegment: '预估分段数',
     segmentCount: '段',

web/models/datasets.ts

@@ -179,6 +179,7 @@ export type CreateDocumentReq = {
   original_document_id?: string
   indexing_technique?: string
   doc_form: 'text_model' | 'qa_model'
+  doc_language: string
   data_source: DataSource
   process_rule: ProcessRule
 }