feat: add status batch

web/app/components/datasets/create/embedding-process/index.module.css

@@ -27,7 +27,7 @@
 }
 
 .cost {
-  @apply w-full flex justify-between items-center text-xs text-gray-700;
+  @apply flex justify-between items-center text-xs text-gray-700;
 }
 .embeddingStatus {
   @apply flex items-center justify-between text-gray-900 font-medium text-sm mr-2;
@@ -36,12 +36,12 @@
   @apply w-3 h-3 mr-1 inline-block align-middle;
 }
 .highIcon {
-  mask-image: url(/../assets/star.svg);
+  mask-image: url(../assets/star.svg);
   @apply bg-orange-500;
 }
 .economyIcon {
   background-color: #444ce7;
-  mask-image: url(/../assets/normal.svg);
+  mask-image: url(../assets/normal.svg);
 }
 .tokens {
   @apply text-xs font-medium px-1;

web/app/components/datasets/create/embedding-process/index.tsx

@@ -10,9 +10,9 @@ import cn from 'classnames'
 import s from './index.module.css'
 import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
 import Button from '@/app/components/base/button'
-import type { FullDocumentDetail, ProcessRuleResponse } from '@/models/datasets'
+import type { FullDocumentDetail, IndexingStatusResponse, ProcessRuleResponse } from '@/models/datasets'
 import { formatNumber } from '@/utils/format'
-import { fetchIndexingStatus as doFetchIndexingStatus, fetchIndexingEstimate, fetchProcessRule } from '@/service/datasets'
+import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchIndexingEstimateBatch, fetchProcessRule } from '@/service/datasets'
 
 type Props = {
   datasetId: string
@@ -79,10 +79,10 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
 
   const getFirstDocument = documents[0]
 
-  const [indexingStatusDetail, setIndexingStatusDetail, getIndexingStatusDetail] = useGetState<any>(null)
+  const [indexingStatusBatchDetail, setIndexingStatusDetail, getIndexingStatusDetail] = useGetState<IndexingStatusResponse[]>([])
   const fetchIndexingStatus = async () => {
-    const status = await doFetchIndexingStatus({ datasetId, documentId: getFirstDocument.id })
-    setIndexingStatusDetail(status)
+    const status = await doFetchIndexingStatus({ datasetId, batchId })
+    setIndexingStatusDetail(status.data)
   }
 
   const [runId, setRunId, getRunId] = useGetState<any>(null)
@@ -93,8 +93,9 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
 
   const startQueryStatus = () => {
     const runId = setInterval(() => {
-      const indexingStatusDetail = getIndexingStatusDetail()
-      if (indexingStatusDetail?.indexing_status === 'completed') {
+      const indexingStatusBatchDetail = getIndexingStatusDetail()
+      const isCompleted = indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error'].includes(indexingStatusDetail.indexing_status))
+      if (isCompleted) {
         stopQueryStatus()
         return
       }
@@ -112,41 +113,42 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
   }, [])
 
   // get rule
-  // get cost
-  // get index status
-
-  const { data: indexingEstimateDetail, error: indexingEstimateErr } = useSWR({
-    action: 'fetchIndexingEstimate',
-    datasetId,
-    documentId: getFirstDocument.id,
-  }, apiParams => fetchIndexingEstimate(omit(apiParams, 'action')), {
-    revalidateOnFocus: false,
-  })
-
   const { data: ruleDetail, error: ruleError } = useSWR({
     action: 'fetchProcessRule',
     params: { documentId: getFirstDocument.id },
   }, apiParams => fetchProcessRule(omit(apiParams, 'action')), {
     revalidateOnFocus: false,
   })
+  // get cost
+  const { data: indexingEstimateDetail, error: indexingEstimateErr } = useSWR({
+    action: 'fetchIndexingEstimateBatch',
+    datasetId,
+    batchId,
+  }, apiParams => fetchIndexingEstimateBatch(omit(apiParams, 'action')), {
+    revalidateOnFocus: false,
+  })
 
   const router = useRouter()
   const navToDocumentList = () => {
     router.push(`/datasets/${datasetId}/documents`)
   }
 
-  const isEmbedding = useMemo(() => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail])
-  const isEmbeddingCompleted = useMemo(() => ['completed'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail])
-  const isEmbeddingPaused = useMemo(() => ['paused'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail])
-  const isEmbeddingError = useMemo(() => ['error'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail])
-  const percent = useMemo(() => {
-    const completedCount = indexingStatusDetail?.completed_segments || 0
-    const totalCount = indexingStatusDetail?.total_segments || 0
-    if (totalCount === 0)
-      return 0
-    const percent = Math.round(completedCount * 100 / totalCount)
-    return percent > 100 ? 100 : percent
-  }, [indexingStatusDetail])
+  const isEmbedding = useMemo(() => {
+    return indexingStatusBatchDetail.some((indexingStatusDetail: { indexing_status: any }) => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''))
+  }, [indexingStatusBatchDetail])
+  const isEmbeddingCompleted = useMemo(() => {
+    return indexingStatusBatchDetail.every((indexingStatusDetail: { indexing_status: any }) => ['completed', 'error'].includes(indexingStatusDetail?.indexing_status || ''))
+  }, [indexingStatusBatchDetail])
+
+  // TODO
+  // const percent = useMemo(() => {
+  //   const completedCount = indexingStatusBatchDetail?.completed_segments || 0
+  //   const totalCount = indexingStatusBatchDetail?.total_segments || 0
+  //   if (totalCount === 0)
+  //     return 0
+  //   const percent = Math.round(completedCount * 100 / totalCount)
+  //   return percent > 100 ? 100 : percent
+  // }, [indexingStatusBatchDetail])
 
   return (
     <>
@@ -154,8 +156,6 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
         <div className={s.embeddingStatus}>
           {isEmbedding && t('datasetDocuments.embedding.processing')}
           {isEmbeddingCompleted && t('datasetDocuments.embedding.completed')}
-          {isEmbeddingPaused && t('datasetDocuments.embedding.paused')}
-          {isEmbeddingError && t('datasetDocuments.embedding.error')}
         </div>
         <div className={s.cost}>
           {indexingType === 'high_quaility' && (
@@ -176,7 +176,7 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
         </div>
       </div>
       {/* TODO progress bar */}
-      <div className={s.progressContainer}>
+      {/* <div className={s.progressContainer}>
         {new Array(10).fill('').map((_, idx) => <div
           key={idx}
           className={cn(s.progressBgItem, isEmbedding ? 'bg-primary-50' : 'bg-gray-100')}
@@ -187,11 +187,11 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
             s.progressBar,
             (isEmbedding || isEmbeddingCompleted) && s.barProcessing,
             (isEmbeddingPaused || isEmbeddingError) && s.barPaused,
-            indexingStatusDetail?.indexing_status === 'completed' && 'rounded-r-md',
+            indexingStatusBatchDetail?.indexing_status === 'completed' && 'rounded-r-md',
           )}
           style={{ width: `${percent}%` }}
         />
-      </div>
+      </div> */}
       <RuleDetail sourceData={ruleDetail} />
       <div className='flex items-center gap-2 mt-10'>
         <Button className='w-fit' type='primary' onClick={navToDocumentList}>

web/models/datasets.ts

@@ -68,6 +68,9 @@ export type IndexingStatusResponse = {
   completed_segments: number
   total_segments: number
 }
+export type IndexingStatusBatchResponse = {
+  data: IndexingStatusResponse[]
+}
 
 export type ProcessMode = 'automatic' | 'custom'
 

web/service/datasets.ts

 import type { Fetcher } from 'swr'
 import qs from 'qs'
 import { del, get, patch, post, put } from './base'
-import type { CreateDocumentReq, DataSet, DataSetListResponse, DocumentDetailResponse, DocumentListResponse, FileIndexingEstimateResponse, HitTestingRecordsResponse, HitTestingResponse, IndexingEstimateResponse, IndexingStatusResponse, InitialDocumentDetail, ProcessRuleResponse, RelatedAppResponse, SegmentsQuery, SegmentsResponse, createDocumentResponse } from '@/models/datasets'
+import type { CreateDocumentReq, DataSet, DataSetListResponse, DocumentDetailResponse, DocumentListResponse, FileIndexingEstimateResponse, HitTestingRecordsResponse, HitTestingResponse, IndexingEstimateResponse, IndexingStatusBatchResponse, IndexingStatusResponse, InitialDocumentDetail, ProcessRuleResponse, RelatedAppResponse, SegmentsQuery, SegmentsResponse, createDocumentResponse } from '@/models/datasets'
 import type { CommonResponse, DataSourceNotionWorkspace } from '@/models/common'
 
 // apis for documents in a dataset
@@ -11,6 +11,11 @@ type CommonDocReq = {
   documentId: string
 }
 
+type BatchReq = {
+  datasetId: string
+  batchId: string
+}
+
 export type SortType = 'created_at' | 'hit_count' | '-created_at' | '-hit_count'
 
 export type MetadataType = 'all' | 'only' | 'without'
@@ -62,11 +67,18 @@ export const createDocument: Fetcher<InitialDocumentDetail, { datasetId: string;
 export const fetchIndexingEstimate: Fetcher<IndexingEstimateResponse, CommonDocReq> = ({ datasetId, documentId }) => {
   return get(`/datasets/${datasetId}/documents/${documentId}/indexing-estimate`, {}) as Promise<IndexingEstimateResponse>
 }
+export const fetchIndexingEstimateBatch: Fetcher<IndexingEstimateResponse, BatchReq> = ({ datasetId, batchId }) => {
+  return get(`/datasets/${datasetId}/batch/${batchId}/indexing-estimate`, {}) as Promise<IndexingEstimateResponse>
+}
 
 export const fetchIndexingStatus: Fetcher<IndexingStatusResponse, CommonDocReq> = ({ datasetId, documentId }) => {
   return get(`/datasets/${datasetId}/documents/${documentId}/indexing-status`, {}) as Promise<IndexingStatusResponse>
 }
 
+export const fetchIndexingStatusBatch: Fetcher<IndexingStatusBatchResponse, BatchReq> = ({ datasetId, batchId }) => {
+  return get(`/datasets/${datasetId}/batch/${batchId}/indexing-status`, {}) as Promise<IndexingStatusBatchResponse>
+}
+
 export const fetchDocumentDetail: Fetcher<DocumentDetailResponse, CommonDocReq & { params: { metadata?: MetadataType } }> = ({ datasetId, documentId, params }) => {
   return get(`/datasets/${datasetId}/documents/${documentId}`, { params }) as Promise<DocumentDetailResponse>
 }