feat: file upload

web/app/components/datasets/create/file-uploader/index.tsx

@@ -10,8 +10,9 @@ import { ToastContext } from '@/app/components/base/toast'
 import { upload } from '@/service/base'
 
 type IFileUploaderProps = {
-  files: FileEntity[]
-  onFileUpdate: (file?: FileEntity) => void
+  fileList: FileEntity[]
+  prepareFileList: (files: any[]) => void
+  onFileUpdate: (fileItem: any, progress: number, list: any[]) => void
   onPreview: (file: FileEntity) => void
 }
 
@@ -27,9 +28,15 @@ const ACCEPTS = [
   '.csv',
 ]
 
-const MAX_SIZE = 10 * 1024 * 1024
+const MAX_SIZE = 15 * 1024 * 1024
+const BATCH_COUNT = 5
 
-const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) => {
+const FileUploader = ({
+  fileList,
+  prepareFileList,
+  onFileUpdate,
+  onPreview,
+}: IFileUploaderProps) => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
   const [dragging, setDragging] = useState(false)
@@ -41,6 +48,9 @@ const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) =>
   const [uploading, setUploading] = useState(false)
   const [percent, setPercent] = useState(0)
 
+  // TODO
+  const fileListRef = useRef<any>(null)
+
   // utils
   const getFileType = (currentFile: File) => {
     if (!currentFile)
@@ -72,6 +82,7 @@ const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) =>
   const onProgress = useCallback((e: ProgressEvent) => {
     if (e.lengthComputable) {
       const percent = Math.floor(e.loaded / e.total * 100)
+      // updateFileItem
       setPercent(percent)
     }
   }, [setPercent])
@@ -79,43 +90,93 @@ const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) =>
     const currentXHR = uploadPromise.current
     currentXHR.abort()
   }
+  const selectHandle = () => {
+    if (fileUploader.current)
+      fileUploader.current.click()
+  }
   // TODO
-  const fileUpload = async (file?: File) => {
-    if (!file)
-      return
+  const removeFile = () => {
+    if (fileUploader.current)
+      fileUploader.current.value = ''
 
-    if (!isValid(file))
-      return
+    setCurrentFile(undefined)
+    // onFileUpdate()
+  }
 
-    setCurrentFile(file)
-    setUploading(true)
+  const fileUpload = (fileItem: any) => {
+    const fileListCopy = fileListRef.current
     const formData = new FormData()
-    formData.append('file', file)
-    // store for abort
-    const currentXHR = new XMLHttpRequest()
-    uploadPromise.current = currentXHR
-    try {
-      const result = await upload({
-        xhr: currentXHR,
-        data: formData,
-        onprogress: onProgress,
-      }) as FileEntity
-      onFileUpdate(result)
-      setUploading(false)
+    formData.append('file', fileItem.file)
+    const onProgress = (e: ProgressEvent) => {
+      if (e.lengthComputable) {
+        const percent = Math.floor(e.loaded / e.total * 100)
+        onFileUpdate(fileItem, percent, fileListCopy)
+      }
     }
-    catch (xhr: any) {
-      setUploading(false)
-      // abort handle
-      if (xhr.readyState === 0 && xhr.status === 0) {
-        if (fileUploader.current)
-          fileUploader.current.value = ''
+    console.log('fff1', fileListCopy)
 
-        setCurrentFile(undefined)
-        return
-      }
-      notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.failed') })
+    return upload({
+      xhr: new XMLHttpRequest(),
+      data: formData,
+      onprogress: onProgress,
+    })
+      .then((res: FileEntity) => {
+        const completeFile = {
+          fileID: fileItem.fileID,
+          file: res,
+        }
+        console.log('fff2', fileListCopy)
+        onFileUpdate(completeFile, 100, fileListCopy)
+        return Promise.resolve({ ...completeFile })
+      })
+      .catch((err) => {
+        console.log(err)
+        notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.failed') })
+        onFileUpdate(fileItem, -2, fileListCopy)
+        return Promise.resolve({ ...fileItem })
+      })
+      .finally()
+  }
+  const uploadBatchFiles = (bFiles: any) => {
+    bFiles.forEach((bf: any) => (bf.progress = 0))
+    return Promise.all(bFiles.map((bFile: any) => fileUpload(bFile)))
+  }
+  const uploadMultipleFiles = async (files: any) => {
+    const length = files.length
+    let start = 0
+    let end = 0
+
+    while (start < length) {
+      if (start + BATCH_COUNT > length)
+        end = length
+      else
+        end = start + BATCH_COUNT
+      const bFiles = files.slice(start, end)
+      await uploadBatchFiles(bFiles)
+      start = end
     }
   }
+  const initialUpload = (files: any) => {
+    if (!files.length)
+      return false
+    const preparedFiles = files.map((file: any, index: number) => {
+      const fileItem = {
+        fileID: `file${index}-${Date.now()}`,
+        file,
+        progress: -1,
+      }
+      return fileItem
+    })
+    prepareFileList(preparedFiles)
+    // TODO fix filelist copy
+    fileListRef.current = preparedFiles
+    uploadMultipleFiles(preparedFiles)
+  }
+  const fileChangeHandle = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const files = [...(e.target.files ?? [])].filter(file => isValid(file))
+    initialUpload(files)
+  }
+
   const handleDragEnter = (e: DragEvent) => {
     e.preventDefault()
     e.stopPropagation()
@@ -147,25 +208,6 @@ const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) =>
     fileUpload(files[0])
   }
 
-  const selectHandle = () => {
-    if (fileUploader.current)
-      fileUploader.current.click()
-  }
-  // TODO
-  const removeFile = () => {
-    if (fileUploader.current)
-      fileUploader.current.value = ''
-
-    setCurrentFile(undefined)
-    onFileUpdate()
-  }
-  // TODO
-  const fileChangeHandle = (e: React.ChangeEvent<HTMLInputElement>) => {
-    const currentFile = e.target.files?.[0]
-    onFileUpdate()
-    fileUpload(currentFile)
-  }
-
   useEffect(() => {
     dropRef.current?.addEventListener('dragenter', handleDragEnter)
     dropRef.current?.addEventListener('dragover', handleDragOver)
@@ -201,6 +243,33 @@ const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) =>
         {dragging && <div ref={dragRef} className={s.draggingCover}/>}
       </div>
       <div className={s.fileList}>
+        {fileList.map(fileItem => (
+          <div
+            // onClick={() => onPreview(currentFile)}
+            className={cn(
+              s.file,
+              fileItem.progress < 100 && s.uploading,
+              // s.active,
+            )}
+          >
+            {fileItem.progress < 100 && (
+              <div className={s.progressbar} style={{ width: `${percent}%` }}/>
+            )}
+            <div className={s.fileInfo}>
+              <div className={cn(s.fileIcon, s[getFileType(fileItem.file)])}/>
+              <div className={s.filename}>{fileItem.file.name}</div>
+              <div className={s.size}>{getFileSize(fileItem.file.size)}</div>
+            </div>
+            <div className={s.actionWrapper}>
+              {fileItem.progress < 100 && (
+                <div className={s.percent}>{`${fileItem.progress}%`}</div>
+              )}
+              {fileItem.progress === 100 && (
+                <div className={s.remove} onClick={removeFile}/>
+              )}
+            </div>
+          </div>
+        ))}
         {currentFile && (
           <div
             // onClick={() => onPreview(currentFile)}
@@ -230,7 +299,7 @@ const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) =>
         )}
       </div>
       {/* TODO */}
-      {!currentFile && files[0] && (
+      {false && !currentFile && fileList[0] && (
         <div
           // onClick={() => onPreview(currentFile)}
           className={cn(
@@ -243,9 +312,9 @@ const FileUploader = ({ files, onFileUpdate, onPreview }: IFileUploaderProps) =>
             <div className={s.progressbar} style={{ width: `${percent}%` }}/>
           )}
           <div className={s.fileInfo}>
-            <div className={cn(s.fileIcon, s[getFileType(files[0])])}/>
-            <div className={s.filename}>{files[0].name}</div>
-            <div className={s.size}>{getFileSize(files[0].size)}</div>
+            <div className={cn(s.fileIcon, s[getFileType(fileList[0])])}/>
+            <div className={s.filename}>{fileList[0].name}</div>
+            <div className={s.size}>{getFileSize(fileList[0].size)}</div>
           </div>
           <div className={s.actionWrapper}>
             {uploading && (

web/app/components/datasets/create/index.tsx

@@ -8,7 +8,7 @@ import StepOne from './step-one'
 import StepTwo from './step-two'
 import StepThree from './step-three'
 import { DataSourceType } from '@/models/datasets'
-import type { DataSet, File, createDocumentResponse } from '@/models/datasets'
+import type { DataSet, createDocumentResponse } from '@/models/datasets'
 import { fetchDataSource, fetchTenantInfo } from '@/service/common'
 import { fetchDataDetail } from '@/service/datasets'
 import type { DataSourceNotionPage } from '@/models/common'
@@ -30,7 +30,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   const [dataSourceType, setDataSourceType] = useState<DataSourceType>(DataSourceType.FILE)
   const [step, setStep] = useState(1)
   const [indexingTypeCache, setIndexTypeCache] = useState('')
-  const [fileList, setFiles] = useState<File[]>([])
+  const [fileList, setFiles] = useState<any[]>([])
   const [result, setResult] = useState<createDocumentResponse | undefined>()
   const [hasError, setHasError] = useState(false)
 
@@ -40,14 +40,19 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   }
 
   // TODO
-  const updateFile = (file?: File) => {
-    if (file) {
-      setFiles([
-        ...fileList,
-        file,
-      ])
+  const updateFileList = (preparedFiles: any) => {
+    console.log('preparedFiles', preparedFiles)
+    setFiles(preparedFiles)
+  }
+  const updateFile = (fileItem: any, progress: number, list: any[]) => {
+    const targetIndex = list.findIndex((file: any) => file.fileID === fileItem.fileID)
+    list[targetIndex] = {
+      ...fileItem,
+      progress,
     }
+    setFiles(list)
   }
+
   const updateIndexingTypeCache = (type: string) => {
     setIndexTypeCache(type)
   }
@@ -111,6 +116,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
           dataSourceTypeDisable={!!detail?.data_source_type}
           changeType={setDataSourceType}
           files={fileList}
+          updateFileList={updateFileList}
           updateFile={updateFile}
           notionPages={notionPages}
           updateNotionPages={updateNotionPages}

web/app/components/datasets/create/step-one/index.tsx

 'use client'
-import React, { useState } from 'react'
+import React, { useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import cn from 'classnames'
 import FilePreview from '../file-preview'
@@ -19,8 +19,9 @@ type IStepOneProps = {
   dataSourceTypeDisable: Boolean
   hasConnection: boolean
   onSetting: () => void
-  files: File[]
-  updateFile: (file?: File) => void
+  files: any[]
+  updateFileList: (files: any[]) => void
+  updateFile: (fileItem: any, progress: number, list: any[]) => void
   notionPages?: any[]
   updateNotionPages: (value: any[]) => void
   onStepChange: () => void
@@ -54,6 +55,7 @@ const StepOne = ({
   onSetting,
   onStepChange,
   files,
+  updateFileList,
   updateFile,
   notionPages = [],
   updateNotionPages,
@@ -80,6 +82,17 @@ const StepOne = ({
     setCurrentNotionPage(undefined)
   }
 
+  // TODO
+  const nextDisabled = useMemo(() => {
+    if (!files.length)
+      return true
+    console.log(files)
+    console.log(files.some(file => file.progress !== 100))
+    if (files.some(file => file.progress !== 100))
+      return true
+    return false
+  }, [files])
+
   return (
     <div className='flex w-full h-full'>
       <div className='grow overflow-y-auto relative'>
@@ -131,9 +144,13 @@ const StepOne = ({
           </div>
           {dataSourceType === DataSourceType.FILE && (
             <>
-              <FileUploader onFileUpdate={updateFile} onPreview={updateCurrentFile} files={files} />
-              {/* TODO */}
-              <Button disabled={!files.length} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
+              <FileUploader
+                fileList={files}
+                prepareFileList={updateFileList}
+                onFileUpdate={updateFile}
+                onPreview={updateCurrentFile}
+              />
+              <Button disabled={nextDisabled} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
             </>
           )}
           {dataSourceType === DataSourceType.NOTION && (