Skip to content

D
dify
Unverified Commit e161c511 authored by KVOJJJin's avatar KVOJJJin Committed by GitHub
Browse files
Options

Feat:csv & docx support (#1139) 

Co-authored-by: 's avatarjyong <jyong@dify.ai>
parent f29e8268
Show whitespace changes
Inline Side-by-side
Showing with 68 additions and 39 deletions
api/controllers/console/datasets/file.py
View file @ e161c511
...@@ -26,7 +26,7 @@ from models.model import UploadFile ...@@ -26,7 +26,7 @@ from models.model import UploadFile
cache = TTLCache(maxsize=None, ttl=30) cache = TTLCache(maxsize=None, ttl=30)
ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx'] ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx', 'docx', 'csv']
PREVIEW_WORDS_LIMIT = 3000 PREVIEW_WORDS_LIMIT = 3000
......
web/app/components/datasets/create/assets/docx.svg 0 → 100644
View file @ e161c511
<svg width="24" height="26" viewBox="0 0 24 26" fill="none" xmlns="http://www.w3.org/2000/svg">
<g filter="url(#filter0_d_6785_286)">
<path d="M3 5.8C3 4.11984 3 3.27976 3.32698 2.63803C3.6146 2.07354 4.07354 1.6146 4.63803 1.32698C5.27976 1 6.11984 1 7.8 1H14L21 8V18.2C21 19.8802 21 20.7202 20.673 21.362C20.3854 21.9265 19.9265 22.3854 19.362 22.673C18.7202 23 17.8802 23 16.2 23H7.8C6.11984 23 5.27976 23 4.63803 22.673C4.07354 22.3854 3.6146 21.9265 3.32698 21.362C3 20.7202 3 19.8802 3 18.2V5.8Z" fill="#2349A9"/>
</g>
<path opacity="0.5" d="M14 1L21 8H16C14.8954 8 14 7.10457 14 6V1Z" fill="white"/>
<g opacity="0.96">
<path d="M8.13338 16C8.13338 16.6188 8.18659 17.1215 7.69589 17.5801C7.4062 17.8508 6.99827 17.9669 6.56078 17.9669H5V14.0331H6.56078C6.99827 14.0331 7.4062 14.1492 7.69589 14.4199C8.18659 14.8785 8.13338 15.3812 8.13338 16ZM7.09877 16C7.09877 15.337 7.06921 15.2265 6.98644 15.116C6.89185 14.9834 6.74996 14.895 6.48983 14.895H6.03461V17.105H6.48983C6.74996 17.105 6.89185 17.0166 6.98644 16.884C7.06921 16.7735 7.09877 16.6685 7.09877 16Z" fill="white"/>
<path d="M11.9192 16C11.9192 16.5912 11.937 17.1436 11.4936 17.558C11.1862 17.8453 10.8314 18 10.3171 18C9.80274 18 9.44802 17.8453 9.14059 17.558C8.69719 17.1436 8.71493 16.5912 8.71493 16C8.71493 15.4088 8.69719 14.8564 9.14059 14.442C9.44802 14.1547 9.80274 14 10.3171 14C10.8314 14 11.1862 14.1547 11.4936 14.442C11.937 14.8564 11.9192 15.4088 11.9192 16ZM10.8846 16C10.8846 15.2818 10.8255 15.1492 10.7309 15.0331C10.6541 14.9392 10.5063 14.8619 10.3171 14.8619C10.1279 14.8619 9.9801 14.9392 9.90325 15.0331C9.80865 15.1492 9.74953 15.2818 9.74953 16C9.74953 16.7182 9.80865 16.8453 9.90325 16.9613C9.9801 17.0552 10.1279 17.1381 10.3171 17.1381C10.5063 17.1381 10.6541 17.0552 10.7309 16.9613C10.8255 16.8453 10.8846 16.7182 10.8846 16Z" fill="white"/>
<path d="M15.689 16.7182C15.5353 17.5856 14.8909 18 14.0928 18C13.6021 18 13.2296 17.8453 12.9222 17.558C12.4788 17.1436 12.4965 16.5912 12.4965 16C12.4965 15.4088 12.4788 14.8564 12.9222 14.442C13.2296 14.1547 13.6021 14 14.0928 14C14.8909 14 15.5353 14.4144 15.689 15.2818H14.6367C14.5717 15.0608 14.4416 14.8619 14.0987 14.8619C13.9095 14.8619 13.7676 14.9337 13.6908 15.0276C13.5962 15.1436 13.5312 15.2818 13.5312 16C13.5312 16.7182 13.5962 16.8564 13.6908 16.9724C13.7676 17.0663 13.9095 17.1381 14.0987 17.1381C14.4416 17.1381 14.5717 16.9392 14.6367 16.7182H15.689Z" fill="white"/>
<path d="M19.5 17.9669H18.3176L17.6259 16.7569L16.9342 17.9669H15.7518L17.0642 15.9503L15.8345 14.0331H17.011L17.6259 15.1436L18.2407 14.0331H19.4172L18.1875 15.9503L19.5 17.9669Z" fill="white"/>
</g>
<defs>
<filter id="filter0_d_6785_286" x="1" y="0" width="22" height="26" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
<feFlood flood-opacity="0" result="BackgroundImageFix"/>
<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
<feOffset dy="1"/>
<feGaussianBlur stdDeviation="1"/>
<feColorMatrix type="matrix" values="0 0 0 0 0.0627451 0 0 0 0 0.0941176 0 0 0 0 0.156863 0 0 0 0.05 0"/>
<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_6785_286"/>
<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_6785_286" result="shape"/>
</filter>
</defs>
</svg>
web/app/components/datasets/create/embedding-process/index.module.css
View file @ e161c511
...@@ -89,7 +89,9 @@ ...@@ -89,7 +89,9 @@
.fileIcon.csv { .fileIcon.csv {
background-image: url(../assets/csv.svg); background-image: url(../assets/csv.svg);
} }
.fileIcon.docx {
background-image: url(../assets/docx.svg);
}
.fileIcon.xlsx, .fileIcon.xlsx,
.fileIcon.xls { .fileIcon.xls {
background-image: url(../assets/xlsx.svg); background-image: url(../assets/xlsx.svg);
......
web/app/components/datasets/create/embedding-process/index.tsx
View file @ e161c511
...@@ -194,7 +194,7 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index ...@@ -194,7 +194,7 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
</div> </div>
<div className={s.progressContainer}> <div className={s.progressContainer}>
{indexingStatusBatchDetail.map(indexingStatusDetail => ( {indexingStatusBatchDetail.map(indexingStatusDetail => (
<div className={cn( <div key={indexingStatusDetail.id} className={cn(
s.sourceItem, s.sourceItem,
indexingStatusDetail.indexing_status === 'error' && s.error, indexingStatusDetail.indexing_status === 'error' && s.error,
indexingStatusDetail.indexing_status === 'completed' && s.success, indexingStatusDetail.indexing_status === 'completed' && s.success,
......
web/app/components/datasets/create/file-uploader/index.module.css
View file @ e161c511
...@@ -98,7 +98,9 @@ ...@@ -98,7 +98,9 @@
.fileIcon.csv { .fileIcon.csv {
background-image: url(../assets/csv.svg); background-image: url(../assets/csv.svg);
} }
.fileIcon.docx {
background-image: url(../assets/docx.svg);
}
.fileIcon.xlsx, .fileIcon.xlsx,
.fileIcon.xls { .fileIcon.xls {
background-image: url(../assets/xlsx.svg); background-image: url(../assets/xlsx.svg);
......
web/app/components/datasets/create/file-uploader/index.tsx
View file @ e161c511
...@@ -29,9 +29,9 @@ const ACCEPTS = [ ...@@ -29,9 +29,9 @@ const ACCEPTS = [
'.md', '.md',
'.markdown', '.markdown',
'.txt', '.txt',
// '.xls',
'.xlsx', '.xlsx',
// '.csv', '.csv',
'.docx',
] ]
const FileUploader = ({ const FileUploader = ({
...@@ -250,7 +250,6 @@ const FileUploader = ({ ...@@ -250,7 +250,6 @@ const FileUploader = ({
className={cn( className={cn(
s.file, s.file,
fileItem.progress < 100 && s.uploading, fileItem.progress < 100 && s.uploading,
// s.active,
)} )}
> >
{fileItem.progress < 100 && ( {fileItem.progress < 100 && (
...@@ -274,33 +273,6 @@ const FileUploader = ({ ...@@ -274,33 +273,6 @@ const FileUploader = ({
</div> </div>
</div> </div>
))} ))}
{/* {currentFile && (
<div
// onClick={() => onPreview(currentFile)}
className={cn(
s.file,
uploading && s.uploading,
// s.active,
)}
>
{uploading && (
<div className={s.progressbar} style={{ width: `${percent}%` }}/>
)}
<div className={s.fileInfo}>
<div className={cn(s.fileIcon, s[getFileType(currentFile)])}/>
<div className={s.filename}>{currentFile.name}</div>
<div className={s.size}>{getFileSize(currentFile.size)}</div>
</div>
<div className={s.actionWrapper}>
{uploading && (
<div className={s.percent}>{`${percent}%`}</div>
)}
{!uploading && (
<div className={s.remove} onClick={() => removeFile(index)}/>
)}
</div>
</div>
)} */}
</div> </div>
</div> </div>
) )
......
web/app/components/datasets/create/step-one/index.tsx
View file @ e161c511
...@@ -65,7 +65,7 @@ const StepOne = ({ ...@@ -65,7 +65,7 @@ const StepOne = ({
const { dataset } = useDatasetDetailContext() const { dataset } = useDatasetDetailContext()
const [showModal, setShowModal] = useState(false) const [showModal, setShowModal] = useState(false)
const [currentFile, setCurrentFile] = useState<File | undefined>() const [currentFile, setCurrentFile] = useState<File | undefined>()
const [currentNotionPage, setCurrentNotionPage] = useState<Page | undefined>() const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
const { t } = useTranslation() const { t } = useTranslation()
const modalShowHandle = () => setShowModal(true) const modalShowHandle = () => setShowModal(true)
...@@ -78,7 +78,7 @@ const StepOne = ({ ...@@ -78,7 +78,7 @@ const StepOne = ({
setCurrentFile(undefined) setCurrentFile(undefined)
} }
const updateCurrentPage = (page: Page) => { const updateCurrentPage = (page: NotionPage) => {
setCurrentNotionPage(page) setCurrentNotionPage(page)
} }
......
web/app/components/datasets/create/step-two/index.module.css
View file @ e161c511
...@@ -290,6 +290,10 @@ ...@@ -290,6 +290,10 @@
background-image: url(../assets/csv.svg); background-image: url(../assets/csv.svg);
} }
.fileIcon.docx {
background-image: url(../assets/docx.svg);
}
.fileIcon.xlsx, .fileIcon.xlsx,
.fileIcon.xls { .fileIcon.xls {
background-image: url(../assets/xlsx.svg); background-image: url(../assets/xlsx.svg);
......
web/app/components/datasets/documents/detail/completed/SegmentCard.tsx
View file @ e161c511
...@@ -43,7 +43,7 @@ type ISegmentCardProps = { ...@@ -43,7 +43,7 @@ type ISegmentCardProps = {
scene?: UsageScene scene?: UsageScene
className?: string className?: string
archived?: boolean archived?: boolean
embeddingAvailable: boolean embeddingAvailable?: boolean
} }
const SegmentCard: FC<ISegmentCardProps> = ({ const SegmentCard: FC<ISegmentCardProps> = ({
......
web/app/components/datasets/documents/style.module.css
View file @ e161c511
...@@ -87,6 +87,9 @@ ...@@ -87,6 +87,9 @@
.csvIcon { .csvIcon {
background-image: url(~@/assets/csv.svg); background-image: url(~@/assets/csv.svg);
} }
.docxIcon {
background-image: url(~@/assets/docx.svg);
}
.statusItemDetail { .statusItemDetail {
@apply h-8 font-medium border border-gray-200 inline-flex items-center rounded-lg pl-3 pr-4 mr-2; @apply h-8 font-medium border border-gray-200 inline-flex items-center rounded-lg pl-3 pr-4 mr-2;
} }
......
web/assets/docx.svg 0 → 100644
View file @ e161c511
<svg width="24" height="26" viewBox="0 0 24 26" fill="none" xmlns="http://www.w3.org/2000/svg">
<g filter="url(#filter0_d_6785_286)">
<path d="M3 5.8C3 4.11984 3 3.27976 3.32698 2.63803C3.6146 2.07354 4.07354 1.6146 4.63803 1.32698C5.27976 1 6.11984 1 7.8 1H14L21 8V18.2C21 19.8802 21 20.7202 20.673 21.362C20.3854 21.9265 19.9265 22.3854 19.362 22.673C18.7202 23 17.8802 23 16.2 23H7.8C6.11984 23 5.27976 23 4.63803 22.673C4.07354 22.3854 3.6146 21.9265 3.32698 21.362C3 20.7202 3 19.8802 3 18.2V5.8Z" fill="#2349A9"/>
</g>
<path opacity="0.5" d="M14 1L21 8H16C14.8954 8 14 7.10457 14 6V1Z" fill="white"/>
<g opacity="0.96">
<path d="M8.13338 16C8.13338 16.6188 8.18659 17.1215 7.69589 17.5801C7.4062 17.8508 6.99827 17.9669 6.56078 17.9669H5V14.0331H6.56078C6.99827 14.0331 7.4062 14.1492 7.69589 14.4199C8.18659 14.8785 8.13338 15.3812 8.13338 16ZM7.09877 16C7.09877 15.337 7.06921 15.2265 6.98644 15.116C6.89185 14.9834 6.74996 14.895 6.48983 14.895H6.03461V17.105H6.48983C6.74996 17.105 6.89185 17.0166 6.98644 16.884C7.06921 16.7735 7.09877 16.6685 7.09877 16Z" fill="white"/>
<path d="M11.9192 16C11.9192 16.5912 11.937 17.1436 11.4936 17.558C11.1862 17.8453 10.8314 18 10.3171 18C9.80274 18 9.44802 17.8453 9.14059 17.558C8.69719 17.1436 8.71493 16.5912 8.71493 16C8.71493 15.4088 8.69719 14.8564 9.14059 14.442C9.44802 14.1547 9.80274 14 10.3171 14C10.8314 14 11.1862 14.1547 11.4936 14.442C11.937 14.8564 11.9192 15.4088 11.9192 16ZM10.8846 16C10.8846 15.2818 10.8255 15.1492 10.7309 15.0331C10.6541 14.9392 10.5063 14.8619 10.3171 14.8619C10.1279 14.8619 9.9801 14.9392 9.90325 15.0331C9.80865 15.1492 9.74953 15.2818 9.74953 16C9.74953 16.7182 9.80865 16.8453 9.90325 16.9613C9.9801 17.0552 10.1279 17.1381 10.3171 17.1381C10.5063 17.1381 10.6541 17.0552 10.7309 16.9613C10.8255 16.8453 10.8846 16.7182 10.8846 16Z" fill="white"/>
<path d="M15.689 16.7182C15.5353 17.5856 14.8909 18 14.0928 18C13.6021 18 13.2296 17.8453 12.9222 17.558C12.4788 17.1436 12.4965 16.5912 12.4965 16C12.4965 15.4088 12.4788 14.8564 12.9222 14.442C13.2296 14.1547 13.6021 14 14.0928 14C14.8909 14 15.5353 14.4144 15.689 15.2818H14.6367C14.5717 15.0608 14.4416 14.8619 14.0987 14.8619C13.9095 14.8619 13.7676 14.9337 13.6908 15.0276C13.5962 15.1436 13.5312 15.2818 13.5312 16C13.5312 16.7182 13.5962 16.8564 13.6908 16.9724C13.7676 17.0663 13.9095 17.1381 14.0987 17.1381C14.4416 17.1381 14.5717 16.9392 14.6367 16.7182H15.689Z" fill="white"/>
<path d="M19.5 17.9669H18.3176L17.6259 16.7569L16.9342 17.9669H15.7518L17.0642 15.9503L15.8345 14.0331H17.011L17.6259 15.1436L18.2407 14.0331H19.4172L18.1875 15.9503L19.5 17.9669Z" fill="white"/>
</g>
<defs>
<filter id="filter0_d_6785_286" x="1" y="0" width="22" height="26" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
<feFlood flood-opacity="0" result="BackgroundImageFix"/>
<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
<feOffset dy="1"/>
<feGaussianBlur stdDeviation="1"/>
<feColorMatrix type="matrix" values="0 0 0 0 0.0627451 0 0 0 0 0.0941176 0 0 0 0 0.156863 0 0 0 0.05 0"/>
<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_6785_286"/>
<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_6785_286" result="shape"/>
</filter>
</defs>
</svg>
web/i18n/lang/dataset-creation.en.ts
View file @ e161c511
...@@ -23,7 +23,7 @@ const translation = { ...@@ -23,7 +23,7 @@ const translation = {
title: 'Upload text file', title: 'Upload text file',
button: 'Drag and drop file, or', button: 'Drag and drop file, or',
browse: 'Browse', browse: 'Browse',
tip: 'Supports txt, html, markdown, xlsx, and pdf. Max {{size}}MB each.', tip: 'Supports txt, html, markdown, xlsx, csv, docx and pdf. Max {{size}}MB each.',
validation: { validation: {
typeError: 'File type not supported', typeError: 'File type not supported',
size: 'File too large. Maximum is {{size}}MB', size: 'File too large. Maximum is {{size}}MB',
......
web/i18n/lang/dataset-creation.zh.ts
View file @ e161c511
...@@ -23,7 +23,7 @@ const translation = { ...@@ -23,7 +23,7 @@ const translation = {
title: '上传文本文件', title: '上传文本文件',
button: '拖拽文件至此，或者', button: '拖拽文件至此，或者',
browse: '选择文件', browse: '选择文件',
tip: '已支持 TXT、 HTML、 Markdown、 PDF、 XLSX，每个文件不超过 {{size}}MB。', tip: '已支持 TXT、 HTML、 Markdown、 PDF、 XLSX、CSV、DOCX，每个文件不超过 {{size}}MB。',
validation: { validation: {
typeError: '文件类型不支持', typeError: '文件类型不支持',
size: '文件太大了，不能超过 {{size}}MB', size: '文件太大了，不能超过 {{size}}MB',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment