Skip to content

D
dify
Commit 3b2b8199 authored by JzoNg's avatar JzoNg
Browse files
Options

feat: notion import in step one

parent 761febc7
Show whitespace changes
Inline Side-by-side
Showing with 244 additions and 101 deletions
web/app/components/datasets/create/assets/Icon-3-dots.svg 0 → 100644
View file @ 3b2b8199
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#374151" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
web/app/components/datasets/create/file-preview/index.tsx
View file @ 3b2b8199
'use client'
import React, { useState, useEffect } from 'react'
import React, { useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import type { File } from '@/models/datasets'
import { fetchFilePreview } from '@/service/common'
import cn from 'classnames'
import s from './index.module.css'
import type { File } from '@/models/datasets'
import { fetchFilePreview } from '@/service/common'
type IProps = {
file?: File,
file?: File
hidePreview: () => void
}
const FilePreview = ({
file,
hidePreview,
}: IProps) => {
const { t } = useTranslation()
const [previewContent, setPreviewContent] = useState('')
......@@ -28,23 +29,27 @@ const FilePreview = ({
}
const getFileName = (currentFile?: File) => {
if (!currentFile) {
if (!currentFile)
return ''
}
const arr = currentFile.name.split('.')
return arr.slice(0, -1).join()
}
useEffect(() => {
if (file) {
if (file)
getPreviewContent(file.id)
}
}, [file])
return (
<div className={cn(s.filePreview)}>
<div className={cn(s.previewHeader)}>
<div className={cn(s.title)}>{t('datasetCreation.stepOne.filePreview')}</div>
<div className={cn(s.title)}>
<span>{t('datasetCreation.stepOne.filePreview')}</span>
{false && (
<div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}></div>
)}
</div>
<div className={cn(s.fileName)}>
<span>{getFileName(file)}</span><span className={cn(s.filetype)}>.{file?.extension}</span>
</div>
......
web/app/components/datasets/create/index.tsx
View file @ 3b2b8199
'use client'
import React, { useState, useCallback, useEffect } from 'react'
import React, { useCallback, useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import type { DataSet, File, createDocumentResponse } from '@/models/datasets'
import { fetchTenantInfo } from '@/service/common'
import { fetchDataDetail } from '@/service/datasets'
import AppUnavailable from '../../base/app-unavailable'
import StepsNavBar from './steps-nav-bar'
import StepOne from './step-one'
import StepTwo from './step-two'
import StepThree from './step-three'
import { DataSourceType } from '@/models/datasets'
import type { DataSet, File, createDocumentResponse } from '@/models/datasets'
import { fetchDataSource, fetchTenantInfo } from '@/service/common'
import { fetchDataDetail } from '@/service/datasets'
import AccountSetting from '@/app/components/header/account-setting'
import AppUnavailable from '../../base/app-unavailable'
type DatasetUpdateFormProps = {
datasetId?: string;
datasetId?: string
}
const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
const { t } = useTranslation()
const [hasSetAPIKEY, setHasSetAPIKEY] = useState(true)
const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
const [hasConnection, setHasConnection] = useState(true)
const [isShowDataSourceSetting, { setTrue: showDataSourceSetting, setFalse: hideDataSourceSetting }] = useBoolean()
const [dataSourceType, setDataSourceType] = useState<DataSourceType>(DataSourceType.FILE)
const [step, setStep] = useState(1)
const [indexingTypeCache, setIndexTypeCache] = useState('')
const [file, setFile] = useState<File | undefined>()
const [result, setResult] = useState<createDocumentResponse | undefined>()
const [hasError, setHasError] = useState(false)
// TODO
const [notionPages, setNotionPages] = useState<any>([])
const updateNotionPages = (value: any[]) => {
setNotionPages(value)
}
const updateFile = (file?: File) => {
setFile(file)
}
......@@ -50,9 +60,15 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
const hasSetKey = data.providers.some(({ is_valid }) => is_valid)
setHasSetAPIKEY(hasSetKey)
}
const checkNotionConnection = async () => {
const { data } = await fetchDataSource({ url: '/data-source/integrates' })
const hasConnection = data.filter(item => item.provider === 'notion') || []
setHasConnection(hasConnection.length > 0)
}
useEffect(() => {
checkAPIKey()
checkNotionConnection()
}, [])
const [detail, setDetail] = useState<DataSet | null>(null)
......@@ -62,16 +78,16 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
try {
const detail = await fetchDataDetail(datasetId)
setDetail(detail)
} catch (e) {
}
catch (e) {
setHasError(true)
}
}
})()
}, [datasetId])
if (hasError) {
if (hasError)
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
}
return (
<div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
......@@ -80,9 +96,15 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
</div>
<div className="grow bg-white">
{step === 1 && <StepOne
hasConnection={hasConnection}
onSetting={showDataSourceSetting}
datasetId={datasetId}
dataSourceType={dataSourceType}
changeType={setDataSourceType}
file={file}
updateFile={updateFile}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
onStepChange={nextStep}
/>}
{(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
......@@ -90,7 +112,9 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
onSetting={showSetAPIKey}
indexingType={detail?.indexing_technique || ''}
datasetId={datasetId}
dataSourceType={dataSourceType}
file={file}
notionPages={notionPages}
onStepChange={changeStep}
updateIndexingTypeCache={updateIndexingTypeCache}
updateResultCache={updateResultCache}
......@@ -106,6 +130,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
await checkAPIKey()
hideSetAPIkey()
}} />}
{isShowDataSourceSetting && <AccountSetting activeTab="data-source" onCancel={hideDataSourceSetting}/>}
</div>
)
}
......
web/app/components/datasets/create/step-one/index.module.css
View file @ 3b2b8199
......@@ -107,3 +107,53 @@
background: center no-repeat url(../assets/folder-plus.svg);
background-size: contain;
}
.notionConnectionTip {
display: flex;
flex-direction: column;
align-items: flex-start;
padding: 24px;
max-width: 640px;
background: #F9FAFB;
border-radius: 16px;
}
.notionIcon {
display: flex;
padding: 12px;
width: 48px;
height: 48px;
background: #fff center no-repeat url(../assets/notion.svg);
background-size: 24px;
border: 0.5px solid #EAECF5;
box-shadow: 0px 12px 16px -4px rgba(16, 24, 40, 0.08), 0px 4px 6px -2px rgba(16, 24, 40, 0.03);
border-radius: 12px;
}
.notionConnectionTip .title {
position: relative;
margin: 24px 0 4px;
font-style: normal;
font-weight: 600;
font-size: 16px;
line-height: 24px;
color: #374151;
}
.notionConnectionTip .title::after {
content: '';
position: absolute;
top: -6px;
right: -12px;
width: 16px;
height: 16px;
background: center no-repeat url(../assets/Icon-3-dots.svg);
background-size: contain;
}
.notionConnectionTip .tip {
margin-bottom: 20px;
font-style: normal;
font-weight: 400;
font-size: 13px;
line-height: 18px;
color: #6B7280;
}
web/app/components/datasets/create/step-one/index.tsx
View file @ 3b2b8199
'use client'
import React, { useState } from 'react'
import { useTranslation } from 'react-i18next'
import type { File } from '@/models/datasets'
import cn from 'classnames'
import FilePreview from '../file-preview'
import FileUploader from '../file-uploader'
import EmptyDatasetCreationModal from '../empty-dataset-creation-modal'
import Button from '@/app/components/base/button'
import cn from 'classnames'
import s from './index.module.css'
import type { File } from '@/models/datasets'
import { DataSourceType } from '@/models/datasets'
import Button from '@/app/components/base/button'
import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
type IStepOneProps = {
datasetId?: string,
file?: File,
updateFile: (file?: File) => void,
onStepChange: () => void,
datasetId?: string
dataSourceType: DataSourceType
hasConnection: boolean
onSetting: () => void
file?: File
updateFile: (file?: File) => void
notionPages?: any[]
updateNotionPages: (value: any[]) => void
onStepChange: () => void
changeType: (type: DataSourceType) => void
}
const StepOne = ({
datasetId,
dataSourceType,
changeType,
hasConnection,
onSetting,
onStepChange,
file,
updateFile,
notionPages = [],
}: IStepOneProps) => {
const [dataSourceType, setDataSourceType] = useState('FILE')
const [showModal, setShowModal] = useState(false)
const [showFilePreview, setShowFilePreview] = useState(true)
const { t } = useTranslation()
const hidePreview = () => setShowFilePreview(false)
const modalShowHandle = () => setShowModal(true)
const modalCloseHandle = () => setShowModal(false)
......@@ -38,41 +52,66 @@ const StepOne = ({
<div className={s.form}>
<div className={s.dataSourceTypeList}>
<div
className={cn(s.dataSourceItem, dataSourceType === 'FILE' && s.active)}
onClick={() => setDataSourceType('FILE')}
className={cn(s.dataSourceItem, dataSourceType === DataSourceType.FILE && s.active)}
onClick={() => changeType(DataSourceType.FILE)}
>
<span className={cn(s.datasetIcon)}/>
<span className={cn(s.datasetIcon)} />
{t('datasetCreation.stepOne.dataSourceType.file')}
</div>
<div
className={cn(s.dataSourceItem, s.disabled, dataSourceType === 'notion' && s.active)}
// onClick={() => setDataSourceType('notion')}
className={cn(s.dataSourceItem, dataSourceType === DataSourceType.NOTION && s.active)}
onClick={() => changeType(DataSourceType.NOTION)}
>
<span className={s.comingTag}>Coming soon</span>
<span className={cn(s.datasetIcon, s.notion)}/>
<span className={cn(s.datasetIcon, s.notion)} />
{t('datasetCreation.stepOne.dataSourceType.notion')}
</div>
<div
className={cn(s.dataSourceItem, s.disabled, dataSourceType === 'web' && s.active)}
// onClick={() => setDataSourceType('web')}
className={cn(s.dataSourceItem, s.disabled, dataSourceType === DataSourceType.WEB && s.active)}
// onClick={() => changeType(DataSourceType.WEB)}
>
<span className={s.comingTag}>Coming soon</span>
<span className={cn(s.datasetIcon, s.web)}/>
<span className={cn(s.datasetIcon, s.web)} />
{t('datasetCreation.stepOne.dataSourceType.web')}
</div>
</div>
{dataSourceType === DataSourceType.FILE && (
<>
<FileUploader onFileUpdate={updateFile} file={file} />
<Button disabled={!file} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
</>
)}
{dataSourceType === DataSourceType.NOTION && (
<>
{!hasConnection && (
<div className={s.notionConnectionTip}>
<span className={s.notionIcon}/>
<div className={s.title}>Notion is not connected</div>
<div className={s.tip}>To sync with Notion, connection to Notion must be established first.</div>
<Button className='h-8' type='primary' onClick={onSetting}>Go to connect</Button>
</div>
)}
{hasConnection && (
<>
{/* TODO */}
<div className='mb-8 w-[640px]'>
<NotionPageSelector />
</div>
<Button disabled={!notionPages.length} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
</>
)}
</>
)}
{!datasetId && (
<>
<div className={s.dividerLine}/>
<div className={s.dividerLine} />
<div onClick={modalShowHandle} className={s.OtherCreationOption}>{t('datasetCreation.stepOne.emptyDatasetCreation')}</div>
</>
)}
</div>
<EmptyDatasetCreationModal show={showModal} onHide={modalCloseHandle}/>
<EmptyDatasetCreationModal show={showModal} onHide={modalCloseHandle} />
</div>
{file && <FilePreview file={file} />}
{file && showFilePreview && <FilePreview file={file} hidePreview={hidePreview} />}
{/* TODO notion page preview */}
</div>
)
}
......
web/app/components/datasets/create/step-three/index.tsx
View file @ 3b2b8199
'use client'
import React from 'react'
import { useTranslation } from 'react-i18next'
import type { createDocumentResponse } from '@/models/datasets'
import cn from 'classnames'
import EmbeddingDetail from '../../documents/detail/embedding'
import cn from 'classnames'
import s from './index.module.css'
import type { FullDocumentDetail, createDocumentResponse } from '@/models/datasets'
type StepThreeProps = {
datasetId?: string,
datasetName?: string,
indexingType?: string,
datasetId?: string
datasetName?: string
indexingType?: string
creationCache?: createDocumentResponse
}
......@@ -38,12 +38,13 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
<div className={s.content}>{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}</div>
</div>
)}
{/* TODO multi doc display */}
<EmbeddingDetail
datasetId={datasetId || creationCache?.dataset?.id}
documentId={creationCache?.document.id}
documentId={creationCache?.documents[0].id}
indexingType={indexingType || creationCache?.dataset?.indexing_technique}
stopPosition='bottom'
detail={creationCache?.document}
detail={creationCache?.documents[0] as FullDocumentDetail}
/>
</div>
</div>
......@@ -58,4 +59,4 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
)
}
export default StepThree;
export default StepThree
web/app/components/datasets/create/step-two/index.tsx
View file @ 3b2b8199
......@@ -8,7 +8,7 @@ import cn from 'classnames'
import Link from 'next/link'
import PreviewItem from './preview-item'
import s from './index.module.css'
import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
import type { CreateDocumentReq, DataSourceType, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
import {
createDocument,
createFirstDocument,
......@@ -28,7 +28,9 @@ type StepTwoProps = {
onSetting: () => void
datasetId?: string
indexingType?: string
dataSourceType: DataSourceType
file?: File
notionPages?: any[]
onStepChange?: (delta: number) => void
updateIndexingTypeCache?: (type: string) => void
updateResultCache?: (res: createDocumentResponse) => void
......@@ -52,6 +54,7 @@ const StepTwo = ({
onSetting,
datasetId,
indexingType,
dataSourceType,
file,
onStepChange,
updateIndexingTypeCache,
......@@ -170,8 +173,14 @@ const StepTwo = ({
}
const getFileIndexingEstimateParams = () => {
// TODO
const params = {
info_list: {
data_source_type: dataSourceType,
file_info_list: [{
file_id: file?.id,
}],
},
dataset_id: datasetId,
indexing_technique: getIndexing_technique(),
process_rule: getProcessRule(),
......@@ -190,9 +199,11 @@ const StepTwo = ({
else {
params = {
data_source: {
type: 'upload_file',
info: file?.id,
name: file?.name,
type: dataSourceType,
// name: file?.name,
info: [{
upload_file_id: file?.id,
}],
},
indexing_technique: getIndexing_technique(),
process_rule: getProcessRule(),
......@@ -250,7 +261,7 @@ const StepTwo = ({
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
updateResultCache && updateResultCache({
document: res,
documents: [res],
})
}
onStepChange && onStepChange(+1)
......
web/models/datasets.ts
View file @ 3b2b8199
import { AppMode } from './app'
import type { AppMode } from './app'
export enum DataSourceType {
FILE = 'upload_file',
NOTION = 'notion_import',
WEB = 'web_import',
}
export type DataSet = {
id: string
......@@ -43,9 +49,9 @@ export type IndexingEstimateResponse = {
preview: string[]
}
export interface FileIndexingEstimateResponse extends IndexingEstimateResponse {
export type FileIndexingEstimateResponse = {
total_nodes: number
}
} & IndexingEstimateResponse
export type IndexingStatusResponse = {
id: string
......@@ -98,17 +104,17 @@ export const DocumentIndexingStatusList = [
export type DocumentIndexingStatus = typeof DocumentIndexingStatusList[number]
export const DisplayStatusList = [
"queuing",
"indexing",
"paused",
"error",
"available",
"enabled",
"disabled",
"archived",
] as const;
export type DocumentDisplayStatus = typeof DisplayStatusList[number];
'queuing',
'indexing',
'paused',
'error',
'available',
'enabled',
'disabled',
'archived',
] as const
export type DocumentDisplayStatus = typeof DisplayStatusList[number]
export type DataSourceInfo = {
upload_file: {
......@@ -157,16 +163,19 @@ export type DocumentListResponse = {
export type CreateDocumentReq = {
original_document_id?: string
indexing_technique?: string;
name: string
indexing_technique?: string
data_source: DataSource
process_rule: ProcessRule
}
export type DataSource = {
type: string
info: string // upload_file_id
name: string
// name: string
info: Info[] // upload_file_id
}
export type Info = {
upload_file_id: string
}
export type ProcessRule = {
......@@ -176,7 +185,7 @@ export type ProcessRule = {
export type createDocumentResponse = {
dataset?: DataSet
document: InitialDocumentDetail
documents: InitialDocumentDetail[]
}
export type FullDocumentDetail = SimpleDocumentDetail & {
......@@ -216,20 +225,20 @@ export type DocMetadata = {
}
export const CUSTOMIZABLE_DOC_TYPES = [
"book",
"web_page",
"paper",
"social_media_post",
"personal_document",
"business_document",
"im_chat_log",
] as const;
export const FIXED_DOC_TYPES = ["synced_from_github", "synced_from_notion", "wikipedia_entry"] as const;
export type CustomizableDocType = typeof CUSTOMIZABLE_DOC_TYPES[number];
export type FixedDocType = typeof FIXED_DOC_TYPES[number];
export type DocType = CustomizableDocType | FixedDocType;
'book',
'web_page',
'paper',
'social_media_post',
'personal_document',
'business_document',
'im_chat_log',
] as const
export const FIXED_DOC_TYPES = ['synced_from_github', 'synced_from_notion', 'wikipedia_entry'] as const
export type CustomizableDocType = typeof CUSTOMIZABLE_DOC_TYPES[number]
export type FixedDocType = typeof FIXED_DOC_TYPES[number]
export type DocType = CustomizableDocType | FixedDocType
export type DocumentDetailResponse = FullDocumentDetail
......
web/service/common.ts
View file @ 3b2b8199
......@@ -90,8 +90,8 @@ export const switchWorkspace: Fetcher<CommonResponse & { new_tenant: IWorkspace
return post(url, { body }) as Promise<CommonResponse & { new_tenant: IWorkspace }>
}
export const fetchDataSource: Fetcher<{ data: DataSourceNotion[] }, { url: string; params: Record<string, any> }> = ({ url, params }) => {
return get(url, { params }) as Promise<{ data: DataSourceNotion[] }>
export const fetchDataSource: Fetcher<{ data: DataSourceNotion[] }, { url: string }> = ({ url }) => {
return get(url) as Promise<{ data: DataSourceNotion[] }>
}
export const syncDataSourceNotion: Fetcher<CommonResponse, { url: string }> = ({ url }) => {
......
web/service/datasets.ts
View file @ 3b2b8199
import type { Fetcher } from 'swr'
import { del, get, post, put, patch } from './base'
import qs from 'qs'
import type { RelatedAppResponse, DataSet, HitTestingResponse, HitTestingRecordsResponse, DataSetListResponse, CreateDocumentReq, InitialDocumentDetail, DocumentDetailResponse, DocumentListResponse, IndexingEstimateResponse, FileIndexingEstimateResponse, IndexingStatusResponse, ProcessRuleResponse, SegmentsQuery, SegmentsResponse, createDocumentResponse } from '@/models/datasets'
import { del, get, patch, post, put } from './base'
import type { CreateDocumentReq, DataSet, DataSetListResponse, DocumentDetailResponse, DocumentListResponse, FileIndexingEstimateResponse, HitTestingRecordsResponse, HitTestingResponse, IndexingEstimateResponse, IndexingStatusResponse, InitialDocumentDetail, ProcessRuleResponse, RelatedAppResponse, SegmentsQuery, SegmentsResponse, createDocumentResponse } from '@/models/datasets'
import type { CommonResponse } from '@/models/common'
// apis for documents in a dataset
......@@ -19,17 +19,17 @@ export const fetchDataDetail: Fetcher<DataSet, string> = (datasetId: string) =>
return get(`/datasets/${datasetId}`) as Promise<DataSet>
}
export const updateDatasetSetting: Fetcher<DataSet, { datasetId: string, body: Partial<Pick<DataSet, 'name' | 'description' | 'permission' | 'indexing_technique'>>}> = ({ datasetId, body }) => {
return patch(`/datasets/${datasetId}`, { body } ) as Promise<DataSet>
export const updateDatasetSetting: Fetcher<DataSet, { datasetId: string; body: Partial<Pick<DataSet, 'name' | 'description' | 'permission' | 'indexing_technique'>> }> = ({ datasetId, body }) => {
return patch(`/datasets/${datasetId}`, { body }) as Promise<DataSet>
}
export const fetchDatasetRelatedApps: Fetcher<RelatedAppResponse, string> = (datasetId: string) => {
return get(`/datasets/${datasetId}/related-apps`) as Promise<RelatedAppResponse>
}
export const fetchDatasets: Fetcher<DataSetListResponse, { url: string, params: { page: number, ids?: string[], limit?: number } }> = ({ url, params }) => {
export const fetchDatasets: Fetcher<DataSetListResponse, { url: string; params: { page: number; ids?: string[]; limit?: number } }> = ({ url, params }) => {
const urlParams = qs.stringify(params, { indices: false })
return get(`${url}?${urlParams}`,) as Promise<DataSetListResponse>
return get(`${url}?${urlParams}`) as Promise<DataSetListResponse>
}
export const createEmptyDataset: Fetcher<DataSet, { name: string }> = ({ name }) => {
......@@ -52,7 +52,7 @@ export const fetchDocuments: Fetcher<DocumentListResponse, { datasetId: string;
}
export const createFirstDocument: Fetcher<createDocumentResponse, { body: CreateDocumentReq }> = ({ body }) => {
return post(`/datasets/init`, { body }) as Promise<createDocumentResponse>
return post('/datasets/init', { body }) as Promise<createDocumentResponse>
}
export const createDocument: Fetcher<InitialDocumentDetail, { datasetId: string; body: CreateDocumentReq }> = ({ datasetId, body }) => {
......@@ -123,5 +123,5 @@ export const fetchTestingRecords: Fetcher<HitTestingRecordsResponse, { datasetId
}
export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, any> = (body: any) => {
return post(`/datasets/file-indexing-estimate`, { body }) as Promise<FileIndexingEstimateResponse>
return post('/datasets/indexing-estimate', { body }) as Promise<FileIndexingEstimateResponse>
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment