Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
D
dify
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ai-tech
dify
Commits
3b2b8199
Commit
3b2b8199
authored
Jun 08, 2023
by
JzoNg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: notion import in step one
parent
761febc7
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
244 additions
and
101 deletions
+244
-101
Icon-3-dots.svg
web/app/components/datasets/create/assets/Icon-3-dots.svg
+3
-0
index.tsx
web/app/components/datasets/create/file-preview/index.tsx
+15
-10
index.tsx
web/app/components/datasets/create/index.tsx
+35
-10
index.module.css
web/app/components/datasets/create/step-one/index.module.css
+50
-0
index.tsx
web/app/components/datasets/create/step-one/index.tsx
+63
-24
index.tsx
web/app/components/datasets/create/step-three/index.tsx
+9
-8
index.tsx
web/app/components/datasets/create/step-two/index.tsx
+17
-6
datasets.ts
web/models/datasets.ts
+42
-33
common.ts
web/service/common.ts
+2
-2
datasets.ts
web/service/datasets.ts
+8
-8
No files found.
web/app/components/datasets/create/assets/Icon-3-dots.svg
0 → 100644
View file @
3b2b8199
<svg
width=
"16"
height=
"16"
viewBox=
"0 0 16 16"
fill=
"none"
xmlns=
"http://www.w3.org/2000/svg"
>
<path
d=
"M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103"
stroke=
"#374151"
stroke-width=
"2"
stroke-linecap=
"round"
stroke-linejoin=
"round"
/>
</svg>
web/app/components/datasets/create/file-preview/index.tsx
View file @
3b2b8199
'use client'
'use client'
import
React
,
{
use
State
,
useEffect
}
from
'react'
import
React
,
{
use
Effect
,
useState
}
from
'react'
import
{
useTranslation
}
from
'react-i18next'
import
{
useTranslation
}
from
'react-i18next'
import
type
{
File
}
from
'@/models/datasets'
import
{
fetchFilePreview
}
from
'@/service/common'
import
cn
from
'classnames'
import
cn
from
'classnames'
import
s
from
'./index.module.css'
import
s
from
'./index.module.css'
import
type
{
File
}
from
'@/models/datasets'
import
{
fetchFilePreview
}
from
'@/service/common'
type
IProps
=
{
type
IProps
=
{
file
?:
File
,
file
?:
File
hidePreview
:
()
=>
void
}
}
const
FilePreview
=
({
const
FilePreview
=
({
file
,
file
,
hidePreview
,
}:
IProps
)
=>
{
}:
IProps
)
=>
{
const
{
t
}
=
useTranslation
()
const
{
t
}
=
useTranslation
()
const
[
previewContent
,
setPreviewContent
]
=
useState
(
''
)
const
[
previewContent
,
setPreviewContent
]
=
useState
(
''
)
...
@@ -28,23 +29,27 @@ const FilePreview = ({
...
@@ -28,23 +29,27 @@ const FilePreview = ({
}
}
const
getFileName
=
(
currentFile
?:
File
)
=>
{
const
getFileName
=
(
currentFile
?:
File
)
=>
{
if
(
!
currentFile
)
{
if
(
!
currentFile
)
return
''
return
''
}
const
arr
=
currentFile
.
name
.
split
(
'.'
)
const
arr
=
currentFile
.
name
.
split
(
'.'
)
return
arr
.
slice
(
0
,
-
1
).
join
()
return
arr
.
slice
(
0
,
-
1
).
join
()
}
}
useEffect
(()
=>
{
useEffect
(()
=>
{
if
(
file
)
{
if
(
file
)
getPreviewContent
(
file
.
id
)
getPreviewContent
(
file
.
id
)
}
},
[
file
])
},
[
file
])
return
(
return
(
<
div
className=
{
cn
(
s
.
filePreview
)
}
>
<
div
className=
{
cn
(
s
.
filePreview
)
}
>
<
div
className=
{
cn
(
s
.
previewHeader
)
}
>
<
div
className=
{
cn
(
s
.
previewHeader
)
}
>
<
div
className=
{
cn
(
s
.
title
)
}
>
{
t
(
'datasetCreation.stepOne.filePreview'
)
}
</
div
>
<
div
className=
{
cn
(
s
.
title
)
}
>
<
span
>
{
t
(
'datasetCreation.stepOne.filePreview'
)
}
</
span
>
{
false
&&
(
<
div
className=
'flex items-center justify-center w-6 h-6 cursor-pointer'
onClick=
{
hidePreview
}
></
div
>
)
}
</
div
>
<
div
className=
{
cn
(
s
.
fileName
)
}
>
<
div
className=
{
cn
(
s
.
fileName
)
}
>
<
span
>
{
getFileName
(
file
)
}
</
span
><
span
className=
{
cn
(
s
.
filetype
)
}
>
.
{
file
?.
extension
}
</
span
>
<
span
>
{
getFileName
(
file
)
}
</
span
><
span
className=
{
cn
(
s
.
filetype
)
}
>
.
{
file
?.
extension
}
</
span
>
</
div
>
</
div
>
...
...
web/app/components/datasets/create/index.tsx
View file @
3b2b8199
'use client'
'use client'
import
React
,
{
use
State
,
useCallback
,
useEffect
}
from
'react'
import
React
,
{
use
Callback
,
useEffect
,
useState
}
from
'react'
import
{
useTranslation
}
from
'react-i18next'
import
{
useTranslation
}
from
'react-i18next'
import
{
useBoolean
}
from
'ahooks'
import
{
useBoolean
}
from
'ahooks'
import
type
{
DataSet
,
File
,
createDocumentResponse
}
from
'@/models/datasets'
import
AppUnavailable
from
'../../base/app-unavailable'
import
{
fetchTenantInfo
}
from
'@/service/common'
import
{
fetchDataDetail
}
from
'@/service/datasets'
import
StepsNavBar
from
'./steps-nav-bar'
import
StepsNavBar
from
'./steps-nav-bar'
import
StepOne
from
'./step-one'
import
StepOne
from
'./step-one'
import
StepTwo
from
'./step-two'
import
StepTwo
from
'./step-two'
import
StepThree
from
'./step-three'
import
StepThree
from
'./step-three'
import
{
DataSourceType
}
from
'@/models/datasets'
import
type
{
DataSet
,
File
,
createDocumentResponse
}
from
'@/models/datasets'
import
{
fetchDataSource
,
fetchTenantInfo
}
from
'@/service/common'
import
{
fetchDataDetail
}
from
'@/service/datasets'
import
AccountSetting
from
'@/app/components/header/account-setting'
import
AccountSetting
from
'@/app/components/header/account-setting'
import
AppUnavailable
from
'../../base/app-unavailable'
type
DatasetUpdateFormProps
=
{
type
DatasetUpdateFormProps
=
{
datasetId
?:
string
;
datasetId
?:
string
}
}
const
DatasetUpdateForm
=
({
datasetId
}:
DatasetUpdateFormProps
)
=>
{
const
DatasetUpdateForm
=
({
datasetId
}:
DatasetUpdateFormProps
)
=>
{
const
{
t
}
=
useTranslation
()
const
{
t
}
=
useTranslation
()
const
[
hasSetAPIKEY
,
setHasSetAPIKEY
]
=
useState
(
true
)
const
[
hasSetAPIKEY
,
setHasSetAPIKEY
]
=
useState
(
true
)
const
[
isShowSetAPIKey
,
{
setTrue
:
showSetAPIKey
,
setFalse
:
hideSetAPIkey
}]
=
useBoolean
()
const
[
isShowSetAPIKey
,
{
setTrue
:
showSetAPIKey
,
setFalse
:
hideSetAPIkey
}]
=
useBoolean
()
const
[
hasConnection
,
setHasConnection
]
=
useState
(
true
)
const
[
isShowDataSourceSetting
,
{
setTrue
:
showDataSourceSetting
,
setFalse
:
hideDataSourceSetting
}]
=
useBoolean
()
const
[
dataSourceType
,
setDataSourceType
]
=
useState
<
DataSourceType
>
(
DataSourceType
.
FILE
)
const
[
step
,
setStep
]
=
useState
(
1
)
const
[
step
,
setStep
]
=
useState
(
1
)
const
[
indexingTypeCache
,
setIndexTypeCache
]
=
useState
(
''
)
const
[
indexingTypeCache
,
setIndexTypeCache
]
=
useState
(
''
)
const
[
file
,
setFile
]
=
useState
<
File
|
undefined
>
()
const
[
file
,
setFile
]
=
useState
<
File
|
undefined
>
()
const
[
result
,
setResult
]
=
useState
<
createDocumentResponse
|
undefined
>
()
const
[
result
,
setResult
]
=
useState
<
createDocumentResponse
|
undefined
>
()
const
[
hasError
,
setHasError
]
=
useState
(
false
)
const
[
hasError
,
setHasError
]
=
useState
(
false
)
// TODO
const
[
notionPages
,
setNotionPages
]
=
useState
<
any
>
([])
const
updateNotionPages
=
(
value
:
any
[])
=>
{
setNotionPages
(
value
)
}
const
updateFile
=
(
file
?:
File
)
=>
{
const
updateFile
=
(
file
?:
File
)
=>
{
setFile
(
file
)
setFile
(
file
)
}
}
...
@@ -50,9 +60,15 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
...
@@ -50,9 +60,15 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
const
hasSetKey
=
data
.
providers
.
some
(({
is_valid
})
=>
is_valid
)
const
hasSetKey
=
data
.
providers
.
some
(({
is_valid
})
=>
is_valid
)
setHasSetAPIKEY
(
hasSetKey
)
setHasSetAPIKEY
(
hasSetKey
)
}
}
const
checkNotionConnection
=
async
()
=>
{
const
{
data
}
=
await
fetchDataSource
({
url
:
'/data-source/integrates'
})
const
hasConnection
=
data
.
filter
(
item
=>
item
.
provider
===
'notion'
)
||
[]
setHasConnection
(
hasConnection
.
length
>
0
)
}
useEffect
(()
=>
{
useEffect
(()
=>
{
checkAPIKey
()
checkAPIKey
()
checkNotionConnection
()
},
[])
},
[])
const
[
detail
,
setDetail
]
=
useState
<
DataSet
|
null
>
(
null
)
const
[
detail
,
setDetail
]
=
useState
<
DataSet
|
null
>
(
null
)
...
@@ -62,16 +78,16 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
...
@@ -62,16 +78,16 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
try
{
try
{
const
detail
=
await
fetchDataDetail
(
datasetId
)
const
detail
=
await
fetchDataDetail
(
datasetId
)
setDetail
(
detail
)
setDetail
(
detail
)
}
catch
(
e
)
{
}
catch
(
e
)
{
setHasError
(
true
)
setHasError
(
true
)
}
}
}
}
})()
})()
},
[
datasetId
])
},
[
datasetId
])
if
(
hasError
)
{
if
(
hasError
)
return
<
AppUnavailable
code=
{
500
}
unknownReason=
{
t
(
'datasetCreation.error.unavailable'
)
as
string
}
/>
return
<
AppUnavailable
code=
{
500
}
unknownReason=
{
t
(
'datasetCreation.error.unavailable'
)
as
string
}
/>
}
return
(
return
(
<
div
className=
'flex'
style=
{
{
height
:
'calc(100vh - 56px)'
}
}
>
<
div
className=
'flex'
style=
{
{
height
:
'calc(100vh - 56px)'
}
}
>
...
@@ -80,9 +96,15 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
...
@@ -80,9 +96,15 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
</
div
>
</
div
>
<
div
className=
"grow bg-white"
>
<
div
className=
"grow bg-white"
>
{
step
===
1
&&
<
StepOne
{
step
===
1
&&
<
StepOne
hasConnection=
{
hasConnection
}
onSetting=
{
showDataSourceSetting
}
datasetId=
{
datasetId
}
datasetId=
{
datasetId
}
dataSourceType=
{
dataSourceType
}
changeType=
{
setDataSourceType
}
file=
{
file
}
file=
{
file
}
updateFile=
{
updateFile
}
updateFile=
{
updateFile
}
notionPages=
{
notionPages
}
updateNotionPages=
{
updateNotionPages
}
onStepChange=
{
nextStep
}
onStepChange=
{
nextStep
}
/>
}
/>
}
{
(
step
===
2
&&
(
!
datasetId
||
(
datasetId
&&
!!
detail
)))
&&
<
StepTwo
{
(
step
===
2
&&
(
!
datasetId
||
(
datasetId
&&
!!
detail
)))
&&
<
StepTwo
...
@@ -90,7 +112,9 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
...
@@ -90,7 +112,9 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
onSetting=
{
showSetAPIKey
}
onSetting=
{
showSetAPIKey
}
indexingType=
{
detail
?.
indexing_technique
||
''
}
indexingType=
{
detail
?.
indexing_technique
||
''
}
datasetId=
{
datasetId
}
datasetId=
{
datasetId
}
dataSourceType=
{
dataSourceType
}
file=
{
file
}
file=
{
file
}
notionPages=
{
notionPages
}
onStepChange=
{
changeStep
}
onStepChange=
{
changeStep
}
updateIndexingTypeCache=
{
updateIndexingTypeCache
}
updateIndexingTypeCache=
{
updateIndexingTypeCache
}
updateResultCache=
{
updateResultCache
}
updateResultCache=
{
updateResultCache
}
...
@@ -106,6 +130,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
...
@@ -106,6 +130,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
await
checkAPIKey
()
await
checkAPIKey
()
hideSetAPIkey
()
hideSetAPIkey
()
}
}
/>
}
}
}
/>
}
{
isShowDataSourceSetting
&&
<
AccountSetting
activeTab=
"data-source"
onCancel=
{
hideDataSourceSetting
}
/>
}
</
div
>
</
div
>
)
)
}
}
...
...
web/app/components/datasets/create/step-one/index.module.css
View file @
3b2b8199
...
@@ -107,3 +107,53 @@
...
@@ -107,3 +107,53 @@
background
:
center
no-repeat
url(../assets/folder-plus.svg)
;
background
:
center
no-repeat
url(../assets/folder-plus.svg)
;
background-size
:
contain
;
background-size
:
contain
;
}
}
.notionConnectionTip
{
display
:
flex
;
flex-direction
:
column
;
align-items
:
flex-start
;
padding
:
24px
;
max-width
:
640px
;
background
:
#F9FAFB
;
border-radius
:
16px
;
}
.notionIcon
{
display
:
flex
;
padding
:
12px
;
width
:
48px
;
height
:
48px
;
background
:
#fff
center
no-repeat
url(../assets/notion.svg)
;
background-size
:
24px
;
border
:
0.5px
solid
#EAECF5
;
box-shadow
:
0px
12px
16px
-4px
rgba
(
16
,
24
,
40
,
0.08
),
0px
4px
6px
-2px
rgba
(
16
,
24
,
40
,
0.03
);
border-radius
:
12px
;
}
.notionConnectionTip
.title
{
position
:
relative
;
margin
:
24px
0
4px
;
font-style
:
normal
;
font-weight
:
600
;
font-size
:
16px
;
line-height
:
24px
;
color
:
#374151
;
}
.notionConnectionTip
.title
::after
{
content
:
''
;
position
:
absolute
;
top
:
-6px
;
right
:
-12px
;
width
:
16px
;
height
:
16px
;
background
:
center
no-repeat
url(../assets/Icon-3-dots.svg)
;
background-size
:
contain
;
}
.notionConnectionTip
.tip
{
margin-bottom
:
20px
;
font-style
:
normal
;
font-weight
:
400
;
font-size
:
13px
;
line-height
:
18px
;
color
:
#6B7280
;
}
web/app/components/datasets/create/step-one/index.tsx
View file @
3b2b8199
'use client'
'use client'
import
React
,
{
useState
}
from
'react'
import
React
,
{
useState
}
from
'react'
import
{
useTranslation
}
from
'react-i18next'
import
{
useTranslation
}
from
'react-i18next'
import
type
{
File
}
from
'@/models/dataset
s'
import
cn
from
'classname
s'
import
FilePreview
from
'../file-preview'
import
FilePreview
from
'../file-preview'
import
FileUploader
from
'../file-uploader'
import
FileUploader
from
'../file-uploader'
import
EmptyDatasetCreationModal
from
'../empty-dataset-creation-modal'
import
EmptyDatasetCreationModal
from
'../empty-dataset-creation-modal'
import
Button
from
'@/app/components/base/button'
import
cn
from
'classnames'
import
s
from
'./index.module.css'
import
s
from
'./index.module.css'
import
type
{
File
}
from
'@/models/datasets'
import
{
DataSourceType
}
from
'@/models/datasets'
import
Button
from
'@/app/components/base/button'
import
{
NotionPageSelector
}
from
'@/app/components/base/notion-page-selector'
type
IStepOneProps
=
{
type
IStepOneProps
=
{
datasetId
?:
string
,
datasetId
?:
string
file
?:
File
,
dataSourceType
:
DataSourceType
updateFile
:
(
file
?:
File
)
=>
void
,
hasConnection
:
boolean
onStepChange
:
()
=>
void
,
onSetting
:
()
=>
void
file
?:
File
updateFile
:
(
file
?:
File
)
=>
void
notionPages
?:
any
[]
updateNotionPages
:
(
value
:
any
[])
=>
void
onStepChange
:
()
=>
void
changeType
:
(
type
:
DataSourceType
)
=>
void
}
}
const
StepOne
=
({
const
StepOne
=
({
datasetId
,
datasetId
,
dataSourceType
,
changeType
,
hasConnection
,
onSetting
,
onStepChange
,
onStepChange
,
file
,
file
,
updateFile
,
updateFile
,
notionPages
=
[],
}:
IStepOneProps
)
=>
{
}:
IStepOneProps
)
=>
{
const
[
dataSourceType
,
setDataSourceType
]
=
useState
(
'FILE'
)
const
[
showModal
,
setShowModal
]
=
useState
(
false
)
const
[
showModal
,
setShowModal
]
=
useState
(
false
)
const
[
showFilePreview
,
setShowFilePreview
]
=
useState
(
true
)
const
{
t
}
=
useTranslation
()
const
{
t
}
=
useTranslation
()
const
hidePreview
=
()
=>
setShowFilePreview
(
false
)
const
modalShowHandle
=
()
=>
setShowModal
(
true
)
const
modalShowHandle
=
()
=>
setShowModal
(
true
)
const
modalCloseHandle
=
()
=>
setShowModal
(
false
)
const
modalCloseHandle
=
()
=>
setShowModal
(
false
)
...
@@ -38,41 +52,66 @@ const StepOne = ({
...
@@ -38,41 +52,66 @@ const StepOne = ({
<
div
className=
{
s
.
form
}
>
<
div
className=
{
s
.
form
}
>
<
div
className=
{
s
.
dataSourceTypeList
}
>
<
div
className=
{
s
.
dataSourceTypeList
}
>
<
div
<
div
className=
{
cn
(
s
.
dataSourceItem
,
dataSourceType
===
'FILE'
&&
s
.
active
)
}
className=
{
cn
(
s
.
dataSourceItem
,
dataSourceType
===
DataSourceType
.
FILE
&&
s
.
active
)
}
onClick=
{
()
=>
setDataSourceType
(
'FILE'
)
}
onClick=
{
()
=>
changeType
(
DataSourceType
.
FILE
)
}
>
>
<
span
className=
{
cn
(
s
.
datasetIcon
)
}
/>
<
span
className=
{
cn
(
s
.
datasetIcon
)
}
/>
{
t
(
'datasetCreation.stepOne.dataSourceType.file'
)
}
{
t
(
'datasetCreation.stepOne.dataSourceType.file'
)
}
</
div
>
</
div
>
<
div
<
div
className=
{
cn
(
s
.
dataSourceItem
,
s
.
disabled
,
dataSourceType
===
'notion'
&&
s
.
active
)
}
className=
{
cn
(
s
.
dataSourceItem
,
dataSourceType
===
DataSourceType
.
NOTION
&&
s
.
active
)
}
// onClick={() => setDataSourceType('notion'
)}
onClick=
{
()
=>
changeType
(
DataSourceType
.
NOTION
)
}
>
>
<
span
className=
{
s
.
comingTag
}
>
Coming soon
</
span
>
<
span
className=
{
cn
(
s
.
datasetIcon
,
s
.
notion
)
}
/>
<
span
className=
{
cn
(
s
.
datasetIcon
,
s
.
notion
)
}
/>
{
t
(
'datasetCreation.stepOne.dataSourceType.notion'
)
}
{
t
(
'datasetCreation.stepOne.dataSourceType.notion'
)
}
</
div
>
</
div
>
<
div
<
div
className=
{
cn
(
s
.
dataSourceItem
,
s
.
disabled
,
dataSourceType
===
'web'
&&
s
.
active
)
}
className=
{
cn
(
s
.
dataSourceItem
,
s
.
disabled
,
dataSourceType
===
DataSourceType
.
WEB
&&
s
.
active
)
}
// onClick={() => setDataSourceType('web'
)}
// onClick={() => changeType(DataSourceType.WEB
)}
>
>
<
span
className=
{
s
.
comingTag
}
>
Coming soon
</
span
>
<
span
className=
{
s
.
comingTag
}
>
Coming soon
</
span
>
<
span
className=
{
cn
(
s
.
datasetIcon
,
s
.
web
)
}
/>
<
span
className=
{
cn
(
s
.
datasetIcon
,
s
.
web
)
}
/>
{
t
(
'datasetCreation.stepOne.dataSourceType.web'
)
}
{
t
(
'datasetCreation.stepOne.dataSourceType.web'
)
}
</
div
>
</
div
>
</
div
>
</
div
>
<
FileUploader
onFileUpdate=
{
updateFile
}
file=
{
file
}
/>
{
dataSourceType
===
DataSourceType
.
FILE
&&
(
<
Button
disabled=
{
!
file
}
className=
{
s
.
submitButton
}
type=
'primary'
onClick=
{
onStepChange
}
>
{
t
(
'datasetCreation.stepOne.button'
)
}
</
Button
>
<>
<
FileUploader
onFileUpdate=
{
updateFile
}
file=
{
file
}
/>
<
Button
disabled=
{
!
file
}
className=
{
s
.
submitButton
}
type=
'primary'
onClick=
{
onStepChange
}
>
{
t
(
'datasetCreation.stepOne.button'
)
}
</
Button
>
</>
)
}
{
dataSourceType
===
DataSourceType
.
NOTION
&&
(
<>
{
!
hasConnection
&&
(
<
div
className=
{
s
.
notionConnectionTip
}
>
<
span
className=
{
s
.
notionIcon
}
/>
<
div
className=
{
s
.
title
}
>
Notion is not connected
</
div
>
<
div
className=
{
s
.
tip
}
>
To sync with Notion, connection to Notion must be established first.
</
div
>
<
Button
className=
'h-8'
type=
'primary'
onClick=
{
onSetting
}
>
Go to connect
</
Button
>
</
div
>
)
}
{
hasConnection
&&
(
<>
{
/* TODO */
}
<
div
className=
'mb-8 w-[640px]'
>
<
NotionPageSelector
/>
</
div
>
<
Button
disabled=
{
!
notionPages
.
length
}
className=
{
s
.
submitButton
}
type=
'primary'
onClick=
{
onStepChange
}
>
{
t
(
'datasetCreation.stepOne.button'
)
}
</
Button
>
</>
)
}
</>
)
}
{
!
datasetId
&&
(
{
!
datasetId
&&
(
<>
<>
<
div
className=
{
s
.
dividerLine
}
/>
<
div
className=
{
s
.
dividerLine
}
/>
<
div
onClick=
{
modalShowHandle
}
className=
{
s
.
OtherCreationOption
}
>
{
t
(
'datasetCreation.stepOne.emptyDatasetCreation'
)
}
</
div
>
<
div
onClick=
{
modalShowHandle
}
className=
{
s
.
OtherCreationOption
}
>
{
t
(
'datasetCreation.stepOne.emptyDatasetCreation'
)
}
</
div
>
</>
</>
)
}
)
}
</
div
>
</
div
>
<
EmptyDatasetCreationModal
show=
{
showModal
}
onHide=
{
modalCloseHandle
}
/>
<
EmptyDatasetCreationModal
show=
{
showModal
}
onHide=
{
modalCloseHandle
}
/>
</
div
>
</
div
>
{
file
&&
<
FilePreview
file=
{
file
}
/>
}
{
file
&&
showFilePreview
&&
<
FilePreview
file=
{
file
}
hidePreview=
{
hidePreview
}
/>
}
{
/* TODO notion page preview */
}
</
div
>
</
div
>
)
)
}
}
...
...
web/app/components/datasets/create/step-three/index.tsx
View file @
3b2b8199
'use client'
'use client'
import
React
from
'react'
import
React
from
'react'
import
{
useTranslation
}
from
'react-i18next'
import
{
useTranslation
}
from
'react-i18next'
import
type
{
createDocumentResponse
}
from
'@/models/dataset
s'
import
cn
from
'classname
s'
import
EmbeddingDetail
from
'../../documents/detail/embedding'
import
EmbeddingDetail
from
'../../documents/detail/embedding'
import
cn
from
'classnames'
import
s
from
'./index.module.css'
import
s
from
'./index.module.css'
import
type
{
FullDocumentDetail
,
createDocumentResponse
}
from
'@/models/datasets'
type
StepThreeProps
=
{
type
StepThreeProps
=
{
datasetId
?:
string
,
datasetId
?:
string
datasetName
?:
string
,
datasetName
?:
string
indexingType
?:
string
,
indexingType
?:
string
creationCache
?:
createDocumentResponse
creationCache
?:
createDocumentResponse
}
}
...
@@ -38,12 +38,13 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
...
@@ -38,12 +38,13 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
<
div
className=
{
s
.
content
}
>
{
`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`
}
</
div
>
<
div
className=
{
s
.
content
}
>
{
`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`
}
</
div
>
</
div
>
</
div
>
)
}
)
}
{
/* TODO multi doc display */
}
<
EmbeddingDetail
<
EmbeddingDetail
datasetId=
{
datasetId
||
creationCache
?.
dataset
?.
id
}
datasetId=
{
datasetId
||
creationCache
?.
dataset
?.
id
}
documentId=
{
creationCache
?.
document
.
id
}
documentId=
{
creationCache
?.
document
s
[
0
]
.
id
}
indexingType=
{
indexingType
||
creationCache
?.
dataset
?.
indexing_technique
}
indexingType=
{
indexingType
||
creationCache
?.
dataset
?.
indexing_technique
}
stopPosition=
'bottom'
stopPosition=
'bottom'
detail=
{
creationCache
?.
document
}
detail=
{
creationCache
?.
document
s
[
0
]
as
FullDocumentDetail
}
/>
/>
</
div
>
</
div
>
</
div
>
</
div
>
...
@@ -58,4 +59,4 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
...
@@ -58,4 +59,4 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: Step
)
)
}
}
export
default
StepThree
;
export
default
StepThree
web/app/components/datasets/create/step-two/index.tsx
View file @
3b2b8199
...
@@ -8,7 +8,7 @@ import cn from 'classnames'
...
@@ -8,7 +8,7 @@ import cn from 'classnames'
import
Link
from
'next/link'
import
Link
from
'next/link'
import
PreviewItem
from
'./preview-item'
import
PreviewItem
from
'./preview-item'
import
s
from
'./index.module.css'
import
s
from
'./index.module.css'
import
type
{
CreateDocumentReq
,
File
,
FullDocumentDetail
,
FileIndexingEstimateResponse
as
IndexingEstimateResponse
,
PreProcessingRule
,
Rules
,
createDocumentResponse
}
from
'@/models/datasets'
import
type
{
CreateDocumentReq
,
DataSourceType
,
File
,
FullDocumentDetail
,
FileIndexingEstimateResponse
as
IndexingEstimateResponse
,
PreProcessingRule
,
Rules
,
createDocumentResponse
}
from
'@/models/datasets'
import
{
import
{
createDocument
,
createDocument
,
createFirstDocument
,
createFirstDocument
,
...
@@ -28,7 +28,9 @@ type StepTwoProps = {
...
@@ -28,7 +28,9 @@ type StepTwoProps = {
onSetting
:
()
=>
void
onSetting
:
()
=>
void
datasetId
?:
string
datasetId
?:
string
indexingType
?:
string
indexingType
?:
string
dataSourceType
:
DataSourceType
file
?:
File
file
?:
File
notionPages
?:
any
[]
onStepChange
?:
(
delta
:
number
)
=>
void
onStepChange
?:
(
delta
:
number
)
=>
void
updateIndexingTypeCache
?:
(
type
:
string
)
=>
void
updateIndexingTypeCache
?:
(
type
:
string
)
=>
void
updateResultCache
?:
(
res
:
createDocumentResponse
)
=>
void
updateResultCache
?:
(
res
:
createDocumentResponse
)
=>
void
...
@@ -52,6 +54,7 @@ const StepTwo = ({
...
@@ -52,6 +54,7 @@ const StepTwo = ({
onSetting
,
onSetting
,
datasetId
,
datasetId
,
indexingType
,
indexingType
,
dataSourceType
,
file
,
file
,
onStepChange
,
onStepChange
,
updateIndexingTypeCache
,
updateIndexingTypeCache
,
...
@@ -170,8 +173,14 @@ const StepTwo = ({
...
@@ -170,8 +173,14 @@ const StepTwo = ({
}
}
const
getFileIndexingEstimateParams
=
()
=>
{
const
getFileIndexingEstimateParams
=
()
=>
{
// TODO
const
params
=
{
const
params
=
{
file_id
:
file
?.
id
,
info_list
:
{
data_source_type
:
dataSourceType
,
file_info_list
:
[{
file_id
:
file
?.
id
,
}],
},
dataset_id
:
datasetId
,
dataset_id
:
datasetId
,
indexing_technique
:
getIndexing_technique
(),
indexing_technique
:
getIndexing_technique
(),
process_rule
:
getProcessRule
(),
process_rule
:
getProcessRule
(),
...
@@ -190,9 +199,11 @@ const StepTwo = ({
...
@@ -190,9 +199,11 @@ const StepTwo = ({
else
{
else
{
params
=
{
params
=
{
data_source
:
{
data_source
:
{
type
:
'upload_file'
,
type
:
dataSourceType
,
info
:
file
?.
id
,
// name: file?.name,
name
:
file
?.
name
,
info
:
[{
upload_file_id
:
file
?.
id
,
}],
},
},
indexing_technique
:
getIndexing_technique
(),
indexing_technique
:
getIndexing_technique
(),
process_rule
:
getProcessRule
(),
process_rule
:
getProcessRule
(),
...
@@ -250,7 +261,7 @@ const StepTwo = ({
...
@@ -250,7 +261,7 @@ const StepTwo = ({
})
})
updateIndexingTypeCache
&&
updateIndexingTypeCache
(
indexType
)
updateIndexingTypeCache
&&
updateIndexingTypeCache
(
indexType
)
updateResultCache
&&
updateResultCache
({
updateResultCache
&&
updateResultCache
({
document
:
res
,
document
s
:
[
res
]
,
})
})
}
}
onStepChange
&&
onStepChange
(
+
1
)
onStepChange
&&
onStepChange
(
+
1
)
...
...
web/models/datasets.ts
View file @
3b2b8199
import
{
AppMode
}
from
'./app'
import
type
{
AppMode
}
from
'./app'
export
enum
DataSourceType
{
FILE
=
'upload_file'
,
NOTION
=
'notion_import'
,
WEB
=
'web_import'
,
}
export
type
DataSet
=
{
export
type
DataSet
=
{
id
:
string
id
:
string
...
@@ -43,9 +49,9 @@ export type IndexingEstimateResponse = {
...
@@ -43,9 +49,9 @@ export type IndexingEstimateResponse = {
preview
:
string
[]
preview
:
string
[]
}
}
export
interface
FileIndexingEstimateResponse
extends
IndexingEstimateResponse
{
export
type
FileIndexingEstimateResponse
=
{
total_nodes
:
number
total_nodes
:
number
}
}
&
IndexingEstimateResponse
export
type
IndexingStatusResponse
=
{
export
type
IndexingStatusResponse
=
{
id
:
string
id
:
string
...
@@ -98,17 +104,17 @@ export const DocumentIndexingStatusList = [
...
@@ -98,17 +104,17 @@ export const DocumentIndexingStatusList = [
export
type
DocumentIndexingStatus
=
typeof
DocumentIndexingStatusList
[
number
]
export
type
DocumentIndexingStatus
=
typeof
DocumentIndexingStatusList
[
number
]
export
const
DisplayStatusList
=
[
export
const
DisplayStatusList
=
[
"queuing"
,
'queuing'
,
"indexing"
,
'indexing'
,
"paused"
,
'paused'
,
"error"
,
'error'
,
"available"
,
'available'
,
"enabled"
,
'enabled'
,
"disabled"
,
'disabled'
,
"archived"
,
'archived'
,
]
as
const
;
]
as
const
export
type
DocumentDisplayStatus
=
typeof
DisplayStatusList
[
number
]
;
export
type
DocumentDisplayStatus
=
typeof
DisplayStatusList
[
number
]
export
type
DataSourceInfo
=
{
export
type
DataSourceInfo
=
{
upload_file
:
{
upload_file
:
{
...
@@ -157,16 +163,19 @@ export type DocumentListResponse = {
...
@@ -157,16 +163,19 @@ export type DocumentListResponse = {
export
type
CreateDocumentReq
=
{
export
type
CreateDocumentReq
=
{
original_document_id
?:
string
original_document_id
?:
string
indexing_technique
?:
string
;
indexing_technique
?:
string
name
:
string
data_source
:
DataSource
data_source
:
DataSource
process_rule
:
ProcessRule
process_rule
:
ProcessRule
}
}
export
type
DataSource
=
{
export
type
DataSource
=
{
type
:
string
type
:
string
info
:
string
// upload_file_id
// name: string
name
:
string
info
:
Info
[]
// upload_file_id
}
export
type
Info
=
{
upload_file_id
:
string
}
}
export
type
ProcessRule
=
{
export
type
ProcessRule
=
{
...
@@ -176,7 +185,7 @@ export type ProcessRule = {
...
@@ -176,7 +185,7 @@ export type ProcessRule = {
export
type
createDocumentResponse
=
{
export
type
createDocumentResponse
=
{
dataset
?:
DataSet
dataset
?:
DataSet
document
:
InitialDocumentDetail
document
s
:
InitialDocumentDetail
[]
}
}
export
type
FullDocumentDetail
=
SimpleDocumentDetail
&
{
export
type
FullDocumentDetail
=
SimpleDocumentDetail
&
{
...
@@ -216,20 +225,20 @@ export type DocMetadata = {
...
@@ -216,20 +225,20 @@ export type DocMetadata = {
}
}
export
const
CUSTOMIZABLE_DOC_TYPES
=
[
export
const
CUSTOMIZABLE_DOC_TYPES
=
[
"book"
,
'book'
,
"web_page"
,
'web_page'
,
"paper"
,
'paper'
,
"social_media_post"
,
'social_media_post'
,
"personal_document"
,
'personal_document'
,
"business_document"
,
'business_document'
,
"im_chat_log"
,
'im_chat_log'
,
]
as
const
;
]
as
const
export
const
FIXED_DOC_TYPES
=
[
"synced_from_github"
,
"synced_from_notion"
,
"wikipedia_entry"
]
as
const
;
export
const
FIXED_DOC_TYPES
=
[
'synced_from_github'
,
'synced_from_notion'
,
'wikipedia_entry'
]
as
const
export
type
CustomizableDocType
=
typeof
CUSTOMIZABLE_DOC_TYPES
[
number
]
;
export
type
CustomizableDocType
=
typeof
CUSTOMIZABLE_DOC_TYPES
[
number
]
export
type
FixedDocType
=
typeof
FIXED_DOC_TYPES
[
number
]
;
export
type
FixedDocType
=
typeof
FIXED_DOC_TYPES
[
number
]
export
type
DocType
=
CustomizableDocType
|
FixedDocType
;
export
type
DocType
=
CustomizableDocType
|
FixedDocType
export
type
DocumentDetailResponse
=
FullDocumentDetail
export
type
DocumentDetailResponse
=
FullDocumentDetail
...
...
web/service/common.ts
View file @
3b2b8199
...
@@ -90,8 +90,8 @@ export const switchWorkspace: Fetcher<CommonResponse & { new_tenant: IWorkspace
...
@@ -90,8 +90,8 @@ export const switchWorkspace: Fetcher<CommonResponse & { new_tenant: IWorkspace
return
post
(
url
,
{
body
})
as
Promise
<
CommonResponse
&
{
new_tenant
:
IWorkspace
}
>
return
post
(
url
,
{
body
})
as
Promise
<
CommonResponse
&
{
new_tenant
:
IWorkspace
}
>
}
}
export
const
fetchDataSource
:
Fetcher
<
{
data
:
DataSourceNotion
[]
},
{
url
:
string
;
params
:
Record
<
string
,
any
>
}
>
=
({
url
,
params
})
=>
{
export
const
fetchDataSource
:
Fetcher
<
{
data
:
DataSourceNotion
[]
},
{
url
:
string
}
>
=
({
url
})
=>
{
return
get
(
url
,
{
params
}
)
as
Promise
<
{
data
:
DataSourceNotion
[]
}
>
return
get
(
url
)
as
Promise
<
{
data
:
DataSourceNotion
[]
}
>
}
}
export
const
syncDataSourceNotion
:
Fetcher
<
CommonResponse
,
{
url
:
string
}
>
=
({
url
})
=>
{
export
const
syncDataSourceNotion
:
Fetcher
<
CommonResponse
,
{
url
:
string
}
>
=
({
url
})
=>
{
...
...
web/service/datasets.ts
View file @
3b2b8199
import
type
{
Fetcher
}
from
'swr'
import
type
{
Fetcher
}
from
'swr'
import
{
del
,
get
,
post
,
put
,
patch
}
from
'./base'
import
qs
from
'qs'
import
qs
from
'qs'
import
type
{
RelatedAppResponse
,
DataSet
,
HitTestingResponse
,
HitTestingRecordsResponse
,
DataSetListResponse
,
CreateDocumentReq
,
InitialDocumentDetail
,
DocumentDetailResponse
,
DocumentListResponse
,
IndexingEstimateResponse
,
FileIndexingEstimateResponse
,
IndexingStatusResponse
,
ProcessRuleResponse
,
SegmentsQuery
,
SegmentsResponse
,
createDocumentResponse
}
from
'@/models/datasets'
import
{
del
,
get
,
patch
,
post
,
put
}
from
'./base'
import
type
{
CreateDocumentReq
,
DataSet
,
DataSetListResponse
,
DocumentDetailResponse
,
DocumentListResponse
,
FileIndexingEstimateResponse
,
HitTestingRecordsResponse
,
HitTestingResponse
,
IndexingEstimateResponse
,
IndexingStatusResponse
,
InitialDocumentDetail
,
ProcessRuleResponse
,
RelatedAppResponse
,
SegmentsQuery
,
SegmentsResponse
,
createDocumentResponse
}
from
'@/models/datasets'
import
type
{
CommonResponse
}
from
'@/models/common'
import
type
{
CommonResponse
}
from
'@/models/common'
// apis for documents in a dataset
// apis for documents in a dataset
...
@@ -19,17 +19,17 @@ export const fetchDataDetail: Fetcher<DataSet, string> = (datasetId: string) =>
...
@@ -19,17 +19,17 @@ export const fetchDataDetail: Fetcher<DataSet, string> = (datasetId: string) =>
return
get
(
`/datasets/
${
datasetId
}
`
)
as
Promise
<
DataSet
>
return
get
(
`/datasets/
${
datasetId
}
`
)
as
Promise
<
DataSet
>
}
}
export
const
updateDatasetSetting
:
Fetcher
<
DataSet
,
{
datasetId
:
string
,
body
:
Partial
<
Pick
<
DataSet
,
'name'
|
'description'
|
'permission'
|
'indexing_technique'
>>
}
>
=
({
datasetId
,
body
})
=>
{
export
const
updateDatasetSetting
:
Fetcher
<
DataSet
,
{
datasetId
:
string
;
body
:
Partial
<
Pick
<
DataSet
,
'name'
|
'description'
|
'permission'
|
'indexing_technique'
>>
}
>
=
({
datasetId
,
body
})
=>
{
return
patch
(
`/datasets/
${
datasetId
}
`
,
{
body
}
)
as
Promise
<
DataSet
>
return
patch
(
`/datasets/
${
datasetId
}
`
,
{
body
})
as
Promise
<
DataSet
>
}
}
export
const
fetchDatasetRelatedApps
:
Fetcher
<
RelatedAppResponse
,
string
>
=
(
datasetId
:
string
)
=>
{
export
const
fetchDatasetRelatedApps
:
Fetcher
<
RelatedAppResponse
,
string
>
=
(
datasetId
:
string
)
=>
{
return
get
(
`/datasets/
${
datasetId
}
/related-apps`
)
as
Promise
<
RelatedAppResponse
>
return
get
(
`/datasets/
${
datasetId
}
/related-apps`
)
as
Promise
<
RelatedAppResponse
>
}
}
export
const
fetchDatasets
:
Fetcher
<
DataSetListResponse
,
{
url
:
string
,
params
:
{
page
:
number
,
ids
?:
string
[],
limit
?:
number
}
}
>
=
({
url
,
params
})
=>
{
export
const
fetchDatasets
:
Fetcher
<
DataSetListResponse
,
{
url
:
string
;
params
:
{
page
:
number
;
ids
?:
string
[];
limit
?:
number
}
}
>
=
({
url
,
params
})
=>
{
const
urlParams
=
qs
.
stringify
(
params
,
{
indices
:
false
})
const
urlParams
=
qs
.
stringify
(
params
,
{
indices
:
false
})
return
get
(
`
${
url
}
?
${
urlParams
}
`
,
)
as
Promise
<
DataSetListResponse
>
return
get
(
`
${
url
}
?
${
urlParams
}
`
)
as
Promise
<
DataSetListResponse
>
}
}
export
const
createEmptyDataset
:
Fetcher
<
DataSet
,
{
name
:
string
}
>
=
({
name
})
=>
{
export
const
createEmptyDataset
:
Fetcher
<
DataSet
,
{
name
:
string
}
>
=
({
name
})
=>
{
...
@@ -52,7 +52,7 @@ export const fetchDocuments: Fetcher<DocumentListResponse, { datasetId: string;
...
@@ -52,7 +52,7 @@ export const fetchDocuments: Fetcher<DocumentListResponse, { datasetId: string;
}
}
export
const
createFirstDocument
:
Fetcher
<
createDocumentResponse
,
{
body
:
CreateDocumentReq
}
>
=
({
body
})
=>
{
export
const
createFirstDocument
:
Fetcher
<
createDocumentResponse
,
{
body
:
CreateDocumentReq
}
>
=
({
body
})
=>
{
return
post
(
`/datasets/init`
,
{
body
})
as
Promise
<
createDocumentResponse
>
return
post
(
'/datasets/init'
,
{
body
})
as
Promise
<
createDocumentResponse
>
}
}
export
const
createDocument
:
Fetcher
<
InitialDocumentDetail
,
{
datasetId
:
string
;
body
:
CreateDocumentReq
}
>
=
({
datasetId
,
body
})
=>
{
export
const
createDocument
:
Fetcher
<
InitialDocumentDetail
,
{
datasetId
:
string
;
body
:
CreateDocumentReq
}
>
=
({
datasetId
,
body
})
=>
{
...
@@ -123,5 +123,5 @@ export const fetchTestingRecords: Fetcher<HitTestingRecordsResponse, { datasetId
...
@@ -123,5 +123,5 @@ export const fetchTestingRecords: Fetcher<HitTestingRecordsResponse, { datasetId
}
}
export
const
fetchFileIndexingEstimate
:
Fetcher
<
FileIndexingEstimateResponse
,
any
>
=
(
body
:
any
)
=>
{
export
const
fetchFileIndexingEstimate
:
Fetcher
<
FileIndexingEstimateResponse
,
any
>
=
(
body
:
any
)
=>
{
return
post
(
`/datasets/file-indexing-estimate`
,
{
body
})
as
Promise
<
FileIndexingEstimateResponse
>
return
post
(
'/datasets/indexing-estimate'
,
{
body
})
as
Promise
<
FileIndexingEstimateResponse
>
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment