Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
D
dify
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ai-tech
dify
Commits
75e302eb
Commit
75e302eb
authored
Jun 14, 2023
by
JzoNg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: add proccess of data embedding
parent
3cf48c8f
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
109 additions
and
48 deletions
+109
-48
index.module.css
...onents/datasets/create/embedding-process/index.module.css
+44
-23
index.tsx
...pp/components/datasets/create/embedding-process/index.tsx
+65
-25
No files found.
web/app/components/datasets/create/embedding-process/index.module.css
View file @
75e302eb
.progressBar
{
@apply
absolute
top-0
h-4;
}
.barPaused
{
background
:
linear-gradient
(
270deg
,
rgba
(
208
,
213
,
221
,
0.8
)
-2.21%
,
rgba
(
208
,
213
,
221
,
0.5
)
100%
);
}
.barProcessing
{
background
:
linear-gradient
(
90deg
,
rgba
(
41
,
112
,
255
,
0.9
)
0%
,
rgba
(
21
,
94
,
239
,
0.9
)
100%
);
}
.progressContainer
{
.progressContainer
{
@apply
relative
flex
mb-2
h-4
rounded-md
w-full;
@apply
relative
pb-4
w-full;
}
.sourceItem
{
display
:
flex
;
justify-content
:
space-between
;
align-items
:
center
;
padding
:
0
4px
;
height
:
24px
;
background
:
#EFF4FF
;
border-radius
:
6px
;
overflow
:
hidden
;
}
.sourceItem.error
{
background
:
#FEE4E2
;
}
.sourceItem.success
{
background
:
#D1FADF
;
}
}
.progressBgItem
{
.progressbar
{
@apply
flex-1
border-r
border-r-white
first
:
rounded-l-md
;
position
:
absolute
;
top
:
0
;
left
:
0
;
height
:
100%
;
background-color
:
#B2CCFF
;
}
}
.progressBgItem
:nth-last-child
(
2
)
{
.sourceItem
.info
.name
{
@apply
rounded-r-md;
font-weight
:
500
;
font-size
:
12px
;
line-height
:
18px
;
color
:
#101828
;
}
}
.sourceItem.success
.info
.name
{
color
:
#05603A
;
}
.sourceItem
.percent
{
font-weight
:
500
;
font-size
:
12px
;
line-height
:
18px
;
color
:
#344054
;
}
.sourceItem
.error
{
color
:
#D92D20
;
}
.sourceItem
.success
{
color
:
#05603A
;
}
.cost
{
.cost
{
@apply
flex
justify-between
items-center
text-xs
text-gray-700;
@apply
flex
justify-between
items-center
text-xs
text-gray-700;
...
...
web/app/components/datasets/create/embedding-process/index.tsx
View file @
75e302eb
...
@@ -13,6 +13,8 @@ import Button from '@/app/components/base/button'
...
@@ -13,6 +13,8 @@ import Button from '@/app/components/base/button'
import
type
{
FullDocumentDetail
,
IndexingStatusResponse
,
ProcessRuleResponse
}
from
'@/models/datasets'
import
type
{
FullDocumentDetail
,
IndexingStatusResponse
,
ProcessRuleResponse
}
from
'@/models/datasets'
import
{
formatNumber
}
from
'@/utils/format'
import
{
formatNumber
}
from
'@/utils/format'
import
{
fetchIndexingStatusBatch
as
doFetchIndexingStatus
,
fetchIndexingEstimateBatch
,
fetchProcessRule
}
from
'@/service/datasets'
import
{
fetchIndexingStatusBatch
as
doFetchIndexingStatus
,
fetchIndexingEstimateBatch
,
fetchProcessRule
}
from
'@/service/datasets'
import
{
DataSourceType
}
from
'@/models/datasets'
// import NotionIcon from '@/app/components/base/notion-icon'
type
Props
=
{
type
Props
=
{
datasetId
:
string
datasetId
:
string
...
@@ -140,15 +142,35 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
...
@@ -140,15 +142,35 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
return
indexingStatusBatchDetail
.
every
((
indexingStatusDetail
:
{
indexing_status
:
any
})
=>
[
'completed'
,
'error'
].
includes
(
indexingStatusDetail
?.
indexing_status
||
''
))
return
indexingStatusBatchDetail
.
every
((
indexingStatusDetail
:
{
indexing_status
:
any
})
=>
[
'completed'
,
'error'
].
includes
(
indexingStatusDetail
?.
indexing_status
||
''
))
},
[
indexingStatusBatchDetail
])
},
[
indexingStatusBatchDetail
])
const
getSourceName
=
(
id
:
string
)
=>
{
const
doc
=
documents
.
find
(
document
=>
document
.
id
===
id
)
return
doc
?.
name
}
const
getSourcePercent
=
(
detail
:
IndexingStatusResponse
)
=>
{
const
completedCount
=
detail
.
completed_segments
||
0
const
totalCount
=
detail
.
total_segments
||
0
if
(
totalCount
===
0
)
return
0
const
percent
=
Math
.
round
(
completedCount
*
100
/
totalCount
)
return
percent
>
100
?
100
:
percent
}
const
getSourceType
=
(
id
:
string
)
=>
{
const
doc
=
documents
.
find
(
document
=>
document
.
id
===
id
)
return
doc
?.
data_source_type
as
DataSourceType
}
// TODO
// TODO
// const percent = useMemo(() => {
// const getIcon = (id: string) => {
// const completedCount = indexingStatusBatchDetail?.completed_segments || 0
// const doc = documents.find(document => document.id === id)
// const totalCount = indexingStatusBatchDetail?.total_segments || 0
// let iconSrc
// if (totalCount === 0)
// if (notionPages.length > 0) {
// return 0
// if (notionPages[0].page_icon && notionPages[0].page_icon.type === 'url')
// const percent = Math.round(completedCount * 100 / totalCount)
// iconSrc = notionPages[0].page_icon.url
// return percent > 100 ? 100 : percent
// if (notionPages[0].page_icon && notionPages[0].page_icon.type === 'emoji')
// }, [indexingStatusBatchDetail])
// iconSrc = notionPages[0].page_icon.emoji
// }
// return iconSrc
// }
const
isSourceEmbedding
=
(
detail
:
IndexingStatusResponse
)
=>
[
'indexing'
,
'splitting'
,
'parsing'
,
'cleaning'
,
'waiting'
].
includes
(
detail
.
indexing_status
||
''
)
return
(
return
(
<>
<>
...
@@ -175,23 +197,41 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
...
@@ -175,23 +197,41 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
)
}
)
}
</
div
>
</
div
>
</
div
>
</
div
>
{
/* TODO progress bar */
}
<
div
className=
{
s
.
progressContainer
}
>
{
/* <div className={s.progressContainer}>
{
indexingStatusBatchDetail
.
map
(
indexingStatusDetail
=>
(
{new Array(10).fill('').map((_, idx) => <div
<
div
className=
{
cn
(
key={idx}
s
.
sourceItem
,
className={cn(s.progressBgItem, isEmbedding ? 'bg-primary-50' : 'bg-gray-100')}
indexingStatusDetail
.
indexing_status
===
'error'
&&
s
.
error
,
/>)}
indexingStatusDetail
.
indexing_status
===
'completed'
&&
s
.
success
,
<div
)
}
>
className={cn(
<
div
className=
{
s
.
progressbar
}
style=
{
{
width
:
`${getSourcePercent(indexingStatusDetail)}%`
}
}
/>
'rounded-l-md',
<
div
className=
{
s
.
info
}
>
s.progressBar,
{
getSourceType
(
indexingStatusDetail
.
id
)
===
DataSourceType
.
FILE
&&
(
(isEmbedding || isEmbeddingCompleted) && s.barProcessing,
<
div
className=
{
s
.
type
}
></
div
>
(isEmbeddingPaused || isEmbeddingError) && s.barPaused,
)
}
indexingStatusBatchDetail?.indexing_status === 'completed' && 'rounded-r-md',
{
/* {getSourceType(indexingStatusDetail.id) === DataSourceType.NOTION && (
)}
<NotionIcon
style={{ width: `${percent}%` }}
className='shrink-0 mr-1'
/>
type='page'
</div> */
}
src={getIcon(indexingStatusDetail.id)}
/>
)} */
}
<
div
className=
{
s
.
name
}
>
{
getSourceName
(
indexingStatusDetail
.
id
)
}
</
div
>
</
div
>
<
div
className=
'shrink-0'
>
{
isSourceEmbedding
(
indexingStatusDetail
)
&&
(
<
div
className=
{
s
.
percent
}
>
{
`${getSourcePercent(indexingStatusDetail)}%`
}
</
div
>
)
}
{
indexingStatusDetail
.
indexing_status
===
'error'
&&
(
<
div
className=
{
s
.
error
}
>
Error%
</
div
>
)
}
{
indexingStatusDetail
.
indexing_status
===
'completed'
&&
(
<
div
className=
{
s
.
success
}
>
100%
</
div
>
)
}
</
div
>
</
div
>
))
}
</
div
>
<
RuleDetail
sourceData=
{
ruleDetail
}
/>
<
RuleDetail
sourceData=
{
ruleDetail
}
/>
<
div
className=
'flex items-center gap-2 mt-10'
>
<
div
className=
'flex items-center gap-2 mt-10'
>
<
Button
className=
'w-fit'
type=
'primary'
onClick=
{
navToDocumentList
}
>
<
Button
className=
'w-fit'
type=
'primary'
onClick=
{
navToDocumentList
}
>
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment