Skip to content

D
dify
Commit 1d9f1fdb authored by StyleZhang's avatar StyleZhang
Browse files
Options

fix: notion-page-selector

parent 7e11aab4
Show whitespace changes
Inline Side-by-side
Showing with 150 additions and 119 deletions
web/app/components/base/notion-page-selector/base.tsx
View file @ 1d9f1fdb
import { useCallback, useEffect, useState } from 'react'
import { useCallback, useEffect, useMemo, useState } from 'react'
import useSWR from 'swr'
import cn from 'classnames'
import s from './base.module.css'
import WorkspaceSelector from './workspace-selector'
import SearchInput from './search-input'
import PageSelector from './page-selector'
import { fetchDataSource } from '@/service/common'
import { preImportNotionPages } from '@/service/datasets'
import AccountSetting from '@/app/components/header/account-setting'
import type { DataSourceNotionPage } from '@/models/common'
import type { DataSourceNotionPage, DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
export type NotionPageSelectorValue = DataSourceNotionPage & { workspace_id: string }
type NotionPageSelectorProps = {
value?: string[]
onSelect: (selectedPages: (DataSourceNotionPage & { workspace_id: string })[]) => void
onSelect: (selectedPages: NotionPageSelectorValue[]) => void
canPreview?: boolean
previewPageId?: string
onPreview?: (selectedPage: DataSourceNotionPage & { workspace_id: string }) => void
onPreview?: (selectedPage: NotionPageSelectorValue) => void
datasetId?: string
}
const NotionPageSelector = ({
value,
onSelect,
canPreview,
previewPageId,
onPreview,
datasetId,
}: NotionPageSelectorProps) => {
const { data } = useSWR({ url: '/notion/pre-import/pages', datasetId }, preImportNotionPages)
const [prevData, setPrevData] = useState(data)
const [searchValue, setSearchValue] = useState('')
const [showDataSourceSetting, setShowDataSourceSetting] = useState(false)
const { data } = useSWR({ url: 'data-source/integrates' }, fetchDataSource)
const notionWorkspaces = data?.data.filter(item => item.provider === 'notion') || []
const firstWorkspace = notionWorkspaces[0]?.id
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
const currentWorkspace = notionWorkspaces.find(workspace => workspace.id === currentWorkspaceId)
const notionWorkspaces = useMemo(() => {
return data?.notion_info || []
}, [data?.notion_info])
const firstWorkspaceId = notionWorkspaces[0]?.workspace_id
const currentWorkspace = notionWorkspaces.find(workspace => workspace.workspace_id === currentWorkspaceId)
const getPagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>] = useMemo(() => {
const selectedPagesId = new Set<string>()
const pagesMap = notionWorkspaces.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
next.pages.forEach((page) => {
if (page.is_bound)
selectedPagesId.add(page.page_id)
prev[page.page_id] = {
...page,
workspace_id: next.workspace_id,
}
})
return prev
}, {})
return [pagesMap, selectedPagesId]
}, [notionWorkspaces])
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set([...Array.from(getPagesMapAndSelectedPagesId[1])]))
if (prevData !== data) {
setPrevData(data)
setSelectedPagesId(new Set([...Array.from(getPagesMapAndSelectedPagesId[1])]))
}
const handleSearchValueChange = useCallback((value: string) => {
setSearchValue(value)
......@@ -38,26 +67,20 @@ const NotionPageSelector = ({
const handleSelectWorkspace = useCallback((workspaceId: string) => {
setCurrentWorkspaceId(workspaceId)
}, [])
const handleSelecPages = (selectedPages: DataSourceNotionPage[]) => {
onSelect(selectedPages.map((item) => {
return {
...item,
workspace_id: currentWorkspace?.source_info.workspace_id || '',
}
}))
}
const handlePreviewPage = (previewPage: DataSourceNotionPage) => {
if (onPreview) {
onPreview({
...previewPage,
workspace_id: currentWorkspace?.source_info.workspace_id || '',
})
const handleSelecPages = (selectedPagesId: Set<string>) => {
setSelectedPagesId(new Set(Array.from(selectedPagesId)))
const selectedPages = Array.from(selectedPagesId).map(pageId => getPagesMapAndSelectedPagesId[0][pageId])
onSelect(selectedPages)
}
const handlePreviewPage = (previewPageId: string) => {
if (onPreview)
onPreview(getPagesMapAndSelectedPagesId[0][previewPageId])
}
useEffect(() => {
setCurrentWorkspaceId(firstWorkspace)
}, [firstWorkspace])
setCurrentWorkspaceId(firstWorkspaceId)
}, [firstWorkspaceId])
return (
<div className='bg-gray-25 border border-gray-200 rounded-xl'>
<div className='flex items-center pl-[10px] pr-2 h-11 bg-white border-b border-b-gray-200 rounded-t-xl'>
......@@ -79,10 +102,10 @@ const NotionPageSelector = ({
</div>
<div className='rounded-b-xl overflow-hidden'>
<PageSelector
key={currentWorkspaceId}
value={value}
value={selectedPagesId}
searchValue={searchValue}
list={currentWorkspace?.source_info.pages || []}
list={currentWorkspace?.pages || []}
pagesMap={getPagesMapAndSelectedPagesId[0]}
onSelect={handleSelecPages}
canPreview={canPreview}
previewPageId={previewPageId}
......
web/app/components/base/notion-page-selector/notion-page-selector-modal/index.tsx
View file @ 1d9f1fdb
......@@ -3,29 +3,29 @@ import { useTranslation } from 'react-i18next'
import cn from 'classnames'
import { XMarkIcon } from '@heroicons/react/24/outline'
import NotionPageSelector from '../base'
import type { NotionPageSelectorValue } from '../base'
import s from './index.module.css'
import Modal from '@/app/components/base/modal'
import type { DataSourceNotionPage } from '@/models/common'
type NotionPageSelectorModalProps = {
isShow: boolean
onClose: () => void
onSave: (selectedPages: DataSourceNotionPage[]) => void
value?: string[]
onSave: (selectedPages: NotionPageSelectorValue[]) => void
datasetId: string
}
const NotionPageSelectorModal = ({
isShow,
onClose,
onSave,
value,
datasetId,
}: NotionPageSelectorModalProps) => {
const { t } = useTranslation()
const [selectedPages, setSelectedPages] = useState<DataSourceNotionPage[]>([])
const [selectedPages, setSelectedPages] = useState<NotionPageSelectorValue[]>([])
const handleClose = () => {
onClose()
}
const handleSelectPage = (newSelectedPages: DataSourceNotionPage[]) => {
const handleSelectPage = (newSelectedPages: NotionPageSelectorValue[]) => {
setSelectedPages(newSelectedPages)
}
const handleSave = () => {
......@@ -47,9 +47,9 @@ const NotionPageSelectorModal = ({
</div>
</div>
<NotionPageSelector
value={value}
onSelect={handleSelectPage}
canPreview={false}
datasetId={datasetId}
/>
<div className='mt-8 flex justify-end'>
<div className={s.operate} onClick={handleClose}>{t('common.operation.cancel')}</div>
......
web/app/components/base/notion-page-selector/page-selector/index.tsx
View file @ 1d9f1fdb
import { memo, useState } from 'react'
import { memo, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { FixedSizeList as List, areEqual } from 'react-window'
import type { ListChildComponentProps } from 'react-window'
......@@ -6,18 +6,18 @@ import cn from 'classnames'
import Checkbox from '../../checkbox'
import NotionIcon from '../../notion-icon'
import s from './index.module.css'
import type { DataSourceNotionPage } from '@/models/common'
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
type PageSelectorProps = {
value?: string[]
value: Set<string>
searchValue: string
pagesMap: DataSourceNotionPageMap
list: DataSourceNotionPage[]
onSelect: (selectedPages: DataSourceNotionPage[]) => void
onSelect: (selectedPagesId: Set<string>) => void
canPreview?: boolean
previewPageId?: string
onPreview?: (selectedPage: DataSourceNotionPage) => void
onPreview?: (selectedPageId: string) => void
}
type NotionPageMap = Record<string, DataSourceNotionPage>
type NotionPageTreeItem = {
children: Set<string>
descendants: Set<string>
......@@ -31,7 +31,7 @@ type NotionPageItem = {
} & DataSourceNotionPage
const recursivePushInParentDescendants = (
listMap: Record<string, DataSourceNotionPage>,
pagesMap: DataSourceNotionPageMap,
listTreeMap: NotionPageTreeMap,
current: NotionPageTreeItem,
leafItem: NotionPageTreeItem,
......@@ -45,7 +45,7 @@ const recursivePushInParentDescendants = (
const descendants = new Set([pageId, leafItem.page_id])
listTreeMap[parentId] = {
...listMap[parentId],
...pagesMap[parentId],
children,
descendants,
deepth: 0,
......@@ -61,7 +61,7 @@ const recursivePushInParentDescendants = (
leafItem.ancestors.unshift(listTreeMap[parentId].page_name)
if (listTreeMap[parentId].parent_id !== 'root')
recursivePushInParentDescendants(listMap, listTreeMap, listTreeMap[parentId], leafItem)
recursivePushInParentDescendants(pagesMap, listTreeMap, listTreeMap[parentId], leafItem)
}
}
......@@ -153,6 +153,7 @@ const Item = memo(({ index, style, data }: ListChildComponentProps<{
const PageSelector = ({
value,
searchValue,
pagesMap,
list,
onSelect,
canPreview = true,
......@@ -160,15 +161,19 @@ const PageSelector = ({
onPreview,
}: PageSelectorProps) => {
const { t } = useTranslation()
const [dataList, setDataList] = useState<NotionPageItem[]>(
list.filter(item => item.parent_id === 'root').map((item) => {
const [prevDataList, setPrevDataList] = useState(list)
const [dataList, setDataList] = useState<NotionPageItem[]>([])
const [localPreviewPageId, setLocalPreviewPageId] = useState('')
if (prevDataList !== list) {
setPrevDataList(list)
setDataList(list.filter(item => item.parent_id === 'root').map((item) => {
return {
...item,
expand: false,
deepth: 0,
}
}),
)
}))
}
const searchDataList = list.filter((item) => {
return item.page_name.includes(searchValue)
}).map((item) => {
......@@ -179,23 +184,18 @@ const PageSelector = ({
}
})
const currentDataList = searchValue ? searchDataList : dataList
const [checkedIds, setCheckedIds] = useState<Set<string>>(new Set(value || []))
const [localPreviewPageId, setLocalPreviewPageId] = useState('')
const currentPreviewPageId = previewPageId === undefined ? localPreviewPageId : previewPageId
const listMap = list.reduce((prev: NotionPageMap, next: DataSourceNotionPage) => {
prev[next.page_id] = next
return prev
}, {})
const listMapWithChildrenAndDescendants = list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => {
const listMapWithChildrenAndDescendants = useMemo(() => {
return list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => {
const pageId = next.page_id
if (!prev[pageId])
prev[pageId] = { ...next, children: new Set(), descendants: new Set(), deepth: 0, ancestors: [] }
recursivePushInParentDescendants(listMap, prev, prev[pageId], prev[pageId])
recursivePushInParentDescendants(pagesMap, prev, prev[pageId], prev[pageId])
return prev
}, {})
}, [list, pagesMap])
const handleToggle = (index: number) => {
const current = dataList[index]
......@@ -216,7 +216,7 @@ const PageSelector = ({
newDataList = [
...dataList.slice(0, index + 1),
...childrenIds.map(item => ({
...listMap[item],
...pagesMap[item],
expand: false,
deepth: listMapWithChildrenAndDescendants[item].deepth,
})),
......@@ -230,25 +230,24 @@ const PageSelector = ({
const pageId = current.page_id
const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
if (checkedIds.has(pageId)) {
if (value.has(pageId)) {
if (!searchValue) {
for (const item of currentWithChildrenAndDescendants.descendants)
checkedIds.delete(item)
value.delete(item)
}
checkedIds.delete(pageId)
value.delete(pageId)
}
else {
if (!searchValue) {
for (const item of currentWithChildrenAndDescendants.descendants)
checkedIds.add(item)
value.add(item)
}
checkedIds.add(pageId)
value.add(pageId)
}
setCheckedIds(new Set([...checkedIds]))
onSelect([...checkedIds].map(item => listMap[item]))
onSelect(new Set([...value]))
}
const handlePreview = (index: number) => {
......@@ -258,7 +257,7 @@ const PageSelector = ({
setLocalPreviewPageId(pageId)
if (onPreview)
onPreview(listMap[pageId])
onPreview(pageId)
}
if (!currentDataList.length) {
......@@ -280,7 +279,7 @@ const PageSelector = ({
itemData={{
dataList: currentDataList,
handleToggle,
checkedIds,
checkedIds: value,
handleCheck,
canPreview,
handlePreview,
......
web/app/components/base/notion-page-selector/workspace-selector/index.tsx
View file @ 1d9f1fdb
......@@ -5,11 +5,11 @@ import { Menu, Transition } from '@headlessui/react'
import cn from 'classnames'
import NotionIcon from '../../notion-icon'
import s from './index.module.css'
import type { DataSourceNotion } from '@/models/common'
import type { DataSourceNotionWorkspace } from '@/models/common'
type WorkspaceSelectorProps = {
value: string
items: DataSourceNotion[]
items: Omit<DataSourceNotionWorkspace, 'total'>[]
onSelect: (v: string) => void
}
export default function WorkspaceSelector({
......@@ -18,7 +18,7 @@ export default function WorkspaceSelector({
onSelect,
}: WorkspaceSelectorProps) {
const { t } = useTranslation()
const currentWorkspace = items.find(item => item.id === value)?.source_info
const currentWorkspace = items.find(item => item.workspace_id === value)
return (
<Menu as="div" className="relative inline-block text-left">
......@@ -32,7 +32,7 @@ export default function WorkspaceSelector({
name={currentWorkspace?.workspace_name}
/>
<div className='mr-1 w-[90px] text-left text-sm font-medium text-gray-700 truncate' title={currentWorkspace?.workspace_name}>{currentWorkspace?.workspace_name}</div>
<div className='mr-1 px-1 h-[18px] bg-primary-50 rounded-lg text-xs font-medium text-primary-600'>{currentWorkspace?.total}</div>
<div className='mr-1 px-1 h-[18px] bg-primary-50 rounded-lg text-xs font-medium text-primary-600'>{currentWorkspace?.pages.length}</div>
<div className={cn(s['down-arrow'], 'mr-2 w-3 h-3')} />
</Menu.Button>
<Transition
......@@ -55,19 +55,19 @@ export default function WorkspaceSelector({
<div className="p-1 max-h-50 overflow-auto">
{
items.map(item => (
<Menu.Item key={item.id}>
<Menu.Item key={item.workspace_id}>
<div
className='flex items-center px-3 h-9 hover:bg-gray-50 cursor-pointer'
onClick={() => onSelect(item.id)}
onClick={() => onSelect(item.workspace_id)}
>
<NotionIcon
className='shrink-0 mr-2'
src={item.source_info.workspace_icon}
name={item.source_info.workspace_name}
src={item.workspace_icon}
name={item.workspace_name}
/>
<div className='grow mr-2 text-sm text-gray-700 truncate' title={item.source_info.workspace_name}>{item.source_info.workspace_name}</div>
<div className='grow mr-2 text-sm text-gray-700 truncate' title={item.workspace_name}>{item.workspace_name}</div>
<div className='shrink-0 text-xs font-medium text-primary-600'>
{item.source_info.total} {t('common.dataSource.notion.selector.pageSelected')}
{item.pages.length} {t('common.dataSource.notion.selector.pageSelected')}
</div>
</div>
</Menu.Item>
......
web/app/components/datasets/documents/index.tsx
View file @ 1d9f1fdb
......@@ -4,7 +4,7 @@ import React, { useMemo, useState } from 'react'
import useSWR from 'swr'
import { useTranslation } from 'react-i18next'
import { useRouter } from 'next/navigation'
import { debounce, omit } from 'lodash-es'
import { debounce, groupBy, omit } from 'lodash-es'
// import Link from 'next/link'
import { PlusIcon } from '@heroicons/react/24/solid'
import List from './list'
......@@ -107,15 +107,23 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
const isLoading = !documentsRes && !error
const handleSaveNotionPageSelected = async (selectedPages: (DataSourceNotionPage & { workspace_id: string })[]) => {
const workspacesMap = groupBy(selectedPages, 'workspace_id')
const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
return {
workspaceId,
pages: workspacesMap[workspaceId],
}
})
const params = {
data_source: {
type: dataset?.data_source_type,
info_list: {
data_source_type: dataset?.data_source_type,
notion_info_list: [{
workspace_id: selectedPages[0].workspace_id,
pages: selectedPages.map((selectedPage) => {
const { page_id, page_name, page_icon, type } = selectedPage
notion_info_list: workspaces.map((workspace) => {
return {
workspace_id: workspace.workspaceId,
pages: workspace.pages.map((page) => {
const { page_id, page_name, page_icon, type } = page
return {
page_id,
page_name,
......@@ -123,7 +131,8 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
type,
}
}),
}],
}
}),
},
},
indexing_technique: dataset?.indexing_technique,
......@@ -138,6 +147,7 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
body: params,
})
mutate()
setNotionPageSelectorModalVisible(false)
}
const handleSync = async () => {
......@@ -180,12 +190,10 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
? <Pagination current={currPage} onChange={setCurrPage} total={total} limit={limit} />
: null}
<NotionPageSelectorModal
value={(documentsRes?.data || []).map((doc) => {
return doc.data_source_info.notion_page_id
})}
isShow={notionPageSelectorModalVisible}
onClose={() => setNotionPageSelectorModalVisible(false)}
onSave={handleSaveNotionPageSelected}
datasetId={dataset?.id || ''}
/>
</div>
</div>
......
web/app/components/datasets/documents/list.tsx
View file @ 1d9f1fdb
......@@ -22,9 +22,8 @@ import type { IndicatorProps } from '@/app/components/header/indicator'
import Indicator from '@/app/components/header/indicator'
import { asyncRunSafe } from '@/utils'
import { formatNumber } from '@/utils/format'
import { archiveDocument, deleteDocument, disableDocument, enableDocument } from '@/service/datasets'
import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument } from '@/service/datasets'
import type { DocumentDisplayStatus, DocumentListResponse } from '@/models/datasets'
import { syncDataSourceNotion } from '@/service/common'
import type { CommonResponse } from '@/models/common'
export const SettingsIcon: FC<{ className?: string }> = ({ className }) => {
......@@ -93,17 +92,13 @@ export const OperationAction: FC<{
archived: boolean
id: string
data_source_type: string
data_source_info: {
notion_workspace_id: string
}
}
datasetId: string
onUpdate: () => void
scene?: 'list' | 'detail'
className?: string
onSync: (workspaceId: string) => void
}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '', onSync }) => {
const { id, enabled = false, archived = false, data_source_type, data_source_info } = detail || {}
}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '' }) => {
const { id, enabled = false, archived = false, data_source_type } = detail || {}
const [showModal, setShowModal] = useState(false)
const { notify } = useContext(ToastContext)
const { t } = useTranslation()
......@@ -124,7 +119,7 @@ export const OperationAction: FC<{
opApi = disableDocument
break
case 'sync':
onSync(data_source_info.notion_workspace_id)
opApi = syncDocument
break
default:
opApi = deleteDocument
......@@ -286,12 +281,6 @@ const DocumentList: FC<IDocumentListProps> = ({ documents = [], datasetId, onUpd
}
}
const handleSync = async (workspaceId: string) => {
await syncDataSourceNotion({ url: `/oauth/data-source/notion/${workspaceId}/sync` })
onSync()
}
return (
<>
<table className={`w-full border-collapse border-0 text-sm mt-3 ${s.documentTable}`}>
......@@ -336,9 +325,8 @@ const DocumentList: FC<IDocumentListProps> = ({ documents = [], datasetId, onUpd
<td>
<OperationAction
datasetId={datasetId}
detail={pick(doc, ['enabled', 'archived', 'id', 'data_source_type', 'data_source_info'])}
detail={pick(doc, ['enabled', 'archived', 'id', 'data_source_type'])}
onUpdate={onUpdate}
onSync={handleSync}
/>
</td>
</tr>
......
web/models/common.ts
View file @ 1d9f1fdb
......@@ -110,16 +110,21 @@ export type DataSourceNotionPage = {
page_name: string
parent_id: string
type: string
is_bound: boolean
}
export type DataSourceNotionPageMap = Record<string, DataSourceNotionPage & { workspace_id: string }>
export type DataSourceNotionWorkspace = {
workspace_name: string
workspace_id: string
workspace_icon: string | null
total: number
total?: number
pages: DataSourceNotionPage[]
}
export type DataSourceNotionWorkspaceMap = Record<string, DataSourceNotionWorkspace>
export type DataSourceNotion = {
id: string
provider: string
......
web/service/datasets.ts
View file @ 1d9f1fdb
......@@ -2,7 +2,7 @@ import type { Fetcher } from 'swr'
import qs from 'qs'
import { del, get, patch, post, put } from './base'
import type { CreateDocumentReq, DataSet, DataSetListResponse, DocumentDetailResponse, DocumentListResponse, FileIndexingEstimateResponse, HitTestingRecordsResponse, HitTestingResponse, IndexingEstimateResponse, IndexingStatusResponse, InitialDocumentDetail, ProcessRuleResponse, RelatedAppResponse, SegmentsQuery, SegmentsResponse, createDocumentResponse } from '@/models/datasets'
import type { CommonResponse } from '@/models/common'
import type { CommonResponse, DataSourceNotionWorkspace } from '@/models/common'
// apis for documents in a dataset
......@@ -95,6 +95,14 @@ export const disableDocument: Fetcher<CommonResponse, CommonDocReq> = ({ dataset
return patch(`/datasets/${datasetId}/documents/${documentId}/status/disable`) as Promise<CommonResponse>
}
export const syncDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
return get(`/datasets/${datasetId}/documents/${documentId}/notion/sync`) as Promise<CommonResponse>
}
export const preImportNotionPages: Fetcher<{ notion_info: DataSourceNotionWorkspace[] }, { url: string; datasetId?: string }> = ({ url, datasetId }) => {
return get(url, { params: { dataset_id: datasetId } }) as Promise<{ notion_info: DataSourceNotionWorkspace[] }>
}
export const modifyDocMetadata: Fetcher<CommonResponse, CommonDocReq & { body: { doc_type: string; doc_metadata: Record<string, any> } }> = ({ datasetId, documentId, body }) => {
return put(`/datasets/${datasetId}/documents/${documentId}/metadata`, { body }) as Promise<CommonResponse>
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment