Feat/dataset support api service (#1240)

Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: crazywoola <427733928@qq.com>

web/app/(commonLayout)/datasets/ApiServer.tsx

+'use client'
+
+import type { FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import CopyFeedback from '@/app/components/base/copy-feedback'
+import SecretKeyButton from '@/app/components/develop/secret-key/secret-key-button'
+import { randomString } from '@/utils'
+
+type ApiServerProps = {
+  apiBaseUrl: string
+}
+const ApiServer: FC<ApiServerProps> = ({
+  apiBaseUrl,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex items-center'>
+      <div className='flex items-center mr-2 pl-1.5 pr-1 h-8 bg-white/80 border-[0.5px] border-white rounded-lg'>
+        <div className='mr-0.5 px-1.5 h-5 border border-gray-200 text-[11px] text-gray-500 rounded-md'>{t('appApi.apiServer')}</div>
+        <div className='px-1 w-[248px] text-[13px] font-medium text-gray-800'>{apiBaseUrl}</div>
+        <div className='mx-1 w-[1px] h-[14px] bg-gray-200'></div>
+        <CopyFeedback
+          content={apiBaseUrl}
+          selectorId={randomString(8)}
+          className={'!w-6 !h-6 hover:bg-gray-200'}
+        />
+      </div>
+      <div className='flex items-center mr-2 px-3 h-8 bg-[#ECFDF3] text-xs font-semibold text-[#039855] rounded-lg border-[0.5px] border-[#D1FADF]'>
+        {t('appApi.ok')}
+      </div>
+      <SecretKeyButton
+        className='flex-shrink-0 !h-8 bg-white'
+        textCls='!text-gray-700 font-medium'
+        iconCls='stroke-[1.2px]'
+      />
+    </div>
+  )
+}
+
+export default ApiServer

web/app/(commonLayout)/datasets/Container.tsx

+'use client'
+
+import { useRef, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import useSWR from 'swr'
+import Datasets from './Datasets'
+import DatasetFooter from './DatasetFooter'
+import ApiServer from './ApiServer'
+import Doc from './Doc'
+import TabSlider from '@/app/components/base/tab-slider'
+import { fetchDatasetApiBaseUrl } from '@/service/datasets'
+
+const Container = () => {
+  const { t } = useTranslation()
+  const options = [
+    {
+      value: 'dataset',
+      text: t('dataset.datasets'),
+    },
+    {
+      value: 'api',
+      text: t('dataset.datasetsApi'),
+    },
+  ]
+  const [activeTab, setActiveTab] = useState('dataset')
+  const containerRef = useRef<HTMLDivElement>(null)
+  const { data } = useSWR(activeTab === 'dataset' ? null : '/datasets/api-base-info', fetchDatasetApiBaseUrl)
+
+  return (
+    <div ref={containerRef} className='grow relative flex flex-col bg-gray-100 overflow-y-auto'>
+      <div className='sticky top-0 flex justify-between pt-4 px-12 pb-2 h-14 bg-gray-100 z-10'>
+        <TabSlider
+          value={activeTab}
+          onChange={newActiveTab => setActiveTab(newActiveTab)}
+          options={options}
+        />
+        {
+          activeTab === 'api' && (
+            <ApiServer apiBaseUrl={data?.api_base_url || ''} />
+          )
+        }
+      </div>
+      {
+        activeTab === 'dataset' && (
+          <div className=''>
+            <Datasets containerRef={containerRef}/>
+            <DatasetFooter />
+          </div>
+        )
+      }
+      {
+        activeTab === 'api' && (
+          <Doc apiBaseUrl={data?.api_base_url || ''} />
+        )
+      }
+    </div>
+  )
+}
+
+export default Container

web/app/(commonLayout)/datasets/Datasets.tsx

@@ -7,7 +7,7 @@ import NewDatasetCard from './NewDatasetCard'
 import DatasetCard from './DatasetCard'
 import type { DataSetListResponse } from '@/models/datasets'
 import { fetchDatasets } from '@/service/datasets'
-import { useAppContext, useSelector } from '@/context/app-context'
+import { useAppContext } from '@/context/app-context'
 
 const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
   if (!pageIndex || previousPageData.has_more)
@@ -15,11 +15,16 @@ const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
   return null
 }
 
-const Datasets = () => {
+type Props = {
+  containerRef: React.RefObject<HTMLDivElement>
+}
+
+const Datasets = ({
+  containerRef,
+}: Props) => {
   const { isCurrentWorkspaceManager } = useAppContext()
   const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchDatasets, { revalidateFirstPage: false, revalidateAll: true })
   const loadingStateRef = useRef(false)
-  const pageContainerRef = useSelector(state => state.pageContainerRef)
   const anchorRef = useRef<HTMLAnchorElement>(null)
 
   useEffect(() => {
@@ -29,19 +34,19 @@ const Datasets = () => {
   useEffect(() => {
     const onScroll = debounce(() => {
       if (!loadingStateRef.current) {
-        const { scrollTop, clientHeight } = pageContainerRef.current!
+        const { scrollTop, clientHeight } = containerRef.current!
         const anchorOffset = anchorRef.current!.offsetTop
         if (anchorOffset - scrollTop - clientHeight < 100)
           setSize(size => size + 1)
       }
     }, 50)
 
-    pageContainerRef.current?.addEventListener('scroll', onScroll)
-    return () => pageContainerRef.current?.removeEventListener('scroll', onScroll)
+    containerRef.current?.addEventListener('scroll', onScroll)
+    return () => containerRef.current?.removeEventListener('scroll', onScroll)
   }, [])
 
   return (
-    <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-8 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 grow shrink-0'>
+    <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 grow shrink-0'>
       { isCurrentWorkspaceManager && <NewDatasetCard ref={anchorRef} /> }
       {data?.map(({ data: datasets }) => datasets.map(dataset => (
         <DatasetCard key={dataset.id} dataset={dataset} onDelete={mutate} />),

web/app/(commonLayout)/datasets/Doc.tsx

+'use client'
+
+import type { FC } from 'react'
+import { useContext } from 'use-context-selector'
+import TemplateEn from './template/template.en.mdx'
+import TemplateZh from './template/template.zh.mdx'
+import I18n from '@/context/i18n'
+
+type DocProps = {
+  apiBaseUrl: string
+}
+const Doc: FC<DocProps> = ({
+  apiBaseUrl,
+}) => {
+  const { locale } = useContext(I18n)
+
+  return (
+    <article className='mx-12 pt-16 bg-white rounded-t-xl prose prose-xl'>
+      {
+        locale === 'en'
+          ? <TemplateEn apiBaseUrl={apiBaseUrl} />
+          : <TemplateZh apiBaseUrl={apiBaseUrl} />
+      }
+    </article>
+  )
+}
+
+export default Doc

web/app/(commonLayout)/datasets/page.tsx

-import Datasets from './Datasets'
-import DatasetFooter from './DatasetFooter'
+import Container from './Container'
 
 const AppList = async () => {
   return (
-    <div className='flex flex-col overflow-auto bg-gray-100 shrink-0 grow'>
-      <Datasets />
-      <DatasetFooter />
-    </div >
+    <Container />
   )
 }
 

web/app/components/base/tab-slider/index.tsx

+import type { FC } from 'react'
+
+type Option = {
+  value: string
+  text: string
+}
+type TabSliderProps = {
+  value: string
+  onChange: (v: string) => void
+  options: Option[]
+}
+const TabSlider: FC<TabSliderProps> = ({
+  value,
+  onChange,
+  options,
+}) => {
+  const currentIndex = options.findIndex(option => option.value === value)
+  const current = options[currentIndex]
+
+  return (
+    <div className='relative flex p-0.5 rounded-lg bg-gray-200'>
+      {
+        options.map((option, index) => (
+          <div
+            key={option.value}
+            className={`
+              flex justify-center items-center w-[118px] h-7 text-[13px] 
+              font-semibold text-gray-600 rounded-[7px] cursor-pointer
+              hover:bg-gray-50
+              ${index !== options.length - 1 && 'mr-[1px]'}
+            `}
+            onClick={() => onChange(option.value)}
+          >
+            {option.text}
+          </div>
+        ))
+      }
+      {
+        current && (
+          <div
+            className={`
+              absolute flex justify-center items-center w-[118px] h-7 bg-white text-[13px] font-semibold text-primary-600 
+              border-[0.5px] border-gray-200 rounded-[7px] shadow-xs transition-transform
+            `}
+            style={{ transform: `translateX(${currentIndex * 118 + 1}px)` }}
+          >
+            {current.text}
+          </div>
+        )
+      }
+    </div>
+  )
+}
+
+export default TabSlider

web/app/components/develop/secret-key/secret-key-button.tsx

@@ -7,7 +7,7 @@ import SecretKeyModal from '@/app/components/develop/secret-key/secret-key-modal
 
 type ISecretKeyButtonProps = {
   className?: string
-  appId: string
+  appId?: string
   iconCls?: string
   textCls?: string
 }

web/app/components/develop/secret-key/secret-key-modal.tsx

@@ -12,7 +12,16 @@ import SecretKeyGenerateModal from './secret-key-generate'
 import s from './style.module.css'
 import Modal from '@/app/components/base/modal'
 import Button from '@/app/components/base/button'
-import { createApikey, delApikey, fetchApiKeysList } from '@/service/apps'
+import {
+  createApikey as createAppApikey,
+  delApikey as delAppApikey,
+  fetchApiKeysList as fetchAppApiKeysList,
+} from '@/service/apps'
+import {
+  createApikey as createDatasetApikey,
+  delApikey as delDatasetApikey,
+  fetchApiKeysList as fetchDatasetApiKeysList,
+} from '@/service/datasets'
 import type { CreateApiKeyResponse } from '@/models/app'
 import Tooltip from '@/app/components/base/tooltip'
 import Loading from '@/app/components/base/loading'
@@ -22,7 +31,7 @@ import { useAppContext } from '@/context/app-context'
 
 type ISecretKeyModalProps = {
   isShow: boolean
-  appId: string
+  appId?: string
   onClose: () => void
 }
 
@@ -37,7 +46,10 @@ const SecretKeyModal = ({
   const [isVisible, setVisible] = useState(false)
   const [newKey, setNewKey] = useState<CreateApiKeyResponse | undefined>(undefined)
   const { mutate } = useSWRConfig()
-  const commonParams = { url: `/apps/${appId}/api-keys`, params: {} }
+  const commonParams = appId
+    ? { url: `/apps/${appId}/api-keys`, params: {} }
+    : { url: '/datasets/api-keys', params: {} }
+  const fetchApiKeysList = appId ? fetchAppApiKeysList : fetchDatasetApiKeysList
   const { data: apiKeysList } = useSWR(commonParams, fetchApiKeysList)
 
   const [delKeyID, setDelKeyId] = useState('')
@@ -64,12 +76,20 @@ const SecretKeyModal = ({
     if (!delKeyID)
       return
 
-    await delApikey({ url: `/apps/${appId}/api-keys/${delKeyID}`, params: {} })
+    const delApikey = appId ? delAppApikey : delDatasetApikey
+    const params = appId
+      ? { url: `/apps/${appId}/api-keys/${delKeyID}`, params: {} }
+      : { url: `/datasets/api-keys/${delKeyID}`, params: {} }
+    await delApikey(params)
     mutate(commonParams)
   }
 
   const onCreate = async () => {
-    const res = await createApikey({ url: `/apps/${appId}/api-keys`, body: {} })
+    const params = appId
+      ? { url: `/apps/${appId}/api-keys`, body: {} }
+      : { url: '/datasets/api-keys', body: {} }
+    const createApikey = appId ? createAppApikey : createDatasetApikey
+    const res = await createApikey(params)
     setVisible(true)
     setNewKey(res)
     mutate(commonParams)

web/i18n/lang/app-api.zh.ts

@@ -12,7 +12,7 @@ const translation = {
   never: '从未',
   apiKeyModal: {
     apiSecretKey: 'API 密钥',
-    apiSecretKeyTips: '如果不想你的应用 API 被滥用，请保护好你的 API Key :) 最佳实践是避免在前端代码中明文引用。',
+    apiSecretKeyTips: '如果不想你的 API 被滥用，请保护好你的 API Key :) 最佳实践是避免在前端代码中明文引用。',
     createNewSecretKey: '创建密钥',
     secretKey: '密钥',
     created: '创建时间',

web/i18n/lang/dataset.en.ts

@@ -18,6 +18,8 @@ const translation = {
   intro6: ' as a standalone ChatGPT index plug-in to publish',
   unavailable: 'Unavailable',
   unavailableTip: 'Embedding model is not available, the default embedding model needs to be configured',
+  datasets: 'DATASETS',
+  datasetsApi: 'API',
 }
 
 export default translation

web/i18n/lang/dataset.zh.ts

@@ -18,6 +18,8 @@ const translation = {
   intro6: '为独立的 ChatGPT 插件发布使用',
   unavailable: '不可用',
   unavailableTip: '由于 embedding 模型不可用，需要配置默认 embedding 模型',
+  datasets: '数据集',
+  datasetsApi: 'API',
 }
 
 export default translation

web/service/datasets.ts

@@ -22,6 +22,10 @@ import type {
   createDocumentResponse,
 } from '@/models/datasets'
 import type { CommonResponse, DataSourceNotionWorkspace } from '@/models/common'
+import type {
+  ApikeysListResponse,
+  CreateApiKeyResponse,
+} from '@/models/app'
 
 // apis for documents in a dataset
 
@@ -192,3 +196,19 @@ export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, an
 export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; pageType: string }> = ({ workspaceID, pageID, pageType }) => {
   return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`)
 }
+
+export const fetchApiKeysList: Fetcher<ApikeysListResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
+  return get<ApikeysListResponse>(url, params)
+}
+
+export const delApikey: Fetcher<CommonResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
+  return del<CommonResponse>(url, params)
+}
+
+export const createApikey: Fetcher<CreateApiKeyResponse, { url: string; body: Record<string, any> }> = ({ url, body }) => {
+  return post<CreateApiKeyResponse>(url, body)
+}
+
+export const fetchDatasetApiBaseUrl: Fetcher<{ api_base_url: string }, string> = (url) => {
+  return get<{ api_base_url: string }>(url)
+}