Merge branch 'main' into feat/multi-models

api/controllers/console/app/statistic.py

@@ -398,9 +398,74 @@ class AverageResponseTimeStatistic(Resource):
         })
 
 
+class TokensPerSecondStatistic(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        account = current_user
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
+    CASE 
+        WHEN SUM(provider_response_latency) = 0 THEN 0
+        ELSE (SUM(answer_tokens) / SUM(provider_response_latency))
+    END as tokens_per_second
+FROM messages
+WHERE app_id = :app_id'''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql_query), arg_dict)
+
+        response_data = []
+
+        for i in rs:
+            response_data.append({
+                'date': str(i.date),
+                'tps': round(i.tokens_per_second, 4)
+            })
+
+        return jsonify({
+            'data': response_data
+        })
+
+
 api.add_resource(DailyConversationStatistic, '/apps/<uuid:app_id>/statistics/daily-conversations')
 api.add_resource(DailyTerminalsStatistic, '/apps/<uuid:app_id>/statistics/daily-end-users')
 api.add_resource(DailyTokenCostStatistic, '/apps/<uuid:app_id>/statistics/token-costs')
 api.add_resource(AverageSessionInteractionStatistic, '/apps/<uuid:app_id>/statistics/average-session-interactions')
 api.add_resource(UserSatisfactionRateStatistic, '/apps/<uuid:app_id>/statistics/user-satisfaction-rate')
 api.add_resource(AverageResponseTimeStatistic, '/apps/<uuid:app_id>/statistics/average-response-time')
+api.add_resource(TokensPerSecondStatistic, '/apps/<uuid:app_id>/statistics/tokens-per-second')

api/core/llm/streamable_azure_open_ai.py

-from langchain.callbacks.manager import Callbacks
+from langchain.callbacks.manager import Callbacks, CallbackManagerForLLMRun
 from langchain.llms import AzureOpenAI
+from langchain.llms.openai import _streaming_response_template, completion_with_retry, _update_response, \
+    update_token_usage
 from langchain.schema import LLMResult
 from typing import Optional, List, Dict, Mapping, Any, Union, Tuple
 
@@ -67,3 +69,58 @@ class StreamableAzureOpenAI(AzureOpenAI):
     @classmethod
     def get_kwargs_from_model_params(cls, params: dict):
         return params
+
+    def _generate(
+        self,
+        prompts: List[str],
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> LLMResult:
+        """Call out to OpenAI's endpoint with k unique prompts.
+
+        Args:
+            prompts: The prompts to pass into the model.
+            stop: Optional list of stop words to use when generating.
+
+        Returns:
+            The full LLM output.
+
+        Example:
+            .. code-block:: python
+
+                response = openai.generate(["Tell me a joke."])
+        """
+        params = self._invocation_params
+        params = {**params, **kwargs}
+        sub_prompts = self.get_sub_prompts(params, prompts, stop)
+        choices = []
+        token_usage: Dict[str, int] = {}
+        # Get the token usage from the response.
+        # Includes prompt, completion, and total tokens used.
+        _keys = {"completion_tokens", "prompt_tokens", "total_tokens"}
+        for _prompts in sub_prompts:
+            if self.streaming:
+                if len(_prompts) > 1:
+                    raise ValueError("Cannot stream results with multiple prompts.")
+                params["stream"] = True
+                response = _streaming_response_template()
+                for stream_resp in completion_with_retry(
+                    self, prompt=_prompts, **params
+                ):
+                    if len(stream_resp["choices"]) > 0:
+                        if run_manager:
+                            run_manager.on_llm_new_token(
+                                stream_resp["choices"][0]["text"],
+                                verbose=self.verbose,
+                                logprobs=stream_resp["choices"][0]["logprobs"],
+                            )
+                        _update_response(response, stream_resp)
+                choices.extend(response["choices"])
+            else:
+                response = completion_with_retry(self, prompt=_prompts, **params)
+                choices.extend(response["choices"])
+            if not self.streaming:
+                # Can't update token usage if streaming
+                update_token_usage(_keys, response, token_usage)
+        return self.create_llm_result(choices, prompts, token_usage)
\ No newline at end of file

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chartView.tsx

@@ -6,7 +6,7 @@ import { useTranslation } from 'react-i18next'
 import useSWR from 'swr'
 import { fetchAppDetail } from '@/service/apps'
 import type { PeriodParams } from '@/app/components/app/overview/appChart'
-import { AvgResponseTime, AvgSessionInteractions, ConversationsChart, CostChart, EndUsersChart, UserSatisfactionRate } from '@/app/components/app/overview/appChart'
+import { AvgResponseTime, AvgSessionInteractions, ConversationsChart, CostChart, EndUsersChart, TokenPerSecond, UserSatisfactionRate } from '@/app/components/app/overview/appChart'
 import type { Item } from '@/app/components/base/select'
 import { SimpleSelect } from '@/app/components/base/select'
 import { TIME_PERIOD_LIST } from '@/app/components/app/log/filter'
@@ -64,11 +64,18 @@ export default function ChartView({ appId }: IChartViewProps) {
               <AvgResponseTime period={period} id={appId} />
             )}
         </div>
+        <div className='flex-1 ml-3'>
+          <TokenPerSecond period={period} id={appId} />
+        </div>
+      </div>
+      <div className='flex flex-row w-full mb-6'>
         <div className='flex-1 ml-3'>
           <UserSatisfactionRate period={period} id={appId} />
         </div>
+        <div className='flex-1 ml-3'>
+          <CostChart period={period} id={appId} />
+        </div>
       </div>
-      <CostChart period={period} id={appId} />
     </div>
   )
 }

web/app/components/app-sidebar/basic.tsx

@@ -4,7 +4,6 @@ import {
 } from '@heroicons/react/24/outline'
 import Tooltip from '../base/tooltip'
 import AppIcon from '../base/app-icon'
-
 const chars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_'
 
 export function randomString(length: number) {
@@ -21,6 +20,7 @@ export type IAppBasicProps = {
   type: string | React.ReactNode
   hoverTip?: string
   textStyle?: { main?: string; extra?: string }
+  isExtraInLine?: boolean
 }
 
 const ApiSvg = <svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
@@ -61,7 +61,7 @@ const ICON_MAP = {
   notion: <AppIcon innerIcon={NotionSvg} className='!border-[0.5px] !border-indigo-100 !bg-white' />,
 }
 
-export default function AppBasic({ icon, icon_background, name, type, hoverTip, textStyle, iconType = 'app' }: IAppBasicProps) {
+export default function AppBasic({ icon, icon_background, name, type, hoverTip, textStyle, iconType = 'app', isExtraInLine }: IAppBasicProps) {
   return (
     <div className="flex items-start">
       {icon && icon_background && iconType === 'app' && (

web/app/components/app/chat/index.tsx

@@ -193,7 +193,7 @@ const Chat: FC<IChatProps> = ({
             )}
             {
               isShowSuggestion && (
-                <div className='pt-2 mb-2 '>
+                <div className='pt-2'>
                   <div className='flex items-center justify-center mb-2.5'>
                     <div className='grow h-[1px]'
                       style={{

web/app/components/app/chat/style.module.css

 .answerIcon {
   position: relative;
-  background: url(./icons/robot.svg);
+  background: url(./icons/robot.svg) 100%/100%;
 }
 
 .typeingIcon {

web/app/components/app/overview/appChart.tsx

@@ -231,6 +231,7 @@ const Chart: React.FC<IChartProps> = ({
       </div>
       <div className='mb-4'>
         <Basic
+          isExtraInLine={CHART_TYPE_CONFIG[chartType].showTokens}
           name={chartType !== 'costs' ? (sumData.toLocaleString() + unit) : `${sumData < 1000 ? sumData : (`${formatNumber(Math.round(sumData / 1000))}k`)}`}
           type={!CHART_TYPE_CONFIG[chartType].showTokens
             ? ''
@@ -316,6 +317,23 @@ export const AvgResponseTime: FC<IBizChartProps> = ({ id, period }) => {
   />
 }
 
+export const TokenPerSecond: FC<IBizChartProps> = ({ id, period }) => {
+  const { t } = useTranslation()
+  const { data: response } = useSWR({ url: `/apps/${id}/statistics/tokens-per-second`, params: period.query }, getAppStatistics)
+  if (!response)
+    return <Loading />
+  const noDataFlag = !response.data || response.data.length === 0
+  return <Chart
+    basicInfo={{ title: t('appOverview.analysis.tps.title'), explanation: t('appOverview.analysis.tps.explanation'), timePeriod: period.name }}
+    chartData={!noDataFlag ? response : { data: getDefaultChartData({ ...(period.query ?? defaultPeriod), key: 'tps' }) } as any}
+    valueKey='tps'
+    chartType='conversations'
+    isAvg
+    unit={t('appOverview.analysis.tokenPS') as string}
+    {...(noDataFlag && { yMax: 100 })}
+  />
+}
+
 export const UserSatisfactionRate: FC<IBizChartProps> = ({ id, period }) => {
   const { t } = useTranslation()
   const { data: response } = useSWR({ url: `/apps/${id}/statistics/user-satisfaction-rate`, params: period.query }, getAppStatistics)

web/app/components/datasets/create/step-two/index.module.css

@@ -13,29 +13,6 @@
   z-index: 10;
 }
 
-.fixed {
-  padding-top: 12px;
-  font-size: 12px;
-  line-height: 18px;
-  background: rgba(255, 255, 255, 0.9);
-  border-bottom: 0.5px solid #EAECF0;
-  backdrop-filter: blur(4px);
-  animation: fix 0.5s;
-}
-
-@keyframes fix {
-  from {
-    padding-top: 42px;
-    font-size: 18px;
-    line-height: 28px;
-  }
-  to {
-    padding-top: 12px;
-    font-size: 12px;
-    line-height: 18px;
-  }
-}
-
 .form {
   @apply px-16 pb-8;
 }
@@ -416,3 +393,28 @@
   color: #101828;
   z-index: 10;
 }
+
+/* 
+ * `fixed` must under `previewHeader` because of style override would not work
+ */
+.fixed {
+  padding-top: 12px;
+  font-size: 12px;
+  line-height: 18px;
+  background: rgba(255, 255, 255, 0.9);
+  border-bottom: 0.5px solid #EAECF0;
+  backdrop-filter: blur(4px);
+  animation: fix 0.5s;
+}
+@keyframes fix {
+  from {
+    padding-top: 42px;
+    font-size: 18px;
+    line-height: 28px;
+  }
+  to {
+    padding-top: 12px;
+    font-size: 12px;
+    line-height: 18px;
+  }
+}

web/app/components/explore/app-list/index.tsx

@@ -95,7 +95,7 @@ const Apps: FC = () => {
         onChange={setCurrCategory}
       />
       <div
-        className='flex mt-6 flex-col overflow-auto bg-gray-100 shrink-0 grow'
+        className='flex mt-6 pb-6 flex-col overflow-auto bg-gray-100 shrink-0 grow'
         style={{
           maxHeight: 'calc(100vh - 243px)',
         }}

web/app/components/explore/create-app-modal/index.tsx

@@ -45,7 +45,7 @@ const CreateAppModal = ({
     <>
       <Modal
         isShow={show}
-        onClose={onHide}
+        onClose={() => {}}
         className={cn(s.modal, '!max-w-[480px]', 'px-8')}
       >
         <span className={s.close} onClick={onHide}/>

web/app/components/header/account-dropdown/workplace-selector/index.tsx

@@ -27,10 +27,11 @@ const WorkplaceSelector = () => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
   const { workspaces } = useWorkspacesContext()
-  const currentWrokspace = workspaces.filter(item => item.current)?.[0]
+  const currentWorkspace = workspaces.find(v => v.current)
 
   const handleSwitchWorkspace = async (tenant_id: string) => {
     try {
+      if (currentWorkspace?.id === tenant_id) return
       await switchWorkspace({ url: '/workspaces/switch', body: { tenant_id } })
       notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
       location.assign(`${location.origin}`)
@@ -51,8 +52,8 @@ const WorkplaceSelector = () => {
                 group hover:bg-gray-50 cursor-pointer ${open && 'bg-gray-50'} rounded-lg
               `,
             )}>
-              <div className={itemIconClassName}>{currentWrokspace?.name[0].toLocaleUpperCase()}</div>
-              <div className={`${itemNameClassName} truncate`}>{currentWrokspace?.name}</div>
+              <div className={itemIconClassName}>{currentWorkspace?.name[0].toLocaleUpperCase()}</div>
+              <div className={`${itemNameClassName} truncate`}>{currentWorkspace?.name}</div>
               <ChevronRight className='shrink-0 w-[14px] h-[14px] text-gray-500' />
             </Menu.Button>
             <Transition

web/hooks/use-copy-to-clipboard.ts

@@ -8,11 +8,6 @@ function useCopyToClipboard(): [CopiedValue, CopyFn] {
   const [copiedText, setCopiedText] = useState<CopiedValue>(null)
 
   const copy: CopyFn = useCallback(async (text: string) => {
-    if (!navigator?.clipboard) {
-      console.warn('Clipboard not supported')
-      return false
-    }
-
     try {
       writeText(text)
       setCopiedText(text)

web/i18n/lang/app-overview.en.ts

@@ -81,6 +81,7 @@ const translation = {
   analysis: {
     title: 'Analysis',
     ms: 'ms',
+    tokenPS: 'Token/s',
     totalMessages: {
       title: 'Total Messages',
       explanation: 'Daily AI interactions count; prompt engineering/debugging excluded.',
@@ -106,6 +107,10 @@ const translation = {
       title: 'Avg. Response Time',
       explanation: 'Time (ms) for AI to process/respond; for text-based apps.',
     },
+    tps: {
+      title: 'Token Output Speed',
+      explanation: 'Measure the performance of the LLM. Count the Tokens output speed of LLM from the beginning of the request to the completion of the output.',
+    },
   },
 }
 

web/i18n/lang/app-overview.zh.ts

@@ -81,6 +81,7 @@ const translation = {
   analysis: {
     title: '分析',
     ms: '毫秒',
+    tokenPS: 'Token/秒',
     totalMessages: {
       title: '全部消息数',
       explanation: '反映 AI 每天的互动总次数，每回答用户一个问题算一条 Message。提示词编排和调试的消息不计入。',
@@ -106,6 +107,10 @@ const translation = {
       title: '平均响应时间',
       explanation: '衡量 AI 应用处理和回复用户请求所花费的平均时间，单位为毫秒，反映性能和用户体验。仅在文本型应用提供。',
     },
+    tps: {
+      title: 'Token 输出速度',
+      explanation: '衡量 LLM 的性能。统计 LLM 从请求开始到输出完毕这段期间的 Tokens 输出速度。',
+    },
   },
 }