Merge branch 'fix/json-format' into deploy/dev

api/core/conversation_message_task.py

@@ -156,7 +156,7 @@ class ConversationMessageTask:
         self.message.message = llm_message.prompt
         self.message.message_tokens = message_tokens
         self.message.message_unit_price = message_unit_price
-        self.message.answer = llm_message.completion.strip() if llm_message.completion else ''
+        self.message.answer = PromptBuilder.process_template(llm_message.completion.strip()) if llm_message.completion else ''
         self.message.answer_tokens = answer_tokens
         self.message.answer_unit_price = answer_unit_price
         self.message.provider_response_latency = llm_message.latency

api/core/data_loader/loader/excel.py

@@ -39,6 +39,7 @@ class ExcelLoader(BaseLoader):
                     row_dict = dict(zip(keys, list(map(str, row))))
                     row_dict = {k: v for k, v in row_dict.items() if v}
                     item = ''.join(f'{k}:{v}\n' for k, v in row_dict.items())
-                    data.append(item)
+                    document = Document(page_content=item)
+                    data.append(document)
 
-        return [Document(page_content='\n\n'.join(data))]
+        return data

api/core/data_loader/loader/notion.py

@@ -81,8 +81,8 @@ class NotionLoader(BaseLoader):
         docs = []
         if notion_page_type == 'database':
             # get all the pages in the database
-            page_text = self._get_notion_database_data(notion_obj_id)
-            docs.append(Document(page_content=page_text))
+            page_text_documents = self._get_notion_database_data(notion_obj_id)
+            docs.extend(page_text_documents)
         elif notion_page_type == 'page':
             page_text_list = self._get_notion_block_data(notion_obj_id)
             for page_text in page_text_list:
@@ -94,7 +94,7 @@ class NotionLoader(BaseLoader):
 
     def _get_notion_database_data(
             self, database_id: str, query_dict: Dict[str, Any] = {}
-    ) -> str:
+    ) -> List[Document]:
         """Get all the pages from a Notion database."""
         res = requests.post(
             DATABASE_URL_TMPL.format(database_id=database_id),
@@ -110,7 +110,7 @@ class NotionLoader(BaseLoader):
 
         database_content_list = []
         if 'results' not in data or data["results"] is None:
-            return ""
+            return []
         for result in data["results"]:
             properties = result['properties']
             data = {}
@@ -143,10 +143,10 @@ class NotionLoader(BaseLoader):
                     row_content = row_content + f'{key}:{value_content}\n'
                 else:
                     row_content = row_content + f'{key}:{value}\n'
-            database_content_list.append(row_content)
-            database_content_list.append(json.dumps(data, ensure_ascii=False))
+            document = Document(page_content=row_content)
+            database_content_list.append(document)
 
-        return "\n\n".join(database_content_list)
+        return database_content_list
 
     def _get_notion_block_data(self, page_id: str) -> List[str]:
         result_lines_arr = []