Commit 5c2d4403 authored by jyong's avatar jyong

fix datatime filed

parent ce4dbe6e
...@@ -39,6 +39,7 @@ class ExcelLoader(BaseLoader): ...@@ -39,6 +39,7 @@ class ExcelLoader(BaseLoader):
row_dict = dict(zip(keys, list(map(str, row)))) row_dict = dict(zip(keys, list(map(str, row))))
row_dict = {k: v for k, v in row_dict.items() if v} row_dict = {k: v for k, v in row_dict.items() if v}
item = ''.join(f'{k}:{v}\n' for k, v in row_dict.items()) item = ''.join(f'{k}:{v}\n' for k, v in row_dict.items())
data.append(item) document = Document(page_content=item)
data.append(document)
return [Document(page_content='\n\n'.join(data))] return data
...@@ -81,8 +81,8 @@ class NotionLoader(BaseLoader): ...@@ -81,8 +81,8 @@ class NotionLoader(BaseLoader):
docs = [] docs = []
if notion_page_type == 'database': if notion_page_type == 'database':
# get all the pages in the database # get all the pages in the database
page_text = self._get_notion_database_data(notion_obj_id) page_text_documents = self._get_notion_database_data(notion_obj_id)
docs.append(Document(page_content=page_text)) docs.extend(page_text_documents)
elif notion_page_type == 'page': elif notion_page_type == 'page':
page_text_list = self._get_notion_block_data(notion_obj_id) page_text_list = self._get_notion_block_data(notion_obj_id)
for page_text in page_text_list: for page_text in page_text_list:
...@@ -94,7 +94,7 @@ class NotionLoader(BaseLoader): ...@@ -94,7 +94,7 @@ class NotionLoader(BaseLoader):
def _get_notion_database_data( def _get_notion_database_data(
self, database_id: str, query_dict: Dict[str, Any] = {} self, database_id: str, query_dict: Dict[str, Any] = {}
) -> str: ) -> List[Document]:
"""Get all the pages from a Notion database.""" """Get all the pages from a Notion database."""
res = requests.post( res = requests.post(
DATABASE_URL_TMPL.format(database_id=database_id), DATABASE_URL_TMPL.format(database_id=database_id),
...@@ -110,7 +110,7 @@ class NotionLoader(BaseLoader): ...@@ -110,7 +110,7 @@ class NotionLoader(BaseLoader):
database_content_list = [] database_content_list = []
if 'results' not in data or data["results"] is None: if 'results' not in data or data["results"] is None:
return "" return []
for result in data["results"]: for result in data["results"]:
properties = result['properties'] properties = result['properties']
data = {} data = {}
...@@ -143,10 +143,10 @@ class NotionLoader(BaseLoader): ...@@ -143,10 +143,10 @@ class NotionLoader(BaseLoader):
row_content = row_content + f'{key}:{value_content}\n' row_content = row_content + f'{key}:{value_content}\n'
else: else:
row_content = row_content + f'{key}:{value}\n' row_content = row_content + f'{key}:{value}\n'
database_content_list.append(row_content) document = Document(page_content=row_content)
database_content_list.append(json.dumps(data, ensure_ascii=False)) database_content_list.append(document)
return "\n\n".join(database_content_list) return database_content_list
def _get_notion_block_data(self, page_id: str) -> List[str]: def _get_notion_block_data(self, page_id: str) -> List[str]:
result_lines_arr = [] result_lines_arr = []
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment