fix: remove empty segment in splitter (#68)

0587ff0f · John Wang · GitHub · ce492d13 · 0587ff0f
Unverified Commit 0587ff0f authored May 17, 2023 by John Wang Committed by GitHub May 17, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

indexing_runner.py api/core/indexing_runner.py +1 -1

No files found.
--- a/api/core/indexing_runner.py
+++ b/api/core/indexing_runner.py
@@ -343,7 +343,7 @@ class IndexingRunner:

            # parse document to nodes
            nodes = node_parser.get_nodes_from_documents([text_doc])
-
+            nodes = [node for node in nodes if node.text is not None and node.text.strip()]
            all_nodes.extend(nodes)

        return all_nodes