use sub to operate all (#475)

998f819b · crazywoola · GitHub · 6194b827 · 998f819b
Unverified Commit 998f819b authored Jun 28, 2023 by crazywoola Committed by GitHub Jun 28, 2023
Show whitespace changes
Inline Side-by-side

Showing with 4 additions and 4 deletions

indexing_runner.py api/core/indexing_runner.py +4 -4

No files found.
--- a/api/core/indexing_runner.py
+++ b/api/core/indexing_runner.py
@@ -346,10 +346,10 @@ class IndexingRunner:
        return text_docs
    def filter_string(self, text):
-        text = text.replace('<|', '<')
+        text = re.sub(r'<\|', '<', text)
-        text = text.replace('|>', '>')
+        text = re.sub(r'\|>', '>', text)
-        pattern = re.compile('[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\xFF]')
+        text = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\xFF]', '', text)
-        return pattern.sub('', text)
+        return text
    def _get_splitter(self, processing_rule: DatasetProcessRule) -> TextSplitter:
        """