ascii filter Unicode U+FFFE

f2605278 · jyong · b0e114b9 · f2605278
Commit f2605278 authored Jan 15, 2024 by jyong
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

indexing_runner.py api/core/indexing_runner.py +3 -1

No files found.
--- a/api/core/indexing_runner.py
+++ b/api/core/indexing_runner.py
@@ -528,11 +528,13 @@ class IndexingRunner:

        return text_docs

+    65534
    def filter_string(self, text):
        text = re.sub(r'<\|', '<', text)
        text = re.sub(r'\|>', '>', text)
        text = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xA0-\xFF\xEF\xBF\xBE]', '', text)
-
+        # Unicode  U+FFFE
+        text = re.sub(u'\uFFFE', '', text)
        return text

    def _get_splitter(self, processing_rule: DatasetProcessRule,