Skip to content

D
dify
Unverified Commit cb7a608d authored by Jyong's avatar Jyong Committed by GitHub
Browse files
Options

ascii filter Unicode U+FFFE (#2038) 

Co-authored-by: 's avatarjyong <jyong@dify.ai>
parent bdb0d772
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 1 deletion
api/core/indexing_runner.py
View file @ cb7a608d
......@@ -531,7 +531,9 @@ class IndexingRunner:
def filter_string(self, text):
text = re.sub(r'<\|', '<', text)
text = re.sub(r'\|>', '>', text)
text = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\xFF]', '', text)
text = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]', '', text)
# Unicode U+FFFE
text = re.sub(u'\uFFFE', '', text)
return text
def _get_splitter(self, processing_rule: DatasetProcessRule,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment