Skip to content

D
dify
Commit 374a4a86 authored by jyong's avatar jyong
Browse files
Options

add clean unused dataset command

parent 9d903e4d
Hide whitespace changes
Inline Side-by-side
Showing with 16 additions and 20 deletions
api/commands.py
View file @ 374a4a86
...@@ -225,26 +225,22 @@ def clean_unused_dataset_indexes(): ...@@ -225,26 +225,22 @@ def clean_unused_dataset_indexes():
).all() ).all()
if not documents or len(documents) == 0: if not documents or len(documents) == 0:
try: try:
all_documents = db.session.query(Document).filter( # remove index
Document.dataset_id == dataset.id, vector_index = IndexBuilder.get_index(dataset, 'high_quality')
Document.indexing_status == 'completed', kw_index = IndexBuilder.get_index(dataset, 'economy')
Document.enabled == True, # delete from vector index
Document.archived == False, if vector_index:
).all() vector_index.delete()
if all_documents and len(all_documents)>0: kw_index.delete()
update_params = { # update document
Document.enabled: False update_params = {
} Document.enabled: False
}
Document.query.filter_by(dataset_id=dataset.id).update(update_params)
db.session.commit() Document.query.filter_by(dataset_id=dataset.id).update(update_params)
# remove index db.session.commit()
vector_index = IndexBuilder.get_index(dataset, 'high_quality') click.echo(click.style('Cleaned unused dataset {} from db success!'.format(dataset.id),
kw_index = IndexBuilder.get_index(dataset, 'economy') fg='green'))
# delete from vector index
if vector_index:
vector_index.delete()
kw_index.delete()
except Exception as e: except Exception as e:
click.echo( click.echo(
click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)), click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment