add clean unused dataset command

api/commands.py

@@ -223,21 +223,28 @@ def clean_unused_dataset_indexes():
                     Document.archived == False,
                     Document.updated_at > thirty_days_ago
                 ).all()
-                if not documents and len(documents) > 0:
+                if not documents or len(documents) == 0:
                     try:
-                        update_params = {
-                            Document.enabled: False
-                        }
-
-                        Document.query.filter_by(dataset_id=dataset.id).update(update_params)
-                        db.session.commit()
-                        # remove index
-                        vector_index = IndexBuilder.get_index(dataset, 'high_quality')
-                        kw_index = IndexBuilder.get_index(dataset, 'economy')
-                        # delete from vector index
-                        if vector_index:
-                            vector_index.delete()
-                        kw_index.delete()
+                        all_documents = db.session.query(Document).filter(
+                            Document.dataset_id == dataset.id,
+                            Document.indexing_status == 'completed',
+                            Document.enabled == True,
+                            Document.archived == False,
+                        ).all()
+                        if all_documents and len(all_documents)>0:
+                            update_params = {
+                                Document.enabled: False
+                            }
+
+                            Document.query.filter_by(dataset_id=dataset.id).update(update_params)
+                            db.session.commit()
+                            # remove index
+                            vector_index = IndexBuilder.get_index(dataset, 'high_quality')
+                            kw_index = IndexBuilder.get_index(dataset, 'economy')
+                            # delete from vector index
+                            if vector_index:
+                                vector_index.delete()
+                            kw_index.delete()
                     except Exception as e:
                         click.echo(
                             click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)),