Feat/dify rag (#2528)

Co-authored-by: jyong <jyong@dify.ai>

api/tasks/remove_document_from_index_task.py

@@ -5,7 +5,7 @@ import click
 from celery import shared_task
 from werkzeug.exceptions import NotFound
 
-from core.index.index import IndexBuilder
+from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from models.dataset import Document, DocumentSegment
@@ -37,18 +37,12 @@ def remove_document_from_index_task(document_id: str):
         if not dataset:
             raise Exception('Document has no dataset')
 
-        vector_index = IndexBuilder.get_index(dataset, 'high_quality')
-        kw_index = IndexBuilder.get_index(dataset, 'economy')
+        index_processor = IndexProcessorFactory(document.doc_form).init_index_processor()
 
-        # delete from vector index
-        if vector_index:
-            vector_index.delete_by_document_id(document.id)
-
-        # delete from keyword index
         segments = db.session.query(DocumentSegment).filter(DocumentSegment.document_id == document.id).all()
         index_node_ids = [segment.index_node_id for segment in segments]
         if index_node_ids:
-            kw_index.delete_by_ids(index_node_ids)
+            index_processor.clean(dataset, index_node_ids)
 
         end_at = time.perf_counter()
         logging.info(