Feat/dify rag (#2528)

Co-authored-by: jyong <jyong@dify.ai>

api/tasks/deal_dataset_vector_index_task.py

@@ -3,9 +3,9 @@ import time
 
 import click
 from celery import shared_task
-from langchain.schema import Document
 
-from core.index.index import IndexBuilder
+from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
+from core.rag.models.document import Document
 from extensions.ext_database import db
 from models.dataset import Dataset, DocumentSegment
 from models.dataset import Document as DatasetDocument
@@ -29,10 +29,10 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):
 
         if not dataset:
             raise Exception('Dataset not found')
-
+        index_type = dataset.doc_form
+        index_processor = IndexProcessorFactory(index_type).init_index_processor()
         if action == "remove":
-            index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=True)
-            index.delete_by_group_id(dataset.id)
+            index_processor.clean(dataset, None, with_keywords=False)
         elif action == "add":
             dataset_documents = db.session.query(DatasetDocument).filter(
                 DatasetDocument.dataset_id == dataset_id,
@@ -42,8 +42,6 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):
             ).all()
 
             if dataset_documents:
-                # save vector index
-                index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=False)
                 documents = []
                 for dataset_document in dataset_documents:
                     # delete from vector index
@@ -65,7 +63,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):
                         documents.append(document)
 
                 # save vector index
-                index.create(documents)
+                index_processor.load(dataset, documents, with_keywords=False)
 
         end_at = time.perf_counter()
         logging.info(