Feat/dify rag (#2528)

Co-authored-by: jyong <jyong@dify.ai>

api/tasks/document_indexing_sync_task.py

@@ -6,9 +6,9 @@ import click
 from celery import shared_task
 from werkzeug.exceptions import NotFound
 
-from core.data_loader.loader.notion import NotionLoader
-from core.index.index import IndexBuilder
 from core.indexing_runner import DocumentIsPausedException, IndexingRunner
+from core.rag.extractor.notion_extractor import NotionExtractor
+from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
 from extensions.ext_database import db
 from models.dataset import Dataset, Document, DocumentSegment
 from models.source import DataSourceBinding
@@ -54,11 +54,11 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
         if not data_source_binding:
             raise ValueError('Data source binding not found.')
 
-        loader = NotionLoader(
-            notion_access_token=data_source_binding.access_token,
+        loader = NotionExtractor(
             notion_workspace_id=workspace_id,
             notion_obj_id=page_id,
-            notion_page_type=page_type
+            notion_page_type=page_type,
+            notion_access_token=data_source_binding.access_token
         )
 
         last_edited_time = loader.get_notion_last_edited_time()
@@ -74,20 +74,14 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
                 dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first()
                 if not dataset:
                     raise Exception('Dataset not found')
-
-                vector_index = IndexBuilder.get_index(dataset, 'high_quality')
-                kw_index = IndexBuilder.get_index(dataset, 'economy')
+                index_type = document.doc_form
+                index_processor = IndexProcessorFactory(index_type).init_index_processor()
 
                 segments = db.session.query(DocumentSegment).filter(DocumentSegment.document_id == document_id).all()
                 index_node_ids = [segment.index_node_id for segment in segments]
 
                 # delete from vector index
-                if vector_index:
-                    vector_index.delete_by_document_id(document_id)
-
-                # delete from keyword index
-                if index_node_ids:
-                    kw_index.delete_by_ids(index_node_ids)
+                index_processor.clean(dataset, index_node_ids)
 
                 for segment in segments:
                     db.session.delete(segment)