estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor: use logger = logging.getLogger(__name__) in logging (#24515)

Browse Source 
Co-authored-by: Yongtao Huang <99629139+hyongtao-db@users.noreply.github.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
This commit is contained in:
Yongtao Huang
2025-08-26 18:10:31 +08:00
committed by GitHub
parent 8af2ae973f
commit fa753239ad
102 changed files with 565 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
api/tasks/add_document_to_index_task.py
View File

@@ -13,6 +13,8 @@ from libs.datetime_utils import naive_utc_now
from models.dataset import DatasetAutoDisableLog, DocumentSegment
from models.dataset import Document as DatasetDocument
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def add_document_to_index_task(dataset_document_id: str):
@@ -22,12 +24,12 @@ def add_document_to_index_task(dataset_document_id: str):
Usage: add_document_to_index_task.delay(dataset_document_id)
"""
logging.info(click.style(f"Start add document to index: {dataset_document_id}", fg="green"))
logger.info(click.style(f"Start add document to index: {dataset_document_id}", fg="green"))
start_at = time.perf_counter()
dataset_document = db.session.query(DatasetDocument).where(DatasetDocument.id == dataset_document_id).first()
if not dataset_document:
logging.info(click.style(f"Document not found: {dataset_document_id}", fg="red"))
logger.info(click.style(f"Document not found: {dataset_document_id}", fg="red"))
db.session.close()
return
@@ -101,11 +103,11 @@ def add_document_to_index_task(dataset_document_id: str):
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(f"Document added to index: {dataset_document.id} latency: {end_at - start_at}", fg="green")
)
except Exception as e:
logging.exception("add document to index failed")
logger.exception("add document to index failed")
dataset_document.enabled = False
dataset_document.disabled_at = naive_utc_now()
dataset_document.indexing_status = "error"

8
api/tasks/annotation/add_annotation_to_index_task.py
View File

@@ -10,6 +10,8 @@ from extensions.ext_database import db
from models.dataset import Dataset
from services.dataset_service import DatasetCollectionBindingService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def add_annotation_to_index_task(
@@ -25,7 +27,7 @@ def add_annotation_to_index_task(
Usage: clean_dataset_task.delay(dataset_id, tenant_id, indexing_technique, index_struct)
"""
logging.info(click.style(f"Start build index for annotation: {annotation_id}", fg="green"))
logger.info(click.style(f"Start build index for annotation: {annotation_id}", fg="green"))
start_at = time.perf_counter()
try:
@@ -48,13 +50,13 @@ def add_annotation_to_index_task(
vector.create([document], duplicate_check=True)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Build index successful for annotation: {annotation_id} latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("Build index for annotation failed")
logger.exception("Build index for annotation failed")
finally:
db.session.close()

8
api/tasks/annotation/batch_import_annotations_task.py
View File

@@ -13,6 +13,8 @@ from models.dataset import Dataset
from models.model import App, AppAnnotationSetting, MessageAnnotation
from services.dataset_service import DatasetCollectionBindingService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def batch_import_annotations_task(job_id: str, content_list: list[dict], app_id: str, tenant_id: str, user_id: str):
@@ -25,7 +27,7 @@ def batch_import_annotations_task(job_id: str, content_list: list[dict], app_id:
:param user_id: user_id
"""
logging.info(click.style(f"Start batch import annotation: {job_id}", fg="green"))
logger.info(click.style(f"Start batch import annotation: {job_id}", fg="green"))
start_at = time.perf_counter()
indexing_cache_key = f"app_annotation_batch_import_{str(job_id)}"
# get app info
@@ -74,7 +76,7 @@ def batch_import_annotations_task(job_id: str, content_list: list[dict], app_id:
db.session.commit()
redis_client.setex(indexing_cache_key, 600, "completed")
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
"Build index successful for batch import annotation: {} latency: {}".format(
job_id, end_at - start_at
@@ -87,6 +89,6 @@ def batch_import_annotations_task(job_id: str, content_list: list[dict], app_id:
redis_client.setex(indexing_cache_key, 600, "error")
indexing_error_msg_key = f"app_annotation_batch_import_error_msg_{str(job_id)}"
redis_client.setex(indexing_error_msg_key, 600, str(e))
logging.exception("Build index for batch import annotations failed")
logger.exception("Build index for batch import annotations failed")
finally:
db.session.close()

10
api/tasks/annotation/delete_annotation_index_task.py
View File

@@ -9,13 +9,15 @@ from extensions.ext_database import db
from models.dataset import Dataset
from services.dataset_service import DatasetCollectionBindingService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def delete_annotation_index_task(annotation_id: str, app_id: str, tenant_id: str, collection_binding_id: str):
"""
Async delete annotation index task
"""
logging.info(click.style(f"Start delete app annotation index: {app_id}", fg="green"))
logger.info(click.style(f"Start delete app annotation index: {app_id}", fg="green"))
start_at = time.perf_counter()
try:
dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type(
@@ -33,10 +35,10 @@ def delete_annotation_index_task(annotation_id: str, app_id: str, tenant_id: str
vector = Vector(dataset, attributes=["doc_id", "annotation_id", "app_id"])
vector.delete_by_metadata_field("annotation_id", annotation_id)
except Exception:
logging.exception("Delete annotation index failed when annotation deleted.")
logger.exception("Delete annotation index failed when annotation deleted.")
end_at = time.perf_counter()
logging.info(click.style(f"App annotations index deleted : {app_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"App annotations index deleted : {app_id} latency: {end_at - start_at}", fg="green"))
except Exception as e:
logging.exception("Annotation deleted index failed")
logger.exception("Annotation deleted index failed")
finally:
db.session.close()

14
api/tasks/annotation/disable_annotation_reply_task.py
View File

@@ -10,26 +10,28 @@ from extensions.ext_redis import redis_client
from models.dataset import Dataset
from models.model import App, AppAnnotationSetting, MessageAnnotation
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def disable_annotation_reply_task(job_id: str, app_id: str, tenant_id: str):
"""
Async enable annotation reply task
"""
logging.info(click.style(f"Start delete app annotations index: {app_id}", fg="green"))
logger.info(click.style(f"Start delete app annotations index: {app_id}", fg="green"))
start_at = time.perf_counter()
# get app info
app = db.session.query(App).where(App.id == app_id, App.tenant_id == tenant_id, App.status == "normal").first()
annotations_count = db.session.query(MessageAnnotation).where(MessageAnnotation.app_id == app_id).count()
if not app:
logging.info(click.style(f"App not found: {app_id}", fg="red"))
logger.info(click.style(f"App not found: {app_id}", fg="red"))
db.session.close()
return
app_annotation_setting = db.session.query(AppAnnotationSetting).where(AppAnnotationSetting.app_id == app_id).first()
if not app_annotation_setting:
logging.info(click.style(f"App annotation setting not found: {app_id}", fg="red"))
logger.info(click.style(f"App annotation setting not found: {app_id}", fg="red"))
db.session.close()
return
@@ -49,7 +51,7 @@ def disable_annotation_reply_task(job_id: str, app_id: str, tenant_id: str):
vector = Vector(dataset, attributes=["doc_id", "annotation_id", "app_id"])
vector.delete()
except Exception:
logging.exception("Delete annotation index failed when annotation deleted.")
logger.exception("Delete annotation index failed when annotation deleted.")
redis_client.setex(disable_app_annotation_job_key, 600, "completed")
# delete annotation setting
@@ -57,9 +59,9 @@ def disable_annotation_reply_task(job_id: str, app_id: str, tenant_id: str):
db.session.commit()
end_at = time.perf_counter()
logging.info(click.style(f"App annotations index deleted : {app_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"App annotations index deleted : {app_id} latency: {end_at - start_at}", fg="green"))
except Exception as e:
logging.exception("Annotation batch deleted index failed")
logger.exception("Annotation batch deleted index failed")
redis_client.setex(disable_app_annotation_job_key, 600, "error")
disable_app_annotation_error_key = f"disable_app_annotation_error_{str(job_id)}"
redis_client.setex(disable_app_annotation_error_key, 600, str(e))

14
api/tasks/annotation/enable_annotation_reply_task.py
View File

@@ -13,6 +13,8 @@ from models.dataset import Dataset
from models.model import App, AppAnnotationSetting, MessageAnnotation
from services.dataset_service import DatasetCollectionBindingService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def enable_annotation_reply_task(
@@ -27,13 +29,13 @@ def enable_annotation_reply_task(
"""
Async enable annotation reply task
"""
logging.info(click.style(f"Start add app annotation to index: {app_id}", fg="green"))
logger.info(click.style(f"Start add app annotation to index: {app_id}", fg="green"))
start_at = time.perf_counter()
# get app info
app = db.session.query(App).where(App.id == app_id, App.tenant_id == tenant_id, App.status == "normal").first()
if not app:
logging.info(click.style(f"App not found: {app_id}", fg="red"))
logger.info(click.style(f"App not found: {app_id}", fg="red"))
db.session.close()
return
@@ -68,7 +70,7 @@ def enable_annotation_reply_task(
try:
old_vector.delete()
except Exception as e:
logging.info(click.style(f"Delete annotation index error: {str(e)}", fg="red"))
logger.info(click.style(f"Delete annotation index error: {str(e)}", fg="red"))
annotation_setting.score_threshold = score_threshold
annotation_setting.collection_binding_id = dataset_collection_binding.id
annotation_setting.updated_user_id = user_id
@@ -104,14 +106,14 @@ def enable_annotation_reply_task(
try:
vector.delete_by_metadata_field("app_id", app_id)
except Exception as e:
logging.info(click.style(f"Delete annotation index error: {str(e)}", fg="red"))
logger.info(click.style(f"Delete annotation index error: {str(e)}", fg="red"))
vector.create(documents)
db.session.commit()
redis_client.setex(enable_app_annotation_job_key, 600, "completed")
end_at = time.perf_counter()
logging.info(click.style(f"App annotations added to index: {app_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"App annotations added to index: {app_id} latency: {end_at - start_at}", fg="green"))
except Exception as e:
logging.exception("Annotation batch created index failed")
logger.exception("Annotation batch created index failed")
redis_client.setex(enable_app_annotation_job_key, 600, "error")
enable_app_annotation_error_key = f"enable_app_annotation_error_{str(job_id)}"
redis_client.setex(enable_app_annotation_error_key, 600, str(e))

8
api/tasks/annotation/update_annotation_to_index_task.py
View File

@@ -10,6 +10,8 @@ from extensions.ext_database import db
from models.dataset import Dataset
from services.dataset_service import DatasetCollectionBindingService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def update_annotation_to_index_task(
@@ -25,7 +27,7 @@ def update_annotation_to_index_task(
Usage: clean_dataset_task.delay(dataset_id, tenant_id, indexing_technique, index_struct)
"""
logging.info(click.style(f"Start update index for annotation: {annotation_id}", fg="green"))
logger.info(click.style(f"Start update index for annotation: {annotation_id}", fg="green"))
start_at = time.perf_counter()
try:
@@ -49,13 +51,13 @@ def update_annotation_to_index_task(
vector.delete_by_metadata_field("annotation_id", annotation_id)
vector.add_texts([document])
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Build index successful for annotation: {annotation_id} latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("Build index for annotation failed")
logger.exception("Build index for annotation failed")
finally:
db.session.close()

12
api/tasks/batch_clean_document_task.py
View File

@@ -11,6 +11,8 @@ from extensions.ext_storage import storage
from models.dataset import Dataset, DocumentSegment
from models.model import UploadFile
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form: str, file_ids: list[str]):
@@ -23,7 +25,7 @@ def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form
Usage: batch_clean_document_task.delay(document_ids, dataset_id)
"""
logging.info(click.style("Start batch clean documents when documents deleted", fg="green"))
logger.info(click.style("Start batch clean documents when documents deleted", fg="green"))
start_at = time.perf_counter()
try:
@@ -47,7 +49,7 @@ def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form
if image_file and image_file.key:
storage.delete(image_file.key)
except Exception:
logging.exception(
logger.exception(
"Delete image_files failed when storage deleted, \
image_upload_file_is: %s",
upload_file_id,
@@ -62,18 +64,18 @@ def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form
try:
storage.delete(file.key)
except Exception:
logging.exception("Delete file failed when document deleted, file_id: %s", file.id)
logger.exception("Delete file failed when document deleted, file_id: %s", file.id)
db.session.delete(file)
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Cleaned documents when documents deleted latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("Cleaned documents when documents deleted failed")
logger.exception("Cleaned documents when documents deleted failed")
finally:
db.session.close()

8
api/tasks/batch_create_segment_to_index_task.py
View File

@@ -21,6 +21,8 @@ from models.dataset import Dataset, Document, DocumentSegment
from models.model import UploadFile
from services.vector_service import VectorService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def batch_create_segment_to_index_task(
@@ -42,7 +44,7 @@ def batch_create_segment_to_index_task(
Usage: batch_create_segment_to_index_task.delay(job_id, upload_file_id, dataset_id, document_id, tenant_id, user_id)
"""
logging.info(click.style(f"Start batch create segment jobId: {job_id}", fg="green"))
logger.info(click.style(f"Start batch create segment jobId: {job_id}", fg="green"))
start_at = time.perf_counter()
indexing_cache_key = f"segment_batch_import_{job_id}"
@@ -142,14 +144,14 @@ def batch_create_segment_to_index_task(
db.session.commit()
redis_client.setex(indexing_cache_key, 600, "completed")
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Segment batch created job: {job_id} latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("Segments batch created index failed")
logger.exception("Segments batch created index failed")
redis_client.setex(indexing_cache_key, 600, "error")
finally:
db.session.close()

26
api/tasks/clean_dataset_task.py
View File

@@ -20,6 +20,8 @@ from models.dataset import (
)
from models.model import UploadFile
logger = logging.getLogger(__name__)
# Add import statement for ValueError
@shared_task(queue="dataset")
@@ -42,7 +44,7 @@ def clean_dataset_task(
Usage: clean_dataset_task.delay(dataset_id, tenant_id, indexing_technique, index_struct)
"""
logging.info(click.style(f"Start clean dataset when dataset deleted: {dataset_id}", fg="green"))
logger.info(click.style(f"Start clean dataset when dataset deleted: {dataset_id}", fg="green"))
start_at = time.perf_counter()
try:
@@ -63,7 +65,7 @@ def clean_dataset_task(
from core.rag.index_processor.constant.index_type import IndexType
doc_form = IndexType.PARAGRAPH_INDEX
logging.info(
logger.info(
click.style(f"Invalid doc_form detected, using default index type for cleanup: {doc_form}", fg="yellow")
)
@@ -72,18 +74,18 @@ def clean_dataset_task(
try:
index_processor = IndexProcessorFactory(doc_form).init_index_processor()
index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True)
logging.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))
logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))
except Exception as index_cleanup_error:
logging.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))
logger.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))
# Continue with document and segment deletion even if vector cleanup fails
logging.info(
logger.info(
click.style(f"Continuing with document and segment deletion for dataset: {dataset_id}", fg="yellow")
)
if documents is None or len(documents) == 0:
logging.info(click.style(f"No documents found for dataset: {dataset_id}", fg="green"))
logger.info(click.style(f"No documents found for dataset: {dataset_id}", fg="green"))
else:
logging.info(click.style(f"Cleaning documents for dataset: {dataset_id}", fg="green"))
logger.info(click.style(f"Cleaning documents for dataset: {dataset_id}", fg="green"))
for document in documents:
db.session.delete(document)
@@ -97,7 +99,7 @@ def clean_dataset_task(
try:
storage.delete(image_file.key)
except Exception:
logging.exception(
logger.exception(
"Delete image_files failed when storage deleted, \
image_upload_file_is: %s",
upload_file_id,
@@ -134,7 +136,7 @@ def clean_dataset_task(
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(f"Cleaned dataset when dataset deleted: {dataset_id} latency: {end_at - start_at}", fg="green")
)
except Exception:
@@ -142,10 +144,10 @@ def clean_dataset_task(
# This ensures the database session is properly cleaned up
try:
db.session.rollback()
logging.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))
logger.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))
except Exception as rollback_error:
logging.exception("Failed to rollback database session")
logger.exception("Failed to rollback database session")
logging.exception("Cleaned dataset when dataset deleted failed")
logger.exception("Cleaned dataset when dataset deleted failed")
finally:
db.session.close()

12
api/tasks/clean_document_task.py
View File

@@ -12,6 +12,8 @@ from extensions.ext_storage import storage
from models.dataset import Dataset, DatasetMetadataBinding, DocumentSegment
from models.model import UploadFile
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_id: Optional[str]):
@@ -24,7 +26,7 @@ def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_i
Usage: clean_document_task.delay(document_id, dataset_id)
"""
logging.info(click.style(f"Start clean document when document deleted: {document_id}", fg="green"))
logger.info(click.style(f"Start clean document when document deleted: {document_id}", fg="green"))
start_at = time.perf_counter()
try:
@@ -49,7 +51,7 @@ def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_i
try:
storage.delete(image_file.key)
except Exception:
logging.exception(
logger.exception(
"Delete image_files failed when storage deleted, \
image_upload_file_is: %s",
upload_file_id,
@@ -64,7 +66,7 @@ def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_i
try:
storage.delete(file.key)
except Exception:
logging.exception("Delete file failed when document deleted, file_id: %s", file_id)
logger.exception("Delete file failed when document deleted, file_id: %s", file_id)
db.session.delete(file)
db.session.commit()
@@ -76,13 +78,13 @@ def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_i
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Cleaned document when document deleted: {document_id} latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("Cleaned document when document deleted failed")
logger.exception("Cleaned document when document deleted failed")
finally:
db.session.close()

10
api/tasks/clean_notion_document_task.py
View File

@@ -8,6 +8,8 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFacto
from extensions.ext_database import db
from models.dataset import Dataset, Document, DocumentSegment
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def clean_notion_document_task(document_ids: list[str], dataset_id: str):
@@ -18,9 +20,7 @@ def clean_notion_document_task(document_ids: list[str], dataset_id: str):
Usage: clean_notion_document_task.delay(document_ids, dataset_id)
"""
logging.info(
click.style(f"Start clean document when import form notion document deleted: {dataset_id}", fg="green")
)
logger.info(click.style(f"Start clean document when import form notion document deleted: {dataset_id}", fg="green"))
start_at = time.perf_counter()
try:
@@ -43,7 +43,7 @@ def clean_notion_document_task(document_ids: list[str], dataset_id: str):
db.session.delete(segment)
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
"Clean document when import form notion document deleted end :: {} latency: {}".format(
dataset_id, end_at - start_at
@@ -52,6 +52,6 @@ def clean_notion_document_task(document_ids: list[str], dataset_id: str):
)
)
except Exception:
logging.exception("Cleaned document when import form notion document deleted failed")
logger.exception("Cleaned document when import form notion document deleted failed")
finally:
db.session.close()

16
api/tasks/create_segment_to_index_task.py
View File

@@ -12,6 +12,8 @@ from extensions.ext_redis import redis_client
from libs.datetime_utils import naive_utc_now
from models.dataset import DocumentSegment
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def create_segment_to_index_task(segment_id: str, keywords: Optional[list[str]] = None):
@@ -21,12 +23,12 @@ def create_segment_to_index_task(segment_id: str, keywords: Optional[list[str]]
:param keywords:
Usage: create_segment_to_index_task.delay(segment_id)
"""
logging.info(click.style(f"Start create segment to index: {segment_id}", fg="green"))
logger.info(click.style(f"Start create segment to index: {segment_id}", fg="green"))
start_at = time.perf_counter()
segment = db.session.query(DocumentSegment).where(DocumentSegment.id == segment_id).first()
if not segment:
logging.info(click.style(f"Segment not found: {segment_id}", fg="red"))
logger.info(click.style(f"Segment not found: {segment_id}", fg="red"))
db.session.close()
return
@@ -58,17 +60,17 @@ def create_segment_to_index_task(segment_id: str, keywords: Optional[list[str]]
dataset = segment.dataset
if not dataset:
logging.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))
return
dataset_document = segment.document
if not dataset_document:
logging.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))
return
if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != "completed":
logging.info(click.style(f"Segment {segment.id} document status is invalid, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} document status is invalid, pass.", fg="cyan"))
return
index_type = dataset.doc_form
@@ -85,9 +87,9 @@ def create_segment_to_index_task(segment_id: str, keywords: Optional[list[str]]
db.session.commit()
end_at = time.perf_counter()
logging.info(click.style(f"Segment created to index: {segment.id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Segment created to index: {segment.id} latency: {end_at - start_at}", fg="green"))
except Exception as e:
logging.exception("create segment to index failed")
logger.exception("create segment to index failed")
segment.enabled = False
segment.disabled_at = naive_utc_now()
segment.status = "error"

8
api/tasks/deal_dataset_vector_index_task.py
View File

@@ -12,6 +12,8 @@ from extensions.ext_database import db
from models.dataset import Dataset, DocumentSegment
from models.dataset import Document as DatasetDocument
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def deal_dataset_vector_index_task(dataset_id: str, action: Literal["remove", "add", "update"]):
@@ -21,7 +23,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: Literal["remove", "a
:param action: action
Usage: deal_dataset_vector_index_task.delay(dataset_id, action)
"""
logging.info(click.style(f"Start deal dataset vector index: {dataset_id}", fg="green"))
logger.info(click.style(f"Start deal dataset vector index: {dataset_id}", fg="green"))
start_at = time.perf_counter()
try:
@@ -163,8 +165,8 @@ def deal_dataset_vector_index_task(dataset_id: str, action: Literal["remove", "a
index_processor.clean(dataset, None, with_keywords=False, delete_child_chunks=False)
end_at = time.perf_counter()
logging.info(click.style(f"Deal dataset vector index: {dataset_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Deal dataset vector index: {dataset_id} latency: {end_at - start_at}", fg="green"))
except Exception:
logging.exception("Deal dataset vector index failed")
logger.exception("Deal dataset vector index failed")
finally:
db.session.close()

8
api/tasks/delete_conversation_task.py
View File

@@ -10,6 +10,8 @@ from models.model import Message, MessageAnnotation, MessageFeedback
from models.tools import ToolConversationVariables, ToolFile
from models.web import PinnedConversation
logger = logging.getLogger(__name__)
@shared_task(queue="conversation")
def delete_conversation_related_data(conversation_id: str) -> None:
@@ -20,7 +22,7 @@ def delete_conversation_related_data(conversation_id: str) -> None:
conversation_id: conversation Id
"""
logging.info(
logger.info(
click.style(f"Starting to delete conversation data from db for conversation_id {conversation_id}", fg="green")
)
start_at = time.perf_counter()
@@ -53,7 +55,7 @@ def delete_conversation_related_data(conversation_id: str) -> None:
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Succeeded cleaning data from db for conversation_id {conversation_id} latency: {end_at - start_at}",
fg="green",
@@ -61,7 +63,7 @@ def delete_conversation_related_data(conversation_id: str) -> None:
)
except Exception as e:
logging.exception("Failed to delete data from db for conversation_id: %s failed", conversation_id)
logger.exception("Failed to delete data from db for conversation_id: %s failed", conversation_id)
db.session.rollback()
raise e
finally:

8
api/tasks/delete_segment_from_index_task.py
View File

@@ -8,6 +8,8 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFacto
from extensions.ext_database import db
from models.dataset import Dataset, Document
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def delete_segment_from_index_task(index_node_ids: list, dataset_id: str, document_id: str):
@@ -19,7 +21,7 @@ def delete_segment_from_index_task(index_node_ids: list, dataset_id: str, docume
Usage: delete_segment_from_index_task.delay(index_node_ids, dataset_id, document_id)
"""
logging.info(click.style("Start delete segment from index", fg="green"))
logger.info(click.style("Start delete segment from index", fg="green"))
start_at = time.perf_counter()
try:
dataset = db.session.query(Dataset).where(Dataset.id == dataset_id).first()
@@ -38,8 +40,8 @@ def delete_segment_from_index_task(index_node_ids: list, dataset_id: str, docume
index_processor.clean(dataset, index_node_ids, with_keywords=True, delete_child_chunks=True)
end_at = time.perf_counter()
logging.info(click.style(f"Segment deleted from index latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Segment deleted from index latency: {end_at - start_at}", fg="green"))
except Exception:
logging.exception("delete segment from index failed")
logger.exception("delete segment from index failed")
finally:
db.session.close()

18
api/tasks/disable_segment_from_index_task.py
View File

@@ -9,6 +9,8 @@ from extensions.ext_database import db
from extensions.ext_redis import redis_client
from models.dataset import DocumentSegment
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def disable_segment_from_index_task(segment_id: str):
@@ -18,17 +20,17 @@ def disable_segment_from_index_task(segment_id: str):
Usage: disable_segment_from_index_task.delay(segment_id)
"""
logging.info(click.style(f"Start disable segment from index: {segment_id}", fg="green"))
logger.info(click.style(f"Start disable segment from index: {segment_id}", fg="green"))
start_at = time.perf_counter()
segment = db.session.query(DocumentSegment).where(DocumentSegment.id == segment_id).first()
if not segment:
logging.info(click.style(f"Segment not found: {segment_id}", fg="red"))
logger.info(click.style(f"Segment not found: {segment_id}", fg="red"))
db.session.close()
return
if segment.status != "completed":
logging.info(click.style(f"Segment is not completed, disable is not allowed: {segment_id}", fg="red"))
logger.info(click.style(f"Segment is not completed, disable is not allowed: {segment_id}", fg="red"))
db.session.close()
return
@@ -38,17 +40,17 @@ def disable_segment_from_index_task(segment_id: str):
dataset = segment.dataset
if not dataset:
logging.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))
return
dataset_document = segment.document
if not dataset_document:
logging.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))
return
if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != "completed":
logging.info(click.style(f"Segment {segment.id} document status is invalid, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} document status is invalid, pass.", fg="cyan"))
return
index_type = dataset_document.doc_form
@@ -56,9 +58,9 @@ def disable_segment_from_index_task(segment_id: str):
index_processor.clean(dataset, [segment.index_node_id])
end_at = time.perf_counter()
logging.info(click.style(f"Segment removed from index: {segment.id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Segment removed from index: {segment.id} latency: {end_at - start_at}", fg="green"))
except Exception:
logging.exception("remove segment from index failed")
logger.exception("remove segment from index failed")
segment.enabled = True
db.session.commit()
finally:

10
api/tasks/disable_segments_from_index_task.py
View File

@@ -10,6 +10,8 @@ from extensions.ext_redis import redis_client
from models.dataset import Dataset, DocumentSegment
from models.dataset import Document as DatasetDocument
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def disable_segments_from_index_task(segment_ids: list, dataset_id: str, document_id: str):
@@ -25,18 +27,18 @@ def disable_segments_from_index_task(segment_ids: list, dataset_id: str, documen
dataset = db.session.query(Dataset).where(Dataset.id == dataset_id).first()
if not dataset:
logging.info(click.style(f"Dataset {dataset_id} not found, pass.", fg="cyan"))
logger.info(click.style(f"Dataset {dataset_id} not found, pass.", fg="cyan"))
db.session.close()
return
dataset_document = db.session.query(DatasetDocument).where(DatasetDocument.id == document_id).first()
if not dataset_document:
logging.info(click.style(f"Document {document_id} not found, pass.", fg="cyan"))
logger.info(click.style(f"Document {document_id} not found, pass.", fg="cyan"))
db.session.close()
return
if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != "completed":
logging.info(click.style(f"Document {document_id} status is invalid, pass.", fg="cyan"))
logger.info(click.style(f"Document {document_id} status is invalid, pass.", fg="cyan"))
db.session.close()
return
# sync index processor
@@ -61,7 +63,7 @@ def disable_segments_from_index_task(segment_ids: list, dataset_id: str, documen
index_processor.clean(dataset, index_node_ids, with_keywords=True, delete_child_chunks=False)
end_at = time.perf_counter()
logging.info(click.style(f"Segments removed from index latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Segments removed from index latency: {end_at - start_at}", fg="green"))
except Exception:
# update segment error msg
db.session.query(DocumentSegment).where(

16
api/tasks/document_indexing_sync_task.py
View File

@@ -12,6 +12,8 @@ from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document, DocumentSegment
from models.source import DataSourceOauthBinding
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def document_indexing_sync_task(dataset_id: str, document_id: str):
@@ -22,13 +24,13 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
Usage: document_indexing_sync_task.delay(dataset_id, document_id)
"""
logging.info(click.style(f"Start sync document: {document_id}", fg="green"))
logger.info(click.style(f"Start sync document: {document_id}", fg="green"))
start_at = time.perf_counter()
document = db.session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
if not document:
logging.info(click.style(f"Document not found: {document_id}", fg="red"))
logger.info(click.style(f"Document not found: {document_id}", fg="red"))
db.session.close()
return
@@ -93,7 +95,7 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
db.session.delete(segment)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
"Cleaned document when document update data source or process rule: {} latency: {}".format(
document_id, end_at - start_at
@@ -102,16 +104,16 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
)
)
except Exception:
logging.exception("Cleaned document when document update data source or process rule failed")
logger.exception("Cleaned document when document update data source or process rule failed")
try:
indexing_runner = IndexingRunner()
indexing_runner.run([document])
end_at = time.perf_counter()
logging.info(click.style(f"update document: {document.id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"update document: {document.id} latency: {end_at - start_at}", fg="green"))
except DocumentIsPausedError as ex:
logging.info(click.style(str(ex), fg="yellow"))
logger.info(click.style(str(ex), fg="yellow"))
except Exception:
logging.exception("document_indexing_sync_task failed, document_id: %s", document_id)
logger.exception("document_indexing_sync_task failed, document_id: %s", document_id)
finally:
db.session.close()

12
api/tasks/document_indexing_task.py
View File

@@ -11,6 +11,8 @@ from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document
from services.feature_service import FeatureService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def document_indexing_task(dataset_id: str, document_ids: list):
@@ -26,7 +28,7 @@ def document_indexing_task(dataset_id: str, document_ids: list):
dataset = db.session.query(Dataset).where(Dataset.id == dataset_id).first()
if not dataset:
logging.info(click.style(f"Dataset is not found: {dataset_id}", fg="yellow"))
logger.info(click.style(f"Dataset is not found: {dataset_id}", fg="yellow"))
db.session.close()
return
# check document limit
@@ -60,7 +62,7 @@ def document_indexing_task(dataset_id: str, document_ids: list):
return
for document_id in document_ids:
logging.info(click.style(f"Start process document: {document_id}", fg="green"))
logger.info(click.style(f"Start process document: {document_id}", fg="green"))
document = (
db.session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
@@ -77,10 +79,10 @@ def document_indexing_task(dataset_id: str, document_ids: list):
indexing_runner = IndexingRunner()
indexing_runner.run(documents)
end_at = time.perf_counter()
logging.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
except DocumentIsPausedError as ex:
logging.info(click.style(str(ex), fg="yellow"))
logger.info(click.style(str(ex), fg="yellow"))
except Exception:
logging.exception("Document indexing task failed, dataset_id: %s", dataset_id)
logger.exception("Document indexing task failed, dataset_id: %s", dataset_id)
finally:
db.session.close()

16
api/tasks/document_indexing_update_task.py
View File

@@ -10,6 +10,8 @@ from extensions.ext_database import db
from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document, DocumentSegment
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def document_indexing_update_task(dataset_id: str, document_id: str):
@@ -20,13 +22,13 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
Usage: document_indexing_update_task.delay(dataset_id, document_id)
"""
logging.info(click.style(f"Start update document: {document_id}", fg="green"))
logger.info(click.style(f"Start update document: {document_id}", fg="green"))
start_at = time.perf_counter()
document = db.session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
if not document:
logging.info(click.style(f"Document not found: {document_id}", fg="red"))
logger.info(click.style(f"Document not found: {document_id}", fg="red"))
db.session.close()
return
@@ -54,7 +56,7 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
db.session.delete(segment)
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
"Cleaned document when document update data source or process rule: {} latency: {}".format(
document_id, end_at - start_at
@@ -63,16 +65,16 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
)
)
except Exception:
logging.exception("Cleaned document when document update data source or process rule failed")
logger.exception("Cleaned document when document update data source or process rule failed")
try:
indexing_runner = IndexingRunner()
indexing_runner.run([document])
end_at = time.perf_counter()
logging.info(click.style(f"update document: {document.id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"update document: {document.id} latency: {end_at - start_at}", fg="green"))
except DocumentIsPausedError as ex:
logging.info(click.style(str(ex), fg="yellow"))
logger.info(click.style(str(ex), fg="yellow"))
except Exception:
logging.exception("document_indexing_update_task failed, document_id: %s", document_id)
logger.exception("document_indexing_update_task failed, document_id: %s", document_id)
finally:
db.session.close()

12
api/tasks/duplicate_document_indexing_task.py
View File

@@ -12,6 +12,8 @@ from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document, DocumentSegment
from services.feature_service import FeatureService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def duplicate_document_indexing_task(dataset_id: str, document_ids: list):
@@ -27,7 +29,7 @@ def duplicate_document_indexing_task(dataset_id: str, document_ids: list):
dataset = db.session.query(Dataset).where(Dataset.id == dataset_id).first()
if dataset is None:
logging.info(click.style(f"Dataset not found: {dataset_id}", fg="red"))
logger.info(click.style(f"Dataset not found: {dataset_id}", fg="red"))
db.session.close()
return
@@ -63,7 +65,7 @@ def duplicate_document_indexing_task(dataset_id: str, document_ids: list):
db.session.close()
for document_id in document_ids:
logging.info(click.style(f"Start process document: {document_id}", fg="green"))
logger.info(click.style(f"Start process document: {document_id}", fg="green"))
document = (
db.session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
@@ -95,10 +97,10 @@ def duplicate_document_indexing_task(dataset_id: str, document_ids: list):
indexing_runner = IndexingRunner()
indexing_runner.run(documents)
end_at = time.perf_counter()
logging.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
except DocumentIsPausedError as ex:
logging.info(click.style(str(ex), fg="yellow"))
logger.info(click.style(str(ex), fg="yellow"))
except Exception:
logging.exception("duplicate_document_indexing_task failed, dataset_id: %s", dataset_id)
logger.exception("duplicate_document_indexing_task failed, dataset_id: %s", dataset_id)
finally:
db.session.close()

18
api/tasks/enable_segment_to_index_task.py
View File

@@ -12,6 +12,8 @@ from extensions.ext_redis import redis_client
from libs.datetime_utils import naive_utc_now
from models.dataset import DocumentSegment
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def enable_segment_to_index_task(segment_id: str):
@@ -21,17 +23,17 @@ def enable_segment_to_index_task(segment_id: str):
Usage: enable_segment_to_index_task.delay(segment_id)
"""
logging.info(click.style(f"Start enable segment to index: {segment_id}", fg="green"))
logger.info(click.style(f"Start enable segment to index: {segment_id}", fg="green"))
start_at = time.perf_counter()
segment = db.session.query(DocumentSegment).where(DocumentSegment.id == segment_id).first()
if not segment:
logging.info(click.style(f"Segment not found: {segment_id}", fg="red"))
logger.info(click.style(f"Segment not found: {segment_id}", fg="red"))
db.session.close()
return
if segment.status != "completed":
logging.info(click.style(f"Segment is not completed, enable is not allowed: {segment_id}", fg="red"))
logger.info(click.style(f"Segment is not completed, enable is not allowed: {segment_id}", fg="red"))
db.session.close()
return
@@ -51,17 +53,17 @@ def enable_segment_to_index_task(segment_id: str):
dataset = segment.dataset
if not dataset:
logging.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} has no dataset, pass.", fg="cyan"))
return
dataset_document = segment.document
if not dataset_document:
logging.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} has no document, pass.", fg="cyan"))
return
if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != "completed":
logging.info(click.style(f"Segment {segment.id} document status is invalid, pass.", fg="cyan"))
logger.info(click.style(f"Segment {segment.id} document status is invalid, pass.", fg="cyan"))
return
index_processor = IndexProcessorFactory(dataset_document.doc_form).init_index_processor()
@@ -85,9 +87,9 @@ def enable_segment_to_index_task(segment_id: str):
index_processor.load(dataset, [document])
end_at = time.perf_counter()
logging.info(click.style(f"Segment enabled to index: {segment.id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Segment enabled to index: {segment.id} latency: {end_at - start_at}", fg="green"))
except Exception as e:
logging.exception("enable segment to index failed")
logger.exception("enable segment to index failed")
segment.enabled = False
segment.disabled_at = naive_utc_now()
segment.status = "error"

14
api/tasks/enable_segments_to_index_task.py
View File

@@ -13,6 +13,8 @@ from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, DocumentSegment
from models.dataset import Document as DatasetDocument
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_id: str):
@@ -27,17 +29,17 @@ def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_i
start_at = time.perf_counter()
dataset = db.session.query(Dataset).where(Dataset.id == dataset_id).first()
if not dataset:
logging.info(click.style(f"Dataset {dataset_id} not found, pass.", fg="cyan"))
logger.info(click.style(f"Dataset {dataset_id} not found, pass.", fg="cyan"))
return
dataset_document = db.session.query(DatasetDocument).where(DatasetDocument.id == document_id).first()
if not dataset_document:
logging.info(click.style(f"Document {document_id} not found, pass.", fg="cyan"))
logger.info(click.style(f"Document {document_id} not found, pass.", fg="cyan"))
db.session.close()
return
if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != "completed":
logging.info(click.style(f"Document {document_id} status is invalid, pass.", fg="cyan"))
logger.info(click.style(f"Document {document_id} status is invalid, pass.", fg="cyan"))
db.session.close()
return
# sync index processor
@@ -53,7 +55,7 @@ def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_i
.all()
)
if not segments:
logging.info(click.style(f"Segments not found: {segment_ids}", fg="cyan"))
logger.info(click.style(f"Segments not found: {segment_ids}", fg="cyan"))
db.session.close()
return
@@ -91,9 +93,9 @@ def enable_segments_to_index_task(segment_ids: list, dataset_id: str, document_i
index_processor.load(dataset, documents)
end_at = time.perf_counter()
logging.info(click.style(f"Segments enabled to index latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Segments enabled to index latency: {end_at - start_at}", fg="green"))
except Exception as e:
logging.exception("enable segments to index failed")
logger.exception("enable segments to index failed")
# update segment error msg
db.session.query(DocumentSegment).where(
DocumentSegment.id.in_(segment_ids),

14
api/tasks/mail_account_deletion_task.py
View File

@@ -7,6 +7,8 @@ from celery import shared_task
from extensions.ext_mail import mail
from libs.email_i18n import EmailType, get_email_i18n_service
logger = logging.getLogger(__name__)
@shared_task(queue="mail")
def send_deletion_success_task(to: str, language: str = "en-US") -> None:
@@ -20,7 +22,7 @@ def send_deletion_success_task(to: str, language: str = "en-US") -> None:
if not mail.is_inited():
return
logging.info(click.style(f"Start send account deletion success email to {to}", fg="green"))
logger.info(click.style(f"Start send account deletion success email to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -36,11 +38,11 @@ def send_deletion_success_task(to: str, language: str = "en-US") -> None:
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(f"Send account deletion success email to {to}: latency: {end_at - start_at}", fg="green")
)
except Exception:
logging.exception("Send account deletion success email to %s failed", to)
logger.exception("Send account deletion success email to %s failed", to)
@shared_task(queue="mail")
@@ -56,7 +58,7 @@ def send_account_deletion_verification_code(to: str, code: str, language: str =
if not mail.is_inited():
return
logging.info(click.style(f"Start send account deletion verification code email to {to}", fg="green"))
logger.info(click.style(f"Start send account deletion verification code email to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -72,7 +74,7 @@ def send_account_deletion_verification_code(to: str, code: str, language: str =
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
"Send account deletion verification code email to {} succeeded: latency: {}".format(
to, end_at - start_at
@@ -81,4 +83,4 @@ def send_account_deletion_verification_code(to: str, code: str, language: str =
)
)
except Exception:
logging.exception("Send account deletion verification code email to %s failed", to)
logger.exception("Send account deletion verification code email to %s failed", to)

14
api/tasks/mail_change_mail_task.py
View File

@@ -7,6 +7,8 @@ from celery import shared_task
from extensions.ext_mail import mail
from libs.email_i18n import EmailType, get_email_i18n_service
logger = logging.getLogger(__name__)
@shared_task(queue="mail")
def send_change_mail_task(language: str, to: str, code: str, phase: str) -> None:
@@ -22,7 +24,7 @@ def send_change_mail_task(language: str, to: str, code: str, phase: str) -> None
if not mail.is_inited():
return
logging.info(click.style(f"Start change email mail to {to}", fg="green"))
logger.info(click.style(f"Start change email mail to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -35,9 +37,9 @@ def send_change_mail_task(language: str, to: str, code: str, phase: str) -> None
)
end_at = time.perf_counter()
logging.info(click.style(f"Send change email mail to {to} succeeded: latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Send change email mail to {to} succeeded: latency: {end_at - start_at}", fg="green"))
except Exception:
logging.exception("Send change email mail to %s failed", to)
logger.exception("Send change email mail to %s failed", to)
@shared_task(queue="mail")
@@ -52,7 +54,7 @@ def send_change_mail_completed_notification_task(language: str, to: str) -> None
if not mail.is_inited():
return
logging.info(click.style(f"Start change email completed notify mail to {to}", fg="green"))
logger.info(click.style(f"Start change email completed notify mail to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -68,11 +70,11 @@ def send_change_mail_completed_notification_task(language: str, to: str) -> None
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Send change email completed mail to {to} succeeded: latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("Send change email completed mail to %s failed", to)
logger.exception("Send change email completed mail to %s failed", to)

8
api/tasks/mail_email_code_login.py
View File

@@ -7,6 +7,8 @@ from celery import shared_task
from extensions.ext_mail import mail
from libs.email_i18n import EmailType, get_email_i18n_service
logger = logging.getLogger(__name__)
@shared_task(queue="mail")
def send_email_code_login_mail_task(language: str, to: str, code: str) -> None:
@@ -21,7 +23,7 @@ def send_email_code_login_mail_task(language: str, to: str, code: str) -> None:
if not mail.is_inited():
return
logging.info(click.style(f"Start email code login mail to {to}", fg="green"))
logger.info(click.style(f"Start email code login mail to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -37,8 +39,8 @@ def send_email_code_login_mail_task(language: str, to: str, code: str) -> None:
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(f"Send email code login mail to {to} succeeded: latency: {end_at - start_at}", fg="green")
)
except Exception:
logging.exception("Send email code login mail to %s failed", to)
logger.exception("Send email code login mail to %s failed", to)

8
api/tasks/mail_inner_task.py
View File

@@ -9,13 +9,15 @@ from flask import render_template_string
from extensions.ext_mail import mail
from libs.email_i18n import get_email_i18n_service
logger = logging.getLogger(__name__)
@shared_task(queue="mail")
def send_inner_email_task(to: list[str], subject: str, body: str, substitutions: Mapping[str, str]):
if not mail.is_inited():
return
logging.info(click.style(f"Start enterprise mail to {to} with subject {subject}", fg="green"))
logger.info(click.style(f"Start enterprise mail to {to} with subject {subject}", fg="green"))
start_at = time.perf_counter()
try:
@@ -25,6 +27,6 @@ def send_inner_email_task(to: list[str], subject: str, body: str, substitutions:
email_service.send_raw_email(to=to, subject=subject, html_content=html_content)
end_at = time.perf_counter()
logging.info(click.style(f"Send enterprise mail to {to} succeeded: latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Send enterprise mail to {to} succeeded: latency: {end_at - start_at}", fg="green"))
except Exception:
logging.exception("Send enterprise mail to %s failed", to)
logger.exception("Send enterprise mail to %s failed", to)

10
api/tasks/mail_invite_member_task.py
View File

@@ -8,6 +8,8 @@ from configs import dify_config
from extensions.ext_mail import mail
from libs.email_i18n import EmailType, get_email_i18n_service
logger = logging.getLogger(__name__)
@shared_task(queue="mail")
def send_invite_member_mail_task(language: str, to: str, token: str, inviter_name: str, workspace_name: str) -> None:
@@ -24,7 +26,7 @@ def send_invite_member_mail_task(language: str, to: str, token: str, inviter_nam
if not mail.is_inited():
return
logging.info(click.style(f"Start send invite member mail to {to} in workspace {workspace_name}", fg="green"))
logger.info(click.style(f"Start send invite member mail to {to} in workspace {workspace_name}", fg="green"))
start_at = time.perf_counter()
try:
@@ -43,8 +45,6 @@ def send_invite_member_mail_task(language: str, to: str, token: str, inviter_nam
)
end_at = time.perf_counter()
logging.info(
click.style(f"Send invite member mail to {to} succeeded: latency: {end_at - start_at}", fg="green")
)
logger.info(click.style(f"Send invite member mail to {to} succeeded: latency: {end_at - start_at}", fg="green"))
except Exception:
logging.exception("Send invite member mail to %s failed", to)
logger.exception("Send invite member mail to %s failed", to)

20
api/tasks/mail_owner_transfer_task.py
View File

@@ -7,6 +7,8 @@ from celery import shared_task
from extensions.ext_mail import mail
from libs.email_i18n import EmailType, get_email_i18n_service
logger = logging.getLogger(__name__)
@shared_task(queue="mail")
def send_owner_transfer_confirm_task(language: str, to: str, code: str, workspace: str) -> None:
@@ -22,7 +24,7 @@ def send_owner_transfer_confirm_task(language: str, to: str, code: str, workspac
if not mail.is_inited():
return
logging.info(click.style(f"Start owner transfer confirm mail to {to}", fg="green"))
logger.info(click.style(f"Start owner transfer confirm mail to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -39,14 +41,14 @@ def send_owner_transfer_confirm_task(language: str, to: str, code: str, workspac
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Send owner transfer confirm mail to {to} succeeded: latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("owner transfer confirm email mail to %s failed", to)
logger.exception("owner transfer confirm email mail to %s failed", to)
@shared_task(queue="mail")
@@ -63,7 +65,7 @@ def send_old_owner_transfer_notify_email_task(language: str, to: str, workspace:
if not mail.is_inited():
return
logging.info(click.style(f"Start old owner transfer notify mail to {to}", fg="green"))
logger.info(click.style(f"Start old owner transfer notify mail to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -80,14 +82,14 @@ def send_old_owner_transfer_notify_email_task(language: str, to: str, workspace:
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Send old owner transfer notify mail to {to} succeeded: latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("old owner transfer notify email mail to %s failed", to)
logger.exception("old owner transfer notify email mail to %s failed", to)
@shared_task(queue="mail")
@@ -103,7 +105,7 @@ def send_new_owner_transfer_notify_email_task(language: str, to: str, workspace:
if not mail.is_inited():
return
logging.info(click.style(f"Start new owner transfer notify mail to {to}", fg="green"))
logger.info(click.style(f"Start new owner transfer notify mail to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -119,11 +121,11 @@ def send_new_owner_transfer_notify_email_task(language: str, to: str, workspace:
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(
f"Send new owner transfer notify mail to {to} succeeded: latency: {end_at - start_at}",
fg="green",
)
)
except Exception:
logging.exception("new owner transfer notify email mail to %s failed", to)
logger.exception("new owner transfer notify email mail to %s failed", to)

8
api/tasks/mail_reset_password_task.py
View File

@@ -7,6 +7,8 @@ from celery import shared_task
from extensions.ext_mail import mail
from libs.email_i18n import EmailType, get_email_i18n_service
logger = logging.getLogger(__name__)
@shared_task(queue="mail")
def send_reset_password_mail_task(language: str, to: str, code: str) -> None:
@@ -21,7 +23,7 @@ def send_reset_password_mail_task(language: str, to: str, code: str) -> None:
if not mail.is_inited():
return
logging.info(click.style(f"Start password reset mail to {to}", fg="green"))
logger.info(click.style(f"Start password reset mail to {to}", fg="green"))
start_at = time.perf_counter()
try:
@@ -37,8 +39,8 @@ def send_reset_password_mail_task(language: str, to: str, code: str) -> None:
)
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(f"Send password reset mail to {to} succeeded: latency: {end_at - start_at}", fg="green")
)
except Exception:
logging.exception("Send password reset mail to %s failed", to)
logger.exception("Send password reset mail to %s failed", to)

8
api/tasks/ops_trace_task.py
View File

@@ -12,6 +12,8 @@ from extensions.ext_storage import storage
from models.model import Message
from models.workflow import WorkflowRun
logger = logging.getLogger(__name__)
@shared_task(queue="ops_trace")
def process_trace_tasks(file_info):
@@ -43,11 +45,11 @@ def process_trace_tasks(file_info):
if trace_type:
trace_info = trace_type(**trace_info)
trace_instance.trace(trace_info)
logging.info("Processing trace tasks success, app_id: %s", app_id)
logger.info("Processing trace tasks success, app_id: %s", app_id)
except Exception as e:
logging.info("error:\n\n\n%s\n\n\n\n", e)
logger.info("error:\n\n\n%s\n\n\n\n", e)
failed_key = f"{OPS_TRACE_FAILED_KEY}_{app_id}"
redis_client.incr(failed_key)
logging.info("Processing trace tasks failed, app_id: %s", app_id)
logger.info("Processing trace tasks failed, app_id: %s", app_id)
finally:
storage.delete(file_path)

12
api/tasks/recover_document_indexing_task.py
View File

@@ -8,6 +8,8 @@ from core.indexing_runner import DocumentIsPausedError, IndexingRunner
from extensions.ext_database import db
from models.dataset import Document
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def recover_document_indexing_task(dataset_id: str, document_id: str):
@@ -18,13 +20,13 @@ def recover_document_indexing_task(dataset_id: str, document_id: str):
Usage: recover_document_indexing_task.delay(dataset_id, document_id)
"""
logging.info(click.style(f"Recover document: {document_id}", fg="green"))
logger.info(click.style(f"Recover document: {document_id}", fg="green"))
start_at = time.perf_counter()
document = db.session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
if not document:
logging.info(click.style(f"Document not found: {document_id}", fg="red"))
logger.info(click.style(f"Document not found: {document_id}", fg="red"))
db.session.close()
return
@@ -37,10 +39,10 @@ def recover_document_indexing_task(dataset_id: str, document_id: str):
elif document.indexing_status == "indexing":
indexing_runner.run_in_indexing_status(document)
end_at = time.perf_counter()
logging.info(click.style(f"Processed document: {document.id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Processed document: {document.id} latency: {end_at - start_at}", fg="green"))
except DocumentIsPausedError as ex:
logging.info(click.style(str(ex), fg="yellow"))
logger.info(click.style(str(ex), fg="yellow"))
except Exception:
logging.exception("recover_document_indexing_task failed, document_id: %s", document_id)
logger.exception("recover_document_indexing_task failed, document_id: %s", document_id)
finally:
db.session.close()

26
api/tasks/remove_app_and_related_data_task.py
View File

@@ -40,10 +40,12 @@ from models.workflow import (
)
from repositories.factory import DifyAPIRepositoryFactory
logger = logging.getLogger(__name__)
@shared_task(queue="app_deletion", bind=True, max_retries=3)
def remove_app_and_related_data_task(self, tenant_id: str, app_id: str):
logging.info(click.style(f"Start deleting app and related data: {tenant_id}:{app_id}", fg="green"))
logger.info(click.style(f"Start deleting app and related data: {tenant_id}:{app_id}", fg="green"))
start_at = time.perf_counter()
try:
# Delete related data
@@ -69,14 +71,12 @@ def remove_app_and_related_data_task(self, tenant_id: str, app_id: str):
_delete_draft_variables(app_id)
end_at = time.perf_counter()
logging.info(click.style(f"App and related data deleted: {app_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"App and related data deleted: {app_id} latency: {end_at - start_at}", fg="green"))
except SQLAlchemyError as e:
logging.exception(
click.style(f"Database error occurred while deleting app {app_id} and related data", fg="red")
)
logger.exception(click.style(f"Database error occurred while deleting app {app_id} and related data", fg="red"))
raise self.retry(exc=e, countdown=60) # Retry after 60 seconds
except Exception as e:
logging.exception(click.style(f"Error occurred while deleting app {app_id} and related data", fg="red"))
logger.exception(click.style(f"Error occurred while deleting app {app_id} and related data", fg="red"))
raise self.retry(exc=e, countdown=60) # Retry after 60 seconds
@@ -215,7 +215,7 @@ def _delete_app_workflow_runs(tenant_id: str, app_id: str):
batch_size=1000,
)
logging.info("Deleted %s workflow runs for app %s", deleted_count, app_id)
logger.info("Deleted %s workflow runs for app %s", deleted_count, app_id)
def _delete_app_workflow_node_executions(tenant_id: str, app_id: str):
@@ -229,7 +229,7 @@ def _delete_app_workflow_node_executions(tenant_id: str, app_id: str):
batch_size=1000,
)
logging.info("Deleted %s workflow node executions for app %s", deleted_count, app_id)
logger.info("Deleted %s workflow node executions for app %s", deleted_count, app_id)
def _delete_app_workflow_app_logs(tenant_id: str, app_id: str):
@@ -266,7 +266,7 @@ def _delete_conversation_variables(*, app_id: str):
with db.engine.connect() as conn:
conn.execute(stmt)
conn.commit()
logging.info(click.style(f"Deleted conversation variables for app {app_id}", fg="green"))
logger.info(click.style(f"Deleted conversation variables for app {app_id}", fg="green"))
def _delete_app_messages(tenant_id: str, app_id: str):
@@ -389,9 +389,9 @@ def delete_draft_variables_batch(app_id: str, batch_size: int = 1000) -> int:
batch_deleted = deleted_result.rowcount
total_deleted += batch_deleted
logging.info(click.style(f"Deleted {batch_deleted} draft variables (batch) for app {app_id}", fg="green"))
logger.info(click.style(f"Deleted {batch_deleted} draft variables (batch) for app {app_id}", fg="green"))
logging.info(click.style(f"Deleted {total_deleted} total draft variables for app {app_id}", fg="green"))
logger.info(click.style(f"Deleted {total_deleted} total draft variables for app {app_id}", fg="green"))
return total_deleted
@@ -407,8 +407,8 @@ def _delete_records(query_sql: str, params: dict, delete_func: Callable, name: s
try:
delete_func(record_id)
db.session.commit()
logging.info(click.style(f"Deleted {name} {record_id}", fg="green"))
logger.info(click.style(f"Deleted {name} {record_id}", fg="green"))
except Exception:
logging.exception("Error occurred while deleting %s %s", name, record_id)
logger.exception("Error occurred while deleting %s %s", name, record_id)
continue
rs.close()

16
api/tasks/remove_document_from_index_task.py
View File

@@ -10,6 +10,8 @@ from extensions.ext_redis import redis_client
from libs.datetime_utils import naive_utc_now
from models.dataset import Document, DocumentSegment
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def remove_document_from_index_task(document_id: str):
@@ -19,17 +21,17 @@ def remove_document_from_index_task(document_id: str):
Usage: remove_document_from_index.delay(document_id)
"""
logging.info(click.style(f"Start remove document segments from index: {document_id}", fg="green"))
logger.info(click.style(f"Start remove document segments from index: {document_id}", fg="green"))
start_at = time.perf_counter()
document = db.session.query(Document).where(Document.id == document_id).first()
if not document:
logging.info(click.style(f"Document not found: {document_id}", fg="red"))
logger.info(click.style(f"Document not found: {document_id}", fg="red"))
db.session.close()
return
if document.indexing_status != "completed":
logging.info(click.style(f"Document is not completed, remove is not allowed: {document_id}", fg="red"))
logger.info(click.style(f"Document is not completed, remove is not allowed: {document_id}", fg="red"))
db.session.close()
return
@@ -49,7 +51,7 @@ def remove_document_from_index_task(document_id: str):
try:
index_processor.clean(dataset, index_node_ids, with_keywords=True, delete_child_chunks=False)
except Exception:
logging.exception("clean dataset %s from index failed", dataset.id)
logger.exception("clean dataset %s from index failed", dataset.id)
# update segment to disable
db.session.query(DocumentSegment).where(DocumentSegment.document_id == document.id).update(
{
@@ -62,11 +64,9 @@ def remove_document_from_index_task(document_id: str):
db.session.commit()
end_at = time.perf_counter()
logging.info(
click.style(f"Document removed from index: {document.id} latency: {end_at - start_at}", fg="green")
)
logger.info(click.style(f"Document removed from index: {document.id} latency: {end_at - start_at}", fg="green"))
except Exception:
logging.exception("remove document from index failed")
logger.exception("remove document from index failed")
if not document.archived:
document.enabled = True
db.session.commit()

16
api/tasks/retry_document_indexing_task.py
View File

@@ -12,6 +12,8 @@ from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document, DocumentSegment
from services.feature_service import FeatureService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def retry_document_indexing_task(dataset_id: str, document_ids: list[str]):
@@ -27,7 +29,7 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str]):
try:
dataset = db.session.query(Dataset).where(Dataset.id == dataset_id).first()
if not dataset:
logging.info(click.style(f"Dataset not found: {dataset_id}", fg="red"))
logger.info(click.style(f"Dataset not found: {dataset_id}", fg="red"))
return
tenant_id = dataset.tenant_id
for document_id in document_ids:
@@ -57,12 +59,12 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str]):
redis_client.delete(retry_indexing_cache_key)
return
logging.info(click.style(f"Start retry document: {document_id}", fg="green"))
logger.info(click.style(f"Start retry document: {document_id}", fg="green"))
document = (
db.session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
)
if not document:
logging.info(click.style(f"Document not found: {document_id}", fg="yellow"))
logger.info(click.style(f"Document not found: {document_id}", fg="yellow"))
return
try:
# clean old data
@@ -92,13 +94,13 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str]):
document.stopped_at = naive_utc_now()
db.session.add(document)
db.session.commit()
logging.info(click.style(str(ex), fg="yellow"))
logger.info(click.style(str(ex), fg="yellow"))
redis_client.delete(retry_indexing_cache_key)
logging.exception("retry_document_indexing_task failed, document_id: %s", document_id)
logger.exception("retry_document_indexing_task failed, document_id: %s", document_id)
end_at = time.perf_counter()
logging.info(click.style(f"Retry dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Retry dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
except Exception as e:
logging.exception(
logger.exception(
"retry_document_indexing_task failed, dataset_id: %s, document_ids: %s", dataset_id, document_ids
)
raise e

12
api/tasks/sync_website_document_indexing_task.py
View File

@@ -12,6 +12,8 @@ from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document, DocumentSegment
from services.feature_service import FeatureService
logger = logging.getLogger(__name__)
@shared_task(queue="dataset")
def sync_website_document_indexing_task(dataset_id: str, document_id: str):
@@ -52,10 +54,10 @@ def sync_website_document_indexing_task(dataset_id: str, document_id: str):
redis_client.delete(sync_indexing_cache_key)
return
logging.info(click.style(f"Start sync website document: {document_id}", fg="green"))
logger.info(click.style(f"Start sync website document: {document_id}", fg="green"))
document = db.session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
if not document:
logging.info(click.style(f"Document not found: {document_id}", fg="yellow"))
logger.info(click.style(f"Document not found: {document_id}", fg="yellow"))
return
try:
# clean old data
@@ -85,8 +87,8 @@ def sync_website_document_indexing_task(dataset_id: str, document_id: str):
document.stopped_at = naive_utc_now()
db.session.add(document)
db.session.commit()
logging.info(click.style(str(ex), fg="yellow"))
logger.info(click.style(str(ex), fg="yellow"))
redis_client.delete(sync_indexing_cache_key)
logging.exception("sync_website_document_indexing_task failed, document_id: %s", document_id)
logger.exception("sync_website_document_indexing_task failed, document_id: %s", document_id)
end_at = time.perf_counter()
logging.info(click.style(f"Sync document: {document_id} latency: {end_at - start_at}", fg="green"))
logger.info(click.style(f"Sync document: {document_id} latency: {end_at - start_at}", fg="green"))