estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor: use logger = logging.getLogger(__name__) in logging (#24515)

Browse Source 
Co-authored-by: Yongtao Huang <99629139+hyongtao-db@users.noreply.github.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
This commit is contained in:
Yongtao Huang
2025-08-26 18:10:31 +08:00
committed by GitHub
parent 8af2ae973f
commit fa753239ad
102 changed files with 565 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
api/tasks/clean_dataset_task.py
View File

@@ -20,6 +20,8 @@ from models.dataset import (
)
from models.model import UploadFile
logger = logging.getLogger(__name__)
# Add import statement for ValueError
@shared_task(queue="dataset")
@@ -42,7 +44,7 @@ def clean_dataset_task(
Usage: clean_dataset_task.delay(dataset_id, tenant_id, indexing_technique, index_struct)
"""
logging.info(click.style(f"Start clean dataset when dataset deleted: {dataset_id}", fg="green"))
logger.info(click.style(f"Start clean dataset when dataset deleted: {dataset_id}", fg="green"))
start_at = time.perf_counter()
try:
@@ -63,7 +65,7 @@ def clean_dataset_task(
from core.rag.index_processor.constant.index_type import IndexType
doc_form = IndexType.PARAGRAPH_INDEX
logging.info(
logger.info(
click.style(f"Invalid doc_form detected, using default index type for cleanup: {doc_form}", fg="yellow")
)
@@ -72,18 +74,18 @@ def clean_dataset_task(
try:
index_processor = IndexProcessorFactory(doc_form).init_index_processor()
index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True)
logging.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))
logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))
except Exception as index_cleanup_error:
logging.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))
logger.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))
# Continue with document and segment deletion even if vector cleanup fails
logging.info(
logger.info(
click.style(f"Continuing with document and segment deletion for dataset: {dataset_id}", fg="yellow")
)
if documents is None or len(documents) == 0:
logging.info(click.style(f"No documents found for dataset: {dataset_id}", fg="green"))
logger.info(click.style(f"No documents found for dataset: {dataset_id}", fg="green"))
else:
logging.info(click.style(f"Cleaning documents for dataset: {dataset_id}", fg="green"))
logger.info(click.style(f"Cleaning documents for dataset: {dataset_id}", fg="green"))
for document in documents:
db.session.delete(document)
@@ -97,7 +99,7 @@ def clean_dataset_task(
try:
storage.delete(image_file.key)
except Exception:
logging.exception(
logger.exception(
"Delete image_files failed when storage deleted, \
image_upload_file_is: %s",
upload_file_id,
@@ -134,7 +136,7 @@ def clean_dataset_task(
db.session.commit()
end_at = time.perf_counter()
logging.info(
logger.info(
click.style(f"Cleaned dataset when dataset deleted: {dataset_id} latency: {end_at - start_at}", fg="green")
)
except Exception:
@@ -142,10 +144,10 @@ def clean_dataset_task(
# This ensures the database session is properly cleaned up
try:
db.session.rollback()
logging.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))
logger.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))
except Exception as rollback_error:
logging.exception("Failed to rollback database session")
logger.exception("Failed to rollback database session")
logging.exception("Cleaned dataset when dataset deleted failed")
logger.exception("Cleaned dataset when dataset deleted failed")
finally:
db.session.close()