feat: add administrative commands to free up storage space by removing unused files (#18835)

api/extensions/ext_commands.py

@@ -5,6 +5,7 @@ def init_app(app: DifyApp):
     from commands import (
         add_qdrant_index,
         clear_free_plan_tenant_expired_logs,
+        clear_orphaned_file_records,
         convert_to_agent_apps,
         create_tenant,
         extract_plugins,
@@ -13,6 +14,7 @@ def init_app(app: DifyApp):
         install_plugins,
         migrate_data_for_plugin,
         old_metadata_migration,
+        remove_orphaned_files_on_storage,
         reset_email,
         reset_encrypt_key_pair,
         reset_password,
@@ -36,6 +38,8 @@ def init_app(app: DifyApp):
         install_plugins,
         old_metadata_migration,
         clear_free_plan_tenant_expired_logs,
+        clear_orphaned_file_records,
+        remove_orphaned_files_on_storage,
     ]
     for cmd in cmds_to_register:
         app.cli.add_command(cmd)

api/extensions/ext_storage.py

@@ -102,6 +102,9 @@ class Storage:
     def delete(self, filename):
         return self.storage_runner.delete(filename)
 
+    def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+        return self.storage_runner.scan(path, files=files, directories=directories)
+
 
 storage = Storage()
 

api/extensions/storage/base_storage.py

@@ -30,3 +30,11 @@ class BaseStorage(ABC):
     @abstractmethod
     def delete(self, filename):
         raise NotImplementedError
+
+    def scan(self, path, files=True, directories=False) -> list[str]:
+        """
+        Scan files and directories in the given path.
+        This method is implemented only in some storage backends.
+        If a storage backend doesn't support scanning, it will raise NotImplementedError.
+        """
+        raise NotImplementedError("This storage backend doesn't support scanning")

api/extensions/storage/opendal_storage.py

@@ -80,3 +80,20 @@ class OpenDALStorage(BaseStorage):
             logger.debug(f"file {filename} deleted")
             return
         logger.debug(f"file {filename} not found, skip delete")
+
+    def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+        if not self.exists(path):
+            raise FileNotFoundError("Path not found")
+
+        all_files = self.op.scan(path=path)
+        if files and directories:
+            logger.debug(f"files and directories on {path} scanned")
+            return [f.path for f in all_files]
+        if files:
+            logger.debug(f"files on {path} scanned")
+            return [f.path for f in all_files if not f.path.endswith("/")]
+        elif directories:
+            logger.debug(f"directories on {path} scanned")
+            return [f.path for f in all_files if f.path.endswith("/")]
+        else:
+            raise ValueError("At least one of files or directories must be True")