fix: respect resolution settings for vision for basic chatbot, text generator, and parameter extractor node (#16041)

api/core/app/apps/base_app_runner.py

@@ -17,7 +17,11 @@ from core.external_data_tool.external_data_fetch import ExternalDataFetch
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage
+from core.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    ImagePromptMessageContent,
+    PromptMessage,
+)
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.errors.invoke import InvokeBadRequestError
 from core.moderation.input_moderation import InputModeration
@@ -141,6 +145,7 @@ class AppRunner:
         query: Optional[str] = None,
         context: Optional[str] = None,
         memory: Optional[TokenBufferMemory] = None,
+        image_detail_config: Optional[ImagePromptMessageContent.DETAIL] = None,
     ) -> tuple[list[PromptMessage], Optional[list[str]]]:
         """
         Organize prompt messages
@@ -167,6 +172,7 @@ class AppRunner:
                 context=context,
                 memory=memory,
                 model_config=model_config,
+                image_detail_config=image_detail_config,
             )
         else:
             memory_config = MemoryConfig(window=MemoryConfig.WindowConfig(enabled=False))
@@ -201,6 +207,7 @@ class AppRunner:
                 memory_config=memory_config,
                 memory=memory,
                 model_config=model_config,
+                image_detail_config=image_detail_config,
             )
             stop = model_config.stop
 

api/core/app/apps/chat/app_runner.py

@@ -11,6 +11,7 @@ from core.app.entities.queue_entities import QueueAnnotationReplyEvent
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance
+from core.model_runtime.entities.message_entities import ImagePromptMessageContent
 from core.moderation.base import ModerationError
 from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
 from extensions.ext_database import db
@@ -50,6 +51,16 @@ class ChatAppRunner(AppRunner):
         query = application_generate_entity.query
         files = application_generate_entity.files
 
+        image_detail_config = (
+            application_generate_entity.file_upload_config.image_config.detail
+            if (
+                application_generate_entity.file_upload_config
+                and application_generate_entity.file_upload_config.image_config
+            )
+            else None
+        )
+        image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
+
         # Pre-calculate the number of tokens of the prompt messages,
         # and return the rest number of tokens by model context token size limit and max token size limit.
         # If the rest number of tokens is not enough, raise exception.
@@ -85,6 +96,7 @@ class ChatAppRunner(AppRunner):
             files=files,
             query=query,
             memory=memory,
+            image_detail_config=image_detail_config,
         )
 
         # moderation
@@ -182,6 +194,7 @@ class ChatAppRunner(AppRunner):
             query=query,
             context=context,
             memory=memory,
+            image_detail_config=image_detail_config,
         )
 
         # check hosting moderation

api/core/app/apps/completion/app_runner.py

@@ -9,6 +9,7 @@ from core.app.entities.app_invoke_entities import (
 )
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
 from core.model_manager import ModelInstance
+from core.model_runtime.entities.message_entities import ImagePromptMessageContent
 from core.moderation.base import ModerationError
 from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
 from extensions.ext_database import db
@@ -43,6 +44,16 @@ class CompletionAppRunner(AppRunner):
         query = application_generate_entity.query
         files = application_generate_entity.files
 
+        image_detail_config = (
+            application_generate_entity.file_upload_config.image_config.detail
+            if (
+                application_generate_entity.file_upload_config
+                and application_generate_entity.file_upload_config.image_config
+            )
+            else None
+        )
+        image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
+
         # Pre-calculate the number of tokens of the prompt messages,
         # and return the rest number of tokens by model context token size limit and max token size limit.
         # If the rest number of tokens is not enough, raise exception.
@@ -66,6 +77,7 @@ class CompletionAppRunner(AppRunner):
             inputs=inputs,
             files=files,
             query=query,
+            image_detail_config=image_detail_config,
         )
 
         # moderation
@@ -140,6 +152,7 @@ class CompletionAppRunner(AppRunner):
             files=files,
             query=query,
             context=context,
+            image_detail_config=image_detail_config,
         )
 
         # check hosting moderation