feat/enhance the multi-modal support (#8818)

api/core/model_runtime/entities/__init__.py

@@ -0,0 +1,38 @@
+from .llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from .message_entities import (
+    AssistantPromptMessage,
+    AudioPromptMessageContent,
+    ImagePromptMessageContent,
+    PromptMessage,
+    PromptMessageContent,
+    PromptMessageContentType,
+    PromptMessageRole,
+    PromptMessageTool,
+    SystemPromptMessage,
+    TextPromptMessageContent,
+    ToolPromptMessage,
+    UserPromptMessage,
+)
+from .model_entities import ModelPropertyKey
+
+__all__ = [
+    "ImagePromptMessageContent",
+    "PromptMessage",
+    "PromptMessageRole",
+    "LLMUsage",
+    "ModelPropertyKey",
+    "AssistantPromptMessage",
+    "PromptMessage",
+    "PromptMessageContent",
+    "PromptMessageRole",
+    "SystemPromptMessage",
+    "TextPromptMessageContent",
+    "UserPromptMessage",
+    "PromptMessageTool",
+    "ToolPromptMessage",
+    "PromptMessageContentType",
+    "LLMResult",
+    "LLMResultChunk",
+    "LLMResultChunkDelta",
+    "AudioPromptMessageContent",
+]

api/core/model_runtime/entities/message_entities.py

@@ -2,7 +2,7 @@ from abc import ABC
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, field_validator
+from pydantic import BaseModel, Field, field_validator
 
 
 class PromptMessageRole(Enum):
@@ -55,6 +55,7 @@ class PromptMessageContentType(Enum):
 
     TEXT = "text"
     IMAGE = "image"
+    AUDIO = "audio"
 
 
 class PromptMessageContent(BaseModel):
@@ -74,12 +75,18 @@ class TextPromptMessageContent(PromptMessageContent):
     type: PromptMessageContentType = PromptMessageContentType.TEXT
 
 
+class AudioPromptMessageContent(PromptMessageContent):
+    type: PromptMessageContentType = PromptMessageContentType.AUDIO
+    data: str = Field(..., description="Base64 encoded audio data")
+    format: str = Field(..., description="Audio format")
+
+
 class ImagePromptMessageContent(PromptMessageContent):
     """
     Model class for image prompt message content.
     """
 
-    class DETAIL(Enum):
+    class DETAIL(str, Enum):
         LOW = "low"
         HIGH = "high"