feat/enhance the multi-modal support (#8818)

api/fields/conversation_fields.py

@@ -3,6 +3,8 @@ from flask_restful import fields
 from fields.member_fields import simple_account_fields
 from libs.helper import TimestampField
 
+from .raws import FilesContainedField
+
 
 class MessageTextField(fields.Raw):
     def format(self, value):
@@ -33,8 +35,12 @@ annotation_hit_history_fields = {
 
 message_file_fields = {
     "id": fields.String,
+    "filename": fields.String,
     "type": fields.String,
     "url": fields.String,
+    "mime_type": fields.String,
+    "size": fields.Integer,
+    "transfer_method": fields.String,
     "belongs_to": fields.String(default="user"),
 }
 
@@ -55,7 +61,7 @@ agent_thought_fields = {
 message_detail_fields = {
     "id": fields.String,
     "conversation_id": fields.String,
-    "inputs": fields.Raw,
+    "inputs": FilesContainedField,
     "query": fields.String,
     "message": fields.Raw,
     "message_tokens": fields.Integer,
@@ -71,7 +77,7 @@ message_detail_fields = {
     "annotation_hit_history": fields.Nested(annotation_hit_history_fields, allow_null=True),
     "created_at": TimestampField,
     "agent_thoughts": fields.List(fields.Nested(agent_thought_fields)),
-    "message_files": fields.List(fields.Nested(message_file_fields), attribute="files"),
+    "message_files": fields.List(fields.Nested(message_file_fields)),
     "metadata": fields.Raw(attribute="message_metadata_dict"),
     "status": fields.String,
     "error": fields.String,
@@ -99,7 +105,7 @@ simple_model_config_fields = {
 }
 
 simple_message_detail_fields = {
-    "inputs": fields.Raw,
+    "inputs": FilesContainedField,
     "query": fields.String,
     "message": MessageTextField,
     "answer": fields.String,
@@ -187,7 +193,7 @@ conversation_detail_fields = {
 simple_conversation_fields = {
     "id": fields.String,
     "name": fields.String,
-    "inputs": fields.Raw,
+    "inputs": FilesContainedField,
     "status": fields.String,
     "introduction": fields.String,
     "created_at": TimestampField,

api/fields/file_fields.py

@@ -17,3 +17,8 @@ file_fields = {
     "created_by": fields.String,
     "created_at": TimestampField,
 }
+
+remote_file_info_fields = {
+    "file_type": fields.String(attribute="file_type"),
+    "file_length": fields.Integer(attribute="file_length"),
+}

api/fields/message_fields.py

@@ -3,6 +3,8 @@ from flask_restful import fields
 from fields.conversation_fields import message_file_fields
 from libs.helper import TimestampField
 
+from .raws import FilesContainedField
+
 feedback_fields = {"rating": fields.String}
 
 retriever_resource_fields = {
@@ -63,14 +65,14 @@ message_fields = {
     "id": fields.String,
     "conversation_id": fields.String,
     "parent_message_id": fields.String,
-    "inputs": fields.Raw,
+    "inputs": FilesContainedField,
     "query": fields.String,
     "answer": fields.String(attribute="re_sign_file_url_answer"),
     "feedback": fields.Nested(feedback_fields, attribute="user_feedback", allow_null=True),
     "retriever_resources": fields.List(fields.Nested(retriever_resource_fields)),
     "created_at": TimestampField,
     "agent_thoughts": fields.List(fields.Nested(agent_thought_fields)),
-    "message_files": fields.List(fields.Nested(message_file_fields), attribute="files"),
+    "message_files": fields.List(fields.Nested(message_file_fields)),
     "status": fields.String,
     "error": fields.String,
 }

api/fields/raws.py

@@ -0,0 +1,17 @@
+from flask_restful import fields
+
+from core.file import File
+
+
+class FilesContainedField(fields.Raw):
+    def format(self, value):
+        return self._format_file_object(value)
+
+    def _format_file_object(self, v):
+        if isinstance(v, File):
+            return v.model_dump()
+        if isinstance(v, dict):
+            return {k: self._format_file_object(vv) for k, vv in v.items()}
+        if isinstance(v, list):
+            return [self._format_file_object(vv) for vv in v]
+        return v

api/fields/workflow_fields.py

@@ -1,7 +1,7 @@
 from flask_restful import fields
 
-from core.app.segments import SecretVariable, SegmentType, Variable
 from core.helper import encrypter
+from core.variables import SecretVariable, SegmentType, Variable
 from fields.member_fields import simple_account_fields
 from libs.helper import TimestampField