FEAT: NEW WORKFLOW ENGINE (#3160)

Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Yeuoly <admin@srmxy.cn>
Co-authored-by: JzoNg <jzongcode@gmail.com>
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
Co-authored-by: jyong <jyong@dify.ai>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: jyong <718720800@qq.com>

api/core/prompt/utils/prompt_message_util.py

@@ -0,0 +1,85 @@
+from typing import cast
+
+from core.model_runtime.entities.message_entities import (
+    ImagePromptMessageContent,
+    PromptMessage,
+    PromptMessageContentType,
+    PromptMessageRole,
+    TextPromptMessageContent,
+)
+from core.prompt.simple_prompt_transform import ModelMode
+
+
+class PromptMessageUtil:
+    @staticmethod
+    def prompt_messages_to_prompt_for_saving(model_mode: str, prompt_messages: list[PromptMessage]) -> list[dict]:
+        """
+        Prompt messages to prompt for saving.
+        :param model_mode: model mode
+        :param prompt_messages: prompt messages
+        :return:
+        """
+        prompts = []
+        if model_mode == ModelMode.CHAT.value:
+            for prompt_message in prompt_messages:
+                if prompt_message.role == PromptMessageRole.USER:
+                    role = 'user'
+                elif prompt_message.role == PromptMessageRole.ASSISTANT:
+                    role = 'assistant'
+                elif prompt_message.role == PromptMessageRole.SYSTEM:
+                    role = 'system'
+                else:
+                    continue
+
+                text = ''
+                files = []
+                if isinstance(prompt_message.content, list):
+                    for content in prompt_message.content:
+                        if content.type == PromptMessageContentType.TEXT:
+                            content = cast(TextPromptMessageContent, content)
+                            text += content.data
+                        else:
+                            content = cast(ImagePromptMessageContent, content)
+                            files.append({
+                                "type": 'image',
+                                "data": content.data[:10] + '...[TRUNCATED]...' + content.data[-10:],
+                                "detail": content.detail.value
+                            })
+                else:
+                    text = prompt_message.content
+
+                prompts.append({
+                    "role": role,
+                    "text": text,
+                    "files": files
+                })
+        else:
+            prompt_message = prompt_messages[0]
+            text = ''
+            files = []
+            if isinstance(prompt_message.content, list):
+                for content in prompt_message.content:
+                    if content.type == PromptMessageContentType.TEXT:
+                        content = cast(TextPromptMessageContent, content)
+                        text += content.data
+                    else:
+                        content = cast(ImagePromptMessageContent, content)
+                        files.append({
+                            "type": 'image',
+                            "data": content.data[:10] + '...[TRUNCATED]...' + content.data[-10:],
+                            "detail": content.detail.value
+                        })
+            else:
+                text = prompt_message.content
+
+            params = {
+                "role": 'user',
+                "text": text,
+            }
+
+            if files:
+                params['files'] = files
+
+            prompts.append(params)
+
+        return prompts

api/core/prompt/utils/prompt_template_parser.py

@@ -0,0 +1,45 @@
+import re
+
+REGEX = re.compile(r"\{\{([a-zA-Z_][a-zA-Z0-9_]{0,29}|#histories#|#query#|#context#)\}\}")
+WITH_VARIABLE_TMPL_REGEX = re.compile(
+    r"\{\{([a-zA-Z_][a-zA-Z0-9_]{0,29}|#[a-zA-Z0-9_]{1,50}\.[a-zA-Z0-9_\.]{1,100}#|#histories#|#query#|#context#)\}\}"
+)
+
+
+class PromptTemplateParser:
+    """
+    Rules:
+
+    1. Template variables must be enclosed in `{{}}`.
+    2. The template variable Key can only be: letters + numbers + underscore, with a maximum length of 16 characters,
+       and can only start with letters and underscores.
+    3. The template variable Key cannot contain new lines or spaces, and must comply with rule 2.
+    4. In addition to the above, 3 types of special template variable Keys are accepted:
+       `{{#histories#}}` `{{#query#}}` `{{#context#}}`. No other `{{##}}` template variables are allowed.
+    """
+
+    def __init__(self, template: str, with_variable_tmpl: bool = False):
+        self.template = template
+        self.with_variable_tmpl = with_variable_tmpl
+        self.regex = WITH_VARIABLE_TMPL_REGEX if with_variable_tmpl else REGEX
+        self.variable_keys = self.extract()
+
+    def extract(self) -> list:
+        # Regular expression to match the template rules
+        return re.findall(self.regex, self.template)
+
+    def format(self, inputs: dict, remove_template_variables: bool = True) -> str:
+        def replacer(match):
+            key = match.group(1)
+            value = inputs.get(key, match.group(0))  # return original matched string if key not found
+
+            if remove_template_variables:
+                return PromptTemplateParser.remove_template_variables(value, self.with_variable_tmpl)
+            return value
+
+        prompt = re.sub(self.regex, replacer, self.template)
+        return re.sub(r'<\|.*?\|>', '', prompt)
+
+    @classmethod
+    def remove_template_variables(cls, text: str, with_variable_tmpl: bool = False):
+        return re.sub(WITH_VARIABLE_TMPL_REGEX if with_variable_tmpl else REGEX, r'{\1}', text)