estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Feat/assistant app (#2086)

Browse Source 
Co-authored-by: chenhe <guchenhe@gmail.com>
Co-authored-by: Pascal M <11357019+perzeuss@users.noreply.github.com>
This commit is contained in:
Yeuoly
2024-01-23 19:58:23 +08:00
committed by GitHub
parent 7bbe12b2bd
commit 86286e1ac8
175 changed files with 11619 additions and 1235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
api/models/model.py
View File

@@ -1,11 +1,13 @@
import json
import uuid
from core.file.upload_file_parser import UploadFileParser
from core.file.tool_file_parser import ToolFileParser
from extensions.ext_database import db
from flask import current_app, request
from flask_login import UserMixin
from libs.helper import generate_string
from sqlalchemy import Float
from sqlalchemy import Float, text
from sqlalchemy.dialects.postgresql import UUID
from .account import Account, Tenant
@@ -66,7 +68,65 @@ class App(db.Model):
def tenant(self):
tenant = db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first()
return tenant
@property
def is_agent(self) -> bool:
app_model_config = self.app_model_config
if not app_model_config:
return False
if not app_model_config.agent_mode:
return False
if self.app_model_config.agent_mode_dict.get('enabled', False) \
and self.app_model_config.agent_mode_dict.get('strategy', '') in ['function_call', 'react']:
return True
return False
@property
def deleted_tools(self) -> list:
# get agent mode tools
app_model_config = self.app_model_config
if not app_model_config:
return []
if not app_model_config.agent_mode:
return []
agent_mode = app_model_config.agent_mode_dict
tools = agent_mode.get('tools', [])
provider_ids = []
for tool in tools:
keys = list(tool.keys())
if len(keys) >= 4:
provider_type = tool.get('provider_type', '')
provider_id = tool.get('provider_id', '')
if provider_type == 'api':
# check if provider id is a uuid string, if not, skip
try:
uuid.UUID(provider_id)
except Exception:
continue
provider_ids.append(provider_id)
if not provider_ids:
return []
api_providers = db.session.execute(
text('SELECT id FROM tool_api_providers WHERE id IN :provider_ids'),
{'provider_ids': tuple(provider_ids)}
).fetchall()
deleted_tools = []
current_api_provider_ids = [str(api_provider.id) for api_provider in api_providers]
for tool in tools:
keys = list(tool.keys())
if len(keys) >= 4:
provider_type = tool.get('provider_type', '')
provider_id = tool.get('provider_id', '')
if provider_type == 'api' and provider_id not in current_api_provider_ids:
deleted_tools.append(tool['tool_name'])
return deleted_tools
class AppModelConfig(db.Model):
__tablename__ = 'app_model_configs'
@@ -168,7 +228,7 @@ class AppModelConfig(db.Model):
@property
def agent_mode_dict(self) -> dict:
return json.loads(self.agent_mode) if self.agent_mode else {"enabled": False, "strategy": None, "tools": []}
return json.loads(self.agent_mode) if self.agent_mode else {"enabled": False, "strategy": None, "tools": [], "prompt": None}
@property
def chat_prompt_config_dict(self) -> dict:
@@ -337,6 +397,12 @@ class InstalledApp(db.Model):
tenant = db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first()
return tenant
@property
def is_agent(self) -> bool:
app = self.app
if not app:
return False
return app.is_agent
class Conversation(db.Model):
__tablename__ = 'conversations'
@@ -582,11 +648,22 @@ class Message(db.Model):
upload_file=upload_file,
force_url=True
)
if message_file.transfer_method == 'tool_file':
# get extension
if '.' in message_file.url:
extension = f'.{message_file.url.split(".")[-1]}'
if len(extension) > 10:
extension = '.bin'
else:
extension = '.bin'
# add sign url
url = ToolFileParser.get_tool_file_manager().sign_file(file_id=message_file.id, extension=extension)
files.append({
'id': message_file.id,
'type': message_file.type,
'url': url
'url': url,
'belongs_to': message_file.belongs_to if message_file.belongs_to else 'user'
})
return files
@@ -632,12 +709,12 @@ class MessageFile(db.Model):
type = db.Column(db.String(255), nullable=False)
transfer_method = db.Column(db.String(255), nullable=False)
url = db.Column(db.Text, nullable=True)
belongs_to = db.Column(db.String(255), nullable=True)
upload_file_id = db.Column(UUID, nullable=True)
created_by_role = db.Column(db.String(255), nullable=False)
created_by = db.Column(UUID, nullable=False)
created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
class MessageAnnotation(db.Model):
__tablename__ = 'message_annotations'
__table_args__ = (
@@ -912,7 +989,7 @@ class MessageAgentThought(db.Model):
id = db.Column(UUID, nullable=False, server_default=db.text('uuid_generate_v4()'))
message_id = db.Column(UUID, nullable=False)
message_chain_id = db.Column(UUID, nullable=False)
message_chain_id = db.Column(UUID, nullable=True)
position = db.Column(db.Integer, nullable=False)
thought = db.Column(db.Text, nullable=True)
tool = db.Column(db.Text, nullable=True)
@@ -924,6 +1001,7 @@ class MessageAgentThought(db.Model):
message_token = db.Column(db.Integer, nullable=True)
message_unit_price = db.Column(db.Numeric, nullable=True)
message_price_unit = db.Column(db.Numeric(10, 7), nullable=False, server_default=db.text('0.001'))
message_files = db.Column(db.Text, nullable=True)
answer = db.Column(db.Text, nullable=True)
answer_token = db.Column(db.Integer, nullable=True)
answer_unit_price = db.Column(db.Numeric, nullable=True)
@@ -936,6 +1014,12 @@ class MessageAgentThought(db.Model):
created_by = db.Column(UUID, nullable=False)
created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
@property
def files(self) -> list:
if self.message_files:
return json.loads(self.message_files)
else:
return []
class DatasetRetrieverResource(db.Model):
__tablename__ = 'dataset_retriever_resources'

227
api/models/tools.py Normal file
View File

@@ -0,0 +1,227 @@
import json
from enum import Enum
from typing import List
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy import ForeignKey
from extensions.ext_database import db
from core.tools.entities.tool_bundle import ApiBasedToolBundle
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolRuntimeVariablePool
from models.model import Tenant, Account, App
class BuiltinToolProvider(db.Model):
"""
This table stores the tool provider information for built-in tools for each tenant.
"""
__tablename__ = 'tool_builtin_providers'
__table_args__ = (
db.PrimaryKeyConstraint('id', name='tool_builtin_provider_pkey'),
# one tenant can only have one tool provider with the same name
db.UniqueConstraint('tenant_id', 'provider', name='unique_builtin_tool_provider')
)
# id of the tool provider
id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
# id of the tenant
tenant_id = db.Column(UUID, nullable=True)
# who created this tool provider
user_id = db.Column(UUID, nullable=False)
# name of the tool provider
provider = db.Column(db.String(40), nullable=False)
# credential of the tool provider
encrypted_credentials = db.Column(db.Text, nullable=True)
created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
@property
def credentials(self) -> dict:
return json.loads(self.encrypted_credentials)
class PublishedAppTool(db.Model):
"""
The table stores the apps published as a tool for each person.
"""
__tablename__ = 'tool_published_apps'
__table_args__ = (
db.PrimaryKeyConstraint('id', name='published_app_tool_pkey'),
db.UniqueConstraint('app_id', 'user_id', name='unique_published_app_tool')
)
# id of the tool provider
id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
# id of the app
app_id = db.Column(UUID, ForeignKey('apps.id'), nullable=False)
# who published this tool
user_id = db.Column(UUID, nullable=False)
# description of the tool, stored in i18n format, for human
description = db.Column(db.Text, nullable=False)
# llm_description of the tool, for LLM
llm_description = db.Column(db.Text, nullable=False)
# query decription, query will be seem as a parameter of the tool, to describe this parameter to llm, we need this field
query_description = db.Column(db.Text, nullable=False)
# query name, the name of the query parameter
query_name = db.Column(db.String(40), nullable=False)
# name of the tool provider
tool_name = db.Column(db.String(40), nullable=False)
# author
author = db.Column(db.String(40), nullable=False)
created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
@property
def description_i18n(self) -> I18nObject:
return I18nObject(**json.loads(self.description))
@property
def app(self) -> App:
return db.session.query(App).filter(App.id == self.app_id).first()
class ApiToolProvider(db.Model):
"""
The table stores the api providers.
"""
__tablename__ = 'tool_api_providers'
__table_args__ = (
db.PrimaryKeyConstraint('id', name='tool_api_provider_pkey'),
)
id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
# name of the api provider
name = db.Column(db.String(40), nullable=False)
# icon
icon = db.Column(db.String(255), nullable=False)
# original schema
schema = db.Column(db.Text, nullable=False)
schema_type_str = db.Column(db.String(40), nullable=False)
# who created this tool
user_id = db.Column(UUID, nullable=False)
# tanent id
tenant_id = db.Column(UUID, nullable=False)
# description of the provider
description = db.Column(db.Text, nullable=False)
# json format tools
tools_str = db.Column(db.Text, nullable=False)
# json format credentials
credentials_str = db.Column(db.Text, nullable=False)
# privacy policy
privacy_policy = db.Column(db.String(255), nullable=True)
created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
@property
def schema_type(self) -> ApiProviderSchemaType:
return ApiProviderSchemaType.value_of(self.schema_type_str)
@property
def tools(self) -> List[ApiBasedToolBundle]:
return [ApiBasedToolBundle(**tool) for tool in json.loads(self.tools_str)]
@property
def credentials(self) -> dict:
return json.loads(self.credentials_str)
@property
def is_taned(self) -> bool:
return self.tenant_id is not None
@property
def user(self) -> Account:
return db.session.query(Account).filter(Account.id == self.user_id).first()
@property
def tanent(self) -> Tenant:
return db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first()
class ToolModelInvoke(db.Model):
"""
store the invoke logs from tool invoke
"""
__tablename__ = "tool_model_invokes"
__table_args__ = (
db.PrimaryKeyConstraint('id', name='tool_model_invoke_pkey'),
)
id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
# who invoke this tool
user_id = db.Column(UUID, nullable=False)
# tanent id
tenant_id = db.Column(UUID, nullable=False)
# provider
provider = db.Column(db.String(40), nullable=False)
# type
tool_type = db.Column(db.String(40), nullable=False)
# tool name
tool_name = db.Column(db.String(40), nullable=False)
# invoke parameters
model_parameters = db.Column(db.Text, nullable=False)
# prompt messages
prompt_messages = db.Column(db.Text, nullable=False)
# invoke response
model_response = db.Column(db.Text, nullable=False)
prompt_tokens = db.Column(db.Integer, nullable=False, server_default=db.text('0'))
answer_tokens = db.Column(db.Integer, nullable=False, server_default=db.text('0'))
answer_unit_price = db.Column(db.Numeric(10, 4), nullable=False)
answer_price_unit = db.Column(db.Numeric(10, 7), nullable=False, server_default=db.text('0.001'))
provider_response_latency = db.Column(db.Float, nullable=False, server_default=db.text('0'))
total_price = db.Column(db.Numeric(10, 7))
currency = db.Column(db.String(255), nullable=False)
created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
class ToolConversationVariables(db.Model):
"""
store the conversation variables from tool invoke
"""
__tablename__ = "tool_conversation_variables"
__table_args__ = (
db.PrimaryKeyConstraint('id', name='tool_conversation_variables_pkey'),
# add index for user_id and conversation_id
db.Index('user_id_idx', 'user_id'),
db.Index('conversation_id_idx', 'conversation_id'),
)
id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
# conversation user id
user_id = db.Column(UUID, nullable=False)
# tanent id
tenant_id = db.Column(UUID, nullable=False)
# conversation id
conversation_id = db.Column(UUID, nullable=False)
# variables pool
variables_str = db.Column(db.Text, nullable=False)
created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
@property
def variables(self) -> dict:
return json.loads(self.variables_str)
class ToolFile(db.Model):
"""
store the file created by agent
"""
__tablename__ = "tool_files"
__table_args__ = (
db.PrimaryKeyConstraint('id', name='tool_file_pkey'),
)
id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
# conversation user id
user_id = db.Column(UUID, nullable=False)
# tanent id
tenant_id = db.Column(UUID, nullable=False)
# conversation id
conversation_id = db.Column(UUID, nullable=False)
# file key
file_key = db.Column(db.String(255), nullable=False)
# mime type
mimetype = db.Column(db.String(255), nullable=False)
# original url
original_url = db.Column(db.String(255), nullable=True)