Initial commit

api/models/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

api/models/account.py

@@ -0,0 +1,180 @@
+import enum
+from typing import List
+
+from flask_login import UserMixin
+from extensions.ext_database import db
+from sqlalchemy.dialects.postgresql import UUID
+
+
+class AccountStatus(str, enum.Enum):
+    PENDING = 'pending'
+    UNINITIALIZED = 'uninitialized'
+    ACTIVE = 'active'
+    BANNED = 'banned'
+    CLOSED = 'closed'
+
+
+class Account(UserMixin, db.Model):
+    __tablename__ = 'accounts'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='account_pkey'),
+        db.Index('account_email_idx', 'email')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    name = db.Column(db.String(255), nullable=False)
+    email = db.Column(db.String(255), nullable=False)
+    password = db.Column(db.String(255), nullable=True)
+    password_salt = db.Column(db.String(255), nullable=True)
+    avatar = db.Column(db.String(255))
+    interface_language = db.Column(db.String(255))
+    interface_theme = db.Column(db.String(255))
+    timezone = db.Column(db.String(255))
+    last_login_at = db.Column(db.DateTime)
+    last_login_ip = db.Column(db.String(255))
+    status = db.Column(db.String(16), nullable=False, server_default=db.text("'active'::character varying"))
+    initialized_at = db.Column(db.DateTime)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    _current_tenant: db.Model = None
+
+    @property
+    def current_tenant(self):
+        return self._current_tenant
+
+    @current_tenant.setter
+    def current_tenant(self, value):
+        tenant = value
+        ta = TenantAccountJoin.query.filter_by(tenant_id=tenant.id, account_id=self.id).first()
+        if ta:
+            tenant.current_role = ta.role
+        else:
+            tenant = None
+        self._current_tenant = tenant
+
+    @property
+    def current_tenant_id(self):
+        return self._current_tenant.id
+
+    @current_tenant_id.setter
+    def current_tenant_id(self, value):
+        try:
+            tenant_account_join = db.session.query(Tenant, TenantAccountJoin) \
+                .filter(Tenant.id == value) \
+                .filter(TenantAccountJoin.tenant_id == Tenant.id) \
+                .filter(TenantAccountJoin.account_id == self.id) \
+                .one_or_none()
+
+            if tenant_account_join:
+                tenant, ta = tenant_account_join
+                tenant.current_role = ta.role
+            else:
+                tenant = None
+        except:
+            tenant = None
+
+        self._current_tenant = tenant
+
+    def get_status(self) -> AccountStatus:
+        status_str = self.status
+        return AccountStatus(status_str)
+
+    @classmethod
+    def get_by_openid(cls, provider: str, open_id: str) -> db.Model:
+        account_integrate = db.session.query(AccountIntegrate). \
+            filter(AccountIntegrate.provider == provider, AccountIntegrate.open_id == open_id). \
+            one_or_none()
+        if account_integrate:
+            return db.session.query(Account). \
+                filter(Account.id == account_integrate.account_id). \
+                one_or_none()
+        return None
+
+    def get_integrates(self) -> List[db.Model]:
+        ai = db.Model
+        return db.session.query(ai).filter(
+            ai.account_id == self.id
+        ).all()
+
+
+class Tenant(db.Model):
+    __tablename__ = 'tenants'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='tenant_pkey'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    name = db.Column(db.String(255), nullable=False)
+    encrypt_public_key = db.Column(db.Text)
+    plan = db.Column(db.String(255), nullable=False, server_default=db.text("'basic'::character varying"))
+    status = db.Column(db.String(255), nullable=False, server_default=db.text("'normal'::character varying"))
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    def get_accounts(self) -> List[db.Model]:
+        Account = db.Model
+        return db.session.query(Account).filter(
+            Account.id == TenantAccountJoin.account_id,
+            TenantAccountJoin.tenant_id == self.id
+        ).all()
+
+
+class TenantAccountJoinRole(enum.Enum):
+    OWNER = 'owner'
+    ADMIN = 'admin'
+    NORMAL = 'normal'
+
+
+class TenantAccountJoin(db.Model):
+    __tablename__ = 'tenant_account_joins'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='tenant_account_join_pkey'),
+        db.Index('tenant_account_join_account_id_idx', 'account_id'),
+        db.Index('tenant_account_join_tenant_id_idx', 'tenant_id'),
+        db.UniqueConstraint('tenant_id', 'account_id', name='unique_tenant_account_join')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    account_id = db.Column(UUID, nullable=False)
+    role = db.Column(db.String(16), nullable=False, server_default='normal')
+    invited_by = db.Column(UUID, nullable=True)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class AccountIntegrate(db.Model):
+    __tablename__ = 'account_integrates'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='account_integrate_pkey'),
+        db.UniqueConstraint('account_id', 'provider', name='unique_account_provider'),
+        db.UniqueConstraint('provider', 'open_id', name='unique_provider_open_id')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    account_id = db.Column(UUID, nullable=False)
+    provider = db.Column(db.String(16), nullable=False)
+    open_id = db.Column(db.String(255), nullable=False)
+    encrypted_token = db.Column(db.String(255), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class InvitationCode(db.Model):
+    __tablename__ = 'invitation_codes'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='invitation_code_pkey'),
+        db.Index('invitation_codes_batch_idx', 'batch'),
+        db.Index('invitation_codes_code_idx', 'code', 'status')
+    )
+
+    id = db.Column(db.Integer, nullable=False)
+    batch = db.Column(db.String(255), nullable=False)
+    code = db.Column(db.String(32), nullable=False)
+    status = db.Column(db.String(16), nullable=False, server_default=db.text("'unused'::character varying"))
+    used_at = db.Column(db.DateTime)
+    used_by_tenant_id = db.Column(UUID)
+    used_by_account_id = db.Column(UUID)
+    deprecated_at = db.Column(db.DateTime)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))

api/models/dataset.py

@@ -0,0 +1,415 @@
+import json
+import pickle
+from json import JSONDecodeError
+
+from sqlalchemy import func
+from sqlalchemy.dialects.postgresql import UUID
+
+from extensions.ext_database import db
+from models.account import Account
+from models.model import App, UploadFile
+
+class Dataset(db.Model):
+    __tablename__ = 'datasets'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='dataset_pkey'),
+        db.Index('dataset_tenant_idx', 'tenant_id'),
+    )
+
+    INDEXING_TECHNIQUE_LIST = ['high_quality', 'economy']
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    description = db.Column(db.Text, nullable=True)
+    provider = db.Column(db.String(255), nullable=False,
+                         server_default=db.text("'vendor'::character varying"))
+    permission = db.Column(db.String(255), nullable=False,
+                           server_default=db.text("'only_me'::character varying"))
+    data_source_type = db.Column(db.String(255))
+    indexing_technique = db.Column(db.String(255), nullable=True)
+    index_struct = db.Column(db.Text, nullable=True)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False,
+                           server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_by = db.Column(UUID, nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False,
+                           server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @property
+    def dataset_keyword_table(self):
+        dataset_keyword_table = db.session.query(DatasetKeywordTable).filter(
+            DatasetKeywordTable.dataset_id == self.id).first()
+        if dataset_keyword_table:
+            return dataset_keyword_table
+
+        return None
+
+    @property
+    def index_struct_dict(self):
+        return json.loads(self.index_struct) if self.index_struct else None
+
+    @property
+    def created_by_account(self):
+        return Account.query.get(self.created_by)
+
+    @property
+    def latest_process_rule(self):
+        return DatasetProcessRule.query.filter(DatasetProcessRule.dataset_id == self.id) \
+            .order_by(DatasetProcessRule.created_at.desc()).first()
+
+    @property
+    def app_count(self):
+        return db.session.query(func.count(AppDatasetJoin.id)).filter(AppDatasetJoin.dataset_id == self.id).scalar()
+
+    @property
+    def document_count(self):
+        return db.session.query(func.count(Document.id)).filter(Document.dataset_id == self.id).scalar()
+
+    @property
+    def word_count(self):
+        return Document.query.with_entities(func.coalesce(func.sum(Document.word_count))) \
+            .filter(Document.dataset_id == self.id).scalar()
+
+
+class DatasetProcessRule(db.Model):
+    __tablename__ = 'dataset_process_rules'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='dataset_process_rule_pkey'),
+        db.Index('dataset_process_rule_dataset_id_idx', 'dataset_id'),
+    )
+
+    id = db.Column(UUID, nullable=False,
+                   server_default=db.text('uuid_generate_v4()'))
+    dataset_id = db.Column(UUID, nullable=False)
+    mode = db.Column(db.String(255), nullable=False,
+                     server_default=db.text("'automatic'::character varying"))
+    rules = db.Column(db.Text, nullable=True)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False,
+                           server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    MODES = ['automatic', 'custom']
+    PRE_PROCESSING_RULES = ['remove_stopwords', 'remove_extra_spaces', 'remove_urls_emails']
+    AUTOMATIC_RULES = {
+        'pre_processing_rules': [
+            {'id': 'remove_extra_spaces', 'enabled': True},
+            {'id': 'remove_urls_emails', 'enabled': False}
+        ],
+        'segmentation': {
+            'delimiter': '\n',
+            'max_tokens': 1000
+        }
+    }
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'dataset_id': self.dataset_id,
+            'mode': self.mode,
+            'rules': self.rules_dict,
+            'created_by': self.created_by,
+            'created_at': self.created_at,
+        }
+
+    @property
+    def rules_dict(self):
+        try:
+            return json.loads(self.rules) if self.rules else None
+        except JSONDecodeError:
+            return None
+
+
+class Document(db.Model):
+    __tablename__ = 'documents'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='document_pkey'),
+        db.Index('document_dataset_id_idx', 'dataset_id'),
+        db.Index('document_is_paused_idx', 'is_paused'),
+    )
+
+    # initial fields
+    id = db.Column(UUID, nullable=False,
+                   server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    dataset_id = db.Column(UUID, nullable=False)
+    position = db.Column(db.Integer, nullable=False)
+    data_source_type = db.Column(db.String(255), nullable=False)
+    data_source_info = db.Column(db.Text, nullable=True)
+    dataset_process_rule_id = db.Column(UUID, nullable=True)
+    batch = db.Column(db.String(255), nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    created_from = db.Column(db.String(255), nullable=False)
+    created_by = db.Column(UUID, nullable=False)
+    created_api_request_id = db.Column(UUID, nullable=True)
+    created_at = db.Column(db.DateTime, nullable=False,
+                           server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    # start processing
+    processing_started_at = db.Column(db.DateTime, nullable=True)
+
+    # parsing
+    file_id = db.Column(db.Text, nullable=True)
+    word_count = db.Column(db.Integer, nullable=True)
+    parsing_completed_at = db.Column(db.DateTime, nullable=True)
+
+    # cleaning
+    cleaning_completed_at = db.Column(db.DateTime, nullable=True)
+
+    # split
+    splitting_completed_at = db.Column(db.DateTime, nullable=True)
+
+    # indexing
+    tokens = db.Column(db.Integer, nullable=True)
+    indexing_latency = db.Column(db.Float, nullable=True)
+    completed_at = db.Column(db.DateTime, nullable=True)
+
+    # pause
+    is_paused = db.Column(db.Boolean, nullable=True, server_default=db.text('false'))
+    paused_by = db.Column(UUID, nullable=True)
+    paused_at = db.Column(db.DateTime, nullable=True)
+
+    # error
+    error = db.Column(db.Text, nullable=True)
+    stopped_at = db.Column(db.DateTime, nullable=True)
+
+    # basic fields
+    indexing_status = db.Column(db.String(
+        255), nullable=False, server_default=db.text("'waiting'::character varying"))
+    enabled = db.Column(db.Boolean, nullable=False,
+                        server_default=db.text('true'))
+    disabled_at = db.Column(db.DateTime, nullable=True)
+    disabled_by = db.Column(UUID, nullable=True)
+    archived = db.Column(db.Boolean, nullable=False,
+                         server_default=db.text('false'))
+    archived_reason = db.Column(db.String(255), nullable=True)
+    archived_by = db.Column(UUID, nullable=True)
+    archived_at = db.Column(db.DateTime, nullable=True)
+    updated_at = db.Column(db.DateTime, nullable=False,
+                           server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    doc_type = db.Column(db.String(40), nullable=True)
+    doc_metadata = db.Column(db.JSON, nullable=True)
+
+    DATA_SOURCES = ['upload_file']
+
+    @property
+    def display_status(self):
+        status = None
+        if self.indexing_status == 'waiting':
+            status = 'queuing'
+        elif self.indexing_status not in ['completed', 'error', 'waiting'] and self.is_paused:
+            status = 'paused'
+        elif self.indexing_status in ['parsing', 'cleaning', 'splitting', 'indexing']:
+            status = 'indexing'
+        elif self.indexing_status == 'error':
+            status = 'error'
+        elif self.indexing_status == 'completed' and not self.archived and self.enabled:
+            status = 'available'
+        elif self.indexing_status == 'completed' and not self.archived and not self.enabled:
+            status = 'disabled'
+        elif self.indexing_status == 'completed' and self.archived:
+            status = 'archived'
+        return status
+
+    @property
+    def data_source_info_dict(self):
+        if self.data_source_info:
+            try:
+                data_source_info_dict = json.loads(self.data_source_info)
+            except JSONDecodeError:
+                data_source_info_dict = {}
+
+            return data_source_info_dict
+        return None
+
+    @property
+    def data_source_detail_dict(self):
+        if self.data_source_info:
+            if self.data_source_type == 'upload_file':
+                data_source_info_dict = json.loads(self.data_source_info)
+                file_detail = db.session.query(UploadFile). \
+                    filter(UploadFile.id == data_source_info_dict['upload_file_id']). \
+                    one_or_none()
+                if file_detail:
+                    return {
+                        'upload_file': {
+                            'id': file_detail.id,
+                            'name': file_detail.name,
+                            'size': file_detail.size,
+                            'extension': file_detail.extension,
+                            'mime_type': file_detail.mime_type,
+                            'created_by': file_detail.created_by,
+                            'created_at': file_detail.created_at.timestamp()
+                        }
+                    }
+        return {}
+
+    @property
+    def average_segment_length(self):
+        if self.word_count and self.word_count != 0 and self.segment_count and self.segment_count != 0:
+            return self.word_count//self.segment_count
+        return 0
+
+    @property
+    def dataset_process_rule(self):
+        if self.dataset_process_rule_id:
+            return DatasetProcessRule.query.get(self.dataset_process_rule_id)
+        return None
+
+    @property
+    def dataset(self):
+        return Dataset.query.get(self.dataset_id)
+
+    @property
+    def segment_count(self):
+        return DocumentSegment.query.filter(DocumentSegment.document_id == self.id).count()
+
+    @property
+    def hit_count(self):
+        return DocumentSegment.query.with_entities(func.coalesce(func.sum(DocumentSegment.hit_count))) \
+            .filter(DocumentSegment.document_id == self.id).scalar()
+
+
+class DocumentSegment(db.Model):
+    __tablename__ = 'document_segments'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='document_segment_pkey'),
+        db.Index('document_segment_dataset_id_idx', 'dataset_id'),
+        db.Index('document_segment_document_id_idx', 'document_id'),
+        db.Index('document_segment_tenant_dataset_idx', 'dataset_id', 'tenant_id'),
+        db.Index('document_segment_tenant_document_idx', 'document_id', 'tenant_id'),
+        db.Index('document_segment_dataset_node_idx', 'dataset_id', 'index_node_id'),
+    )
+
+    # initial fields
+    id = db.Column(UUID, nullable=False,
+                   server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    dataset_id = db.Column(UUID, nullable=False)
+    document_id = db.Column(UUID, nullable=False)
+    position = db.Column(db.Integer, nullable=False)
+    content = db.Column(db.Text, nullable=False)
+    word_count = db.Column(db.Integer, nullable=False)
+    tokens = db.Column(db.Integer, nullable=False)
+
+    # indexing fields
+    keywords = db.Column(db.JSON, nullable=True)
+    index_node_id = db.Column(db.String(255), nullable=True)
+    index_node_hash = db.Column(db.String(255), nullable=True)
+
+    # basic fields
+    hit_count = db.Column(db.Integer, nullable=False, default=0)
+    enabled = db.Column(db.Boolean, nullable=False,
+                        server_default=db.text('true'))
+    disabled_at = db.Column(db.DateTime, nullable=True)
+    disabled_by = db.Column(UUID, nullable=True)
+    status = db.Column(db.String(255), nullable=False,
+                       server_default=db.text("'waiting'::character varying"))
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False,
+                           server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    indexing_at = db.Column(db.DateTime, nullable=True)
+    completed_at = db.Column(db.DateTime, nullable=True)
+    error = db.Column(db.Text, nullable=True)
+    stopped_at = db.Column(db.DateTime, nullable=True)
+
+    @property
+    def dataset(self):
+        return db.session.query(Dataset).filter(Dataset.id == self.dataset_id).first()
+
+    @property
+    def document(self):
+        return db.session.query(Document).filter(Document.id == self.document_id).first()
+
+    @property
+    def embedding(self):
+        embedding = db.session.query(Embedding).filter(Embedding.hash == self.index_node_hash).first() \
+            if self.index_node_hash else None
+
+        if embedding:
+            return embedding.embedding
+
+        return None
+
+    @property
+    def previous_segment(self):
+        return db.session.query(DocumentSegment).filter(
+            DocumentSegment.document_id == self.document_id,
+            DocumentSegment.position == self.position - 1
+        ).first()
+
+    @property
+    def next_segment(self):
+        return db.session.query(DocumentSegment).filter(
+            DocumentSegment.document_id == self.document_id,
+            DocumentSegment.position == self.position + 1
+        ).first()
+
+
+class AppDatasetJoin(db.Model):
+    __tablename__ = 'app_dataset_joins'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='app_dataset_join_pkey'),
+        db.Index('app_dataset_join_app_dataset_idx', 'dataset_id', 'app_id'),
+    )
+
+    id = db.Column(UUID, primary_key=True, nullable=False, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    dataset_id = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+
+    @property
+    def app(self):
+        return App.query.get(self.app_id)
+
+
+class DatasetQuery(db.Model):
+    __tablename__ = 'dataset_queries'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='dataset_query_pkey'),
+        db.Index('dataset_query_dataset_id_idx', 'dataset_id'),
+    )
+
+    id = db.Column(UUID, primary_key=True, nullable=False, server_default=db.text('uuid_generate_v4()'))
+    dataset_id = db.Column(UUID, nullable=False)
+    content = db.Column(db.Text, nullable=False)
+    source = db.Column(db.String(255), nullable=False)
+    source_app_id = db.Column(UUID, nullable=True)
+    created_by_role = db.Column(db.String, nullable=False)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+
+
+class DatasetKeywordTable(db.Model):
+    __tablename__ = 'dataset_keyword_tables'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='dataset_keyword_table_pkey'),
+        db.Index('dataset_keyword_table_dataset_id_idx', 'dataset_id'),
+    )
+
+    id = db.Column(UUID, primary_key=True, server_default=db.text('uuid_generate_v4()'))
+    dataset_id = db.Column(UUID, nullable=False, unique=True)
+    keyword_table = db.Column(db.Text, nullable=False)
+
+    @property
+    def keyword_table_dict(self):
+        return json.loads(self.keyword_table) if self.keyword_table else None
+
+
+class Embedding(db.Model):
+    __tablename__ = 'embeddings'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='embedding_pkey'),
+        db.UniqueConstraint('hash', name='embedding_hash_idx')
+    )
+
+    id = db.Column(UUID, primary_key=True, server_default=db.text('uuid_generate_v4()'))
+    hash = db.Column(db.String(64), nullable=False)
+    embedding = db.Column(db.LargeBinary, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    def set_embedding(self, embedding_data: list[float]):
+        self.embedding = pickle.dumps(embedding_data, protocol=pickle.HIGHEST_PROTOCOL)
+
+    def get_embedding(self) -> list[float]:
+        return pickle.loads(self.embedding)

api/models/model.py

@@ -0,0 +1,622 @@
+import json
+
+from flask import current_app
+from flask_login import UserMixin
+from sqlalchemy.dialects.postgresql import UUID
+
+from libs.helper import generate_string
+from extensions.ext_database import db
+from .account import Account, Tenant
+
+
+class DifySetup(db.Model):
+    __tablename__ = 'dify_setups'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('version', name='dify_setup_pkey'),
+    )
+
+    version = db.Column(db.String(255), nullable=False)
+    setup_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class App(db.Model):
+    __tablename__ = 'apps'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='app_pkey'),
+        db.Index('app_tenant_id_idx', 'tenant_id')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    mode = db.Column(db.String(255), nullable=False)
+    icon = db.Column(db.String(255))
+    icon_background = db.Column(db.String(255))
+    app_model_config_id = db.Column(UUID, nullable=True)
+    status = db.Column(db.String(255), nullable=False, server_default=db.text("'normal'::character varying"))
+    enable_site = db.Column(db.Boolean, nullable=False)
+    enable_api = db.Column(db.Boolean, nullable=False)
+    api_rpm = db.Column(db.Integer, nullable=False)
+    api_rph = db.Column(db.Integer, nullable=False)
+    is_demo = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+    is_public = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @property
+    def site(self):
+        site = db.session.query(Site).filter(Site.app_id == self.id).first()
+        return site
+
+    @property
+    def app_model_config(self):
+        app_model_config = db.session.query(AppModelConfig).filter(
+            AppModelConfig.id == self.app_model_config_id).first()
+        return app_model_config
+
+    @property
+    def api_base_url(self):
+        return current_app.config['API_URL'] + '/v1'
+
+    @property
+    def tenant(self):
+        tenant = db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first()
+        return tenant
+
+
+class AppModelConfig(db.Model):
+    __tablename__ = 'app_model_configs'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='app_model_config_pkey'),
+        db.Index('app_app_id_idx', 'app_id')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    provider = db.Column(db.String(255), nullable=False)
+    model_id = db.Column(db.String(255), nullable=False)
+    configs = db.Column(db.JSON, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    opening_statement = db.Column(db.Text)
+    suggested_questions = db.Column(db.Text)
+    suggested_questions_after_answer = db.Column(db.Text)
+    more_like_this = db.Column(db.Text)
+    model = db.Column(db.Text)
+    user_input_form = db.Column(db.Text)
+    pre_prompt = db.Column(db.Text)
+    agent_mode = db.Column(db.Text)
+
+    @property
+    def app(self):
+        app = db.session.query(App).filter(App.id == self.app_id).first()
+        return app
+
+    @property
+    def model_dict(self) -> dict:
+        return json.loads(self.model) if self.model else None
+
+    @property
+    def suggested_questions_list(self) -> list:
+        return json.loads(self.suggested_questions) if self.suggested_questions else []
+
+    @property
+    def suggested_questions_after_answer_dict(self) -> dict:
+        return json.loads(self.suggested_questions_after_answer) if self.suggested_questions_after_answer \
+            else {"enabled": False}
+
+    @property
+    def more_like_this_dict(self) -> dict:
+        return json.loads(self.more_like_this) if self.more_like_this else {"enabled": False}
+
+    @property
+    def user_input_form_list(self) -> dict:
+        return json.loads(self.user_input_form) if self.user_input_form else []
+
+    @property
+    def agent_mode_dict(self) -> dict:
+        return json.loads(self.agent_mode) if self.agent_mode else {"enabled": False, "tools": []}
+
+
+class RecommendedApp(db.Model):
+    __tablename__ = 'recommended_apps'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='recommended_app_pkey'),
+        db.Index('recommended_app_app_id_idx', 'app_id'),
+        db.Index('recommended_app_is_listed_idx', 'is_listed')
+    )
+
+    id = db.Column(UUID, primary_key=True, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    description = db.Column(db.JSON, nullable=False)
+    copyright = db.Column(db.String(255), nullable=False)
+    privacy_policy = db.Column(db.String(255), nullable=False)
+    category = db.Column(db.String(255), nullable=False)
+    position = db.Column(db.Integer, nullable=False, default=0)
+    is_listed = db.Column(db.Boolean, nullable=False, default=True)
+    install_count = db.Column(db.Integer, nullable=False, default=0)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @property
+    def app(self):
+        app = db.session.query(App).filter(App.id == self.app_id).first()
+        return app
+
+    # def set_description(self, lang, desc):
+    #     if self.description is None:
+    #         self.description = {}
+    #     self.description[lang] = desc
+
+    def get_description(self, lang):
+        if self.description and lang in self.description:
+            return self.description[lang]
+        else:
+            return self.description.get('en')
+
+
+class InstalledApp(db.Model):
+    __tablename__ = 'installed_apps'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='installed_app_pkey'),
+        db.Index('installed_app_tenant_id_idx', 'tenant_id'),
+        db.Index('installed_app_app_id_idx', 'app_id'),
+        db.UniqueConstraint('tenant_id', 'app_id', name='unique_tenant_app')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    app_id = db.Column(UUID, nullable=False)
+    app_owner_tenant_id = db.Column(UUID, nullable=False)
+    position = db.Column(db.Integer, nullable=False, default=0)
+    is_pinned = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+    last_used_at = db.Column(db.DateTime, nullable=True)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @property
+    def app(self):
+        app = db.session.query(App).filter(App.id == self.app_id).first()
+        return app
+
+    @property
+    def tenant(self):
+        tenant = db.session.query(Tenant).filter(Tenant.id == self.tenant_id).first()
+        return tenant
+
+
+class Conversation(db.Model):
+    __tablename__ = 'conversations'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='conversation_pkey'),
+        db.Index('conversation_app_from_user_idx', 'app_id', 'from_source', 'from_end_user_id')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    app_model_config_id = db.Column(UUID, nullable=False)
+    model_provider = db.Column(db.String(255), nullable=False)
+    override_model_configs = db.Column(db.Text)
+    model_id = db.Column(db.String(255), nullable=False)
+    mode = db.Column(db.String(255), nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    summary = db.Column(db.Text)
+    inputs = db.Column(db.JSON)
+    introduction = db.Column(db.Text)
+    system_instruction = db.Column(db.Text)
+    system_instruction_tokens = db.Column(db.Integer, nullable=False, server_default=db.text('0'))
+    status = db.Column(db.String(255), nullable=False)
+    from_source = db.Column(db.String(255), nullable=False)
+    from_end_user_id = db.Column(UUID)
+    from_account_id = db.Column(UUID)
+    read_at = db.Column(db.DateTime)
+    read_account_id = db.Column(UUID)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    messages = db.relationship("Message", backref="conversation", lazy='select', passive_deletes="all")
+    message_annotations = db.relationship("MessageAnnotation", backref="conversation", lazy='select', passive_deletes="all")
+
+    @property
+    def model_config(self):
+        model_config = {}
+        if self.override_model_configs:
+            override_model_configs = json.loads(self.override_model_configs)
+
+            if 'model' in override_model_configs:
+                model_config['model'] = override_model_configs['model']
+                model_config['pre_prompt'] = override_model_configs['pre_prompt']
+                model_config['agent_mode'] = override_model_configs['agent_mode']
+                model_config['opening_statement'] = override_model_configs['opening_statement']
+                model_config['suggested_questions'] = override_model_configs['suggested_questions']
+                model_config['suggested_questions_after_answer'] = override_model_configs[
+                    'suggested_questions_after_answer'] \
+                    if 'suggested_questions_after_answer' in override_model_configs else {"enabled": False}
+                model_config['more_like_this'] = override_model_configs['more_like_this'] \
+                    if 'more_like_this' in override_model_configs else {"enabled": False}
+                model_config['user_input_form'] = override_model_configs['user_input_form']
+            else:
+                model_config['configs'] = override_model_configs
+        else:
+            app_model_config = db.session.query(AppModelConfig).filter(
+                AppModelConfig.id == self.app_model_config_id).first()
+
+            model_config['configs'] = app_model_config.configs
+            model_config['model'] = app_model_config.model_dict
+            model_config['pre_prompt'] = app_model_config.pre_prompt
+            model_config['agent_mode'] = app_model_config.agent_mode_dict
+            model_config['opening_statement'] = app_model_config.opening_statement
+            model_config['suggested_questions'] = app_model_config.suggested_questions_list
+            model_config['suggested_questions_after_answer'] = app_model_config.suggested_questions_after_answer_dict
+            model_config['more_like_this'] = app_model_config.more_like_this_dict
+            model_config['user_input_form'] = app_model_config.user_input_form_list
+
+        model_config['model_id'] = self.model_id
+        model_config['provider'] = self.model_provider
+
+        return model_config
+
+    @property
+    def summary_or_query(self):
+        if self.summary:
+            return self.summary
+        else:
+            first_message = self.first_message
+            if first_message:
+                return first_message.query
+            else:
+                return ''
+
+    @property
+    def annotated(self):
+        return db.session.query(MessageAnnotation).filter(MessageAnnotation.conversation_id == self.id).count() > 0
+
+    @property
+    def annotation(self):
+        return db.session.query(MessageAnnotation).filter(MessageAnnotation.conversation_id == self.id).first()
+
+    @property
+    def message_count(self):
+        return db.session.query(Message).filter(Message.conversation_id == self.id).count()
+
+    @property
+    def user_feedback_stats(self):
+        like = db.session.query(MessageFeedback) \
+            .filter(MessageFeedback.conversation_id == self.id,
+                    MessageFeedback.from_source == 'user',
+                    MessageFeedback.rating == 'like').count()
+
+        dislike = db.session.query(MessageFeedback) \
+            .filter(MessageFeedback.conversation_id == self.id,
+                    MessageFeedback.from_source == 'user',
+                    MessageFeedback.rating == 'dislike').count()
+
+        return {'like': like, 'dislike': dislike}
+
+    @property
+    def admin_feedback_stats(self):
+        like = db.session.query(MessageFeedback) \
+            .filter(MessageFeedback.conversation_id == self.id,
+                    MessageFeedback.from_source == 'admin',
+                    MessageFeedback.rating == 'like').count()
+
+        dislike = db.session.query(MessageFeedback) \
+            .filter(MessageFeedback.conversation_id == self.id,
+                    MessageFeedback.from_source == 'admin',
+                    MessageFeedback.rating == 'dislike').count()
+
+        return {'like': like, 'dislike': dislike}
+
+    @property
+    def first_message(self):
+        return db.session.query(Message).filter(Message.conversation_id == self.id).first()
+
+    @property
+    def app(self):
+        return db.session.query(App).filter(App.id == self.app_id).first()
+
+
+class Message(db.Model):
+    __tablename__ = 'messages'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='message_pkey'),
+        db.Index('message_app_id_idx', 'app_id', 'created_at'),
+        db.Index('message_conversation_id_idx', 'conversation_id'),
+        db.Index('message_end_user_idx', 'app_id', 'from_source', 'from_end_user_id'),
+        db.Index('message_account_idx', 'app_id', 'from_source', 'from_account_id'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    model_provider = db.Column(db.String(255), nullable=False)
+    model_id = db.Column(db.String(255), nullable=False)
+    override_model_configs = db.Column(db.Text)
+    conversation_id = db.Column(UUID, db.ForeignKey('conversations.id'), nullable=False)
+    inputs = db.Column(db.JSON)
+    query = db.Column(db.Text, nullable=False)
+    message = db.Column(db.JSON, nullable=False)
+    message_tokens = db.Column(db.Integer, nullable=False, server_default=db.text('0'))
+    message_unit_price = db.Column(db.Numeric(10, 4), nullable=False)
+    answer = db.Column(db.Text, nullable=False)
+    answer_tokens = db.Column(db.Integer, nullable=False, server_default=db.text('0'))
+    answer_unit_price = db.Column(db.Numeric(10, 4), nullable=False)
+    provider_response_latency = db.Column(db.Float, nullable=False, server_default=db.text('0'))
+    total_price = db.Column(db.Numeric(10, 7))
+    currency = db.Column(db.String(255), nullable=False)
+    from_source = db.Column(db.String(255), nullable=False)
+    from_end_user_id = db.Column(UUID)
+    from_account_id = db.Column(UUID)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    agent_based = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+
+    @property
+    def user_feedback(self):
+        feedback = db.session.query(MessageFeedback).filter(MessageFeedback.message_id == self.id,
+                                                            MessageFeedback.from_source == 'user').first()
+        return feedback
+
+    @property
+    def admin_feedback(self):
+        feedback = db.session.query(MessageFeedback).filter(MessageFeedback.message_id == self.id,
+                                                            MessageFeedback.from_source == 'admin').first()
+        return feedback
+
+    @property
+    def feedbacks(self):
+        feedbacks = db.session.query(MessageFeedback).filter(MessageFeedback.message_id == self.id).all()
+        return feedbacks
+
+    @property
+    def annotation(self):
+        annotation = db.session.query(MessageAnnotation).filter(MessageAnnotation.message_id == self.id).first()
+        return annotation
+
+    @property
+    def app_model_config(self):
+        conversation = db.session.query(Conversation).filter(Conversation.id == self.conversation_id).first()
+        if conversation:
+            return db.session.query(AppModelConfig).filter(
+                AppModelConfig.id == conversation.app_model_config_id).first()
+
+        return None
+
+
+class MessageFeedback(db.Model):
+    __tablename__ = 'message_feedbacks'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='message_feedback_pkey'),
+        db.Index('message_feedback_app_idx', 'app_id'),
+        db.Index('message_feedback_message_idx', 'message_id', 'from_source'),
+        db.Index('message_feedback_conversation_idx', 'conversation_id', 'from_source', 'rating')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    conversation_id = db.Column(UUID, nullable=False)
+    message_id = db.Column(UUID, nullable=False)
+    rating = db.Column(db.String(255), nullable=False)
+    content = db.Column(db.Text)
+    from_source = db.Column(db.String(255), nullable=False)
+    from_end_user_id = db.Column(UUID)
+    from_account_id = db.Column(UUID)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @property
+    def from_account(self):
+        account = db.session.query(Account).filter(Account.id == self.from_account_id).first()
+        return account
+
+
+class MessageAnnotation(db.Model):
+    __tablename__ = 'message_annotations'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='message_annotation_pkey'),
+        db.Index('message_annotation_app_idx', 'app_id'),
+        db.Index('message_annotation_conversation_idx', 'conversation_id'),
+        db.Index('message_annotation_message_idx', 'message_id')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    conversation_id = db.Column(UUID, db.ForeignKey('conversations.id'), nullable=False)
+    message_id = db.Column(UUID, nullable=False)
+    content = db.Column(db.Text, nullable=False)
+    account_id = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @property
+    def account(self):
+        account = db.session.query(Account).filter(Account.id == self.account_id).first()
+        return account
+
+
+class OperationLog(db.Model):
+    __tablename__ = 'operation_logs'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='operation_log_pkey'),
+        db.Index('operation_log_account_action_idx', 'tenant_id', 'account_id', 'action')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    account_id = db.Column(UUID, nullable=False)
+    action = db.Column(db.String(255), nullable=False)
+    content = db.Column(db.JSON)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    created_ip = db.Column(db.String(255), nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class EndUser(UserMixin, db.Model):
+    __tablename__ = 'end_users'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='end_user_pkey'),
+        db.Index('end_user_session_id_idx', 'session_id', 'type'),
+        db.Index('end_user_tenant_session_id_idx', 'tenant_id', 'session_id', 'type'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    app_id = db.Column(UUID, nullable=True)
+    type = db.Column(db.String(255), nullable=False)
+    external_user_id = db.Column(db.String(255), nullable=True)
+    name = db.Column(db.String(255))
+    is_anonymous = db.Column(db.Boolean, nullable=False, server_default=db.text('true'))
+    session_id = db.Column(db.String(255), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class Site(db.Model):
+    __tablename__ = 'sites'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='site_pkey'),
+        db.Index('site_app_id_idx', 'app_id'),
+        db.Index('site_code_idx', 'code', 'status')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    title = db.Column(db.String(255), nullable=False)
+    icon = db.Column(db.String(255))
+    icon_background = db.Column(db.String(255))
+    description = db.Column(db.String(255))
+    default_language = db.Column(db.String(255), nullable=False)
+    copyright = db.Column(db.String(255))
+    privacy_policy = db.Column(db.String(255))
+    customize_domain = db.Column(db.String(255))
+    customize_token_strategy = db.Column(db.String(255), nullable=False)
+    prompt_public = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+    status = db.Column(db.String(255), nullable=False, server_default=db.text("'normal'::character varying"))
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    code = db.Column(db.String(255))
+
+    @staticmethod
+    def generate_code(n):
+        while True:
+            result = generate_string(n)
+            while db.session.query(Site).filter(Site.code == result).count() > 0:
+                result = generate_string(n)
+
+            return result
+
+    @property
+    def app_base_url(self):
+        return current_app.config['APP_URL']
+
+
+class ApiToken(db.Model):
+    __tablename__ = 'api_tokens'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='api_token_pkey'),
+        db.Index('api_token_app_id_type_idx', 'app_id', 'type'),
+        db.Index('api_token_token_idx', 'token', 'type')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=True)
+    dataset_id = db.Column(UUID, nullable=True)
+    type = db.Column(db.String(16), nullable=False)
+    token = db.Column(db.String(255), nullable=False)
+    last_used_at = db.Column(db.DateTime, nullable=True)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @staticmethod
+    def generate_api_key(prefix, n):
+        while True:
+            result = prefix + generate_string(n)
+            while db.session.query(ApiToken).filter(ApiToken.token == result).count() > 0:
+                result = prefix + generate_string(n)
+
+            return result
+
+
+class UploadFile(db.Model):
+    __tablename__ = 'upload_files'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='upload_file_pkey'),
+        db.Index('upload_file_tenant_idx', 'tenant_id')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    storage_type = db.Column(db.String(255), nullable=False)
+    key = db.Column(db.String(255), nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    size = db.Column(db.Integer, nullable=False)
+    extension = db.Column(db.String(255), nullable=False)
+    mime_type = db.Column(db.String(255), nullable=True)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    used = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+    used_by = db.Column(UUID, nullable=True)
+    used_at = db.Column(db.DateTime, nullable=True)
+    hash = db.Column(db.String(255), nullable=True)
+
+
+class ApiRequest(db.Model):
+    __tablename__ = 'api_requests'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='api_request_pkey'),
+        db.Index('api_request_token_idx', 'tenant_id', 'api_token_id')
+    )
+
+    id = db.Column(UUID, nullable=False, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    api_token_id = db.Column(UUID, nullable=False)
+    path = db.Column(db.String(255), nullable=False)
+    request = db.Column(db.Text, nullable=True)
+    response = db.Column(db.Text, nullable=True)
+    ip = db.Column(db.String(255), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class MessageChain(db.Model):
+    __tablename__ = 'message_chains'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='message_chain_pkey'),
+        db.Index('message_chain_message_id_idx', 'message_id')
+    )
+
+    id = db.Column(UUID, nullable=False, server_default=db.text('uuid_generate_v4()'))
+    message_id = db.Column(UUID, nullable=False)
+    type = db.Column(db.String(255), nullable=False)
+    input = db.Column(db.Text, nullable=True)
+    output = db.Column(db.Text, nullable=True)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+
+
+class MessageAgentThought(db.Model):
+    __tablename__ = 'message_agent_thoughts'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='message_agent_thought_pkey'),
+        db.Index('message_agent_thought_message_id_idx', 'message_id'),
+        db.Index('message_agent_thought_message_chain_id_idx', 'message_chain_id'),
+    )
+
+    id = db.Column(UUID, nullable=False, server_default=db.text('uuid_generate_v4()'))
+    message_id = db.Column(UUID, nullable=False)
+    message_chain_id = db.Column(UUID, nullable=False)
+    position = db.Column(db.Integer, nullable=False)
+    thought = db.Column(db.Text, nullable=True)
+    tool = db.Column(db.Text, nullable=True)
+    tool_input = db.Column(db.Text, nullable=True)
+    observation = db.Column(db.Text, nullable=True)
+    # plugin_id = db.Column(UUID, nullable=True)  ## for future design
+    tool_process_data = db.Column(db.Text, nullable=True)
+    message = db.Column(db.Text, nullable=True)
+    message_token = db.Column(db.Integer, nullable=True)
+    message_unit_price = db.Column(db.Numeric, nullable=True)
+    answer = db.Column(db.Text, nullable=True)
+    answer_token = db.Column(db.Integer, nullable=True)
+    answer_unit_price = db.Column(db.Numeric, nullable=True)
+    tokens = db.Column(db.Integer, nullable=True)
+    total_price = db.Column(db.Numeric, nullable=True)
+    currency = db.Column(db.String, nullable=True)
+    latency = db.Column(db.Float, nullable=True)
+    created_by_role = db.Column(db.String, nullable=False)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())

api/models/provider.py

@@ -0,0 +1,77 @@
+from enum import Enum
+
+from sqlalchemy.dialects.postgresql import UUID
+
+from extensions.ext_database import db
+
+
+class ProviderType(Enum):
+    CUSTOM = 'custom'
+    SYSTEM = 'system'
+
+
+class ProviderName(Enum):
+    OPENAI = 'openai'
+    AZURE_OPENAI = 'azure_openai'
+    ANTHROPIC = 'anthropic'
+    COHERE = 'cohere'
+    HUGGINGFACEHUB = 'huggingfacehub'
+
+    @staticmethod
+    def value_of(value):
+        for member in ProviderName:
+            if member.value == value:
+                return member
+        raise ValueError(f"No matching enum found for value '{value}'")
+
+
+class ProviderQuotaType(Enum):
+    MONTHLY = 'monthly'
+    TRIAL = 'trial'
+
+
+class Provider(db.Model):
+    """
+    Provider model representing the API providers and their configurations.
+    """
+    __tablename__ = 'providers'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='provider_pkey'),
+        db.Index('provider_tenant_id_provider_idx', 'tenant_id', 'provider_name'),
+        db.UniqueConstraint('tenant_id', 'provider_name', 'provider_type', 'quota_type', name='unique_provider_name_type_quota')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    provider_name = db.Column(db.String(40), nullable=False)
+    provider_type = db.Column(db.String(40), nullable=False, server_default=db.text("'custom'::character varying"))
+    encrypted_config = db.Column(db.Text, nullable=True)
+    is_valid = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+    last_used = db.Column(db.DateTime, nullable=True)
+
+    quota_type = db.Column(db.String(40), nullable=True, server_default=db.text("''::character varying"))
+    quota_limit = db.Column(db.Integer, nullable=True)
+    quota_used = db.Column(db.Integer, default=0)
+
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    def __repr__(self):
+        return f"<Provider(id={self.id}, tenant_id={self.tenant_id}, provider_name='{self.provider_name}', provider_type='{self.provider_type}')>"
+
+    @property
+    def token_is_set(self):
+        """
+         Returns True if the encrypted_config is not None, indicating that the token is set.
+         """
+        return self.encrypted_config is not None
+
+    @property
+    def is_enabled(self):
+        """
+        Returns True if the provider is enabled.
+        """
+        if self.provider_type == ProviderType.SYSTEM.value:
+            return self.is_valid
+        else:
+            return self.is_valid and self.token_is_set

api/models/task.py

@@ -0,0 +1,37 @@
+from extensions.ext_database import db
+from celery import states
+from datetime import datetime
+
+
+class CeleryTask(db.Model):
+    """Task result/status."""
+
+    __tablename__ = 'celery_taskmeta'
+
+    id = db.Column(db.Integer, db.Sequence('task_id_sequence'),
+                   primary_key=True, autoincrement=True)
+    task_id = db.Column(db.String(155), unique=True)
+    status = db.Column(db.String(50), default=states.PENDING)
+    result = db.Column(db.PickleType, nullable=True)
+    date_done = db.Column(db.DateTime, default=datetime.utcnow,
+                          onupdate=datetime.utcnow, nullable=True)
+    traceback = db.Column(db.Text, nullable=True)
+    name = db.Column(db.String(155), nullable=True)
+    args = db.Column(db.LargeBinary, nullable=True)
+    kwargs = db.Column(db.LargeBinary, nullable=True)
+    worker = db.Column(db.String(155), nullable=True)
+    retries = db.Column(db.Integer, nullable=True)
+    queue = db.Column(db.String(155), nullable=True)
+
+
+class CeleryTaskSet(db.Model):
+    """TaskSet result."""
+
+    __tablename__ = 'celery_tasksetmeta'
+
+    id = db.Column(db.Integer, db.Sequence('taskset_id_sequence'),
+                   autoincrement=True, primary_key=True)
+    taskset_id = db.Column(db.String(155), unique=True)
+    result = db.Column(db.PickleType, nullable=True)
+    date_done = db.Column(db.DateTime, default=datetime.utcnow,
+                          nullable=True)

api/models/web.py

@@ -0,0 +1,36 @@
+from sqlalchemy.dialects.postgresql import UUID
+
+from extensions.ext_database import db
+from models.model import Message
+
+
+class SavedMessage(db.Model):
+    __tablename__ = 'saved_messages'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='saved_message_pkey'),
+        db.Index('saved_message_message_idx', 'app_id', 'message_id', 'created_by'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    message_id = db.Column(UUID, nullable=False)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+    @property
+    def message(self):
+        return db.session.query(Message).filter(Message.id == self.message_id).first()
+
+
+class PinnedConversation(db.Model):
+    __tablename__ = 'pinned_conversations'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='pinned_conversation_pkey'),
+        db.Index('pinned_conversation_conversation_idx', 'app_id', 'conversation_id', 'created_by'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    app_id = db.Column(UUID, nullable=False)
+    conversation_id = db.Column(UUID, nullable=False)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))