Feat/firecrawl data source (#5232)

Co-authored-by: Nicolas <nicolascamara29@gmail.com>
Co-authored-by: chenhe <guchenhe@gmail.com>
Co-authored-by: takatost <takatost@gmail.com>

api/models/dataset.py

@@ -270,7 +270,7 @@ class Document(db.Model):
         255), nullable=False, server_default=db.text("'text_model'::character varying"))
     doc_language = db.Column(db.String(255), nullable=True)
 
-    DATA_SOURCES = ['upload_file', 'notion_import']
+    DATA_SOURCES = ['upload_file', 'notion_import', 'website_crawl']
 
     @property
     def display_status(self):
@@ -322,7 +322,7 @@ class Document(db.Model):
                             'created_at': file_detail.created_at.timestamp()
                         }
                     }
-            elif self.data_source_type == 'notion_import':
+            elif self.data_source_type == 'notion_import' or self.data_source_type == 'website_crawl':
                 return json.loads(self.data_source_info)
         return {}
 

api/models/source.py

@@ -1,11 +1,13 @@
+import json
+
 from sqlalchemy.dialects.postgresql import JSONB
 
 from extensions.ext_database import db
 from models import StringUUID
 
 
-class DataSourceBinding(db.Model):
-    __tablename__ = 'data_source_bindings'
+class DataSourceOauthBinding(db.Model):
+    __tablename__ = 'data_source_oauth_bindings'
     __table_args__ = (
         db.PrimaryKeyConstraint('id', name='source_binding_pkey'),
         db.Index('source_binding_tenant_id_idx', 'tenant_id'),
@@ -20,3 +22,33 @@ class DataSourceBinding(db.Model):
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
     disabled = db.Column(db.Boolean, nullable=True, server_default=db.text('false'))
+
+
+class DataSourceApiKeyAuthBinding(db.Model):
+    __tablename__ = 'data_source_api_key_auth_bindings'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='data_source_api_key_auth_binding_pkey'),
+        db.Index('data_source_api_key_auth_binding_tenant_id_idx', 'tenant_id'),
+        db.Index('data_source_api_key_auth_binding_provider_idx', 'provider'),
+    )
+
+    id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(StringUUID, nullable=False)
+    category = db.Column(db.String(255), nullable=False)
+    provider = db.Column(db.String(255), nullable=False)
+    credentials = db.Column(db.Text, nullable=True)  # JSON
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    disabled = db.Column(db.Boolean, nullable=True, server_default=db.text('false'))
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'tenant_id': self.tenant_id,
+            'category': self.category,
+            'provider': self.provider,
+            'credentials': json.loads(self.credentials),
+            'created_at': self.created_at.timestamp(),
+            'updated_at': self.updated_at.timestamp(),
+            'disabled': self.disabled
+        }