Feat add testcontainers test (#23269)
This commit is contained in:
NeatGuyCoding
328
api/tests/test_containers_integration_tests/conftest.py
Normal file
328
api/tests/test_containers_integration_tests/conftest.py
Normal file
@@ -0,0 +1,328 @@
|
||||
"""
|
||||
TestContainers-based integration test configuration for Dify API.
|
||||
|
||||
This module provides containerized test infrastructure using TestContainers library
|
||||
to spin up real database and service instances for integration testing. This approach
|
||||
ensures tests run against actual service implementations rather than mocks, providing
|
||||
more reliable and realistic test scenarios.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from flask.testing import FlaskClient
|
||||
from sqlalchemy.orm import Session
|
||||
from testcontainers.core.container import DockerContainer
|
||||
from testcontainers.core.waiting_utils import wait_for_logs
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
from testcontainers.redis import RedisContainer
|
||||
|
||||
from app_factory import create_app
|
||||
from models import db
|
||||
|
||||
# Configure logging for test containers
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DifyTestContainers:
|
||||
"""
|
||||
Manages all test containers required for Dify integration tests.
|
||||
|
||||
This class provides a centralized way to manage multiple containers
|
||||
needed for comprehensive integration testing, including databases,
|
||||
caches, and search engines.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize container management with default configurations."""
|
||||
self.postgres: Optional[PostgresContainer] = None
|
||||
self.redis: Optional[RedisContainer] = None
|
||||
self.dify_sandbox: Optional[DockerContainer] = None
|
||||
self._containers_started = False
|
||||
logger.info("DifyTestContainers initialized - ready to manage test containers")
|
||||
|
||||
def start_containers_with_env(self) -> None:
|
||||
"""
|
||||
Start all required containers for integration testing.
|
||||
|
||||
This method initializes and starts PostgreSQL, Redis
|
||||
containers with appropriate configurations for Dify testing. Containers
|
||||
are started in dependency order to ensure proper initialization.
|
||||
"""
|
||||
if self._containers_started:
|
||||
logger.info("Containers already started - skipping container startup")
|
||||
return
|
||||
|
||||
logger.info("Starting test containers for Dify integration tests...")
|
||||
|
||||
# Start PostgreSQL container for main application database
|
||||
# PostgreSQL is used for storing user data, workflows, and application state
|
||||
logger.info("Initializing PostgreSQL container...")
|
||||
self.postgres = PostgresContainer(
|
||||
image="postgres:16-alpine",
|
||||
)
|
||||
self.postgres.start()
|
||||
db_host = self.postgres.get_container_host_ip()
|
||||
db_port = self.postgres.get_exposed_port(5432)
|
||||
os.environ["DB_HOST"] = db_host
|
||||
os.environ["DB_PORT"] = str(db_port)
|
||||
os.environ["DB_USERNAME"] = self.postgres.username
|
||||
os.environ["DB_PASSWORD"] = self.postgres.password
|
||||
os.environ["DB_DATABASE"] = self.postgres.dbname
|
||||
logger.info(
|
||||
"PostgreSQL container started successfully - Host: %s, Port: %s User: %s, Database: %s",
|
||||
db_host,
|
||||
db_port,
|
||||
self.postgres.username,
|
||||
self.postgres.dbname,
|
||||
)
|
||||
|
||||
# Wait for PostgreSQL to be ready
|
||||
logger.info("Waiting for PostgreSQL to be ready to accept connections...")
|
||||
wait_for_logs(self.postgres, "is ready to accept connections", timeout=30)
|
||||
logger.info("PostgreSQL container is ready and accepting connections")
|
||||
|
||||
# Install uuid-ossp extension for UUID generation
|
||||
logger.info("Installing uuid-ossp extension...")
|
||||
try:
|
||||
import psycopg2
|
||||
|
||||
conn = psycopg2.connect(
|
||||
host=db_host,
|
||||
port=db_port,
|
||||
user=self.postgres.username,
|
||||
password=self.postgres.password,
|
||||
database=self.postgres.dbname,
|
||||
)
|
||||
conn.autocommit = True
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";')
|
||||
cursor.close()
|
||||
conn.close()
|
||||
logger.info("uuid-ossp extension installed successfully")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to install uuid-ossp extension: %s", e)
|
||||
|
||||
# Set up storage environment variables
|
||||
os.environ["STORAGE_TYPE"] = "opendal"
|
||||
os.environ["OPENDAL_SCHEME"] = "fs"
|
||||
os.environ["OPENDAL_FS_ROOT"] = "storage"
|
||||
|
||||
# Start Redis container for caching and session management
|
||||
# Redis is used for storing session data, cache entries, and temporary data
|
||||
logger.info("Initializing Redis container...")
|
||||
self.redis = RedisContainer(image="redis:latest", port=6379)
|
||||
self.redis.start()
|
||||
redis_host = self.redis.get_container_host_ip()
|
||||
redis_port = self.redis.get_exposed_port(6379)
|
||||
os.environ["REDIS_HOST"] = redis_host
|
||||
os.environ["REDIS_PORT"] = str(redis_port)
|
||||
logger.info("Redis container started successfully - Host: %s, Port: %s", redis_host, redis_port)
|
||||
|
||||
# Wait for Redis to be ready
|
||||
logger.info("Waiting for Redis to be ready to accept connections...")
|
||||
wait_for_logs(self.redis, "Ready to accept connections", timeout=30)
|
||||
logger.info("Redis container is ready and accepting connections")
|
||||
|
||||
# Start Dify Sandbox container for code execution environment
|
||||
# Dify Sandbox provides a secure environment for executing user code
|
||||
logger.info("Initializing Dify Sandbox container...")
|
||||
self.dify_sandbox = DockerContainer(image="langgenius/dify-sandbox:latest")
|
||||
self.dify_sandbox.with_exposed_ports(8194)
|
||||
self.dify_sandbox.env = {
|
||||
"API_KEY": "test_api_key",
|
||||
}
|
||||
self.dify_sandbox.start()
|
||||
sandbox_host = self.dify_sandbox.get_container_host_ip()
|
||||
sandbox_port = self.dify_sandbox.get_exposed_port(8194)
|
||||
os.environ["CODE_EXECUTION_ENDPOINT"] = f"http://{sandbox_host}:{sandbox_port}"
|
||||
os.environ["CODE_EXECUTION_API_KEY"] = "test_api_key"
|
||||
logger.info("Dify Sandbox container started successfully - Host: %s, Port: %s", sandbox_host, sandbox_port)
|
||||
|
||||
# Wait for Dify Sandbox to be ready
|
||||
logger.info("Waiting for Dify Sandbox to be ready to accept connections...")
|
||||
wait_for_logs(self.dify_sandbox, "config init success", timeout=60)
|
||||
logger.info("Dify Sandbox container is ready and accepting connections")
|
||||
|
||||
self._containers_started = True
|
||||
logger.info("All test containers started successfully")
|
||||
|
||||
def stop_containers(self) -> None:
|
||||
"""
|
||||
Stop and clean up all test containers.
|
||||
|
||||
This method ensures proper cleanup of all containers to prevent
|
||||
resource leaks and conflicts between test runs.
|
||||
"""
|
||||
if not self._containers_started:
|
||||
logger.info("No containers to stop - containers were not started")
|
||||
return
|
||||
|
||||
logger.info("Stopping and cleaning up test containers...")
|
||||
containers = [self.redis, self.postgres, self.dify_sandbox]
|
||||
for container in containers:
|
||||
if container:
|
||||
try:
|
||||
container_name = container.image
|
||||
logger.info("Stopping container: %s", container_name)
|
||||
container.stop()
|
||||
logger.info("Successfully stopped container: %s", container_name)
|
||||
except Exception as e:
|
||||
# Log error but don't fail the test cleanup
|
||||
logger.warning("Failed to stop container %s: %s", container, e)
|
||||
|
||||
self._containers_started = False
|
||||
logger.info("All test containers stopped and cleaned up successfully")
|
||||
|
||||
|
||||
# Global container manager instance
|
||||
_container_manager = DifyTestContainers()
|
||||
|
||||
|
||||
def _create_app_with_containers() -> Flask:
|
||||
"""
|
||||
Create Flask application configured to use test containers.
|
||||
|
||||
This function creates a Flask application instance that is configured
|
||||
to connect to the test containers instead of the default development
|
||||
or production databases.
|
||||
|
||||
Returns:
|
||||
Flask: Configured Flask application for containerized testing
|
||||
"""
|
||||
logger.info("Creating Flask application with test container configuration...")
|
||||
|
||||
# Re-create the config after environment variables have been set
|
||||
from configs import dify_config
|
||||
|
||||
# Force re-creation of config with new environment variables
|
||||
dify_config.__dict__.clear()
|
||||
dify_config.__init__()
|
||||
|
||||
# Create and configure the Flask application
|
||||
logger.info("Initializing Flask application...")
|
||||
app = create_app()
|
||||
logger.info("Flask application created successfully")
|
||||
|
||||
# Initialize database schema
|
||||
logger.info("Creating database schema...")
|
||||
with app.app_context():
|
||||
db.create_all()
|
||||
logger.info("Database schema created successfully")
|
||||
|
||||
logger.info("Flask application configured and ready for testing")
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def set_up_containers_and_env() -> Generator[DifyTestContainers, None, None]:
|
||||
"""
|
||||
Session-scoped fixture to manage test containers.
|
||||
|
||||
This fixture ensures containers are started once per test session
|
||||
and properly cleaned up when all tests are complete. This approach
|
||||
improves test performance by reusing containers across multiple tests.
|
||||
|
||||
Yields:
|
||||
DifyTestContainers: Container manager instance
|
||||
"""
|
||||
logger.info("=== Starting test session container management ===")
|
||||
_container_manager.start_containers_with_env()
|
||||
logger.info("Test containers ready for session")
|
||||
yield _container_manager
|
||||
logger.info("=== Cleaning up test session containers ===")
|
||||
_container_manager.stop_containers()
|
||||
logger.info("Test session container cleanup completed")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def flask_app_with_containers(set_up_containers_and_env) -> Flask:
|
||||
"""
|
||||
Session-scoped Flask application fixture using test containers.
|
||||
|
||||
This fixture provides a Flask application instance that is configured
|
||||
to use the test containers for all database and service connections.
|
||||
|
||||
Args:
|
||||
containers: Container manager fixture
|
||||
|
||||
Returns:
|
||||
Flask: Configured Flask application
|
||||
"""
|
||||
logger.info("=== Creating session-scoped Flask application ===")
|
||||
app = _create_app_with_containers()
|
||||
logger.info("Session-scoped Flask application created successfully")
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def flask_req_ctx_with_containers(flask_app_with_containers) -> Generator[None, None, None]:
|
||||
"""
|
||||
Request context fixture for containerized Flask application.
|
||||
|
||||
This fixture provides a Flask request context for tests that need
|
||||
to interact with the Flask application within a request scope.
|
||||
|
||||
Args:
|
||||
flask_app_with_containers: Flask application fixture
|
||||
|
||||
Yields:
|
||||
None: Request context is active during yield
|
||||
"""
|
||||
logger.debug("Creating Flask request context...")
|
||||
with flask_app_with_containers.test_request_context():
|
||||
logger.debug("Flask request context active")
|
||||
yield
|
||||
logger.debug("Flask request context closed")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_client_with_containers(flask_app_with_containers) -> Generator[FlaskClient, None, None]:
|
||||
"""
|
||||
Test client fixture for containerized Flask application.
|
||||
|
||||
This fixture provides a Flask test client that can be used to make
|
||||
HTTP requests to the containerized application for integration testing.
|
||||
|
||||
Args:
|
||||
flask_app_with_containers: Flask application fixture
|
||||
|
||||
Yields:
|
||||
FlaskClient: Test client instance
|
||||
"""
|
||||
logger.debug("Creating Flask test client...")
|
||||
with flask_app_with_containers.test_client() as client:
|
||||
logger.debug("Flask test client ready")
|
||||
yield client
|
||||
logger.debug("Flask test client closed")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_session_with_containers(flask_app_with_containers) -> Generator[Session, None, None]:
|
||||
"""
|
||||
Database session fixture for containerized testing.
|
||||
|
||||
This fixture provides a SQLAlchemy database session that is connected
|
||||
to the test PostgreSQL container, allowing tests to interact with
|
||||
the database directly.
|
||||
|
||||
Args:
|
||||
flask_app_with_containers: Flask application fixture
|
||||
|
||||
Yields:
|
||||
Session: Database session instance
|
||||
"""
|
||||
logger.debug("Creating database session...")
|
||||
with flask_app_with_containers.app_context():
|
||||
session = db.session()
|
||||
logger.debug("Database session created and ready")
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
logger.debug("Database session closed")
|
||||
Reference in New Issue
Block a user