diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index 4751b9601..591c24cbe 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -300,8 +300,7 @@ class DatasetQueueMonitorConfig(BaseSettings): class MiddlewareConfig( # place the configs in alphabet order - CeleryConfig, - DatabaseConfig, + CeleryConfig, # Note: CeleryConfig already inherits from DatabaseConfig KeywordStoreConfig, RedisConfig, # configs of storage and storage providers diff --git a/api/configs/middleware/vdb/clickzetta_config.py b/api/configs/middleware/vdb/clickzetta_config.py index 04f81e25f..61bc01202 100644 --- a/api/configs/middleware/vdb/clickzetta_config.py +++ b/api/configs/middleware/vdb/clickzetta_config.py @@ -1,9 +1,10 @@ from typing import Optional -from pydantic import BaseModel, Field +from pydantic import Field +from pydantic_settings import BaseSettings -class ClickzettaConfig(BaseModel): +class ClickzettaConfig(BaseSettings): """ Clickzetta Lakehouse vector database configuration """ diff --git a/api/configs/middleware/vdb/matrixone_config.py b/api/configs/middleware/vdb/matrixone_config.py index 9400612d8..3e7ce7b67 100644 --- a/api/configs/middleware/vdb/matrixone_config.py +++ b/api/configs/middleware/vdb/matrixone_config.py @@ -1,7 +1,8 @@ -from pydantic import BaseModel, Field +from pydantic import Field +from pydantic_settings import BaseSettings -class MatrixoneConfig(BaseModel): +class MatrixoneConfig(BaseSettings): """Matrixone vector database configuration.""" MATRIXONE_HOST: str = Field(default="localhost", description="Host address of the Matrixone server") diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py index f511e20e6..b8d723ef4 100644 --- a/api/configs/packaging/__init__.py +++ b/api/configs/packaging/__init__.py @@ -1,6 +1,6 @@ from pydantic import Field -from configs.packaging.pyproject import PyProjectConfig, PyProjectTomlConfig +from configs.packaging.pyproject import PyProjectTomlConfig class PackagingInfo(PyProjectTomlConfig): diff --git a/api/configs/remote_settings_sources/apollo/client.py b/api/configs/remote_settings_sources/apollo/client.py index 877ff8409..e30e6218a 100644 --- a/api/configs/remote_settings_sources/apollo/client.py +++ b/api/configs/remote_settings_sources/apollo/client.py @@ -4,8 +4,9 @@ import logging import os import threading import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path +from typing import Any from .python_3x import http_request, makedirs_wrapper from .utils import ( @@ -25,13 +26,13 @@ logger = logging.getLogger(__name__) class ApolloClient: def __init__( self, - config_url, - app_id, - cluster="default", - secret="", - start_hot_update=True, - change_listener=None, - _notification_map=None, + config_url: str, + app_id: str, + cluster: str = "default", + secret: str = "", + start_hot_update: bool = True, + change_listener: Callable[[str, str, str, Any], None] | None = None, + _notification_map: dict[str, int] | None = None, ): # Core routing parameters self.config_url = config_url @@ -47,17 +48,17 @@ class ApolloClient: # Private control variables self._cycle_time = 5 self._stopping = False - self._cache = {} - self._no_key = {} - self._hash = {} + self._cache: dict[str, dict[str, Any]] = {} + self._no_key: dict[str, str] = {} + self._hash: dict[str, str] = {} self._pull_timeout = 75 self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/" - self._long_poll_thread = None + self._long_poll_thread: threading.Thread | None = None self._change_listener = change_listener # "add" "delete" "update" if _notification_map is None: _notification_map = {"application": -1} self._notification_map = _notification_map - self.last_release_key = None + self.last_release_key: str | None = None # Private startup method self._path_checker() if start_hot_update: @@ -68,7 +69,7 @@ class ApolloClient: heartbeat.daemon = True heartbeat.start() - def get_json_from_net(self, namespace="application"): + def get_json_from_net(self, namespace: str = "application") -> dict[str, Any] | None: url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format( self.config_url, self.app_id, self.cluster, namespace, "", self.ip ) @@ -88,7 +89,7 @@ class ApolloClient: logger.exception("an error occurred in get_json_from_net") return None - def get_value(self, key, default_val=None, namespace="application"): + def get_value(self, key: str, default_val: Any = None, namespace: str = "application") -> Any: try: # read memory configuration namespace_cache = self._cache.get(namespace) @@ -104,7 +105,8 @@ class ApolloClient: namespace_data = self.get_json_from_net(namespace) val = get_value_from_dict(namespace_data, key) if val is not None: - self._update_cache_and_file(namespace_data, namespace) + if namespace_data is not None: + self._update_cache_and_file(namespace_data, namespace) return val # read the file configuration @@ -126,23 +128,23 @@ class ApolloClient: # to ensure the real-time correctness of the function call. # If the user does not have the same default val twice # and the default val is used here, there may be a problem. - def _set_local_cache_none(self, namespace, key): + def _set_local_cache_none(self, namespace: str, key: str) -> None: no_key = no_key_cache_key(namespace, key) self._no_key[no_key] = key - def _start_hot_update(self): + def _start_hot_update(self) -> None: self._long_poll_thread = threading.Thread(target=self._listener) # When the asynchronous thread is started, the daemon thread will automatically exit # when the main thread is launched. self._long_poll_thread.daemon = True self._long_poll_thread.start() - def stop(self): + def stop(self) -> None: self._stopping = True logger.info("Stopping listener...") # Call the set callback function, and if it is abnormal, try it out - def _call_listener(self, namespace, old_kv, new_kv): + def _call_listener(self, namespace: str, old_kv: dict[str, Any] | None, new_kv: dict[str, Any] | None) -> None: if self._change_listener is None: return if old_kv is None: @@ -168,12 +170,12 @@ class ApolloClient: except BaseException as e: logger.warning(str(e)) - def _path_checker(self): + def _path_checker(self) -> None: if not os.path.isdir(self._cache_file_path): makedirs_wrapper(self._cache_file_path) # update the local cache and file cache - def _update_cache_and_file(self, namespace_data, namespace="application"): + def _update_cache_and_file(self, namespace_data: dict[str, Any], namespace: str = "application") -> None: # update the local cache self._cache[namespace] = namespace_data # update the file cache @@ -187,7 +189,7 @@ class ApolloClient: self._hash[namespace] = new_hash # get the configuration from the local file - def _get_local_cache(self, namespace="application"): + def _get_local_cache(self, namespace: str = "application") -> dict[str, Any]: cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt") if os.path.isfile(cache_file_path): with open(cache_file_path) as f: @@ -195,8 +197,8 @@ class ApolloClient: return result return {} - def _long_poll(self): - notifications = [] + def _long_poll(self) -> None: + notifications: list[dict[str, Any]] = [] for key in self._cache: namespace_data = self._cache[key] notification_id = -1 @@ -236,7 +238,7 @@ class ApolloClient: except Exception as e: logger.warning(str(e)) - def _get_net_and_set_local(self, namespace, n_id, call_change=False): + def _get_net_and_set_local(self, namespace: str, n_id: int, call_change: bool = False) -> None: namespace_data = self.get_json_from_net(namespace) if not namespace_data: return @@ -248,7 +250,7 @@ class ApolloClient: new_kv = namespace_data.get(CONFIGURATIONS) self._call_listener(namespace, old_kv, new_kv) - def _listener(self): + def _listener(self) -> None: logger.info("start long_poll") while not self._stopping: self._long_poll() @@ -266,13 +268,13 @@ class ApolloClient: headers["Timestamp"] = time_unix_now return headers - def _heart_beat(self): + def _heart_beat(self) -> None: while not self._stopping: for namespace in self._notification_map: self._do_heart_beat(namespace) time.sleep(60 * 10) # 10 minutes - def _do_heart_beat(self, namespace): + def _do_heart_beat(self, namespace: str) -> None: url = f"{self.config_url}/configs/{self.app_id}/{self.cluster}/{namespace}?ip={self.ip}" try: code, body = http_request(url, timeout=3, headers=self._sign_headers(url)) @@ -292,7 +294,7 @@ class ApolloClient: logger.exception("an error occurred in _do_heart_beat") return None - def get_all_dicts(self, namespace): + def get_all_dicts(self, namespace: str) -> dict[str, Any] | None: namespace_data = self._cache.get(namespace) if namespace_data is None: net_namespace_data = self.get_json_from_net(namespace) diff --git a/api/configs/remote_settings_sources/apollo/python_3x.py b/api/configs/remote_settings_sources/apollo/python_3x.py index 6a5f38199..d21e0ecff 100644 --- a/api/configs/remote_settings_sources/apollo/python_3x.py +++ b/api/configs/remote_settings_sources/apollo/python_3x.py @@ -2,6 +2,8 @@ import logging import os import ssl import urllib.request +from collections.abc import Mapping +from typing import Any from urllib import parse from urllib.error import HTTPError @@ -19,9 +21,9 @@ urllib.request.install_opener(opener) logger = logging.getLogger(__name__) -def http_request(url, timeout, headers={}): +def http_request(url: str, timeout: int | float, headers: Mapping[str, str] = {}) -> tuple[int, str | None]: try: - request = urllib.request.Request(url, headers=headers) + request = urllib.request.Request(url, headers=dict(headers)) res = urllib.request.urlopen(request, timeout=timeout) body = res.read().decode("utf-8") return res.code, body @@ -33,9 +35,9 @@ def http_request(url, timeout, headers={}): raise e -def url_encode(params): +def url_encode(params: dict[str, Any]) -> str: return parse.urlencode(params) -def makedirs_wrapper(path): +def makedirs_wrapper(path: str) -> None: os.makedirs(path, exist_ok=True) diff --git a/api/configs/remote_settings_sources/apollo/utils.py b/api/configs/remote_settings_sources/apollo/utils.py index f5b82908e..cff187954 100644 --- a/api/configs/remote_settings_sources/apollo/utils.py +++ b/api/configs/remote_settings_sources/apollo/utils.py @@ -1,5 +1,6 @@ import hashlib import socket +from typing import Any from .python_3x import url_encode @@ -10,7 +11,7 @@ NAMESPACE_NAME = "namespaceName" # add timestamps uris and keys -def signature(timestamp, uri, secret): +def signature(timestamp: str, uri: str, secret: str) -> str: import base64 import hmac @@ -19,16 +20,16 @@ def signature(timestamp, uri, secret): return base64.b64encode(hmac_code).decode() -def url_encode_wrapper(params): +def url_encode_wrapper(params: dict[str, Any]) -> str: return url_encode(params) -def no_key_cache_key(namespace, key): +def no_key_cache_key(namespace: str, key: str) -> str: return f"{namespace}{len(namespace)}{key}" # Returns whether the obtained value is obtained, and None if it does not -def get_value_from_dict(namespace_cache, key): +def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any | None: if namespace_cache: kv_data = namespace_cache.get(CONFIGURATIONS) if kv_data is None: @@ -38,7 +39,7 @@ def get_value_from_dict(namespace_cache, key): return None -def init_ip(): +def init_ip() -> str: ip = "" s = None try: diff --git a/api/configs/remote_settings_sources/nacos/__init__.py b/api/configs/remote_settings_sources/nacos/__init__.py index c6efd6f3a..f3e630675 100644 --- a/api/configs/remote_settings_sources/nacos/__init__.py +++ b/api/configs/remote_settings_sources/nacos/__init__.py @@ -11,16 +11,16 @@ logger = logging.getLogger(__name__) from configs.remote_settings_sources.base import RemoteSettingsSource -from .utils import _parse_config +from .utils import parse_config class NacosSettingsSource(RemoteSettingsSource): def __init__(self, configs: Mapping[str, Any]): self.configs = configs - self.remote_configs: dict[str, Any] = {} + self.remote_configs: dict[str, str] = {} self.async_init() - def async_init(self): + def async_init(self) -> None: data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties") group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify") tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "") @@ -33,18 +33,15 @@ class NacosSettingsSource(RemoteSettingsSource): logger.exception("[get-access-token] exception occurred") raise - def _parse_config(self, content: str): + def _parse_config(self, content: str) -> dict[str, str]: if not content: return {} try: - return _parse_config(self, content) + return parse_config(content) except Exception as e: raise RuntimeError(f"Failed to parse config: {e}") def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: - if not isinstance(self.remote_configs, dict): - raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}") - field_value = self.remote_configs.get(field_name) if field_value is None: return None, field_name, False diff --git a/api/configs/remote_settings_sources/nacos/http_request.py b/api/configs/remote_settings_sources/nacos/http_request.py index db9db84a8..6401c5830 100644 --- a/api/configs/remote_settings_sources/nacos/http_request.py +++ b/api/configs/remote_settings_sources/nacos/http_request.py @@ -17,11 +17,17 @@ class NacosHttpClient: self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY") self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY") self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848") - self.token = None + self.token: str | None = None self.token_ttl = 18000 self.token_expire_time: float = 0 - def http_request(self, url, method="GET", headers=None, params=None): + def http_request( + self, url: str, method: str = "GET", headers: dict[str, str] | None = None, params: dict[str, str] | None = None + ) -> str: + if headers is None: + headers = {} + if params is None: + params = {} try: self._inject_auth_info(headers, params) response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params) @@ -30,7 +36,7 @@ class NacosHttpClient: except requests.RequestException as e: return f"Request to Nacos failed: {e}" - def _inject_auth_info(self, headers, params, module="config"): + def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None: headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"}) if module == "login": @@ -45,16 +51,17 @@ class NacosHttpClient: headers["timeStamp"] = ts if self.username and self.password: self.get_access_token(force_refresh=False) - params["accessToken"] = self.token + if self.token is not None: + params["accessToken"] = self.token - def __do_sign(self, sign_str, sk): + def __do_sign(self, sign_str: str, sk: str) -> str: return ( base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest()) .decode() .strip() ) - def get_sign_str(self, group, tenant, ts): + def get_sign_str(self, group: str, tenant: str, ts: str) -> str: sign_str = "" if tenant: sign_str = tenant + "+" @@ -63,7 +70,7 @@ class NacosHttpClient: sign_str += ts # Directly concatenate ts without conditional checks, because the nacos auth header forced it. return sign_str - def get_access_token(self, force_refresh=False): + def get_access_token(self, force_refresh: bool = False) -> str | None: current_time = time.time() if self.token and not force_refresh and self.token_expire_time > current_time: return self.token @@ -77,6 +84,7 @@ class NacosHttpClient: self.token = response_data.get("accessToken") self.token_ttl = response_data.get("tokenTtl", 18000) self.token_expire_time = current_time + self.token_ttl - 10 + return self.token except Exception: logger.exception("[get-access-token] exception occur") raise diff --git a/api/configs/remote_settings_sources/nacos/utils.py b/api/configs/remote_settings_sources/nacos/utils.py index f3372563b..2d52b46af 100644 --- a/api/configs/remote_settings_sources/nacos/utils.py +++ b/api/configs/remote_settings_sources/nacos/utils.py @@ -1,4 +1,4 @@ -def _parse_config(self, content: str) -> dict[str, str]: +def parse_config(content: str) -> dict[str, str]: config: dict[str, str] = {} if not content: return config diff --git a/api/pyrightconfig.json b/api/pyrightconfig.json index dfffdb8cf..8694f44fa 100644 --- a/api/pyrightconfig.json +++ b/api/pyrightconfig.json @@ -1,5 +1,7 @@ { - "include": ["."], + "include": [ + "." + ], "exclude": [ "tests/", "migrations/", @@ -19,10 +21,9 @@ "events/", "contexts/", "constants/", - "configs/", "commands.py" ], "typeCheckingMode": "strict", "pythonVersion": "3.11", "pythonPlatform": "All" -} +} \ No newline at end of file