feat: backend model load balancing support (#4927)

2024-06-05 00:13:04 +08:00
parent 52ec152dd3
commit d1dbbc1e33
47 changed files with 2191 additions and 256 deletions
--- a/api/core/helper/model_provider_cache.py
+++ b/api/core/helper/model_provider_cache.py
@@ -9,6 +9,7 @@ from extensions.ext_redis import redis_client
 class ProviderCredentialsCacheType(Enum):
    PROVIDER = "provider"
    MODEL = "provider_model"
+    LOAD_BALANCING_MODEL = "load_balancing_provider_model"


 class ProviderCredentialsCache: