feat: add cohere llm and embedding (#2115)

2024-01-21 20:52:56 +08:00
parent 8438d820ad
commit a18dde9b0d
27 changed files with 1689 additions and 3 deletions
--- a/api/core/spiltter/fixed_text_splitter.py
+++ b/api/core/spiltter/fixed_text_splitter.py
@@ -24,6 +24,9 @@ class EnhanceRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter):
            **kwargs: Any,
    ):
        def _token_encoder(text: str) -> int:
+            if not text:
+                return 0
+
            if embedding_model_instance:
                embedding_model_type_instance = embedding_model_instance.model_type_instance
                embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_type_instance)