estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix: Wrong cutoff length lead to empty input in openai compatible embedding model. (#7133)

Browse Source
This commit is contained in:
Yanyi Liu
2024-08-09 19:11:57 +08:00
committed by GitHub
parent 2c188a45c8
commit 4cbeb6815b
3 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py
View File

@@ -72,7 +72,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text) num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size: if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens))) cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start # if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff]) inputs.append(text[0: cutoff])
else: else:

2
api/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py
View File

@@ -76,7 +76,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text) num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size: if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens))) cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start # if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff]) inputs.append(text[0: cutoff])
else: else:

2
api/core/model_runtime/model_providers/perfxcloud/text_embedding/text_embedding.py
View File

@@ -79,7 +79,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text) num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size: if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens))) cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start # if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff]) inputs.append(text[0: cutoff])
else: else: