estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

external knowledge api (#8913)

Browse Source 
Co-authored-by: Yi <yxiaoisme@gmail.com>
This commit is contained in:
Jyong
2024-09-30 15:38:43 +08:00
committed by GitHub
parent 77aef9ff1d
commit 9d221a5e19
90 changed files with 4623 additions and 1171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
web/app/components/workflow/nodes/knowledge-retrieval/utils.ts
View File

@@ -21,6 +21,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
let allHighQualityFullTextSearch = true
let allEconomic = true
let mixtureHighQualityAndEconomic = true
let allExternal = true
let allInternal = true
let mixtureInternalAndExternal = true
let inconsistentEmbeddingModel = false
if (!datasets.length) {
allHighQuality = false
@@ -29,6 +32,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
allEconomic = false
mixtureHighQualityAndEconomic = false
inconsistentEmbeddingModel = false
allExternal = false
allInternal = false
mixtureInternalAndExternal = false
}
datasets.forEach((dataset) => {
if (dataset.indexing_technique === 'economy') {
@@ -45,8 +51,21 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.fullText)
allHighQualityFullTextSearch = false
}
if (dataset.provider !== 'external') {
allExternal = false
}
else {
allInternal = false
allHighQuality = false
allHighQualityVectorSearch = false
allHighQualityFullTextSearch = false
mixtureHighQualityAndEconomic = false
}
})
if (allExternal || allInternal)
mixtureInternalAndExternal = false
if (allHighQuality || allEconomic)
mixtureHighQualityAndEconomic = false
@@ -59,6 +78,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
allHighQualityFullTextSearch,
allEconomic,
mixtureHighQualityAndEconomic,
allInternal,
allExternal,
mixtureInternalAndExternal,
inconsistentEmbeddingModel,
} as SelectedDatasetsMode
}
@@ -70,6 +92,9 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
allHighQualityFullTextSearch,
allEconomic,
mixtureHighQualityAndEconomic,
allInternal,
allExternal,
mixtureInternalAndExternal,
inconsistentEmbeddingModel,
} = getSelectedDatasetsMode(selectedDatasets)
@@ -91,13 +116,13 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
reranking_enable: allEconomic ? reranking_enable : true,
}
if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel || allExternal || mixtureInternalAndExternal)
result.reranking_mode = RerankingModeEnum.RerankingModel
if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined)
if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined && allInternal)
result.reranking_mode = RerankingModeEnum.WeightedScore
if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && !weights) {
if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && allInternal && !weights) {
result.weights = {
vector_setting: {
vector_weight: allHighQualityVectorSearch