estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

external knowledge api (#8913)

Browse Source 
Co-authored-by: Yi <yxiaoisme@gmail.com>
This commit is contained in:
Jyong
2024-09-30 15:38:43 +08:00
committed by GitHub
parent 77aef9ff1d
commit 9d221a5e19
90 changed files with 4623 additions and 1171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx
View File

@@ -1,13 +1,15 @@
'use client'
import type { FC } from 'react'
import React, { useCallback } from 'react'
import React, { useCallback, useState } from 'react'
import { useBoolean } from 'ahooks'
import {
RiDeleteBinLine,
RiEditLine,
} from '@remixicon/react'
import { useTranslation } from 'react-i18next'
import type { DataSet } from '@/models/datasets'
import { DataSourceType } from '@/models/datasets'
import ActionButton, { ActionButtonState } from '@/app/components/base/action-button'
import FileIcon from '@/app/components/base/file-icon'
import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
import SettingsModal from '@/app/components/app/configuration/dataset-config/settings-modal'
@@ -30,8 +32,10 @@ const DatasetItem: FC<Props> = ({
readonly,
}) => {
const media = useBreakpoints()
const { t } = useTranslation()
const isMobile = media === MediaType.mobile
const { formatIndexingTechniqueAndMethod } = useKnowledge()
const [isDeleteHovered, setIsDeleteHovered] = useState(false)
const [isShowSettingsModal, {
setTrue: showSettingsModal,
@@ -43,8 +47,18 @@ const DatasetItem: FC<Props> = ({
hideSettingsModal()
}, [hideSettingsModal, onChange])
const handleRemove = useCallback((e: React.MouseEvent) => {
e.stopPropagation()
onRemove()
}, [onRemove])
return (
<div className='flex items-center h-10 justify-between rounded-xl px-2 bg-white border border-gray-200 cursor-pointer group/dataset-item'>
<div className={`flex items-center h-10 justify-between rounded-xl px-2 border-[0.5px]
border-components-panel-border-subtle cursor-pointer group/dataset-item
${isDeleteHovered
? 'bg-state-destructive-hover border-state-destructive-border'
: 'bg-components-panel-on-panel-item-bg hover:bg-components-panel-on-panel-item-bg-hover'
}`}>
<div className='w-0 grow flex items-center space-x-1.5'>
{
payload.data_source_type === DataSourceType.NOTION
@@ -61,24 +75,36 @@ const DatasetItem: FC<Props> = ({
</div>
{!readonly && (
<div className='hidden group-hover/dataset-item:flex shrink-0 ml-2 items-center space-x-1'>
<div
className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
onClick={showSettingsModal}
<ActionButton
onClick={(e) => {
e.stopPropagation()
showSettingsModal()
}}
>
<RiEditLine className='w-4 h-4 text-gray-500' />
</div>
<div
className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
onClick={onRemove}
<RiEditLine className='w-4 h-4 flex-shrink-0 text-text-tertiary' />
</ActionButton>
<ActionButton
onClick={handleRemove}
state={ActionButtonState.Destructive}
onMouseEnter={() => setIsDeleteHovered(true)}
onMouseLeave={() => setIsDeleteHovered(false)}
>
<RiDeleteBinLine className='w-4 h-4 text-gray-500' />
</div>
<RiDeleteBinLine className={`w-4 h-4 flex-shrink-0 ${isDeleteHovered ? 'text-text-destructive' : 'text-text-tertiary'}`} />
</ActionButton>
</div>
)}
<Badge
className='group-hover/dataset-item:hidden shrink-0'
text={formatIndexingTechniqueAndMethod(payload.indexing_technique, payload.retrieval_model_dict?.search_method)}
/>
{
payload.indexing_technique && <Badge
className='group-hover/dataset-item:hidden shrink-0'
text={formatIndexingTechniqueAndMethod(payload.indexing_technique, payload.retrieval_model_dict?.search_method)}
/>
}
{
payload.provider === 'external' && <Badge
className='group-hover/dataset-item:hidden shrink-0'
text={t('dataset.externalTag')}
/>
}
{isShowSettingsModal && (
<Drawer isOpen={isShowSettingsModal} onClose={hideSettingsModal} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'>

10
web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts
View File

@@ -207,9 +207,11 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
const handleOnDatasetsChange = useCallback((newDatasets: DataSet[]) => {
const {
allEconomic,
mixtureHighQualityAndEconomic,
mixtureInternalAndExternal,
inconsistentEmbeddingModel,
allInternal,
allExternal,
} = getSelectedDatasetsMode(newDatasets)
const newInputs = produce(inputs, (draft) => {
draft.dataset_ids = newDatasets.map(d => d.id)
@@ -222,7 +224,11 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
setInputs(newInputs)
setSelectedDatasets(newDatasets)
if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
if (
(allInternal && (mixtureHighQualityAndEconomic || inconsistentEmbeddingModel))
|| mixtureInternalAndExternal
|| (allExternal && newDatasets.length > 1)
)
setRerankModelOpen(true)
}, [inputs, setInputs, payload.retrieval_mode])

31
web/app/components/workflow/nodes/knowledge-retrieval/utils.ts
View File

@@ -21,6 +21,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
let allHighQualityFullTextSearch = true
let allEconomic = true
let mixtureHighQualityAndEconomic = true
let allExternal = true
let allInternal = true
let mixtureInternalAndExternal = true
let inconsistentEmbeddingModel = false
if (!datasets.length) {
allHighQuality = false
@@ -29,6 +32,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
allEconomic = false
mixtureHighQualityAndEconomic = false
inconsistentEmbeddingModel = false
allExternal = false
allInternal = false
mixtureInternalAndExternal = false
}
datasets.forEach((dataset) => {
if (dataset.indexing_technique === 'economy') {
@@ -45,8 +51,21 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.fullText)
allHighQualityFullTextSearch = false
}
if (dataset.provider !== 'external') {
allExternal = false
}
else {
allInternal = false
allHighQuality = false
allHighQualityVectorSearch = false
allHighQualityFullTextSearch = false
mixtureHighQualityAndEconomic = false
}
})
if (allExternal || allInternal)
mixtureInternalAndExternal = false
if (allHighQuality || allEconomic)
mixtureHighQualityAndEconomic = false
@@ -59,6 +78,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
allHighQualityFullTextSearch,
allEconomic,
mixtureHighQualityAndEconomic,
allInternal,
allExternal,
mixtureInternalAndExternal,
inconsistentEmbeddingModel,
} as SelectedDatasetsMode
}
@@ -70,6 +92,9 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
allHighQualityFullTextSearch,
allEconomic,
mixtureHighQualityAndEconomic,
allInternal,
allExternal,
mixtureInternalAndExternal,
inconsistentEmbeddingModel,
} = getSelectedDatasetsMode(selectedDatasets)
@@ -91,13 +116,13 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
reranking_enable: allEconomic ? reranking_enable : true,
}
if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel || allExternal || mixtureInternalAndExternal)
result.reranking_mode = RerankingModeEnum.RerankingModel
if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined)
if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined && allInternal)
result.reranking_mode = RerankingModeEnum.WeightedScore
if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && !weights) {
if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && allInternal && !weights) {
result.weights = {
vector_setting: {
vector_weight: allHighQualityVectorSearch