feat: add retriever rank fe (#1557)

Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
2023-11-18 11:53:35 +08:00
parent e017eff5e4
commit 888e8c6dac
80 changed files with 2757 additions and 467 deletions
--- a/web/app/components/datasets/common/check-rerank-model.ts
+++ b/web/app/components/datasets/common/check-rerank-model.ts
@@ -0,0 +1,50 @@
+import type { BackendModel } from '../../header/account-setting/model-page/declarations'
+import { RETRIEVE_METHOD, type RetrievalConfig } from '@/types/app'
+
+export const isReRankModelSelected = ({
+  rerankDefaultModel,
+  isRerankDefaultModelVaild,
+  retrievalConfig,
+  indexMethod,
+}: {
+  rerankDefaultModel?: BackendModel
+  isRerankDefaultModelVaild: boolean
+  retrievalConfig: RetrievalConfig
+  indexMethod?: string
+}) => {
+  const rerankModel = (retrievalConfig.reranking_model?.reranking_model_name ? retrievalConfig.reranking_model : undefined) || (isRerankDefaultModelVaild ? rerankDefaultModel : undefined)
+  if (
+    indexMethod === 'high_quality'
+    && (retrievalConfig.reranking_enable || retrievalConfig.search_method === RETRIEVE_METHOD.fullText)
+    && !rerankModel
+  )
+    return false
+
+  return true
+}
+
+export const ensureRerankModelSelected = ({
+  rerankDefaultModel,
+  indexMethod,
+  retrievalConfig,
+}: {
+  rerankDefaultModel: BackendModel
+  retrievalConfig: RetrievalConfig
+  indexMethod?: string
+}) => {
+  const rerankModel = retrievalConfig.reranking_model?.reranking_model_name ? retrievalConfig.reranking_model : undefined
+  if (
+    indexMethod === 'high_quality'
+    && (retrievalConfig.reranking_enable || retrievalConfig.search_method === RETRIEVE_METHOD.fullText)
+    && !rerankModel
+  ) {
+    return {
+      ...retrievalConfig,
+      reranking_model: {
+        reranking_provider_name: rerankDefaultModel.model_provider.provider_name,
+        reranking_model_name: rerankDefaultModel.model_name,
+      },
+    }
+  }
+  return retrievalConfig
+}
--- a/web/app/components/datasets/common/economical-retrieval-method-config/index.tsx
+++ b/web/app/components/datasets/common/economical-retrieval-method-config/index.tsx
@@ -0,0 +1,40 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import RetrievalParamConfig from '../retrieval-param-config'
+import { RETRIEVE_METHOD } from '@/types/app'
+import RadioCard from '@/app/components/base/radio-card'
+import { HighPriority } from '@/app/components/base/icons/src/vender/solid/arrows'
+import type { RetrievalConfig } from '@/types/app'
+
+type Props = {
+  value: RetrievalConfig
+  onChange: (value: RetrievalConfig) => void
+}
+
+const EconomicalRetrievalMethodConfig: FC<Props> = ({
+  value,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='space-y-2'>
+      <RadioCard
+        icon={<HighPriority className='w-4 h-4 text-[#7839EE]' />}
+        title={t('dataset.retrieval.invertedIndex.title')}
+        description={t('dataset.retrieval.invertedIndex.description')}
+        noRadio
+        chosenConfig={
+          <RetrievalParamConfig
+            type={RETRIEVE_METHOD.invertedIndex}
+            value={value}
+            onChange={onChange}
+          />
+        }
+      />
+    </div>
+  )
+}
+export default React.memo(EconomicalRetrievalMethodConfig)
--- a/web/app/components/datasets/common/retrieval-method-config/index.tsx
+++ b/web/app/components/datasets/common/retrieval-method-config/index.tsx
@@ -0,0 +1,91 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import RetrievalParamConfig from '../retrieval-param-config'
+import type { RetrievalConfig } from '@/types/app'
+import { RETRIEVE_METHOD } from '@/types/app'
+import RadioCard from '@/app/components/base/radio-card'
+import { PatternRecognition, Semantic } from '@/app/components/base/icons/src/vender/solid/development'
+import { FileSearch02 } from '@/app/components/base/icons/src/vender/solid/files'
+import { useProviderContext } from '@/context/provider-context'
+
+type Props = {
+  value: RetrievalConfig
+  onChange: (value: RetrievalConfig) => void
+}
+
+const RetrievalMethodConfig: FC<Props> = ({
+  value,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+  const { supportRetrievalMethods } = useProviderContext()
+  return (
+    <div className='space-y-2'>
+      {supportRetrievalMethods.includes(RETRIEVE_METHOD.semantic) && (
+        <RadioCard
+          icon={<Semantic className='w-4 h-4 text-[#7839EE]' />}
+          title={t('dataset.retrieval.semantic_search.title')}
+          description={t('dataset.retrieval.semantic_search.description')}
+          isChosen={value.search_method === RETRIEVE_METHOD.semantic}
+          onChosen={() => onChange({
+            ...value,
+            search_method: RETRIEVE_METHOD.semantic,
+          })}
+          chosenConfig={
+            <RetrievalParamConfig
+              type={RETRIEVE_METHOD.semantic}
+              value={value}
+              onChange={onChange}
+            />
+          }
+        />
+      )}
+      {supportRetrievalMethods.includes(RETRIEVE_METHOD.semantic) && (
+        <RadioCard
+          icon={<FileSearch02 className='w-4 h-4 text-[#7839EE]' />}
+          title={t('dataset.retrieval.full_text_search.title')}
+          description={t('dataset.retrieval.full_text_search.description')}
+          isChosen={value.search_method === RETRIEVE_METHOD.fullText}
+          onChosen={() => onChange({
+            ...value,
+            search_method: RETRIEVE_METHOD.fullText,
+          })}
+          chosenConfig={
+            <RetrievalParamConfig
+              type={RETRIEVE_METHOD.fullText}
+              value={value}
+              onChange={onChange}
+            />
+          }
+        />
+      )}
+      {supportRetrievalMethods.includes(RETRIEVE_METHOD.semantic) && (
+        <RadioCard
+          icon={<PatternRecognition className='w-4 h-4 text-[#7839EE]' />}
+          title={
+            <div className='flex items-center space-x-1'>
+              <div>{t('dataset.retrieval.hybrid_search.title')}</div>
+              <div className='flex h-full items-center px-1.5 rounded-md border border-[#E0EAFF] text-xs font-medium text-[#444CE7]'>{t('dataset.retrieval.hybrid_search.recommend')}</div>
+            </div>
+          }
+          description={t('dataset.retrieval.hybrid_search.description')}
+          isChosen={value.search_method === RETRIEVE_METHOD.hybrid}
+          onChosen={() => onChange({
+            ...value,
+            search_method: RETRIEVE_METHOD.hybrid,
+          })}
+          chosenConfig={
+            <RetrievalParamConfig
+              type={RETRIEVE_METHOD.hybrid}
+              value={value}
+              onChange={onChange}
+            />
+          }
+        />
+      )}
+    </div>
+  )
+}
+export default React.memo(RetrievalMethodConfig)
--- a/web/app/components/datasets/common/retrieval-method-info/index.tsx
+++ b/web/app/components/datasets/common/retrieval-method-info/index.tsx
@@ -0,0 +1,64 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import type { RetrievalConfig } from '@/types/app'
+import { RETRIEVE_METHOD } from '@/types/app'
+import RadioCard from '@/app/components/base/radio-card'
+import { HighPriority } from '@/app/components/base/icons/src/vender/solid/arrows'
+import { PatternRecognition, Semantic } from '@/app/components/base/icons/src/vender/solid/development'
+import { FileSearch02 } from '@/app/components/base/icons/src/vender/solid/files'
+
+type Props = {
+  value: RetrievalConfig
+}
+
+export const getIcon = (type: RETRIEVE_METHOD) => {
+  return ({
+    [RETRIEVE_METHOD.semantic]: Semantic,
+    [RETRIEVE_METHOD.fullText]: FileSearch02,
+    [RETRIEVE_METHOD.hybrid]: PatternRecognition,
+    [RETRIEVE_METHOD.invertedIndex]: HighPriority,
+  })[type] || FileSearch02
+}
+
+const EconomicalRetrievalMethodConfig: FC<Props> = ({
+  // type,
+  value,
+}) => {
+  const { t } = useTranslation()
+  const type = value.search_method
+  const Icon = getIcon(type)
+  return (
+    <div className='space-y-2'>
+      <RadioCard
+        icon={<Icon className='w-4 h-4 text-[#7839EE]' />}
+        title={t(`dataset.retrieval.${type}.title`)}
+        description={t(`dataset.retrieval.${type}.description`)}
+        noRadio
+        chosenConfigWrapClassName='!pb-3'
+        chosenConfig={
+          <div className='flex flex-wrap leading-[18px] text-xs font-normal'>
+            {value.reranking_model.reranking_model_name && (
+              <div className='mr-8 flex space-x-1'>
+                <div className='text-gray-500'>{t('common.modelProvider.rerankModel.key')}</div>
+                <div className='font-medium text-gray-800'>{value.reranking_model.reranking_model_name}</div>
+              </div>
+            )}
+
+            <div className='mr-8 flex space-x-1'>
+              <div className='text-gray-500'>{t('appDebug.datasetConfig.top_k')}</div>
+              <div className='font-medium text-gray-800'>{value.top_k}</div>
+            </div>
+
+            <div className='mr-8 flex space-x-1'>
+              <div className='text-gray-500'>{t('appDebug.datasetConfig.score_threshold')}</div>
+              <div className='font-medium text-gray-800'>{value.score_threshold}</div>
+            </div>
+          </div>
+        }
+      />
+    </div>
+  )
+}
+export default React.memo(EconomicalRetrievalMethodConfig)
--- a/web/app/components/datasets/common/retrieval-param-config/index.tsx
+++ b/web/app/components/datasets/common/retrieval-param-config/index.tsx
@@ -0,0 +1,131 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
+import TopKItem from '@/app/components/base/param-item/top-k-item'
+import ScoreThresholdItem from '@/app/components/base/param-item/score-threshold-item'
+import { RETRIEVE_METHOD } from '@/types/app'
+import Switch from '@/app/components/base/switch'
+import Tooltip from '@/app/components/base/tooltip-plus'
+import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general'
+import ModelSelector from '@/app/components/header/account-setting/model-page/model-selector'
+import { ModelType } from '@/app/components/header/account-setting/model-page/declarations'
+import type { RetrievalConfig } from '@/types/app'
+import { useProviderContext } from '@/context/provider-context'
+
+type Props = {
+  type: RETRIEVE_METHOD
+  value: RetrievalConfig
+  onChange: (value: RetrievalConfig) => void
+}
+
+const RetrievalParamConfig: FC<Props> = ({
+  type,
+  value,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+  const canToggleRerankModalEnable = type !== RETRIEVE_METHOD.hybrid
+  const isEconomical = type === RETRIEVE_METHOD.invertedIndex
+  const {
+    rerankDefaultModel,
+  } = useProviderContext()
+
+  const rerankModel = (() => {
+    if (value.reranking_model) {
+      return {
+        provider_name: value.reranking_model.reranking_provider_name,
+        model_name: value.reranking_model.reranking_model_name,
+      }
+    }
+    else if (rerankDefaultModel) {
+      return {
+        provider_name: rerankDefaultModel.model_provider.provider_name,
+        model_name: rerankDefaultModel.model_name,
+      }
+    }
+  })()
+
+  return (
+    <div>
+      {!isEconomical && (
+        <div>
+          <div className='flex h-8 items-center text-[13px] font-medium text-gray-900 space-x-2'>
+            {canToggleRerankModalEnable && (
+              <Switch
+                size='md'
+                defaultValue={value.reranking_enable}
+                onChange={(v) => {
+                  onChange({
+                    ...value,
+                    reranking_enable: v,
+                  })
+                }}
+              />
+            )}
+            <div className='flex items-center'>
+              <span className='mr-0.5'>{t('common.modelProvider.rerankModel.key')}</span>
+              <Tooltip popupContent={<div className="w-[200px]">{t('common.modelProvider.rerankModel.tip')}</div>}>
+                <HelpCircle className='w-[14px] h-[14px] text-gray-400' />
+              </Tooltip>
+            </div>
+          </div>
+          <div>
+            <ModelSelector
+              whenEmptyGoToSetting
+              popClassName='!max-w-[100%] !w-full'
+              value={rerankModel && { providerName: rerankModel.provider_name, modelName: rerankModel.model_name } as any}
+              modelType={ModelType.reranking}
+              readonly={!value.reranking_enable && type !== RETRIEVE_METHOD.hybrid}
+              onChange={(v) => {
+                onChange({
+                  ...value,
+                  reranking_model: {
+                    reranking_provider_name: v.model_provider.provider_name,
+                    reranking_model_name: v.model_name,
+                  },
+                })
+              }}
+            />
+          </div>
+        </div>
+      )}
+
+      <div className={cn(!isEconomical && 'mt-4', 'flex space-between space-x-6')}>
+        <TopKItem
+          className='grow'
+          value={value.top_k}
+          onChange={(_key, v) => {
+            onChange({
+              ...value,
+              top_k: v,
+            })
+          }}
+          enable={true}
+        />
+        {(!isEconomical && !(value.search_method === RETRIEVE_METHOD.fullText && !value.reranking_enable)) && (
+          <ScoreThresholdItem
+            className='grow'
+            value={value.score_threshold}
+            onChange={(_key, v) => {
+              onChange({
+                ...value,
+                score_threshold: v,
+              })
+            }}
+            enable={value.score_threshold_enable}
+            hasSwitch={true}
+            onSwitchChange={(_key, v) => {
+              onChange({
+                ...value,
+                score_threshold_enable: v,
+              })
+            }}
+          />
+        )}
+      </div>
+    </div>
+  )
+}
+export default React.memo(RetrievalParamConfig)