Feature/mutil embedding model (#908)

Co-authored-by: JzoNg <jzongcode@gmail.com> Co-authored-by: jyong <jyong@dify.ai> Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
2023-08-18 17:37:31 +08:00
parent 4420281d96
commit db7156dafd
54 changed files with 1704 additions and 278 deletions
--- a/web/app/components/datasets/create/step-two/index.tsx
+++ b/web/app/components/datasets/create/step-two/index.tsx
@@ -2,12 +2,14 @@
 'use client'
 import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
 import { useBoolean } from 'ahooks'
 import { XMarkIcon } from '@heroicons/react/20/solid'
 import cn from 'classnames'
 import Link from 'next/link'
 import { groupBy } from 'lodash-es'
 import PreviewItem, { PreviewType } from './preview-item'
+import LanguageSelect from './language-select'
 import s from './index.module.css'
 import type { CreateDocumentReq, CustomFile, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
 import {
@@ -22,11 +24,13 @@ import Loading from '@/app/components/base/loading'
 import Toast from '@/app/components/base/toast'
 import { formatNumber } from '@/utils/format'
 import type { DataSourceNotionPage } from '@/models/common'
-import { DataSourceType } from '@/models/datasets'
+import { DataSourceType, DocForm } from '@/models/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import Switch from '@/app/components/base/switch'
 import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
+import { XClose } from '@/app/components/base/icons/src/vender/line/general'
 import { useDatasetDetailContext } from '@/context/dataset-detail'
+import I18n from '@/context/i18n'
 import { IS_CE_EDITION } from '@/config'

 type Page = DataSourceNotionPage & { workspace_id: string }
@@ -56,10 +60,6 @@ enum IndexingType {
  QUALIFIED = 'high_quality',
  ECONOMICAL = 'economy',
 }
-enum DocForm {
-  TEXT = 'text_model',
-  QA = 'qa_model',
-}

 const StepTwo = ({
  isSetting,
@@ -78,6 +78,8 @@ const StepTwo = ({
  onCancel,
 }: StepTwoProps) => {
  const { t } = useTranslation()
+  const { locale } = useContext(I18n)
+
  const { mutateDatasetRes } = useDatasetDetailContext()
  const scrollRef = useRef<HTMLDivElement>(null)
  const [scrolled, setScrolled] = useState(false)
@@ -98,6 +100,8 @@ const StepTwo = ({
  const [docForm, setDocForm] = useState<DocForm | string>(
    datasetId && documentDetail ? documentDetail.doc_form : DocForm.TEXT,
  )
+  const [docLanguage, setDocLanguage] = useState<string>(locale === 'en' ? 'English' : 'Chinese')
+  const [QATipHide, setQATipHide] = useState(false)
  const [previewSwitched, setPreviewSwitched] = useState(false)
  const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
  const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
@@ -230,6 +234,8 @@ const StepTwo = ({
        indexing_technique: getIndexing_technique(),
        process_rule: getProcessRule(),
        doc_form: docForm,
+        doc_language: docLanguage,
+        dataset_id: datasetId,
      }
    }
    if (dataSourceType === DataSourceType.NOTION) {
@@ -241,6 +247,8 @@ const StepTwo = ({
        indexing_technique: getIndexing_technique(),
        process_rule: getProcessRule(),
        doc_form: docForm,
+        doc_language: docLanguage,
+        dataset_id: datasetId,
      }
    }
    return params
@@ -252,6 +260,7 @@ const StepTwo = ({
      params = {
        original_document_id: documentDetail?.id,
        doc_form: docForm,
+        doc_language: docLanguage,
        process_rule: getProcessRule(),
      } as CreateDocumentReq
    }
@@ -266,6 +275,7 @@ const StepTwo = ({
        indexing_technique: getIndexing_technique(),
        process_rule: getProcessRule(),
        doc_form: docForm,
+        doc_language: docLanguage,
      } as CreateDocumentReq
      if (dataSourceType === DataSourceType.FILE) {
        params.data_source.info_list.file_info_list = {
@@ -348,6 +358,10 @@ const StepTwo = ({
      setDocForm(DocForm.TEXT)
  }

+  const handleSelect = (language: string) => {
+    setDocLanguage(language)
+  }
+
  const changeToEconomicalType = () => {
    if (!hasSetIndexType) {
      setIndexType(IndexingType.ECONOMICAL)
@@ -574,21 +588,32 @@ const StepTwo = ({
              </div>
            )}
            {IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
-              <div className='flex justify-between items-center mt-3 px-5 py-4 rounded-xl bg-gray-50 border border-gray-100'>
-                <div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
-                  <MessageChatSquare className='w-4 h-4' />
-                </div>
-                <div className='grow mx-3'>
-                  <div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
-                  <div className='text-[13px] leading-[18px] text-gray-500'>{t('datasetCreation.stepTwo.QATip')}</div>
-                </div>
-                <div className='shrink-0'>
-                  <Switch
-                    defaultValue={docForm === DocForm.QA}
-                    onChange={handleSwitch}
-                    size='md'
-                  />
+              <div className='mt-3 rounded-xl bg-gray-50 border border-gray-100'>
+                <div className='flex justify-between items-center px-5 py-4'>
+                  <div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
+                    <MessageChatSquare className='w-4 h-4' />
+                  </div>
+                  <div className='grow mx-3'>
+                    <div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
+                    <div className='inline-flex items-center text-[13px] leading-[18px] text-gray-500'>
+                      <span className='pr-1'>{t('datasetCreation.stepTwo.QALanguage')}</span>
+                      <LanguageSelect currentLanguage={docLanguage} onSelect={handleSelect} />
+                    </div>
+                  </div>
+                  <div className='shrink-0'>
+                    <Switch
+                      defaultValue={docForm === DocForm.QA}
+                      onChange={handleSwitch}
+                      size='md'
+                    />
+                  </div>
                </div>
+                {docForm === DocForm.QA && !QATipHide && (
+                  <div className='flex justify-between items-center px-5 py-2 bg-orange-50 border-t border-amber-100 rounded-b-xl text-[13px] leading-[18px] text-medium text-amber-500'>
+                    {t('datasetCreation.stepTwo.QATip')}
+                    <XClose className='w-4 h-4 text-gray-500 cursor-pointer' onClick={() => setQATipHide(true)} />
+                  </div>
+                )}
              </div>
            )}
            <div className={s.source}>
--- a/web/app/components/datasets/create/step-two/language-select/index.tsx
+++ b/web/app/components/datasets/create/step-two/language-select/index.tsx
@@ -0,0 +1,38 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from 'classnames'
+import { ChevronDown } from '@/app/components/base/icons/src/vender/line/arrows'
+import Popover from '@/app/components/base/popover'
+
+export type ILanguageSelectProps = {
+  currentLanguage: string
+  onSelect: (language: string) => void
+}
+
+const LanguageSelect: FC<ILanguageSelectProps> = ({
+  currentLanguage,
+  onSelect,
+}) => {
+  return (
+    <Popover
+      manualClose
+      trigger='click'
+      htmlContent={
+        <div className='w-full py-1'>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('English')}>English</div>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('Chinese')}>简体中文</div>
+        </div>
+      }
+      btnElement={
+        <div className='inline-flex items-center'>
+          <span className='pr-[2px] text-xs leading-[18px] font-medium'>{currentLanguage === 'English' ? 'English' : '简体中文'}</span>
+          <ChevronDown className='w-3 h-3 opacity-60' />
+        </div>
+      }
+      btnClassName={open => cn('!border-0 !px-0 !py-0 !bg-inherit !hover:bg-inherit', open ? 'text-blue-600' : 'text-gray-500')}
+      className='!w-[120px] h-fit !z-20 !translate-x-0 !left-[-16px]'
+    />
+  )
+}
+export default React.memo(LanguageSelect)