estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Feat/auto rule generate (#300)

Browse Source
This commit is contained in:
Joel
2023-06-06 10:52:02 +08:00
committed by GitHub
parent e61c84ca72
commit 6483beb096
14 changed files with 808 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
web/app/components/datasets/create/step-two/index.tsx
View File

@@ -1,36 +1,36 @@
/* eslint-disable no-mixed-operators */
'use client'
import React, { useState, useRef, useEffect, useLayoutEffect } from 'react'
import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import type { File, PreProcessingRule, Rules, FileIndexingEstimateResponse as IndexingEstimateResponse } from '@/models/datasets'
import {
fetchDefaultProcessRule,
createFirstDocument,
createDocument,
fetchFileIndexingEstimate as didFetchFileIndexingEstimate,
} from '@/service/datasets'
import type { CreateDocumentReq, createDocumentResponse, FullDocumentDetail } from '@/models/datasets'
import Button from '@/app/components/base/button'
import PreviewItem from './preview-item'
import Loading from '@/app/components/base/loading'
import { XMarkIcon } from '@heroicons/react/20/solid'
import cn from 'classnames'
import s from './index.module.css'
import Link from 'next/link'
import PreviewItem from './preview-item'
import s from './index.module.css'
import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
import {
createDocument,
createFirstDocument,
fetchFileIndexingEstimate as didFetchFileIndexingEstimate,
fetchDefaultProcessRule,
} from '@/service/datasets'
import Button from '@/app/components/base/button'
import Loading from '@/app/components/base/loading'
import Toast from '@/app/components/base/toast'
import { formatNumber } from '@/utils/format'
type StepTwoProps = {
isSetting?: boolean,
isSetting?: boolean
documentDetail?: FullDocumentDetail
hasSetAPIKEY: boolean,
onSetting: () => void,
datasetId?: string,
indexingType?: string,
file?: File,
onStepChange?: (delta: number) => void,
updateIndexingTypeCache?: (type: string) => void,
hasSetAPIKEY: boolean
onSetting: () => void
datasetId?: string
indexingType?: string
file?: File
onStepChange?: (delta: number) => void
updateIndexingTypeCache?: (type: string) => void
updateResultCache?: (res: createDocumentResponse) => void
onSave?: () => void
onCancel?: () => void
@@ -71,8 +71,10 @@ const StepTwo = ({
const [defaultConfig, setDefaultConfig] = useState<Rules>()
const hasSetIndexType = !!indexingType
const [indexType, setIndexType] = useState<IndexingType>(
indexingType ||
hasSetAPIKEY ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL
indexingType
|| hasSetAPIKEY
? IndexingType.QUALIFIED
: IndexingType.ECONOMICAL,
)
const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
@@ -82,19 +84,19 @@ const StepTwo = ({
})()
const scrollHandle = (e: any) => {
if (e.target.scrollTop > 0) {
if (e.target.scrollTop > 0)
setScrolled(true)
} else {
else
setScrolled(false)
}
}
const previewScrollHandle = (e: any) => {
if (e.target.scrollTop > 0) {
if (e.target.scrollTop > 0)
setPreviewScrolled(true)
} else {
else
setPreviewScrolled(false)
}
}
const getFileName = (name: string) => {
const arr = name.split('.')
@@ -102,18 +104,17 @@ const StepTwo = ({
}
const getRuleName = (key: string) => {
if (key === 'remove_extra_spaces') {
if (key === 'remove_extra_spaces')
return t('datasetCreation.stepTwo.removeExtraSpaces')
}
if (key === 'remove_urls_emails') {
if (key === 'remove_urls_emails')
return t('datasetCreation.stepTwo.removeUrlEmails')
}
if (key === 'remove_stopwords') {
if (key === 'remove_stopwords')
return t('datasetCreation.stepTwo.removeStopwords')
}
}
const ruleChangeHandle = (id: string) => {
const newRules = rules.map(rule => {
const newRules = rules.map((rule) => {
if (rule.id === id) {
return {
id: rule.id,
@@ -132,13 +133,23 @@ const StepTwo = ({
}
}
const fetchFileIndexingEstimate = async () => {
// eslint-disable-next-line @typescript-eslint/no-use-before-define
const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams())
if (segmentationType === SegmentType.CUSTOM)
setCustomFileIndexingEstimate(res)
else
setAutomaticFileIndexingEstimate(res)
}
const confirmChangeCustomConfig = async () => {
setCustomFileIndexingEstimate(null)
setShowPreview()
await fetchFileIndexingEstimate()
}
const getIndexing_technique = () => indexingType ? indexingType : indexType
const getIndexing_technique = () => indexingType || indexType
const getProcessRule = () => {
const processRule: any = {
@@ -168,16 +179,6 @@ const StepTwo = ({
return params
}
const fetchFileIndexingEstimate = async () => {
const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams())
if (segmentationType === SegmentType.CUSTOM) {
setCustomFileIndexingEstimate(res)
}
else {
setAutomaticFileIndexingEstimate(res)
}
}
const getCreationParams = () => {
let params
if (isSetting) {
@@ -185,7 +186,8 @@ const StepTwo = ({
original_document_id: documentDetail?.id,
process_rule: getProcessRule(),
} as CreateDocumentReq
} else {
}
else {
params = {
data_source: {
type: 'upload_file',
@@ -226,25 +228,25 @@ const StepTwo = ({
}
const getDefaultMode = () => {
if (documentDetail) {
if (documentDetail)
setSegmentationType(documentDetail.dataset_process_rule.mode)
}
}
const createHandle = async () => {
try {
let res;
let res
const params = getCreationParams()
if (!datasetId) {
res = await createFirstDocument({
body: params
body: params,
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
updateResultCache && updateResultCache(res)
} else {
}
else {
res = await createDocument({
datasetId,
body: params
body: params,
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
updateResultCache && updateResultCache({
@@ -257,7 +259,7 @@ const StepTwo = ({
catch (err) {
Toast.notify({
type: 'error',
message: err + '',
message: `${err}`,
})
}
}
@@ -266,35 +268,36 @@ const StepTwo = ({
// fetch rules
if (!isSetting) {
getRules()
} else {
}
else {
getRulesFromDetail()
getDefaultMode()
}
}, [])
useEffect(() => {
scrollRef.current?.addEventListener('scroll', scrollHandle);
scrollRef.current?.addEventListener('scroll', scrollHandle)
return () => {
scrollRef.current?.removeEventListener('scroll', scrollHandle);
scrollRef.current?.removeEventListener('scroll', scrollHandle)
}
}, [])
useLayoutEffect(() => {
if (showPreview) {
previewScrollRef.current?.addEventListener('scroll', previewScrollHandle);
previewScrollRef.current?.addEventListener('scroll', previewScrollHandle)
return () => {
previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle);
previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle)
}
}
}, [showPreview])
useEffect(() => {
// get indexing type by props
if (indexingType) {
if (indexingType)
setIndexType(indexingType as IndexingType)
} else {
else
setIndexType(hasSetAPIKEY ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL)
}
}, [hasSetAPIKEY, indexingType, datasetId])
useEffect(() => {
@@ -302,7 +305,8 @@ const StepTwo = ({
setAutomaticFileIndexingEstimate(null)
setShowPreview()
fetchFileIndexingEstimate()
} else {
}
else {
hidePreview()
setCustomFileIndexingEstimate(null)
}
@@ -320,7 +324,7 @@ const StepTwo = ({
className={cn(
s.radioItem,
s.segmentationItem,
segmentationType === SegmentType.AUTO && s.active
segmentationType === SegmentType.AUTO && s.active,
)}
onClick={() => setSegmentationType(SegmentType.AUTO)}
>
@@ -355,7 +359,7 @@ const StepTwo = ({
type="text"
className={s.input}
placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''} value={segmentIdentifier}
onChange={(e) => setSegmentIdentifier(e.target.value)}
onChange={e => setSegmentIdentifier(e.target.value)}
/>
</div>
</div>
@@ -366,7 +370,7 @@ const StepTwo = ({
type="number"
className={s.input}
placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''} value={max}
onChange={(e) => setMax(Number(e.target.value))}
onChange={e => setMax(Number(e.target.value))}
/>
</div>
</div>
@@ -403,9 +407,8 @@ const StepTwo = ({
hasSetIndexType && '!w-full',
)}
onClick={() => {
if (hasSetAPIKEY) {
if (hasSetAPIKEY)
setIndexType(IndexingType.QUALIFIED)
}
}}
>
<span className={cn(s.typeIcon, s.qualified)} />
@@ -418,11 +421,13 @@ const StepTwo = ({
<div className={s.tip}>{t('datasetCreation.stepTwo.qualifiedTip')}</div>
<div className='pb-0.5 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateCost')}</div>
{
!!fileIndexingEstimate ? (
<div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.tokens)} tokens(<span className='text-yellow-500'>${formatNumber(fileIndexingEstimate.total_price)}</span>)</div>
) : (
<div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
)
fileIndexingEstimate
? (
<div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.tokens)} tokens(<span className='text-yellow-500'>${formatNumber(fileIndexingEstimate.total_price)}</span>)</div>
)
: (
<div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
)
}
</div>
{!hasSetAPIKEY && (
@@ -434,7 +439,6 @@ const StepTwo = ({
</div>
)}
{(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
<div
className={cn(
@@ -476,51 +480,58 @@ const StepTwo = ({
<div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateSegment')}</div>
<div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
{
!!fileIndexingEstimate ? (
<div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.total_segments)} </div>
) : (
<div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
)
fileIndexingEstimate
? (
<div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.total_segments)} </div>
)
: (
<div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
)
}
</div>
</div>
</div>
{!isSetting ? (
<div className='flex items-center mt-8 py-2'>
<Button onClick={() => onStepChange && onStepChange(-1)}>{t('datasetCreation.stepTwo.lastStep')}</Button>
<div className={s.divider} />
<Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.nextStep')}</Button>
</div>
) : (
<div className='flex items-center mt-8 py-2'>
<Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
<Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
</div>
)}
{!isSetting
? (
<div className='flex items-center mt-8 py-2'>
<Button onClick={() => onStepChange && onStepChange(-1)}>{t('datasetCreation.stepTwo.lastStep')}</Button>
<div className={s.divider} />
<Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.nextStep')}</Button>
</div>
)
: (
<div className='flex items-center mt-8 py-2'>
<Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
<Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
</div>
)}
</div>
</div>
</div>
{(showPreview) ? (
<div ref={previewScrollRef} className={cn(s.previewWrap, 'relativeh-full overflow-y-scroll border-l border-[#F2F4F7]')}>
<div className={cn(s.previewHeader, previewScrolled && `${s.fixed} pb-3`, ' flex items-center justify-between px-8')}>
<span>{t('datasetCreation.stepTwo.previewTitle')}</span>
<div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
<XMarkIcon className='h-4 w-4'></XMarkIcon>
{(showPreview)
? (
<div ref={previewScrollRef} className={cn(s.previewWrap, 'relativeh-full overflow-y-scroll border-l border-[#F2F4F7]')}>
<div className={cn(s.previewHeader, previewScrolled && `${s.fixed} pb-3`, ' flex items-center justify-between px-8')}>
<span>{t('datasetCreation.stepTwo.previewTitle')}</span>
<div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
<XMarkIcon className='h-4 w-4'></XMarkIcon>
</div>
</div>
<div className='my-4 px-8 space-y-4'>
{fileIndexingEstimate?.preview
? (
<>
{fileIndexingEstimate?.preview.map((item, index) => (
<PreviewItem key={item} content={item} index={index + 1} />
))}
</>
)
: <div className='flex items-center justify-center h-[200px]'><Loading type='area'></Loading></div>
}
</div>
</div>
<div className='my-4 px-8 space-y-4'>
{fileIndexingEstimate?.preview ? (
<>
{fileIndexingEstimate?.preview.map((item, index) => (
<PreviewItem key={item} content={item} index={index + 1} />
))}
</>
) : <div className='flex items-center justify-center h-[200px]'><Loading type='area'></Loading></div>
}
</div>
</div>
) :
(<div className={cn(s.sideTip)}>
)
: (<div className={cn(s.sideTip)}>
<div className={s.tipCard}>
<span className={s.icon} />
<div className={s.title}>{t('datasetCreation.stepTwo.sideTipTitle')}</div>

17
web/app/components/datasets/documents/detail/settings/index.tsx
View File

@@ -1,5 +1,5 @@
'use client'
import React, { useState, useCallback, useEffect } from 'react'
import React, { useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useBoolean } from 'ahooks'
import { useContext } from 'use-context-selector'
@@ -7,7 +7,8 @@ import { useRouter } from 'next/navigation'
import DatasetDetailContext from '@/context/dataset-detail'
import type { FullDocumentDetail } from '@/models/datasets'
import { fetchTenantInfo } from '@/service/common'
import { fetchDocumentDetail, MetadataType } from '@/service/datasets'
import type { MetadataType } from '@/service/datasets'
import { fetchDocumentDetail } from '@/service/datasets'
import Loading from '@/app/components/base/loading'
import StepTwo from '@/app/components/datasets/create/step-two'
@@ -15,8 +16,8 @@ import AccountSetting from '@/app/components/header/account-setting'
import AppUnavailable from '@/app/components/base/app-unavailable'
type DocumentSettingsProps = {
datasetId: string;
documentId: string;
datasetId: string
documentId: string
}
const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
@@ -48,18 +49,18 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
const detail = await fetchDocumentDetail({
datasetId,
documentId,
params: { metadata: 'without' as MetadataType }
params: { metadata: 'without' as MetadataType },
})
setDocumentDetail(detail)
} catch (e) {
}
catch (e) {
setHasError(true)
}
})()
}, [datasetId, documentId])
if (hasError) {
if (hasError)
return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
}
return (
<div className='flex' style={{ height: 'calc(100vh - 56px)' }}>