estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: support config chunk length by env (#12925)

Browse Source
This commit is contained in:
Joel
2025-01-22 10:43:40 +08:00
committed by GitHub
parent e23f4b0265
commit e09f6e4987
7 changed files with 17 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
web/app/components/datasets/create/step-two/index.tsx
View File

@@ -98,6 +98,7 @@ export enum IndexingType {
const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n'
const DEFAULT_MAXIMUM_CHUNK_LENGTH = 500
const DEFAULT_OVERLAP = 50
const MAXIMUM_CHUNK_TOKEN_LENGTH = parseInt(globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', 10)
type ParentChildConfig = {
chunkForContext: ParentMode
@@ -163,7 +164,7 @@ const StepTwo = ({
doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER))
}, [])
const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXIMUM_CHUNK_LENGTH) // default chunk length
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000)
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH)
const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
const [rules, setRules] = useState<PreProcessingRule[]>([])
const [defaultConfig, setDefaultConfig] = useState<Rules>()
@@ -342,8 +343,8 @@ const StepTwo = ({
}
const updatePreview = () => {
if (segmentationType === ProcessMode.general && maxChunkLength > 4000) {
Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
if (segmentationType === ProcessMode.general && maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) {
Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: MAXIMUM_CHUNK_TOKEN_LENGTH }) })
return
}
fetchEstimate()
@@ -393,7 +394,7 @@ const StepTwo = ({
score_threshold_enabled: false,
score_threshold: 0.5,
})
// eslint-disable-next-line react-hooks/exhaustive-deps
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [rerankDefaultModel, isRerankDefaultModelValid])
const getCreationParams = () => {

6
web/app/components/datasets/create/step-two/inputs.tsx
View File

@@ -39,6 +39,8 @@ export const DelimiterInput: FC<InputProps & { tooltip?: string }> = (props) =>
}
export const MaxLengthInput: FC<InputNumberProps> = (props) => {
const maxValue = parseInt(globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', 10)
const { t } = useTranslation()
return <FormField label={<div className='system-sm-semibold mb-1'>
{t('datasetCreation.stepTwo.maxLength')}
@@ -46,8 +48,8 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
<InputNumber
type="number"
className='h-9'
placeholder={'≤ 4000'}
max={4000}
placeholder={`≤ ${maxValue}`}
max={maxValue}
min={1}
{...props}
/>