add tidb on qdrant type (#9831)

Co-authored-by: Zhaofeng Miao <522856232@qq.com>

web/app/components/datasets/create/website/jina-reader/base/checkbox-with-label.tsx

@@ -0,0 +1,40 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from '@/utils/classnames'
+import Checkbox from '@/app/components/base/checkbox'
+import Tooltip from '@/app/components/base/tooltip'
+
+type Props = {
+  className?: string
+  isChecked: boolean
+  onChange: (isChecked: boolean) => void
+  label: string
+  labelClassName?: string
+  tooltip?: string
+}
+
+const CheckboxWithLabel: FC<Props> = ({
+  className = '',
+  isChecked,
+  onChange,
+  label,
+  labelClassName,
+  tooltip,
+}) => {
+  return (
+    <label className={cn(className, 'flex items-center h-7 space-x-2')}>
+      <Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
+      <div className={cn(labelClassName, 'text-sm font-normal text-gray-800')}>{label}</div>
+      {tooltip && (
+        <Tooltip
+          popupContent={
+            <div className='w-[200px]'>{tooltip}</div>
+          }
+          triggerClassName='ml-0.5 w-4 h-4'
+        />
+      )}
+    </label>
+  )
+}
+export default React.memo(CheckboxWithLabel)

web/app/components/datasets/create/website/jina-reader/base/error-message.tsx

@@ -0,0 +1,30 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from '@/utils/classnames'
+import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+
+type Props = {
+  className?: string
+  title: string
+  errorMsg?: string
+}
+
+const ErrorMessage: FC<Props> = ({
+  className,
+  title,
+  errorMsg,
+}) => {
+  return (
+    <div className={cn(className, 'py-2 px-4 border-t border-gray-200 bg-[#FFFAEB]')}>
+      <div className='flex items-center h-5'>
+        <AlertTriangle className='mr-2 w-4 h-4 text-[#F79009]' />
+        <div className='text-sm font-medium text-[#DC6803]'>{title}</div>
+      </div>
+      {errorMsg && (
+        <div className='mt-1 pl-6 leading-[18px] text-xs font-normal text-gray-700'>{errorMsg}</div>
+      )}
+    </div>
+  )
+}
+export default React.memo(ErrorMessage)

web/app/components/datasets/create/website/jina-reader/base/field.tsx

@@ -0,0 +1,54 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import Input from './input'
+import cn from '@/utils/classnames'
+import Tooltip from '@/app/components/base/tooltip'
+
+type Props = {
+  className?: string
+  label: string
+  labelClassName?: string
+  value: string | number
+  onChange: (value: string | number) => void
+  isRequired?: boolean
+  placeholder?: string
+  isNumber?: boolean
+  tooltip?: string
+}
+
+const Field: FC<Props> = ({
+  className,
+  label,
+  labelClassName,
+  value,
+  onChange,
+  isRequired = false,
+  placeholder = '',
+  isNumber = false,
+  tooltip,
+}) => {
+  return (
+    <div className={cn(className)}>
+      <div className='flex py-[7px]'>
+        <div className={cn(labelClassName, 'flex items-center h-[18px] text-[13px] font-medium text-gray-900')}>{label} </div>
+        {isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
+        {tooltip && (
+          <Tooltip
+            popupContent={
+              <div className='w-[200px]'>{tooltip}</div>
+            }
+            triggerClassName='ml-0.5 w-4 h-4'
+          />
+        )}
+      </div>
+      <Input
+        value={value}
+        onChange={onChange}
+        placeholder={placeholder}
+        isNumber={isNumber}
+      />
+    </div>
+  )
+}
+export default React.memo(Field)

web/app/components/datasets/create/website/jina-reader/base/input.tsx

@@ -0,0 +1,58 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+
+type Props = {
+  value: string | number
+  onChange: (value: string | number) => void
+  placeholder?: string
+  isNumber?: boolean
+}
+
+const MIN_VALUE = 0
+
+const Input: FC<Props> = ({
+  value,
+  onChange,
+  placeholder = '',
+  isNumber = false,
+}) => {
+  const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
+    const value = e.target.value
+    if (isNumber) {
+      let numberValue = parseInt(value, 10) // integer only
+      if (isNaN(numberValue)) {
+        onChange('')
+        return
+      }
+      if (numberValue < MIN_VALUE)
+        numberValue = MIN_VALUE
+
+      onChange(numberValue)
+      return
+    }
+    onChange(value)
+  }, [isNumber, onChange])
+
+  const otherOption = (() => {
+    if (isNumber) {
+      return {
+        min: MIN_VALUE,
+      }
+    }
+    return {
+
+    }
+  })()
+  return (
+    <input
+      type={isNumber ? 'number' : 'text'}
+      {...otherOption}
+      value={value}
+      onChange={handleChange}
+      className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400'
+      placeholder={placeholder}
+    />
+  )
+}
+export default React.memo(Input)

web/app/components/datasets/create/website/jina-reader/base/options-wrap.tsx

@@ -0,0 +1,55 @@
+'use client'
+import { useBoolean } from 'ahooks'
+import type { FC } from 'react'
+import React, { useEffect } from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from '@/utils/classnames'
+import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
+import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  children: React.ReactNode
+  controlFoldOptions?: number
+}
+
+const OptionsWrap: FC<Props> = ({
+  className = '',
+  children,
+  controlFoldOptions,
+}) => {
+  const { t } = useTranslation()
+
+  const [fold, {
+    toggle: foldToggle,
+    setTrue: foldHide,
+  }] = useBoolean(false)
+
+  useEffect(() => {
+    if (controlFoldOptions)
+      foldHide()
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [controlFoldOptions])
+  return (
+    <div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
+      <div
+        className='flex justify-between items-center h-[26px] py-1 cursor-pointer select-none'
+        onClick={foldToggle}
+      >
+        <div className='flex items-center text-gray-700'>
+          <Settings04 className='mr-1 w-4 h-4' />
+          <div className='text-[13px] font-semibold text-gray-800 uppercase'>{t(`${I18N_PREFIX}.options`)}</div>
+        </div>
+        <ChevronRight className={cn(!fold && 'rotate-90', 'w-4 h-4 text-gray-500')} />
+      </div>
+      {!fold && (
+        <div className='mb-4'>
+          {children}
+        </div>
+      )}
+
+    </div>
+  )
+}
+export default React.memo(OptionsWrap)

web/app/components/datasets/create/website/jina-reader/base/url-input.tsx

@@ -0,0 +1,48 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import Input from './input'
+import Button from '@/app/components/base/button'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  isRunning: boolean
+  onRun: (url: string) => void
+}
+
+const UrlInput: FC<Props> = ({
+  isRunning,
+  onRun,
+}) => {
+  const { t } = useTranslation()
+  const [url, setUrl] = useState('')
+  const handleUrlChange = useCallback((url: string | number) => {
+    setUrl(url as string)
+  }, [])
+  const handleOnRun = useCallback(() => {
+    if (isRunning)
+      return
+    onRun(url)
+  }, [isRunning, onRun, url])
+
+  return (
+    <div className='flex items-center justify-between'>
+      <Input
+        value={url}
+        onChange={handleUrlChange}
+        placeholder='https://docs.dify.ai'
+      />
+      <Button
+        variant='primary'
+        onClick={handleOnRun}
+        className='ml-2'
+        loading={isRunning}
+      >
+        {!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
+      </Button>
+    </div>
+  )
+}
+export default React.memo(UrlInput)

web/app/components/datasets/create/website/jina-reader/crawled-result-item.tsx

@@ -0,0 +1,40 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from '@/utils/classnames'
+import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
+import Checkbox from '@/app/components/base/checkbox'
+
+type Props = {
+  payload: CrawlResultItemType
+  isChecked: boolean
+  isPreview: boolean
+  onCheckChange: (checked: boolean) => void
+  onPreview: () => void
+}
+
+const CrawledResultItem: FC<Props> = ({
+  isPreview,
+  payload,
+  isChecked,
+  onCheckChange,
+  onPreview,
+}) => {
+  const { t } = useTranslation()
+
+  const handleCheckChange = useCallback(() => {
+    onCheckChange(!isChecked)
+  }, [isChecked, onCheckChange])
+  return (
+    <div className={cn(isPreview ? 'border-[#D1E0FF] bg-primary-50 shadow-xs' : 'group hover:bg-gray-100', 'rounded-md px-2 py-[5px] cursor-pointer border border-transparent')}>
+      <div className='flex items-center h-5'>
+        <Checkbox className='group-hover:border-2 group-hover:border-primary-600 mr-2 shrink-0' checked={isChecked} onCheck={handleCheckChange} />
+        <div className='grow w-0 truncate text-sm font-medium text-gray-700' title={payload.title}>{payload.title}</div>
+        <div onClick={onPreview} className='hidden group-hover:flex items-center h-6 px-2 text-xs rounded-md font-medium text-gray-500 uppercase hover:bg-gray-50'>{t('datasetCreation.stepOne.website.preview')}</div>
+      </div>
+      <div className='mt-0.5 truncate pl-6 leading-[18px] text-xs font-normal text-gray-500' title={payload.source_url}>{payload.source_url}</div>
+    </div>
+  )
+}
+export default React.memo(CrawledResultItem)

web/app/components/datasets/create/website/jina-reader/crawled-result.tsx

@@ -0,0 +1,87 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import CheckboxWithLabel from './base/checkbox-with-label'
+import CrawledResultItem from './crawled-result-item'
+import cn from '@/utils/classnames'
+import type { CrawlResultItem } from '@/models/datasets'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  list: CrawlResultItem[]
+  checkedList: CrawlResultItem[]
+  onSelectedChange: (selected: CrawlResultItem[]) => void
+  onPreview: (payload: CrawlResultItem) => void
+  usedTime: number
+}
+
+const CrawledResult: FC<Props> = ({
+  className = '',
+  list,
+  checkedList,
+  onSelectedChange,
+  onPreview,
+  usedTime,
+}) => {
+  const { t } = useTranslation()
+
+  const isCheckAll = checkedList.length === list.length
+
+  const handleCheckedAll = useCallback(() => {
+    if (!isCheckAll)
+      onSelectedChange(list)
+
+    else
+      onSelectedChange([])
+  }, [isCheckAll, list, onSelectedChange])
+
+  const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
+    return (checked: boolean) => {
+      if (checked)
+        onSelectedChange([...checkedList, item])
+
+      else
+        onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
+    }
+  }, [checkedList, onSelectedChange])
+
+  const [previewIndex, setPreviewIndex] = React.useState<number>(-1)
+  const handlePreview = useCallback((index: number) => {
+    return () => {
+      setPreviewIndex(index)
+      onPreview(list[index])
+    }
+  }, [list, onPreview])
+
+  return (
+    <div className={cn(className, 'border-t border-gray-200')}>
+      <div className='flex items-center justify-between h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
+        <CheckboxWithLabel
+          isChecked={isCheckAll}
+          onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
+          labelClassName='!font-medium'
+        />
+        <div>{t(`${I18N_PREFIX}.scrapTimeInfo`, {
+          total: list.length,
+          time: usedTime.toFixed(1),
+        })}</div>
+      </div>
+      <div className='p-2'>
+        {list.map((item, index) => (
+          <CrawledResultItem
+            key={item.source_url}
+            isPreview={index === previewIndex}
+            onPreview={handlePreview(index)}
+            payload={item}
+            isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
+            onCheckChange={handleItemCheckChange(item)}
+          />
+        ))}
+      </div>
+    </div>
+  )
+}
+export default React.memo(CrawledResult)

web/app/components/datasets/create/website/jina-reader/crawling.tsx

@@ -0,0 +1,37 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from '@/utils/classnames'
+import { RowStruct } from '@/app/components/base/icons/src/public/other'
+
+type Props = {
+  className?: string
+  crawledNum: number
+  totalNum: number
+}
+
+const Crawling: FC<Props> = ({
+  className = '',
+  crawledNum,
+  totalNum,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className={cn(className, 'border-t border-gray-200')}>
+      <div className='flex items-center h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
+        {t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum}
+      </div>
+
+      <div className='p-2'>
+        {['', '', '', ''].map((item, index) => (
+          <div className='py-[5px]' key={index}>
+            <RowStruct />
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+export default React.memo(Crawling)

web/app/components/datasets/create/website/jina-reader/mock-crawl-result.ts

@@ -0,0 +1,24 @@
+import type { CrawlResultItem } from '@/models/datasets'
+
+const result: CrawlResultItem[] = [
+  {
+    title: 'Start the frontend Docker container separately',
+    markdown: 'Markdown 1',
+    description: 'Description 1',
+    source_url: 'https://example.com/1',
+  },
+  {
+    title: 'Advanced Tool Integration',
+    markdown: 'Markdown 2',
+    description: 'Description 2',
+    source_url: 'https://example.com/2',
+  },
+  {
+    title: 'Local Source Code Start | English | Dify',
+    markdown: 'Markdown 3',
+    description: 'Description 3',
+    source_url: 'https://example.com/3',
+  },
+]
+
+export default result