feat: Integrate WaterCrawl.dev as a new knowledge base provider (#16396)
Co-authored-by: crazywoola <427733928@qq.com>
This commit is contained in:
Amir Mohsen Asaran
committed by
GitHub
GitHub
parent
0afad94378
commit
f54905e685
@@ -0,0 +1,161 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import {
|
||||
PortalToFollowElem,
|
||||
PortalToFollowElemContent,
|
||||
} from '@/app/components/base/portal-to-follow-elem'
|
||||
import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
|
||||
import Button from '@/app/components/base/button'
|
||||
import type { WatercrawlConfig } from '@/models/common'
|
||||
import Field from '@/app/components/datasets/create/website/base/field'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { createDataSourceApiKeyBinding } from '@/service/datasets'
|
||||
import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
|
||||
type Props = {
|
||||
onCancel: () => void
|
||||
onSaved: () => void
|
||||
}
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.watercrawl'
|
||||
|
||||
const DEFAULT_BASE_URL = 'https://app.watercrawl.dev'
|
||||
|
||||
const ConfigWatercrawlModal: FC<Props> = ({
|
||||
onCancel,
|
||||
onSaved,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const [isSaving, setIsSaving] = useState(false)
|
||||
const [config, setConfig] = useState<WatercrawlConfig>({
|
||||
api_key: '',
|
||||
base_url: '',
|
||||
})
|
||||
|
||||
const handleConfigChange = useCallback((key: string) => {
|
||||
return (value: string | number) => {
|
||||
setConfig(prev => ({ ...prev, [key]: value as string }))
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleSave = useCallback(async () => {
|
||||
if (isSaving)
|
||||
return
|
||||
let errorMsg = ''
|
||||
if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
|
||||
errorMsg = t('common.errorMsg.urlError')
|
||||
if (!errorMsg) {
|
||||
if (!config.api_key) {
|
||||
errorMsg = t('common.errorMsg.fieldRequired', {
|
||||
field: 'API Key',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (errorMsg) {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: errorMsg,
|
||||
})
|
||||
return
|
||||
}
|
||||
const postData = {
|
||||
category: 'website',
|
||||
provider: 'watercrawl',
|
||||
credentials: {
|
||||
auth_type: 'x-api-key',
|
||||
config: {
|
||||
api_key: config.api_key,
|
||||
base_url: config.base_url || DEFAULT_BASE_URL,
|
||||
},
|
||||
},
|
||||
}
|
||||
try {
|
||||
setIsSaving(true)
|
||||
await createDataSourceApiKeyBinding(postData)
|
||||
Toast.notify({
|
||||
type: 'success',
|
||||
message: t('common.api.success'),
|
||||
})
|
||||
}
|
||||
finally {
|
||||
setIsSaving(false)
|
||||
}
|
||||
|
||||
onSaved()
|
||||
}, [config.api_key, config.base_url, onSaved, t, isSaving])
|
||||
|
||||
return (
|
||||
<PortalToFollowElem open>
|
||||
<PortalToFollowElemContent className='w-full h-full z-[60]'>
|
||||
<div className='fixed inset-0 flex items-center justify-center bg-background-overlay'>
|
||||
<div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-components-panel-bg shadow-xl rounded-2xl overflow-y-auto'>
|
||||
<div className='px-8 pt-8'>
|
||||
<div className='flex justify-between items-center mb-4'>
|
||||
<div className='system-xl-semibold text-text-primary'>{t(`${I18N_PREFIX}.configWatercrawl`)}</div>
|
||||
</div>
|
||||
|
||||
<div className='space-y-4'>
|
||||
<Field
|
||||
label='API Key'
|
||||
labelClassName='!text-sm'
|
||||
isRequired
|
||||
value={config.api_key}
|
||||
onChange={handleConfigChange('api_key')}
|
||||
placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
|
||||
/>
|
||||
<Field
|
||||
label='Base URL'
|
||||
labelClassName='!text-sm'
|
||||
value={config.base_url}
|
||||
onChange={handleConfigChange('base_url')}
|
||||
placeholder={DEFAULT_BASE_URL}
|
||||
/>
|
||||
</div>
|
||||
<div className='my-8 flex justify-between items-center h-8'>
|
||||
<a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-text-accent' target='_blank' href='https://app.watercrawl.dev/'>
|
||||
<span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
|
||||
<LinkExternal02 className='w-3 h-3' />
|
||||
</a>
|
||||
<div className='flex'>
|
||||
<Button
|
||||
size='large'
|
||||
className='mr-2'
|
||||
onClick={onCancel}
|
||||
>
|
||||
{t('common.operation.cancel')}
|
||||
</Button>
|
||||
<Button
|
||||
variant='primary'
|
||||
size='large'
|
||||
onClick={handleSave}
|
||||
loading={isSaving}
|
||||
>
|
||||
{t('common.operation.save')}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div className='border-t-[0.5px] border-t-divider-regular'>
|
||||
<div className='flex justify-center items-center py-3 bg-background-section-burn text-xs text-text-tertiary'>
|
||||
<Lock01 className='mr-1 w-3 h-3 text-text-tertiary' />
|
||||
{t('common.modelProvider.encrypted.front')}
|
||||
<a
|
||||
className='text-text-accent mx-1'
|
||||
target='_blank' rel='noopener noreferrer'
|
||||
href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
|
||||
>
|
||||
PKCS1_OAEP
|
||||
</a>
|
||||
{t('common.modelProvider.encrypted.back')}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</PortalToFollowElemContent>
|
||||
</PortalToFollowElem>
|
||||
)
|
||||
}
|
||||
export default React.memo(ConfigWatercrawlModal)
|
||||
@@ -5,19 +5,15 @@ import { useTranslation } from 'react-i18next'
|
||||
import Panel from '../panel'
|
||||
import { DataSourceType } from '../panel/types'
|
||||
import ConfigFirecrawlModal from './config-firecrawl-modal'
|
||||
import ConfigWatercrawlModal from './config-watercrawl-modal'
|
||||
import ConfigJinaReaderModal from './config-jina-reader-modal'
|
||||
import cn from '@/utils/classnames'
|
||||
import s from '@/app/components/datasets/create/website/index.module.css'
|
||||
import { fetchDataSources, removeDataSourceApiKeyBinding } from '@/service/datasets'
|
||||
|
||||
import type {
|
||||
DataSourceItem,
|
||||
} from '@/models/common'
|
||||
import type { DataSourceItem } from '@/models/common'
|
||||
import { DataSourceProvider } from '@/models/common'
|
||||
import { useAppContext } from '@/context/app-context'
|
||||
|
||||
import {
|
||||
DataSourceProvider,
|
||||
} from '@/models/common'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
|
||||
type Props = {
|
||||
@@ -58,6 +54,16 @@ const DataSourceWebsite: FC<Props> = ({ provider }) => {
|
||||
return source?.id
|
||||
}
|
||||
|
||||
const getProviderName = (provider: DataSourceProvider): string => {
|
||||
if (provider === DataSourceProvider.fireCrawl)
|
||||
return 'Firecrawl'
|
||||
|
||||
if (provider === DataSourceProvider.waterCrawl)
|
||||
return 'WaterCrawl'
|
||||
|
||||
return 'Jina Reader'
|
||||
}
|
||||
|
||||
const handleRemove = useCallback((provider: DataSourceProvider) => {
|
||||
return async () => {
|
||||
const dataSourceId = getIdByProvider(provider)
|
||||
@@ -82,27 +88,42 @@ const DataSourceWebsite: FC<Props> = ({ provider }) => {
|
||||
readOnly={!isCurrentWorkspaceManager}
|
||||
configuredList={sources.filter(item => item.provider === provider).map(item => ({
|
||||
id: item.id,
|
||||
logo: ({ className }: { className: string }) => (
|
||||
item.provider === DataSourceProvider.fireCrawl
|
||||
? (
|
||||
<div className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>🔥</div>
|
||||
logo: ({ className }: { className: string }) => {
|
||||
if (item.provider === DataSourceProvider.fireCrawl) {
|
||||
return (
|
||||
<div
|
||||
className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>🔥</div>
|
||||
)
|
||||
: (
|
||||
<div className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>
|
||||
<span className={s.jinaLogo} />
|
||||
}
|
||||
|
||||
if (item.provider === DataSourceProvider.waterCrawl) {
|
||||
return (
|
||||
<div
|
||||
className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>
|
||||
<span className={s.watercrawlLogo}/>
|
||||
</div>
|
||||
)
|
||||
),
|
||||
name: item.provider === DataSourceProvider.fireCrawl ? 'Firecrawl' : 'Jina Reader',
|
||||
}
|
||||
return (
|
||||
<div
|
||||
className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>
|
||||
<span className={s.jinaLogo}/>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
name: getProviderName(item.provider),
|
||||
isActive: true,
|
||||
}))}
|
||||
onRemove={handleRemove(provider)}
|
||||
/>
|
||||
{configTarget === DataSourceProvider.fireCrawl && (
|
||||
<ConfigFirecrawlModal onSaved={handleAdded} onCancel={hideConfig} />
|
||||
<ConfigFirecrawlModal onSaved={handleAdded} onCancel={hideConfig}/>
|
||||
)}
|
||||
{configTarget === DataSourceProvider.waterCrawl && (
|
||||
<ConfigWatercrawlModal onSaved={handleAdded} onCancel={hideConfig}/>
|
||||
)}
|
||||
{configTarget === DataSourceProvider.jinaReader && (
|
||||
<ConfigJinaReaderModal onSaved={handleAdded} onCancel={hideConfig} />
|
||||
<ConfigJinaReaderModal onSaved={handleAdded} onCancel={hideConfig}/>
|
||||
)}
|
||||
</>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user