estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: [frontend] support vision (#1518)

Browse Source 
Co-authored-by: Joel <iamjoel007@gmail.com>
This commit is contained in:
zxhlyh
2023-11-13 22:32:39 +08:00
committed by GitHub
parent 41d0a8b295
commit 6b15827246
74 changed files with 3159 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
web/app/components/share/chat/index.tsx
View File

@@ -3,6 +3,7 @@
import type { FC } from 'react'
import React, { useEffect, useRef, useState } from 'react'
import cn from 'classnames'
import useSWR from 'swr'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import produce from 'immer'
@@ -22,6 +23,7 @@ import {
fetchChatList,
fetchConversations,
fetchSuggestedQuestions,
generationConversationName,
pinConversation,
sendChatMessage,
stopChatMessageResponding,
@@ -39,6 +41,9 @@ import { replaceStringWithValues } from '@/app/components/app/configuration/prom
import { userInputsFormToPromptVariables } from '@/utils/model-config'
import type { InstalledApp } from '@/models/explore'
import Confirm from '@/app/components/base/confirm'
import type { VisionFile, VisionSettings } from '@/types/app'
import { Resolution, TransferMethod } from '@/types/app'
import { fetchFileUploadConfig } from '@/service/common'
export type IMainProps = {
isInstalledApp?: boolean
@@ -244,6 +249,7 @@ const Main: FC<IMainProps> = ({
id: `question-${item.id}`,
content: item.query,
isAnswer: false,
message_files: item.message_files,
})
newChatList.push({
id: item.id,
@@ -351,6 +357,8 @@ const Main: FC<IMainProps> = ({
: fetchAppInfo(), fetchAllConversations(), fetchAppParams(isInstalledApp, installedAppInfo?.id)])
}
const { data: fileUploadConfigResponse } = useSWR(isInstalledApp ? { url: '/files/upload' } : null, fetchFileUploadConfig)
// init
useEffect(() => {
(async () => {
@@ -368,7 +376,11 @@ const Main: FC<IMainProps> = ({
const isNotNewConversation = allConversations.some(item => item.id === _conversationId)
setAllConversationList(allConversations)
// fetch new conversation info
const { user_input_form, opening_statement: introduction, suggested_questions_after_answer, speech_to_text, retriever_resource, sensitive_word_avoidance }: any = appParams
const { user_input_form, opening_statement: introduction, suggested_questions_after_answer, speech_to_text, retriever_resource, file_upload, sensitive_word_avoidance }: any = appParams
setVisionConfig({
...file_upload.image,
image_file_size_limit: appParams?.system_parameters?.image_file_size_limit,
})
const prompt_variables = userInputsFormToPromptVariables(user_input_form)
if (siteInfo.default_language)
changeLanguage(siteInfo.default_language)
@@ -448,24 +460,48 @@ const Main: FC<IMainProps> = ({
const [messageTaskId, setMessageTaskId] = useState('')
const [hasStopResponded, setHasStopResponded, getHasStopResponded] = useGetState(false)
const [isResponsingConIsCurrCon, setIsResponsingConCurrCon, getIsResponsingConIsCurrCon] = useGetState(true)
const handleSend = async (message: string) => {
const [visionConfig, setVisionConfig] = useState<VisionSettings>({
enabled: false,
number_limits: 2,
detail: Resolution.low,
transfer_methods: [TransferMethod.local_file],
})
const handleSend = async (message: string, files?: VisionFile[]) => {
if (isResponsing) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForResponse') })
return
}
const data = {
if (files?.find(item => item.transfer_method === TransferMethod.local_file && !item.upload_file_id)) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForImgUpload') })
return false
}
const data: Record<string, any> = {
inputs: currInputs,
query: message,
conversation_id: isNewConversation ? null : currConversationId,
}
if (visionConfig.enabled && files && files?.length > 0) {
data.files = files.map((item) => {
if (item.transfer_method === TransferMethod.local_file) {
return {
...item,
url: '',
}
}
return item
})
}
// qustion
const questionId = `question-${Date.now()}`
const questionItem = {
id: questionId,
content: message,
isAnswer: false,
message_files: files,
}
const placeholderAnswerId = `answer-placeholder-${Date.now()}`
@@ -519,13 +555,17 @@ const Main: FC<IMainProps> = ({
setChatList(newListWithAnswer)
},
async onCompleted(hasError?: boolean) {
setResponsingFalse()
if (hasError)
return
if (getConversationIdChangeBecauseOfNew()) {
const { data: allConversations }: any = await fetchAllConversations()
setAllConversationList(allConversations)
const newItem: any = await generationConversationName(isInstalledApp, installedAppInfo?.id, allConversations[0].id)
const newAllConversations = produce(allConversations, (draft: any) => {
draft[0].name = newItem.name
})
setAllConversationList(newAllConversations as any)
noticeUpdateList()
}
setConversationIdChangeBecauseOfNew(false)
@@ -537,6 +577,7 @@ const Main: FC<IMainProps> = ({
setSuggestQuestions(data)
setIsShowSuggestion(true)
}
setResponsingFalse()
},
onMessageEnd: isInstalledApp
? (messageEnd) => {
@@ -717,6 +758,10 @@ const Main: FC<IMainProps> = ({
suggestionList={suggestQuestions}
isShowSpeechToText={speechToTextConfig?.enabled}
isShowCitation={citationConfig?.enabled && isInstalledApp}
visionConfig={{
...visionConfig,
image_file_size_limit: fileUploadConfigResponse ? fileUploadConfigResponse.image_file_size_limit : visionConfig.image_file_size_limit,
}}
/>
</div>
</div>)

48
web/app/components/share/chatbot/index.tsx
View File

@@ -14,7 +14,7 @@ import s from './style.module.css'
import { ToastContext } from '@/app/components/base/toast'
import ConfigScene from '@/app/components/share/chatbot/config-scence'
import Header from '@/app/components/share/header'
import { fetchAppInfo, fetchAppParams, fetchChatList, fetchConversations, fetchSuggestedQuestions, sendChatMessage, stopChatMessageResponding, updateFeedback } from '@/service/share'
import { fetchAppInfo, fetchAppParams, fetchChatList, fetchConversations, fetchSuggestedQuestions, generationConversationName, sendChatMessage, stopChatMessageResponding, updateFeedback } from '@/service/share'
import type { ConversationItem, SiteInfo } from '@/models/share'
import type { PromptConfig, SuggestedQuestionsAfterAnswerConfig } from '@/models/debug'
import type { Feedbacktype, IChatItem } from '@/app/components/app/chat/type'
@@ -28,6 +28,8 @@ import type { InstalledApp } from '@/models/explore'
import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
import LogoHeader from '@/app/components/base/logo/logo-embeded-chat-header'
import LogoAvatar from '@/app/components/base/logo/logo-embeded-chat-avatar'
import type { VisionFile, VisionSettings } from '@/types/app'
import { Resolution, TransferMethod } from '@/types/app'
export type IMainProps = {
isInstalledApp?: boolean
@@ -184,6 +186,7 @@ const Main: FC<IMainProps> = ({
id: `question-${item.id}`,
content: item.query,
isAnswer: false,
message_files: item.message_files,
})
newChatList.push({
id: item.id,
@@ -292,7 +295,11 @@ const Main: FC<IMainProps> = ({
const isNotNewConversation = allConversations.some(item => item.id === _conversationId)
setAllConversationList(allConversations)
// fetch new conversation info
const { user_input_form, opening_statement: introduction, suggested_questions_after_answer, speech_to_text, sensitive_word_avoidance }: any = appParams
const { user_input_form, opening_statement: introduction, suggested_questions_after_answer, speech_to_text, file_upload, sensitive_word_avoidance }: any = appParams
setVisionConfig({
...file_upload.image,
image_file_size_limit: appParams?.system_parameters?.image_file_size_limit,
})
const prompt_variables = userInputsFormToPromptVariables(user_input_form)
if (siteInfo.default_language)
changeLanguage(siteInfo.default_language)
@@ -371,24 +378,48 @@ const Main: FC<IMainProps> = ({
const [messageTaskId, setMessageTaskId] = useState('')
const [hasStopResponded, setHasStopResponded, getHasStopResponded] = useGetState(false)
const [shouldReload, setShouldReload] = useState(false)
const [visionConfig, setVisionConfig] = useState<VisionSettings>({
enabled: false,
number_limits: 2,
detail: Resolution.low,
transfer_methods: [TransferMethod.local_file],
})
const handleSend = async (message: string) => {
const handleSend = async (message: string, files?: VisionFile[]) => {
if (isResponsing) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForResponse') })
return
}
const data = {
if (files?.find(item => item.transfer_method === TransferMethod.local_file && !item.upload_file_id)) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForImgUpload') })
return false
}
const data: Record<string, any> = {
inputs: currInputs,
query: message,
conversation_id: isNewConversation ? null : currConversationId,
}
if (visionConfig.enabled && files && files?.length > 0) {
data.files = files.map((item) => {
if (item.transfer_method === TransferMethod.local_file) {
return {
...item,
url: '',
}
}
return item
})
}
// qustion
const questionId = `question-${Date.now()}`
const questionItem = {
id: questionId,
content: message,
isAnswer: false,
message_files: files,
}
const placeholderAnswerId = `answer-placeholder-${Date.now()}`
@@ -436,13 +467,16 @@ const Main: FC<IMainProps> = ({
setChatList(newListWithAnswer)
},
async onCompleted(hasError?: boolean) {
setResponsingFalse()
if (hasError)
return
if (getConversationIdChangeBecauseOfNew()) {
const { data: allConversations }: any = await fetchAllConversations()
setAllConversationList(allConversations)
const newItem: any = await generationConversationName(isInstalledApp, installedAppInfo?.id, allConversations[0].id)
const newAllConversations = produce(allConversations, (draft: any) => {
draft[0].name = newItem.name
})
setAllConversationList(newAllConversations as any)
noticeUpdateList()
}
setConversationIdChangeBecauseOfNew(false)
@@ -454,6 +488,7 @@ const Main: FC<IMainProps> = ({
setSuggestQuestions(data)
setIsShowSuggestion(true)
}
setResponsingFalse()
},
onMessageReplace: (messageReplace) => {
setChatList(produce(
@@ -581,6 +616,7 @@ const Main: FC<IMainProps> = ({
displayScene='web'
isShowSpeechToText={speechToTextConfig?.enabled}
answerIcon={<LogoAvatar className='relative shrink-0' />}
visionConfig={visionConfig}
/>
</div>
</div>)

20
web/app/components/share/text-generation/index.tsx
View File

@@ -26,6 +26,8 @@ import SavedItems from '@/app/components/app/text-generate/saved-items'
import type { InstalledApp } from '@/models/explore'
import { DEFAULT_VALUE_MAX_LEN, appDefaultIconBackground } from '@/config'
import Toast from '@/app/components/base/toast'
import type { VisionFile, VisionSettings } from '@/types/app'
import { Resolution, TransferMethod } from '@/types/app'
const GROUP_SIZE = 5 // to avoid RPM(Request per minute) limit. The group task finished then the next group.
enum TaskStatus {
@@ -92,6 +94,14 @@ const TextGeneration: FC<IMainProps> = ({
// send message task
const [controlSend, setControlSend] = useState(0)
const [controlStopResponding, setControlStopResponding] = useState(0)
const [visionConfig, setVisionConfig] = useState<VisionSettings>({
enabled: false,
number_limits: 2,
detail: Resolution.low,
transfer_methods: [TransferMethod.local_file],
})
const [completionFiles, setCompletionFiles] = useState<VisionFile[]>([])
const handleSend = () => {
setIsCallBatchAPI(false)
setControlSend(Date.now())
@@ -338,7 +348,11 @@ const TextGeneration: FC<IMainProps> = ({
setSiteInfo(siteInfo as SiteInfo)
changeLanguage(siteInfo.default_language)
const { user_input_form, more_like_this, sensitive_word_avoidance }: any = appParams
const { user_input_form, more_like_this, file_upload, sensitive_word_avoidance }: any = appParams
setVisionConfig({
...file_upload.image,
image_file_size_limit: appParams?.system_parameters?.image_file_size_limit,
})
const prompt_variables = userInputsFormToPromptVariables(user_input_form)
setPromptConfig({
prompt_template: '', // placeholder for feture
@@ -378,6 +392,8 @@ const TextGeneration: FC<IMainProps> = ({
handleSaveMessage={handleSaveMessage}
taskId={task?.id}
onCompleted={handleCompleted}
visionConfig={visionConfig}
completionFiles={completionFiles}
/>)
const renderBatchRes = () => {
@@ -512,6 +528,8 @@ const TextGeneration: FC<IMainProps> = ({
onInputsChange={setInputs}
promptConfig={promptConfig}
onSend={handleSend}
visionConfig={visionConfig}
onVisionFilesChange={setCompletionFiles}
/>
</div>
<div className={cn(isInBatchTab ? 'block' : 'hidden')}>

24
web/app/components/share/text-generation/result/index.tsx
View File

@@ -13,6 +13,7 @@ import Loading from '@/app/components/base/loading'
import type { PromptConfig } from '@/models/debug'
import type { InstalledApp } from '@/models/explore'
import type { ModerationService } from '@/models/common'
import { TransferMethod, type VisionFile, type VisionSettings } from '@/types/app'
export type IResultProps = {
isCallBatchAPI: boolean
isPC: boolean
@@ -32,6 +33,8 @@ export type IResultProps = {
onCompleted: (completionRes: string, taskId?: number, success?: boolean) => void
enableModeration?: boolean
moderationService?: (text: string) => ReturnType<ModerationService>
visionConfig: VisionSettings
completionFiles: VisionFile[]
}
const Result: FC<IResultProps> = ({
@@ -51,6 +54,8 @@ const Result: FC<IResultProps> = ({
handleSaveMessage,
taskId,
onCompleted,
visionConfig,
completionFiles,
}) => {
const [isResponsing, { setTrue: setResponsingTrue, setFalse: setResponsingFalse }] = useBoolean(false)
useEffect(() => {
@@ -108,6 +113,11 @@ const Result: FC<IResultProps> = ({
logError(t('appDebug.errorMessage.valueOfVarRequired', { key: hasEmptyInput }))
return false
}
if (completionFiles.find(item => item.transfer_method === TransferMethod.local_file && !item.upload_file_id)) {
notify({ type: 'info', message: t('appDebug.errorMessage.waitForImgUpload') })
return false
}
return !hasEmptyInput
}
@@ -120,9 +130,20 @@ const Result: FC<IResultProps> = ({
if (!checkCanSend())
return
const data = {
const data: Record<string, any> = {
inputs,
}
if (visionConfig.enabled && completionFiles && completionFiles?.length > 0) {
data.files = completionFiles.map((item) => {
if (item.transfer_method === TransferMethod.local_file) {
return {
...item,
url: '',
}
}
return item
})
}
setMessageId(null)
setFeedback({
@@ -145,7 +166,6 @@ const Result: FC<IResultProps> = ({
setResponsingFalse()
onCompleted(getCompletionRes(), taskId, false)
isTimeout = true
console.log(`[#${taskId}]: timeout`)
}
}, 1000)
sendCompletionMessage(data, {

24
web/app/components/share/text-generation/run-once/index.tsx
View File

@@ -9,6 +9,8 @@ import type { SiteInfo } from '@/models/share'
import type { PromptConfig } from '@/models/debug'
import Button from '@/app/components/base/button'
import { DEFAULT_VALUE_MAX_LEN } from '@/config'
import TextGenerationImageUploader from '@/app/components/base/image-uploader/text-generation-image-uploader'
import type { VisionFile, VisionSettings } from '@/types/app'
export type IRunOnceProps = {
siteInfo: SiteInfo
@@ -16,12 +18,16 @@ export type IRunOnceProps = {
inputs: Record<string, any>
onInputsChange: (inputs: Record<string, any>) => void
onSend: () => void
visionConfig: VisionSettings
onVisionFilesChange: (files: VisionFile[]) => void
}
const RunOnce: FC<IRunOnceProps> = ({
promptConfig,
inputs,
onInputsChange,
onSend,
visionConfig,
onVisionFilesChange,
}) => {
const { t } = useTranslation()
@@ -73,6 +79,24 @@ const RunOnce: FC<IRunOnceProps> = ({
</div>
</div>
))}
{
visionConfig?.enabled && (
<div className="w-full mt-4">
<div className="text-gray-900 text-sm font-medium">Image Upload</div>
<div className='mt-2'>
<TextGenerationImageUploader
settings={visionConfig}
onFilesChange={files => onVisionFilesChange(files.filter(file => file.progress !== -1).map(fileItem => ({
type: 'image',
transfer_method: fileItem.type,
url: fileItem.url,
upload_file_id: fileItem.fileId,
})))}
/>
</div>
</div>
)
}
{promptConfig.prompt_variables.length > 0 && (
<div className='mt-4 h-[1px] bg-gray-100'></div>
)}