estel/dify
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: [backend] vision support (#1510)

Browse Source 
Co-authored-by: Garfield Dai <dai.hai@foxmail.com>
This commit is contained in:
takatost
2023-11-13 22:05:46 +08:00
committed by GitHub
parent d0e1ea8f06
commit 41d0a8b295
61 changed files with 1563 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/controllers/console/app/completion.py
View File

@@ -40,12 +40,14 @@ class CompletionMessageApi(Resource):
parser = reqparse.RequestParser()
parser.add_argument('inputs', type=dict, required=True, location='json')
parser.add_argument('query', type=str, location='json', default='')
parser.add_argument('files', type=list, required=False, location='json')
parser.add_argument('model_config', type=dict, required=True, location='json')
parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
parser.add_argument('retriever_from', type=str, required=False, default='dev', location='json')
args = parser.parse_args()
streaming = args['response_mode'] != 'blocking'
args['auto_generate_name'] = False
account = flask_login.current_user
@@ -113,6 +115,7 @@ class ChatMessageApi(Resource):
parser = reqparse.RequestParser()
parser.add_argument('inputs', type=dict, required=True, location='json')
parser.add_argument('query', type=str, required=True, location='json')
parser.add_argument('files', type=list, required=False, location='json')
parser.add_argument('model_config', type=dict, required=True, location='json')
parser.add_argument('conversation_id', type=uuid_value, location='json')
parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
@@ -120,6 +123,7 @@ class ChatMessageApi(Resource):
args = parser.parse_args()
streaming = args['response_mode'] != 'blocking'
args['auto_generate_name'] = False
account = flask_login.current_user

6
api/controllers/console/app/conversation.py
View File

@@ -108,7 +108,7 @@ class CompletionConversationDetailApi(Resource):
conversation_id = str(conversation_id)
return _get_conversation(app_id, conversation_id, 'completion')
@setup_required
@login_required
@account_initialization_required
@@ -230,7 +230,7 @@ class ChatConversationDetailApi(Resource):
conversation_id = str(conversation_id)
return _get_conversation(app_id, conversation_id, 'chat')
@setup_required
@login_required
@account_initialization_required
@@ -253,8 +253,6 @@ class ChatConversationDetailApi(Resource):
return {'result': 'success'}, 204
api.add_resource(CompletionConversationApi, '/apps/<uuid:app_id>/completion-conversations')
api.add_resource(CompletionConversationDetailApi, '/apps/<uuid:app_id>/completion-conversations/<uuid:conversation_id>')
api.add_resource(ChatConversationApi, '/apps/<uuid:app_id>/chat-conversations')

3
api/controllers/console/datasets/data_source.py
View File

@@ -1,7 +1,6 @@
import datetime
import json
from cachetools import TTLCache
from flask import request
from flask_login import current_user
from libs.login import login_required
@@ -20,8 +19,6 @@ from models.source import DataSourceBinding
from services.dataset_service import DatasetService, DocumentService
from tasks.document_indexing_sync_task import document_indexing_sync_task
cache = TTLCache(maxsize=None, ttl=30)
class DataSourceApi(Resource):

11
api/controllers/console/datasets/file.py
View File

@@ -1,5 +1,5 @@
from cachetools import TTLCache
from flask import request, current_app
from flask_login import current_user
import services
from libs.login import login_required
@@ -15,9 +15,6 @@ from fields.file_fields import upload_config_fields, file_fields
from services.file_service import FileService
cache = TTLCache(maxsize=None, ttl=30)
ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx', 'docx', 'csv']
PREVIEW_WORDS_LIMIT = 3000
@@ -30,9 +27,11 @@ class FileApi(Resource):
def get(self):
file_size_limit = current_app.config.get("UPLOAD_FILE_SIZE_LIMIT")
batch_count_limit = current_app.config.get("UPLOAD_FILE_BATCH_LIMIT")
image_file_size_limit = current_app.config.get("UPLOAD_IMAGE_FILE_SIZE_LIMIT")
return {
'file_size_limit': file_size_limit,
'batch_count_limit': batch_count_limit
'batch_count_limit': batch_count_limit,
'image_file_size_limit': image_file_size_limit
}, 200
@setup_required
@@ -51,7 +50,7 @@ class FileApi(Resource):
if len(request.files) > 1:
raise TooManyFilesError()
try:
upload_file = FileService.upload_file(file)
upload_file = FileService.upload_file(file, current_user)
except services.errors.file.FileTooLargeError as file_too_large_error:
raise FileTooLargeError(file_too_large_error.description)
except services.errors.file.UnsupportedFileTypeError:

12
api/controllers/console/explore/completion.py
View File

@@ -1,6 +1,7 @@
# -*- coding:utf-8 -*-
import json
import logging
from datetime import datetime
from typing import Generator, Union
from flask import Response, stream_with_context
@@ -17,6 +18,7 @@ from controllers.console.explore.wraps import InstalledAppResource
from core.conversation_message_task import PubHandler
from core.model_providers.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
from extensions.ext_database import db
from libs.helper import uuid_value
from services.completion_service import CompletionService
@@ -32,11 +34,16 @@ class CompletionApi(InstalledAppResource):
parser = reqparse.RequestParser()
parser.add_argument('inputs', type=dict, required=True, location='json')
parser.add_argument('query', type=str, location='json', default='')
parser.add_argument('files', type=list, required=False, location='json')
parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
parser.add_argument('retriever_from', type=str, required=False, default='explore_app', location='json')
args = parser.parse_args()
streaming = args['response_mode'] == 'streaming'
args['auto_generate_name'] = False
installed_app.last_used_at = datetime.utcnow()
db.session.commit()
try:
response = CompletionService.completion(
@@ -91,12 +98,17 @@ class ChatApi(InstalledAppResource):
parser = reqparse.RequestParser()
parser.add_argument('inputs', type=dict, required=True, location='json')
parser.add_argument('query', type=str, required=True, location='json')
parser.add_argument('files', type=list, required=False, location='json')
parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
parser.add_argument('conversation_id', type=uuid_value, location='json')
parser.add_argument('retriever_from', type=str, required=False, default='explore_app', location='json')
args = parser.parse_args()
streaming = args['response_mode'] == 'streaming'
args['auto_generate_name'] = False
installed_app.last_used_at = datetime.utcnow()
db.session.commit()
try:
response = CompletionService.completion(

14
api/controllers/console/explore/conversation.py
View File

@@ -38,7 +38,8 @@ class ConversationListApi(InstalledAppResource):
user=current_user,
last_id=args['last_id'],
limit=args['limit'],
pinned=pinned
pinned=pinned,
exclude_debug_conversation=True
)
except LastConversationNotExistsError:
raise NotFound("Last Conversation Not Exists.")
@@ -71,11 +72,18 @@ class ConversationRenameApi(InstalledAppResource):
conversation_id = str(c_id)
parser = reqparse.RequestParser()
parser.add_argument('name', type=str, required=True, location='json')
parser.add_argument('name', type=str, required=False, location='json')
parser.add_argument('auto_generate', type=bool, required=False, default='False', location='json')
args = parser.parse_args()
try:
return ConversationService.rename(app_model, conversation_id, current_user, args['name'])
return ConversationService.rename(
app_model,
conversation_id,
current_user,
args['name'],
args['auto_generate']
)
except ConversationNotExistsError:
raise NotFound("Conversation Not Exists.")

5
api/controllers/console/explore/installed_app.py
View File

@@ -39,8 +39,9 @@ class InstalledAppsListApi(Resource):
}
for installed_app in installed_apps
]
installed_apps.sort(key=lambda app: (-app['is_pinned'], app['last_used_at']
if app['last_used_at'] is not None else datetime.min))
installed_apps.sort(key=lambda app: (-app['is_pinned'],
app['last_used_at'] is None,
-app['last_used_at'].timestamp() if app['last_used_at'] is not None else 0))
return {'installed_apps': installed_apps}

15
api/controllers/console/explore/parameter.py
View File

@@ -1,5 +1,6 @@
# -*- coding:utf-8 -*-
from flask_restful import marshal_with, fields
from flask import current_app
from controllers.console import api
from controllers.console.explore.wraps import InstalledAppResource
@@ -19,6 +20,10 @@ class AppParameterApi(InstalledAppResource):
'options': fields.List(fields.String)
}
system_parameters_fields = {
'image_file_size_limit': fields.String
}
parameters_fields = {
'opening_statement': fields.String,
'suggested_questions': fields.Raw,
@@ -27,7 +32,9 @@ class AppParameterApi(InstalledAppResource):
'retriever_resource': fields.Raw,
'more_like_this': fields.Raw,
'user_input_form': fields.Raw,
'sensitive_word_avoidance': fields.Raw
'sensitive_word_avoidance': fields.Raw,
'file_upload': fields.Raw,
'system_parameters': fields.Nested(system_parameters_fields)
}
@marshal_with(parameters_fields)
@@ -44,7 +51,11 @@ class AppParameterApi(InstalledAppResource):
'retriever_resource': app_model_config.retriever_resource_dict,
'more_like_this': app_model_config.more_like_this_dict,
'user_input_form': app_model_config.user_input_form_list,
'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict
'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict,
'file_upload': app_model_config.file_upload_dict,
'system_parameters': {
'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT')
}
}

2
api/controllers/console/explore/saved_message.py
View File

@@ -9,6 +9,7 @@ from controllers.console.explore.wraps import InstalledAppResource
from libs.helper import uuid_value, TimestampField
from services.errors.message import MessageNotExistsError
from services.saved_message_service import SavedMessageService
from fields.conversation_fields import message_file_fields
feedback_fields = {
'rating': fields.String
@@ -19,6 +20,7 @@ message_fields = {
'inputs': fields.Raw,
'query': fields.String,
'answer': fields.String,
'message_files': fields.List(fields.Nested(message_file_fields), attribute='files'),
'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
'created_at': TimestampField
}

3
api/controllers/console/universal_chat/chat.py
View File

@@ -25,6 +25,7 @@ class UniversalChatApi(UniversalChatResource):
parser = reqparse.RequestParser()
parser.add_argument('query', type=str, required=True, location='json')
parser.add_argument('files', type=list, required=False, location='json')
parser.add_argument('conversation_id', type=uuid_value, location='json')
parser.add_argument('provider', type=str, required=True, location='json')
parser.add_argument('model', type=str, required=True, location='json')
@@ -60,6 +61,8 @@ class UniversalChatApi(UniversalChatResource):
del args['model']
del args['tools']
args['auto_generate_name'] = False
try:
response = CompletionService.completion(
app_model=app_model,

11
api/controllers/console/universal_chat/conversation.py
View File

@@ -65,11 +65,18 @@ class UniversalChatConversationRenameApi(UniversalChatResource):
conversation_id = str(c_id)
parser = reqparse.RequestParser()
parser.add_argument('name', type=str, required=True, location='json')
parser.add_argument('name', type=str, required=False, location='json')
parser.add_argument('auto_generate', type=bool, required=False, default='False', location='json')
args = parser.parse_args()
try:
return ConversationService.rename(app_model, conversation_id, current_user, args['name'])
return ConversationService.rename(
app_model,
conversation_id,
current_user,
args['name'],
args['auto_generate']
)
except ConversationNotExistsError:
raise NotFound("Conversation Not Exists.")