diff --git a/CONTRIBUTING-ZH.md b/CONTRIBUTING-ZH.md index 25f48387..1211391d 100644 --- a/CONTRIBUTING-ZH.md +++ b/CONTRIBUTING-ZH.md @@ -125,6 +125,7 @@ Flowise 支持不同的环境变量来配置您的实例。您可以在 `package | FLOWISE_PASSWORD | 登录密码 | 字符串 | | | FLOWISE_FILE_SIZE_LIMIT | 上传文件大小限制 | 字符串 | 50mb | | DEBUG | 打印组件的日志 | 布尔值 | | +| BLOB_STORAGE_PATH | 存储位置 | 字符串 | `your-home-dir/.flowise/storage` | | LOG_PATH | 存储日志文件的位置 | 字符串 | `your-path/Flowise/logs` | | LOG_LEVEL | 日志的不同级别 | 枚举字符串: `error`, `info`, `verbose`, `debug` | `info` | | APIKEY_PATH | 存储 API 密钥的位置 | 字符串 | `your-path/Flowise/packages/server` | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57fee8fc..0c09206e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,6 +129,7 @@ Flowise support different environment variables to configure your instance. You | FLOWISE_PASSWORD | Password to login | String | | | FLOWISE_FILE_SIZE_LIMIT | Upload File Size Limit | String | 50mb | | DEBUG | Print logs from components | Boolean | | +| BLOB_STORAGE_PATH | Location where uploaded files are stored | String | `your-home-dir/.flowise/storage` | | LOG_PATH | Location where log files are stored | String | `your-path/Flowise/logs` | | LOG_LEVEL | Different levels of logs | Enum String: `error`, `info`, `verbose`, `debug` | `info` | | APIKEY_PATH | Location where api keys are saved | String | `your-path/Flowise/packages/server` | diff --git a/docker/.env.example b/docker/.env.example index 84019299..ee972c9a 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -3,6 +3,7 @@ DATABASE_PATH=/root/.flowise APIKEY_PATH=/root/.flowise SECRETKEY_PATH=/root/.flowise LOG_PATH=/root/.flowise/logs +BLOB_STORAGE_PATH=/root/.flowise/storage # CORS_ORIGINS="*" # IFRAME_ORIGINS="*" diff --git a/docker/README.md b/docker/README.md index 11b29cf3..49ce57c0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -31,5 +31,6 @@ If you like to persist your data (flows, logs, apikeys, credentials), set these - APIKEY_PATH=/root/.flowise - LOG_PATH=/root/.flowise/logs - SECRETKEY_PATH=/root/.flowise +- BLOB_STORAGE_PATH=/root/.flowise/storage Flowise also support different environment variables to configure your instance. Read [more](https://docs.flowiseai.com/environment-variables) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4da945fe..4bee2e39 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,6 +26,7 @@ services: - FLOWISE_SECRETKEY_OVERWRITE=${FLOWISE_SECRETKEY_OVERWRITE} - LOG_LEVEL=${LOG_LEVEL} - LOG_PATH=${LOG_PATH} + - BLOB_STORAGE_PATH=${BLOB_STORAGE_PATH} - DISABLE_FLOWISE_TELEMETRY=${DISABLE_FLOWISE_TELEMETRY} ports: - '${PORT}:${PORT}' diff --git a/packages/components/credentials/AssemblyAI.credential.ts b/packages/components/credentials/AssemblyAI.credential.ts new file mode 100644 index 00000000..019cd7aa --- /dev/null +++ b/packages/components/credentials/AssemblyAI.credential.ts @@ -0,0 +1,23 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class AssemblyAIApi implements INodeCredential { + label: string + name: string + version: number + inputs: INodeParams[] + + constructor() { + this.label = 'AssemblyAI API' + this.name = 'assemblyAIApi' + this.version = 1.0 + this.inputs = [ + { + label: 'AssemblyAI Api Key', + name: 'assemblyAIApiKey', + type: 'password' + } + ] + } +} + +module.exports = { credClass: AssemblyAIApi } diff --git a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts index e28a4a3f..db6b37c6 100644 --- a/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts +++ b/packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts @@ -4,13 +4,15 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages' import { ChainValues } from '@langchain/core/utils/types' import { AgentStep } from '@langchain/core/agents' -import { renderTemplate } from '@langchain/core/prompts' +import { renderTemplate, MessagesPlaceholder } from '@langchain/core/prompts' import { RunnableSequence } from '@langchain/core/runnables' import { ChatConversationalAgent } from 'langchain/agents' import { getBaseClasses } from '../../../src/utils' import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { AgentExecutor } from '../../../src/agents' +import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' +import { addImagesToMessages } from '../../../src/multiModalUtils' const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI. @@ -81,12 +83,18 @@ class ConversationalAgent_Agents implements INode { } async init(nodeData: INodeData, input: string, options: ICommonObject): Promise { - return prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory) + return prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory) } async run(nodeData: INodeData, input: string, options: ICommonObject): Promise { const memory = nodeData.inputs?.memory as FlowiseMemory - const executor = await prepareAgent(nodeData, { sessionId: this.sessionId, chatId: options.chatId, input }, options.chatHistory) + + const executor = await prepareAgent( + nodeData, + options, + { sessionId: this.sessionId, chatId: options.chatId, input }, + options.chatHistory + ) const loggerHandler = new ConsoleCallbackHandler(options.logger) const callbacks = await additionalCallbacks(nodeData, options) @@ -120,6 +128,7 @@ class ConversationalAgent_Agents implements INode { const prepareAgent = async ( nodeData: INodeData, + options: ICommonObject, flowObj: { sessionId?: string; chatId?: string; input?: string }, chatHistory: IMessage[] = [] ) => { @@ -131,11 +140,6 @@ const prepareAgent = async ( const memoryKey = memory.memoryKey ? memory.memoryKey : 'chat_history' const inputKey = memory.inputKey ? memory.inputKey : 'input' - /** Bind a stop token to the model */ - const modelWithStop = model.bind({ - stop: ['\nObservation'] - }) - const outputParser = ChatConversationalAgent.getDefaultOutputParser({ llm: model, toolNames: tools.map((tool) => tool.name) @@ -146,6 +150,41 @@ const prepareAgent = async ( outputParser }) + if (model instanceof ChatOpenAI) { + let humanImageMessages: HumanMessage[] = [] + const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + + if (messageContent?.length) { + // Change model to gpt-4-vision + model.modelName = 'gpt-4-vision-preview' + + // Change default max token to higher when using gpt-4-vision + model.maxTokens = 1024 + + for (const msg of messageContent) { + humanImageMessages.push(new HumanMessage({ content: [msg] })) + } + + // Pop the `agent_scratchpad` MessagePlaceHolder + let messagePlaceholder = prompt.promptMessages.pop() as MessagesPlaceholder + + // Add the HumanMessage for images + prompt.promptMessages.push(...humanImageMessages) + + // Add the `agent_scratchpad` MessagePlaceHolder back + prompt.promptMessages.push(messagePlaceholder) + } else { + // revert to previous values if image upload is empty + model.modelName = model.configuredModel + model.maxTokens = model.configuredMaxToken + } + } + + /** Bind a stop token to the model */ + const modelWithStop = model.bind({ + stop: ['\nObservation'] + }) + const runnableAgent = RunnableSequence.from([ { [inputKey]: (i: { input: string; steps: AgentStep[] }) => i.input, @@ -166,7 +205,7 @@ const prepareAgent = async ( sessionId: flowObj?.sessionId, chatId: flowObj?.chatId, input: flowObj?.input, - verbose: process.env.DEBUG === 'true' ? true : false + verbose: process.env.DEBUG === 'true' }) return executor diff --git a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts index 7328b986..d59de540 100644 --- a/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts +++ b/packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts @@ -8,6 +8,10 @@ import { additionalCallbacks } from '../../../src/handler' import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { createReactAgent } from '../../../src/agents' +import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' +import { HumanMessage } from '@langchain/core/messages' +import { addImagesToMessages } from '../../../src/multiModalUtils' +import { ChatPromptTemplate, HumanMessagePromptTemplate } from 'langchain/prompts' class MRKLAgentChat_Agents implements INode { label: string @@ -61,18 +65,39 @@ class MRKLAgentChat_Agents implements INode { let tools = nodeData.inputs?.tools as Tool[] tools = flatten(tools) - const promptWithChat = await pull('hwchase17/react-chat') + const prompt = await pull('hwchase17/react-chat') + let chatPromptTemplate = undefined + + if (model instanceof ChatOpenAI) { + const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + + if (messageContent?.length) { + // Change model to gpt-4-vision + model.modelName = 'gpt-4-vision-preview' + + // Change default max token to higher when using gpt-4-vision + model.maxTokens = 1024 + + const oldTemplate = prompt.template as string + chatPromptTemplate = ChatPromptTemplate.fromMessages([HumanMessagePromptTemplate.fromTemplate(oldTemplate)]) + chatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent })) + } else { + // revert to previous values if image upload is empty + model.modelName = model.configuredModel + model.maxTokens = model.configuredMaxToken + } + } const agent = await createReactAgent({ llm: model, tools, - prompt: promptWithChat + prompt: chatPromptTemplate ?? prompt }) const executor = new AgentExecutor({ agent, tools, - verbose: process.env.DEBUG === 'true' ? true : false + verbose: process.env.DEBUG === 'true' }) const callbacks = await additionalCallbacks(nodeData, options) diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index e82d6804..25d80bee 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -1,14 +1,16 @@ +import { ConversationChain } from 'langchain/chains' import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate } from '@langchain/core/prompts' -import { BaseChatModel } from '@langchain/core/language_models/chat_models' import { RunnableSequence } from '@langchain/core/runnables' import { StringOutputParser } from '@langchain/core/output_parsers' +import { HumanMessage } from '@langchain/core/messages' import { ConsoleCallbackHandler as LCConsoleCallbackHandler } from '@langchain/core/tracers/console' -import { ConversationChain } from 'langchain/chains' -import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface' -import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' -import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse } from '../../outputparsers/OutputParserHelpers' +import { addImagesToMessages } from '../../../src/multiModalUtils' +import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' +import { FlowiseMemory, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler' +import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' let systemMessage = `The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.` const inputKey = 'input' @@ -86,12 +88,14 @@ class ConversationChain_Chains implements INode { } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const chain = prepareChain(nodeData, this.sessionId, options.chatHistory) + const chain = prepareChain(nodeData, options, this.sessionId) return chain } async run(nodeData: INodeData, input: string, options: ICommonObject): Promise { const memory = nodeData.inputs?.memory + + const chain = prepareChain(nodeData, options, this.sessionId) const moderations = nodeData.inputs?.inputModeration as Moderation[] if (moderations && moderations.length > 0) { @@ -105,8 +109,6 @@ class ConversationChain_Chains implements INode { } } - const chain = prepareChain(nodeData, this.sessionId, options.chatHistory) - const loggerHandler = new ConsoleCallbackHandler(options.logger) const additionalCallback = await additionalCallbacks(nodeData, options) @@ -143,7 +145,7 @@ class ConversationChain_Chains implements INode { } } -const prepareChatPrompt = (nodeData: INodeData) => { +const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: HumanMessage[]) => { const memory = nodeData.inputs?.memory as FlowiseMemory const prompt = nodeData.inputs?.systemMessagePrompt as string const chatPromptTemplate = nodeData.inputs?.chatPromptTemplate as ChatPromptTemplate @@ -151,12 +153,10 @@ const prepareChatPrompt = (nodeData: INodeData) => { if (chatPromptTemplate && chatPromptTemplate.promptMessages.length) { const sysPrompt = chatPromptTemplate.promptMessages[0] const humanPrompt = chatPromptTemplate.promptMessages[chatPromptTemplate.promptMessages.length - 1] - const chatPrompt = ChatPromptTemplate.fromMessages([ - sysPrompt, - new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), - humanPrompt - ]) + const messages = [sysPrompt, new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), humanPrompt] + if (humanImageMessages.length) messages.push(...humanImageMessages) + const chatPrompt = ChatPromptTemplate.fromMessages(messages) if ((chatPromptTemplate as any).promptValues) { // @ts-ignore chatPrompt.promptValues = (chatPromptTemplate as any).promptValues @@ -165,21 +165,46 @@ const prepareChatPrompt = (nodeData: INodeData) => { return chatPrompt } - const chatPrompt = ChatPromptTemplate.fromMessages([ + const messages = [ SystemMessagePromptTemplate.fromTemplate(prompt ? prompt : systemMessage), new MessagesPlaceholder(memory.memoryKey ?? 'chat_history'), HumanMessagePromptTemplate.fromTemplate(`{${inputKey}}`) - ]) + ] + if (humanImageMessages.length) messages.push(...(humanImageMessages as any[])) + + const chatPrompt = ChatPromptTemplate.fromMessages(messages) return chatPrompt } -const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMessage[] = []) => { - const model = nodeData.inputs?.model as BaseChatModel +const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => { + const chatHistory = options.chatHistory + let model = nodeData.inputs?.model as ChatOpenAI const memory = nodeData.inputs?.memory as FlowiseMemory const memoryKey = memory.memoryKey ?? 'chat_history' - const chatPrompt = prepareChatPrompt(nodeData) + let humanImageMessages: HumanMessage[] = [] + if (model instanceof ChatOpenAI) { + const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + + if (messageContent?.length) { + // Change model to gpt-4-vision + model.modelName = 'gpt-4-vision-preview' + + // Change default max token to higher when using gpt-4-vision + model.maxTokens = 1024 + + for (const msg of messageContent) { + humanImageMessages.push(new HumanMessage({ content: [msg] })) + } + } else { + // revert to previous values if image upload is empty + model.modelName = model.configuredModel + model.maxTokens = model.configuredMaxToken + } + } + + const chatPrompt = prepareChatPrompt(nodeData, humanImageMessages) let promptVariables = {} const promptValuesRaw = (chatPrompt as any).promptValues if (promptValuesRaw) { @@ -203,7 +228,7 @@ const prepareChain = (nodeData: INodeData, sessionId?: string, chatHistory: IMes }, ...promptVariables }, - chatPrompt, + prepareChatPrompt(nodeData, humanImageMessages), model, new StringOutputParser() ]) diff --git a/packages/components/nodes/chains/LLMChain/LLMChain.ts b/packages/components/nodes/chains/LLMChain/LLMChain.ts index 6ad8d526..c60b4b29 100644 --- a/packages/components/nodes/chains/LLMChain/LLMChain.ts +++ b/packages/components/nodes/chains/LLMChain/LLMChain.ts @@ -1,5 +1,6 @@ import { BaseLanguageModel, BaseLanguageModelCallOptions } from '@langchain/core/language_models/base' import { BaseLLMOutputParser, BaseOutputParser } from '@langchain/core/output_parsers' +import { ChatPromptTemplate, FewShotPromptTemplate, PromptTemplate, HumanMessagePromptTemplate } from '@langchain/core/prompts' import { OutputFixingParser } from 'langchain/output_parsers' import { LLMChain } from 'langchain/chains' import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' @@ -7,6 +8,9 @@ import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils' import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation' import { formatResponse, injectOutputParser } from '../../outputparsers/OutputParserHelpers' +import { ChatOpenAI } from '../../chatmodels/ChatOpenAI/FlowiseChatOpenAI' +import { addImagesToMessages } from '../../../src/multiModalUtils' +import { HumanMessage } from 'langchain/schema' class LLMChain_Chains implements INode { label: string @@ -160,12 +164,7 @@ const runPrediction = async ( const socketIO = isStreaming ? options.socketIO : undefined const socketIOClientId = isStreaming ? options.socketIOClientId : '' const moderations = nodeData.inputs?.inputModeration as Moderation[] - /** - * Apply string transformation to reverse converted special chars: - * FROM: { "value": "hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?" } - * TO: { "value": "hello i am ben\n\n\thow are you?" } - */ - const promptValues = handleEscapeCharacters(promptValuesRaw, true) + let model = nodeData.inputs?.model as ChatOpenAI if (moderations && moderations.length > 0) { try { @@ -178,6 +177,46 @@ const runPrediction = async ( } } + /** + * Apply string transformation to reverse converted special chars: + * FROM: { "value": "hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?" } + * TO: { "value": "hello i am ben\n\n\thow are you?" } + */ + const promptValues = handleEscapeCharacters(promptValuesRaw, true) + const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption) + + if (chain.llm instanceof ChatOpenAI) { + const chatOpenAI = chain.llm as ChatOpenAI + if (messageContent?.length) { + // Change model to gpt-4-vision && max token to higher when using gpt-4-vision + chatOpenAI.modelName = 'gpt-4-vision-preview' + chatOpenAI.maxTokens = 1024 + // Add image to the message + if (chain.prompt instanceof PromptTemplate) { + const existingPromptTemplate = chain.prompt.template as string + let newChatPromptTemplate = ChatPromptTemplate.fromMessages([ + HumanMessagePromptTemplate.fromTemplate(existingPromptTemplate) + ]) + newChatPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent })) + chain.prompt = newChatPromptTemplate + } else if (chain.prompt instanceof ChatPromptTemplate) { + chain.prompt.promptMessages.push(new HumanMessage({ content: messageContent })) + } else if (chain.prompt instanceof FewShotPromptTemplate) { + let existingFewShotPromptTemplate = chain.prompt.examplePrompt.template as string + let newFewShotPromptTemplate = ChatPromptTemplate.fromMessages([ + HumanMessagePromptTemplate.fromTemplate(existingFewShotPromptTemplate) + ]) + newFewShotPromptTemplate.promptMessages.push(new HumanMessage({ content: messageContent })) + // @ts-ignore + chain.prompt.examplePrompt = newFewShotPromptTemplate + } + } else { + // revert to previous values if image upload is empty + chatOpenAI.modelName = model.configuredModel + chatOpenAI.maxTokens = model.configuredMaxToken + } + } + if (promptValues && inputVariables.length > 0) { let seen: string[] = [] diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index 8de3e65c..cc0b0efa 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -1,8 +1,10 @@ -import { ChatOpenAI, OpenAIChatInput } from '@langchain/openai' +import type { ClientOptions } from 'openai' +import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, AzureOpenAIInput, LegacyOpenAIInput } from '@langchain/openai' import { BaseCache } from '@langchain/core/caches' -import { BaseLLMParams } from '@langchain/core/language_models/llms' -import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { BaseChatModelParams } from '@langchain/core/language_models/chat_models' +import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { ChatOpenAI } from './FlowiseChatOpenAI' class ChatOpenAI_ChatModels implements INode { label: string @@ -19,12 +21,12 @@ class ChatOpenAI_ChatModels implements INode { constructor() { this.label = 'ChatOpenAI' this.name = 'chatOpenAI' - this.version = 4.0 + this.version = 5.0 this.type = 'ChatOpenAI' this.icon = 'openai.svg' this.category = 'Chat Models' this.description = 'Wrapper around OpenAI large language models that use the Chat endpoint' - this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -168,6 +170,38 @@ class ChatOpenAI_ChatModels implements INode { type: 'json', optional: true, additionalParams: true + }, + { + label: 'Allow Image Uploads', + name: 'allowImageUploads', + type: 'boolean', + description: + 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + default: false, + optional: true + }, + { + label: 'Image Resolution', + description: 'This parameter controls the resolution in which the model views the image.', + name: 'imageResolution', + type: 'options', + options: [ + { + label: 'Low', + name: 'low' + }, + { + label: 'High', + name: 'high' + }, + { + label: 'Auto', + name: 'auto' + } + ], + default: 'low', + optional: false, + additionalParams: true } ] } @@ -184,12 +218,17 @@ class ChatOpenAI_ChatModels implements INode { const basePath = nodeData.inputs?.basepath as string const baseOptions = nodeData.inputs?.baseOptions + const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean + const imageResolution = nodeData.inputs?.imageResolution as string + const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) const cache = nodeData.inputs?.cache as BaseCache - const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { + const obj: Partial & + Partial & + BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -212,10 +251,24 @@ class ChatOpenAI_ChatModels implements INode { throw new Error("Invalid JSON in the ChatOpenAI's BaseOptions: " + exception) } } - const model = new ChatOpenAI(obj, { - basePath, - baseOptions: parsedBaseOptions - }) + + if (basePath || parsedBaseOptions) { + obj.configuration = { + baseURL: basePath, + baseOptions: parsedBaseOptions + } + } + + const multiModalOption: IMultiModalOption = { + image: { + allowImageUploads: allowImageUploads ?? false, + imageResolution + } + } + obj.multiModalOption = multiModalOption + + const model = new ChatOpenAI(nodeData.id, obj) + return model } } diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts new file mode 100644 index 00000000..2fa65375 --- /dev/null +++ b/packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts @@ -0,0 +1,38 @@ +import type { ClientOptions } from 'openai' +import { + ChatOpenAI as LangchainChatOpenAI, + OpenAIChatInput, + LegacyOpenAIInput, + AzureOpenAIInput, + ChatOpenAICallOptions +} from '@langchain/openai' +import { BaseChatModelParams } from '@langchain/core/language_models/chat_models' +import { IMultiModalOption } from '../../../src' +import { BaseMessageLike, LLMResult } from 'langchain/schema' +import { Callbacks } from '@langchain/core/callbacks/manager' + +export class ChatOpenAI extends LangchainChatOpenAI { + configuredModel: string + configuredMaxToken?: number + multiModalOption?: IMultiModalOption + id: string + + constructor( + id: string, + fields?: Partial & + Partial & + BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption }, + /** @deprecated */ + configuration?: ClientOptions & LegacyOpenAIInput + ) { + super(fields, configuration) + this.id = id + this.multiModalOption = fields?.multiModalOption + this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo' + this.configuredMaxToken = fields?.maxTokens + } + + async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise { + return super.generate(messages, options, callbacks) + } +} diff --git a/packages/components/nodes/speechtotext/assemblyai/AssemblyAI.ts b/packages/components/nodes/speechtotext/assemblyai/AssemblyAI.ts new file mode 100644 index 00000000..c5db6619 --- /dev/null +++ b/packages/components/nodes/speechtotext/assemblyai/AssemblyAI.ts @@ -0,0 +1,33 @@ +import { INode, INodeParams } from '../../../src/Interface' + +class AssemblyAI_SpeechToText implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs?: INodeParams[] + credential: INodeParams + + constructor() { + this.label = 'AssemblyAI' + this.name = 'assemblyAI' + this.version = 1.0 + this.type = 'AssemblyAI' + this.icon = 'assemblyai.png' + this.category = 'SpeechToText' + this.baseClasses = [this.type] + this.inputs = [] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['assemblyAIApi'] + } + } +} + +module.exports = { nodeClass: AssemblyAI_SpeechToText } diff --git a/packages/components/nodes/speechtotext/assemblyai/assemblyai.png b/packages/components/nodes/speechtotext/assemblyai/assemblyai.png new file mode 100644 index 00000000..8919cb18 Binary files /dev/null and b/packages/components/nodes/speechtotext/assemblyai/assemblyai.png differ diff --git a/packages/components/package.json b/packages/components/package.json index 59f42fd1..171238bf 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -46,6 +46,7 @@ "@upstash/redis": "1.22.1", "@zilliz/milvus2-sdk-node": "^2.2.24", "apify-client": "^2.7.1", + "assemblyai": "^4.2.2", "axios": "1.6.2", "cheerio": "^1.0.0-rc.12", "chromadb": "^1.5.11", diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 0d85e1e6..0e280dea 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -21,6 +21,8 @@ export type CommonType = string | number | boolean | undefined | null export type MessageType = 'apiMessage' | 'userMessage' +export type ImageDetail = 'auto' | 'low' | 'high' + /** * Others */ @@ -146,6 +148,33 @@ export interface IUsedTool { toolOutput: string | object } +export interface IFileUpload { + data?: string + type: string + name: string + mime: string +} + +export interface IMultiModalOption { + image?: Record + audio?: Record +} + +export type MessageContentText = { + type: 'text' + text: string +} + +export type MessageContentImageUrl = { + type: 'image_url' + image_url: + | string + | { + url: string + detail?: ImageDetail + } +} + /** * Classes */ diff --git a/packages/components/src/index.ts b/packages/components/src/index.ts index ae2e380e..10cd1036 100644 --- a/packages/components/src/index.ts +++ b/packages/components/src/index.ts @@ -6,3 +6,4 @@ dotenv.config({ path: envPath, override: true }) export * from './Interface' export * from './utils' +export * from './speechToText' diff --git a/packages/components/src/multiModalUtils.ts b/packages/components/src/multiModalUtils.ts new file mode 100644 index 00000000..94414e58 --- /dev/null +++ b/packages/components/src/multiModalUtils.ts @@ -0,0 +1,48 @@ +import { ICommonObject, IFileUpload, IMultiModalOption, INodeData, MessageContentImageUrl } from './Interface' +import { ChatOpenAI as LangchainChatOpenAI } from 'langchain/chat_models/openai' +import path from 'path' +import { getStoragePath } from './utils' +import fs from 'fs' + +export const addImagesToMessages = ( + nodeData: INodeData, + options: ICommonObject, + multiModalOption?: IMultiModalOption +): MessageContentImageUrl[] => { + const imageContent: MessageContentImageUrl[] = [] + let model = nodeData.inputs?.model + + if (model instanceof LangchainChatOpenAI && multiModalOption) { + // Image Uploaded + if (multiModalOption.image && multiModalOption.image.allowImageUploads && options?.uploads && options?.uploads.length > 0) { + const imageUploads = getImageUploads(options.uploads) + for (const upload of imageUploads) { + let bf = upload.data + if (upload.type == 'stored-file') { + const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name) + + // as the image is stored in the server, read the file and convert it to base64 + const contents = fs.readFileSync(filePath) + bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64') + + imageContent.push({ + type: 'image_url', + image_url: { + url: bf, + detail: multiModalOption.image.imageResolution ?? 'low' + } + }) + } + } + } + } + return imageContent +} + +export const getAudioUploads = (uploads: IFileUpload[]) => { + return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('audio/')) +} + +export const getImageUploads = (uploads: IFileUpload[]) => { + return uploads.filter((upload: IFileUpload) => upload.mime.startsWith('image/')) +} diff --git a/packages/components/src/speechToText.ts b/packages/components/src/speechToText.ts new file mode 100644 index 00000000..8524b525 --- /dev/null +++ b/packages/components/src/speechToText.ts @@ -0,0 +1,51 @@ +import { ICommonObject, IFileUpload } from './Interface' +import { getCredentialData, getStoragePath } from './utils' +import { type ClientOptions, OpenAIClient } from '@langchain/openai' +import fs from 'fs' +import path from 'path' +import { AssemblyAI } from 'assemblyai' + +export const convertSpeechToText = async (upload: IFileUpload, speechToTextConfig: ICommonObject, options: ICommonObject) => { + if (speechToTextConfig) { + const credentialId = speechToTextConfig.credentialId as string + const credentialData = await getCredentialData(credentialId ?? '', options) + const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name) + + const audio_file = fs.createReadStream(filePath) + + if (speechToTextConfig.name === 'openAIWhisper') { + const openAIClientOptions: ClientOptions = { + apiKey: credentialData.openAIApiKey + } + const openAIClient = new OpenAIClient(openAIClientOptions) + + const transcription = await openAIClient.audio.transcriptions.create({ + file: audio_file, + model: 'whisper-1', + language: speechToTextConfig?.language, + temperature: speechToTextConfig?.temperature ? parseFloat(speechToTextConfig.temperature) : undefined, + prompt: speechToTextConfig?.prompt + }) + if (transcription?.text) { + return transcription.text + } + } else if (speechToTextConfig.name === 'assemblyAiTranscribe') { + const client = new AssemblyAI({ + apiKey: credentialData.assemblyAIApiKey + }) + + const params = { + audio: audio_file, + speaker_labels: false + } + + const transcription = await client.transcripts.transcribe(params) + if (transcription?.text) { + return transcription.text + } + } + } else { + throw new Error('Speech to text is not selected, but found a recorded audio file. Please fix the chain.') + } + return undefined +} diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index feefa6dc..cad67268 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -770,3 +770,10 @@ export const prepareSandboxVars = (variables: IVariable[]) => { } return vars } + +/** + * Prepare storage path + */ +export const getStoragePath = (): string => { + return process.env.BLOB_STORAGE_PATH ? path.join(process.env.BLOB_STORAGE_PATH) : path.join(getUserHome(), '.flowise', 'storage') +} diff --git a/packages/server/.env.example b/packages/server/.env.example index a7a93345..5f22cafd 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -5,6 +5,7 @@ PORT=3000 # APIKEY_PATH=/your_api_key_path/.flowise # SECRETKEY_PATH=/your_api_key_path/.flowise # LOG_PATH=/your_log_path/.flowise/logs +# BLOB_STORAGE_PATH=/your_database_path/.flowise/storage # NUMBER_OF_PROXIES= 1 diff --git a/packages/server/marketplaces/chatflows/API Agent OpenAI.json b/packages/server/marketplaces/chatflows/API Agent OpenAI.json index 4e8727da..691852d6 100644 --- a/packages/server/marketplaces/chatflows/API Agent OpenAI.json +++ b/packages/server/marketplaces/chatflows/API Agent OpenAI.json @@ -90,7 +90,7 @@ "data": { "id": "chatOpenAI_1", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -237,6 +237,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_1-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_1-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_1-input-imageResolution-options" } ], "inputAnchors": [ @@ -257,7 +290,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -437,7 +472,7 @@ "data": { "id": "chatOpenAI_2", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -584,6 +619,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_2-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_2-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_2-input-imageResolution-options" } ], "inputAnchors": [ @@ -604,7 +672,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/API Agent.json b/packages/server/marketplaces/chatflows/API Agent.json index a0fed334..facdcb6b 100644 --- a/packages/server/marketplaces/chatflows/API Agent.json +++ b/packages/server/marketplaces/chatflows/API Agent.json @@ -398,7 +398,7 @@ "data": { "id": "chatOpenAI_2", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -545,6 +545,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_2-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_2-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_2-input-imageResolution-options" } ], "inputAnchors": [ @@ -565,7 +598,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -597,7 +632,7 @@ "data": { "id": "chatOpenAI_1", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -744,6 +779,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_1-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_1-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_1-input-imageResolution-options" } ], "inputAnchors": [ @@ -764,7 +832,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -796,7 +866,7 @@ "data": { "id": "chatOpenAI_3", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -943,6 +1013,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_3-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_3-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_3-input-imageResolution-options" } ], "inputAnchors": [ @@ -963,7 +1066,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Advanced Structured Output Parser.json b/packages/server/marketplaces/chatflows/Advanced Structured Output Parser.json index c6480dd1..8618bf86 100644 --- a/packages/server/marketplaces/chatflows/Advanced Structured Output Parser.json +++ b/packages/server/marketplaces/chatflows/Advanced Structured Output Parser.json @@ -181,7 +181,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -334,6 +334,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -355,7 +388,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Antonym.json b/packages/server/marketplaces/chatflows/Antonym.json index 5cf6d85c..101d0430 100644 --- a/packages/server/marketplaces/chatflows/Antonym.json +++ b/packages/server/marketplaces/chatflows/Antonym.json @@ -177,7 +177,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -324,6 +324,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -344,7 +377,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/AutoGPT.json b/packages/server/marketplaces/chatflows/AutoGPT.json index c3ed0721..bb7c7bdc 100644 --- a/packages/server/marketplaces/chatflows/AutoGPT.json +++ b/packages/server/marketplaces/chatflows/AutoGPT.json @@ -253,7 +253,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -400,6 +400,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -420,7 +453,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/BabyAGI.json b/packages/server/marketplaces/chatflows/BabyAGI.json index cf8df770..8a800046 100644 --- a/packages/server/marketplaces/chatflows/BabyAGI.json +++ b/packages/server/marketplaces/chatflows/BabyAGI.json @@ -346,7 +346,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -499,6 +499,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -520,7 +553,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/CSV Agent.json b/packages/server/marketplaces/chatflows/CSV Agent.json index af2cfdd7..0a0bdce9 100644 --- a/packages/server/marketplaces/chatflows/CSV Agent.json +++ b/packages/server/marketplaces/chatflows/CSV Agent.json @@ -72,7 +72,7 @@ "id": "chatOpenAI_0", "label": "ChatOpenAI", "name": "chatOpenAI", - "version": 4, + "version": 5, "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], "category": "Chat Models", @@ -218,6 +218,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -238,7 +271,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Chat with a Podcast.json b/packages/server/marketplaces/chatflows/Chat with a Podcast.json index b3b3a4a1..6d0344a3 100644 --- a/packages/server/marketplaces/chatflows/Chat with a Podcast.json +++ b/packages/server/marketplaces/chatflows/Chat with a Podcast.json @@ -196,7 +196,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -349,6 +349,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -370,7 +403,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/ChatGPTPlugin.json b/packages/server/marketplaces/chatflows/ChatGPTPlugin.json index 4f0f7f5f..cbdc4634 100644 --- a/packages/server/marketplaces/chatflows/ChatGPTPlugin.json +++ b/packages/server/marketplaces/chatflows/ChatGPTPlugin.json @@ -217,7 +217,7 @@ "id": "chatOpenAI_0", "label": "ChatOpenAI", "name": "chatOpenAI", - "version": 4, + "version": 5, "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], "category": "Chat Models", @@ -363,6 +363,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -383,7 +416,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Conversational Agent.json b/packages/server/marketplaces/chatflows/Conversational Agent.json index c39b4d65..d07047d6 100644 --- a/packages/server/marketplaces/chatflows/Conversational Agent.json +++ b/packages/server/marketplaces/chatflows/Conversational Agent.json @@ -158,7 +158,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -305,6 +305,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -325,7 +358,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Conversational Retrieval Agent.json b/packages/server/marketplaces/chatflows/Conversational Retrieval Agent.json index 1de16cd8..72ac467e 100644 --- a/packages/server/marketplaces/chatflows/Conversational Retrieval Agent.json +++ b/packages/server/marketplaces/chatflows/Conversational Retrieval Agent.json @@ -489,7 +489,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -642,6 +642,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -663,7 +696,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json b/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json index 08e60afc..df3d1389 100644 --- a/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json +++ b/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json @@ -371,7 +371,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -524,6 +524,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -545,7 +578,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Flowise Docs QnA.json b/packages/server/marketplaces/chatflows/Flowise Docs QnA.json index 7f33ca48..62c72595 100644 --- a/packages/server/marketplaces/chatflows/Flowise Docs QnA.json +++ b/packages/server/marketplaces/chatflows/Flowise Docs QnA.json @@ -378,7 +378,7 @@ "id": "chatOpenAI_0", "label": "ChatOpenAI", "name": "chatOpenAI", - "version": 4, + "version": 5, "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], "category": "Chat Models", @@ -524,6 +524,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -544,7 +577,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/IfElse.json b/packages/server/marketplaces/chatflows/IfElse.json index 7516b19f..cdee6d1d 100644 --- a/packages/server/marketplaces/chatflows/IfElse.json +++ b/packages/server/marketplaces/chatflows/IfElse.json @@ -910,7 +910,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -1063,6 +1063,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -1084,7 +1117,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Image Generation.json b/packages/server/marketplaces/chatflows/Image Generation.json index b97682bd..f798b5a3 100644 --- a/packages/server/marketplaces/chatflows/Image Generation.json +++ b/packages/server/marketplaces/chatflows/Image Generation.json @@ -454,7 +454,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -607,6 +607,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -628,7 +661,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Input Moderation.json b/packages/server/marketplaces/chatflows/Input Moderation.json index efc59707..e35a481d 100644 --- a/packages/server/marketplaces/chatflows/Input Moderation.json +++ b/packages/server/marketplaces/chatflows/Input Moderation.json @@ -166,7 +166,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -319,6 +319,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -340,7 +373,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/List Output Parser.json b/packages/server/marketplaces/chatflows/List Output Parser.json index 46b8c058..5e8602f0 100644 --- a/packages/server/marketplaces/chatflows/List Output Parser.json +++ b/packages/server/marketplaces/chatflows/List Output Parser.json @@ -225,7 +225,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -378,6 +378,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -399,7 +432,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Long Term Memory.json b/packages/server/marketplaces/chatflows/Long Term Memory.json index f9ff3d0f..bc3b8a76 100644 --- a/packages/server/marketplaces/chatflows/Long Term Memory.json +++ b/packages/server/marketplaces/chatflows/Long Term Memory.json @@ -508,7 +508,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -661,6 +661,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -682,7 +715,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Metadata Filter.json b/packages/server/marketplaces/chatflows/Metadata Filter.json index b12a6be8..147a8cf6 100644 --- a/packages/server/marketplaces/chatflows/Metadata Filter.json +++ b/packages/server/marketplaces/chatflows/Metadata Filter.json @@ -455,7 +455,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -608,6 +608,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -629,7 +662,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Multi Prompt Chain.json b/packages/server/marketplaces/chatflows/Multi Prompt Chain.json index 12917939..41cd9b17 100644 --- a/packages/server/marketplaces/chatflows/Multi Prompt Chain.json +++ b/packages/server/marketplaces/chatflows/Multi Prompt Chain.json @@ -280,7 +280,7 @@ "id": "chatOpenAI_0", "label": "ChatOpenAI", "name": "chatOpenAI", - "version": 4, + "version": 5, "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], "category": "Chat Models", @@ -426,6 +426,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -446,7 +479,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Multi Retrieval QA Chain.json b/packages/server/marketplaces/chatflows/Multi Retrieval QA Chain.json index d2d345a3..8f762ca9 100644 --- a/packages/server/marketplaces/chatflows/Multi Retrieval QA Chain.json +++ b/packages/server/marketplaces/chatflows/Multi Retrieval QA Chain.json @@ -390,7 +390,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -543,6 +543,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -564,7 +597,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Multiple VectorDB.json b/packages/server/marketplaces/chatflows/Multiple VectorDB.json index 82b6d2d6..db17df54 100644 --- a/packages/server/marketplaces/chatflows/Multiple VectorDB.json +++ b/packages/server/marketplaces/chatflows/Multiple VectorDB.json @@ -487,7 +487,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -640,6 +640,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -661,7 +694,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -1013,7 +1048,7 @@ "data": { "id": "chatOpenAI_1", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -1166,6 +1201,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_1-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_1-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_1-input-imageResolution-options" } ], "inputAnchors": [ @@ -1187,7 +1255,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -1219,7 +1289,7 @@ "data": { "id": "chatOpenAI_2", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -1372,6 +1442,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_2-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_2-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_2-input-imageResolution-options" } ], "inputAnchors": [ @@ -1393,7 +1496,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/OpenAI Agent.json b/packages/server/marketplaces/chatflows/OpenAI Agent.json index 3792e474..f405640c 100644 --- a/packages/server/marketplaces/chatflows/OpenAI Agent.json +++ b/packages/server/marketplaces/chatflows/OpenAI Agent.json @@ -281,7 +281,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -428,6 +428,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -448,7 +481,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json b/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json index a0664390..bb0c284f 100644 --- a/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json +++ b/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json @@ -429,7 +429,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -582,6 +582,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -603,7 +636,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -635,7 +670,7 @@ "data": { "id": "chatOpenAI_1", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -788,6 +823,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_1-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_1-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_1-input-imageResolution-options" } ], "inputAnchors": [ @@ -809,7 +877,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/ReAct Agent.json b/packages/server/marketplaces/chatflows/ReAct Agent.json index a8c55b40..a4989c47 100644 --- a/packages/server/marketplaces/chatflows/ReAct Agent.json +++ b/packages/server/marketplaces/chatflows/ReAct Agent.json @@ -206,7 +206,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -359,6 +359,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -380,7 +413,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/SQL DB Chain.json b/packages/server/marketplaces/chatflows/SQL DB Chain.json index 29ddeb7d..debe4edc 100644 --- a/packages/server/marketplaces/chatflows/SQL DB Chain.json +++ b/packages/server/marketplaces/chatflows/SQL DB Chain.json @@ -15,7 +15,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -162,6 +162,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -182,7 +215,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/SQL Prompt.json b/packages/server/marketplaces/chatflows/SQL Prompt.json index cbad6a97..cfdb317a 100644 --- a/packages/server/marketplaces/chatflows/SQL Prompt.json +++ b/packages/server/marketplaces/chatflows/SQL Prompt.json @@ -175,7 +175,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -328,6 +328,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -349,7 +382,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -381,7 +416,7 @@ "data": { "id": "chatOpenAI_1", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -534,6 +569,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_1-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_1-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_1-input-imageResolution-options" } ], "inputAnchors": [ @@ -555,7 +623,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -1313,7 +1383,7 @@ "data": { "id": "chatOpenAI_2", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -1466,6 +1536,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_2-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_2-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_2-input-imageResolution-options" } ], "inputAnchors": [ @@ -1487,7 +1590,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Simple Conversation Chain.json b/packages/server/marketplaces/chatflows/Simple Conversation Chain.json index ea9a8d39..d3688e0e 100644 --- a/packages/server/marketplaces/chatflows/Simple Conversation Chain.json +++ b/packages/server/marketplaces/chatflows/Simple Conversation Chain.json @@ -16,7 +16,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -169,6 +169,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -190,7 +223,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Structured Output Parser.json b/packages/server/marketplaces/chatflows/Structured Output Parser.json index f23a3011..b1978cc1 100644 --- a/packages/server/marketplaces/chatflows/Structured Output Parser.json +++ b/packages/server/marketplaces/chatflows/Structured Output Parser.json @@ -16,7 +16,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -169,6 +169,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -190,7 +223,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/Translator.json b/packages/server/marketplaces/chatflows/Translator.json index cc83622b..0155ca46 100644 --- a/packages/server/marketplaces/chatflows/Translator.json +++ b/packages/server/marketplaces/chatflows/Translator.json @@ -84,7 +84,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -231,6 +231,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -251,7 +284,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/WebBrowser.json b/packages/server/marketplaces/chatflows/WebBrowser.json index 9ddf595f..d8b7d9f6 100644 --- a/packages/server/marketplaces/chatflows/WebBrowser.json +++ b/packages/server/marketplaces/chatflows/WebBrowser.json @@ -127,7 +127,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -274,6 +274,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -294,7 +327,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { @@ -433,7 +468,7 @@ "data": { "id": "chatOpenAI_1", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel"], @@ -580,6 +615,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_1-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_1-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_1-input-imageResolution-options" } ], "inputAnchors": [ @@ -600,7 +668,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/marketplaces/chatflows/WebPage QnA.json b/packages/server/marketplaces/chatflows/WebPage QnA.json index 126143b8..5ca29ee9 100644 --- a/packages/server/marketplaces/chatflows/WebPage QnA.json +++ b/packages/server/marketplaces/chatflows/WebPage QnA.json @@ -394,7 +394,7 @@ "data": { "id": "chatOpenAI_0", "label": "ChatOpenAI", - "version": 4, + "version": 5, "name": "chatOpenAI", "type": "ChatOpenAI", "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], @@ -547,6 +547,39 @@ "optional": true, "additionalParams": true, "id": "chatOpenAI_0-input-baseOptions-json" + }, + { + "label": "Allow Image Uploads", + "name": "allowImageUploads", + "type": "boolean", + "description": "Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent", + "default": false, + "optional": true, + "id": "chatOpenAI_0-input-allowImageUploads-boolean" + }, + { + "label": "Image Resolution", + "description": "This parameter controls the resolution in which the model views the image.", + "name": "imageResolution", + "type": "options", + "options": [ + { + "label": "Low", + "name": "low" + }, + { + "label": "High", + "name": "high" + }, + { + "label": "Auto", + "name": "auto" + } + ], + "default": "low", + "optional": false, + "additionalParams": true, + "id": "chatOpenAI_0-input-imageResolution-options" } ], "inputAnchors": [ @@ -568,7 +601,9 @@ "presencePenalty": "", "timeout": "", "basepath": "", - "baseOptions": "" + "baseOptions": "", + "allowImageUploads": true, + "imageResolution": "low" }, "outputAnchors": [ { diff --git a/packages/server/package.json b/packages/server/package.json index 3698a216..cff39a89 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -48,6 +48,7 @@ "@oclif/core": "^1.13.10", "async-mutex": "^0.4.0", "axios": "1.6.2", + "content-disposition": "0.5.4", "cors": "^2.8.5", "crypto-js": "^4.1.1", "dotenv": "^16.0.0", @@ -70,6 +71,7 @@ "winston": "^3.9.0" }, "devDependencies": { + "@types/content-disposition": "0.5.8", "@types/cors": "^2.8.12", "@types/crypto-js": "^4.1.1", "@types/multer": "^1.4.7", diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts index 126aac38..0c5fa5a4 100644 --- a/packages/server/src/Interface.ts +++ b/packages/server/src/Interface.ts @@ -1,4 +1,4 @@ -import { ICommonObject, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components' +import { ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components' export type MessageType = 'apiMessage' | 'userMessage' @@ -31,6 +31,7 @@ export interface IChatMessage { sourceDocuments?: string usedTools?: string fileAnnotations?: string + fileUploads?: string chatType: string chatId: string memoryType?: string @@ -176,6 +177,7 @@ export interface IncomingInput { socketIOClientId?: string chatId?: string stopNodeId?: string + uploads?: IFileUpload[] } export interface IActiveChatflows { @@ -212,3 +214,8 @@ export interface ICredentialReqBody { export interface ICredentialReturnResponse extends ICredential { plainDataObj: ICredentialDataDecrypted } + +export interface IUploadFileSizeAndTypes { + fileTypes: string[] + maxUploadSize: number +} diff --git a/packages/server/src/NodesPool.ts b/packages/server/src/NodesPool.ts index 86736d45..82c97f2a 100644 --- a/packages/server/src/NodesPool.ts +++ b/packages/server/src/NodesPool.ts @@ -56,7 +56,7 @@ export class NodesPool { } } - const skipCategories = ['Analytic'] + const skipCategories = ['Analytic', 'SpeechToText'] if (!skipCategories.includes(newNodeInstance.category)) { this.componentNodes[newNodeInstance.name] = newNodeInstance } diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index 7de5a526..f8877e26 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -23,6 +23,7 @@ export default class Start extends Command { CORS_ORIGINS: Flags.string(), IFRAME_ORIGINS: Flags.string(), DEBUG: Flags.string(), + BLOB_STORAGE_PATH: Flags.string(), APIKEY_PATH: Flags.string(), SECRETKEY_PATH: Flags.string(), FLOWISE_SECRETKEY_OVERWRITE: Flags.string(), @@ -92,6 +93,9 @@ export default class Start extends Command { if (flags.FLOWISE_PASSWORD) process.env.FLOWISE_PASSWORD = flags.FLOWISE_PASSWORD if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH + // Storage + if (flags.BLOB_STORAGE_PATH) process.env.BLOB_STORAGE_PATH = flags.BLOB_STORAGE_PATH + //API Configuration if (flags.FLOWISE_FILE_SIZE_LIMIT) process.env.FLOWISE_FILE_SIZE_LIMIT = flags.FLOWISE_FILE_SIZE_LIMIT diff --git a/packages/server/src/database/entities/ChatFlow.ts b/packages/server/src/database/entities/ChatFlow.ts index b3131c2e..b9048bad 100644 --- a/packages/server/src/database/entities/ChatFlow.ts +++ b/packages/server/src/database/entities/ChatFlow.ts @@ -31,6 +31,9 @@ export class ChatFlow implements IChatFlow { @Column({ nullable: true, type: 'text' }) analytic?: string + @Column({ nullable: true, type: 'text' }) + speechToText?: string + @CreateDateColumn() createdDate: Date diff --git a/packages/server/src/database/entities/ChatMessage.ts b/packages/server/src/database/entities/ChatMessage.ts index 4054a26d..c803ce50 100644 --- a/packages/server/src/database/entities/ChatMessage.ts +++ b/packages/server/src/database/entities/ChatMessage.ts @@ -26,6 +26,9 @@ export class ChatMessage implements IChatMessage { @Column({ nullable: true, type: 'text' }) fileAnnotations?: string + @Column({ nullable: true, type: 'text' }) + fileUploads?: string + @Column() chatType: string diff --git a/packages/server/src/database/migrations/mysql/1701788586491-AddFileUploadsToChatMessage.ts b/packages/server/src/database/migrations/mysql/1701788586491-AddFileUploadsToChatMessage.ts new file mode 100644 index 00000000..d896066b --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1701788586491-AddFileUploadsToChatMessage.ts @@ -0,0 +1,12 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const columnExists = await queryRunner.hasColumn('chat_message', 'fileUploads') + if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`fileUploads\` TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`fileUploads\`;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/1706364937060-AddSpeechToText.ts b/packages/server/src/database/migrations/mysql/1706364937060-AddSpeechToText.ts new file mode 100644 index 00000000..ac11be89 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1706364937060-AddSpeechToText.ts @@ -0,0 +1,12 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddSpeechToText1706364937060 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const columnExists = await queryRunner.hasColumn('chat_flow', 'speechToText') + if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`speechToText\` TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`chat_flow\` DROP COLUMN \`speechToText\`;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index a5220ad8..549742a1 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -10,7 +10,9 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage' import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow' import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage' +import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' +import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' export const mysqlMigrations = [ Init1693840429259, @@ -25,5 +27,7 @@ export const mysqlMigrations = [ AddUsedToolsToChatMessage1699481607341, AddCategoryToChatFlow1699900910291, AddFileAnnotationsToChatMessage1700271021237, - AddVariableEntity1699325775451 + AddFileUploadsToChatMessage1701788586491, + AddVariableEntity1699325775451, + AddSpeechToText1706364937060 ] diff --git a/packages/server/src/database/migrations/postgres/1701788586491-AddFileUploadsToChatMessage.ts b/packages/server/src/database/migrations/postgres/1701788586491-AddFileUploadsToChatMessage.ts new file mode 100644 index 00000000..6574ac81 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1701788586491-AddFileUploadsToChatMessage.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "fileUploads" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileUploads";`) + } +} diff --git a/packages/server/src/database/migrations/postgres/1706364937060-AddSpeechToText.ts b/packages/server/src/database/migrations/postgres/1706364937060-AddSpeechToText.ts new file mode 100644 index 00000000..8ce5b672 --- /dev/null +++ b/packages/server/src/database/migrations/postgres/1706364937060-AddSpeechToText.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddSpeechToText1706364937060 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN IF NOT EXISTS "speechToText" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`) + } +} diff --git a/packages/server/src/database/migrations/postgres/index.ts b/packages/server/src/database/migrations/postgres/index.ts index 3c3fa396..bd631903 100644 --- a/packages/server/src/database/migrations/postgres/index.ts +++ b/packages/server/src/database/migrations/postgres/index.ts @@ -10,7 +10,9 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage' import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow' import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage' +import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' +import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' export const postgresMigrations = [ Init1693891895163, @@ -25,5 +27,7 @@ export const postgresMigrations = [ AddUsedToolsToChatMessage1699481607341, AddCategoryToChatFlow1699900910291, AddFileAnnotationsToChatMessage1700271021237, - AddVariableEntity1699325775451 + AddFileUploadsToChatMessage1701788586491, + AddVariableEntity1699325775451, + AddSpeechToText1706364937060 ] diff --git a/packages/server/src/database/migrations/sqlite/1701788586491-AddFileUploadsToChatMessage.ts b/packages/server/src/database/migrations/sqlite/1701788586491-AddFileUploadsToChatMessage.ts new file mode 100644 index 00000000..68e33220 --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1701788586491-AddFileUploadsToChatMessage.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddFileUploadsToChatMessage1701788586491 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "fileAnnotations" text, "fileUploads" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);` + ) + await queryRunner.query( + `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "fileAnnotations", "usedTools", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "usedTools", "fileAnnotations", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";` + ) + await queryRunner.query(`DROP TABLE "chat_message";`) + await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`) + await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`) + await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileUploads";`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/1706364937060-AddSpeechToText.ts b/packages/server/src/database/migrations/sqlite/1706364937060-AddSpeechToText.ts new file mode 100644 index 00000000..1d77ffea --- /dev/null +++ b/packages/server/src/database/migrations/sqlite/1706364937060-AddSpeechToText.ts @@ -0,0 +1,11 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddSpeechToText1706364937060 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN "speechToText" TEXT;`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "speechToText";`) + } +} diff --git a/packages/server/src/database/migrations/sqlite/index.ts b/packages/server/src/database/migrations/sqlite/index.ts index c0ade080..a50b0792 100644 --- a/packages/server/src/database/migrations/sqlite/index.ts +++ b/packages/server/src/database/migrations/sqlite/index.ts @@ -10,7 +10,9 @@ import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEnt import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage' import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow' import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage' +import { AddFileUploadsToChatMessage1701788586491 } from './1701788586491-AddFileUploadsToChatMessage' import { AddVariableEntity1699325775451 } from './1702200925471-AddVariableEntity' +import { AddSpeechToText1706364937060 } from './1706364937060-AddSpeechToText' export const sqliteMigrations = [ Init1693835579790, @@ -25,5 +27,7 @@ export const sqliteMigrations = [ AddUsedToolsToChatMessage1699481607341, AddCategoryToChatFlow1699900910291, AddFileAnnotationsToChatMessage1700271021237, - AddVariableEntity1699325775451 + AddFileUploadsToChatMessage1701788586491, + AddVariableEntity1699325775451, + AddSpeechToText1706364937060 ] diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index aef8494a..d52658b4 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -5,6 +5,7 @@ import cors from 'cors' import http from 'http' import * as fs from 'fs' import basicAuth from 'express-basic-auth' +import contentDisposition from 'content-disposition' import { Server } from 'socket.io' import logger from './utils/logger' import { expressRequestLogger } from './utils/logger' @@ -21,7 +22,8 @@ import { chatType, IChatMessage, IDepthQueue, - INodeDirectedGraph + INodeDirectedGraph, + IUploadFileSizeAndTypes } from './Interface' import { getNodeModulesPackagePath, @@ -46,6 +48,7 @@ import { getAllConnectedNodes, clearSessionMemory, findMemoryNode, + deleteFolderRecursive, getTelemetryFlowObj, getAppVersion } from './utils' @@ -59,7 +62,18 @@ import { Tool } from './database/entities/Tool' import { Assistant } from './database/entities/Assistant' import { ChatflowPool } from './ChatflowPool' import { CachePool } from './CachePool' -import { ICommonObject, IMessage, INodeOptionsValue, handleEscapeCharacters, webCrawl, xmlScrape } from 'flowise-components' +import { + ICommonObject, + IMessage, + INodeOptionsValue, + INodeParams, + handleEscapeCharacters, + convertSpeechToText, + xmlScrape, + webCrawl, + getStoragePath, + IFileUpload +} from 'flowise-components' import { createRateLimiter, getRateLimiter, initializeRateLimiter } from './utils/rateLimit' import { addAPIKey, compareKeys, deleteAPIKey, getApiKey, getAPIKeys, updateAPIKey } from './utils/apiKey' import { sanitizeMiddleware, getCorsOptions, getAllowedIframeOrigins } from './utils/XSS' @@ -168,7 +182,9 @@ export class App { '/api/v1/node-icon/', '/api/v1/components-credentials-icon/', '/api/v1/chatflows-streaming', + '/api/v1/chatflows-uploads', '/api/v1/openai-assistants-file', + '/api/v1/get-upload-file', '/api/v1/ip' ] this.app.use((req, res, next) => { @@ -389,10 +405,13 @@ export class App { id: req.params.id }) if (!chatflow) return res.status(404).send(`Chatflow ${req.params.id} not found`) - if (chatflow.chatbotConfig) { + const uploadsConfig = await this.getUploadsConfig(req.params.id) + // even if chatbotConfig is not set but uploads are enabled + // send uploadsConfig to the chatbot + if (chatflow.chatbotConfig || uploadsConfig) { try { - const parsedConfig = JSON.parse(chatflow.chatbotConfig) - return res.json(parsedConfig) + const parsedConfig = chatflow.chatbotConfig ? JSON.parse(chatflow.chatbotConfig) : {} + return res.json({ ...parsedConfig, uploads: uploadsConfig }) } catch (e) { return res.status(500).send(`Error parsing Chatbot Config for Chatflow ${req.params.id}`) } @@ -452,6 +471,15 @@ export class App { // Delete chatflow via id this.app.delete('/api/v1/chatflows/:id', async (req: Request, res: Response) => { const results = await this.AppDataSource.getRepository(ChatFlow).delete({ id: req.params.id }) + + try { + // Delete all uploads corresponding to this chatflow + const directory = path.join(getStoragePath(), req.params.id) + deleteFolderRecursive(directory) + } catch (e) { + logger.error(`[server]: Error deleting file storage for chatflow ${req.params.id}: ${e}`) + } + return res.json(results) }) @@ -502,6 +530,16 @@ export class App { return res.json(obj) }) + // Check if chatflow valid for uploads + this.app.get('/api/v1/chatflows-uploads/:id', async (req: Request, res: Response) => { + try { + const uploadsConfig = await this.getUploadsConfig(req.params.id) + return res.json(uploadsConfig) + } catch (e) { + return res.status(500).send(e) + } + }) + // ---------------------------------------- // ChatMessage // ---------------------------------------- @@ -599,6 +637,16 @@ export class App { if (sessionId) deleteOptions.sessionId = sessionId if (chatType) deleteOptions.chatType = chatType + // Delete all uploads corresponding to this chatflow/chatId + if (chatId) { + try { + const directory = path.join(getStoragePath(), chatflowid, chatId) + deleteFolderRecursive(directory) + } catch (e) { + logger.error(`[server]: Error deleting file storage for chatflow ${chatflowid}, chatId ${chatId}: ${e}`) + } + } + const results = await this.AppDataSource.getRepository(ChatMessage).delete(deleteOptions) return res.json(results) }) @@ -1081,6 +1129,11 @@ export class App { } }) + function streamFileToUser(res: Response, filePath: string) { + const fileStream = fs.createReadStream(filePath) + fileStream.pipe(res) + } + // Download file from assistant this.app.post('/api/v1/openai-assistants-file', async (req: Request, res: Response) => { const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', req.body.fileName) @@ -1090,9 +1143,48 @@ export class App { if (filePath.includes('..')) return res.status(500).send(`Invalid file path`) //only return from the .flowise openai-assistant folder if (!(filePath.includes('.flowise') && filePath.includes('openai-assistant'))) return res.status(500).send(`Invalid file path`) - res.setHeader('Content-Disposition', 'attachment; filename=' + path.basename(filePath)) - const fileStream = fs.createReadStream(filePath) - fileStream.pipe(res) + + if (fs.existsSync(filePath)) { + res.setHeader('Content-Disposition', contentDisposition(path.basename(filePath))) + streamFileToUser(res, filePath) + } else { + return res.status(404).send(`File ${req.body.fileName} not found`) + } + }) + + this.app.get('/api/v1/get-upload-path', async (req: Request, res: Response) => { + return res.json({ + storagePath: getStoragePath() + }) + }) + + // stream uploaded image + this.app.get('/api/v1/get-upload-file', async (req: Request, res: Response) => { + try { + if (!req.query.chatflowId || !req.query.chatId || !req.query.fileName) { + return res.status(500).send(`Invalid file path`) + } + const chatflowId = req.query.chatflowId as string + const chatId = req.query.chatId as string + const fileName = req.query.fileName as string + + const filePath = path.join(getStoragePath(), chatflowId, chatId, fileName) + //raise error if file path is not absolute + if (!path.isAbsolute(filePath)) return res.status(500).send(`Invalid file path`) + //raise error if file path contains '..' + if (filePath.includes('..')) return res.status(500).send(`Invalid file path`) + //only return from the storage folder + if (!filePath.startsWith(getStoragePath())) return res.status(500).send(`Invalid file path`) + + if (fs.existsSync(filePath)) { + res.setHeader('Content-Disposition', contentDisposition(path.basename(filePath))) + streamFileToUser(res, filePath) + } else { + return res.status(404).send(`File ${fileName} not found`) + } + } catch (error) { + return res.status(500).send(`Invalid file path`) + } }) // ---------------------------------------- @@ -1424,6 +1516,74 @@ export class App { return false } + /** + * Method that checks if uploads are enabled in the chatflow + * @param {string} chatflowid + */ + async getUploadsConfig(chatflowid: string): Promise { + const chatflow = await this.AppDataSource.getRepository(ChatFlow).findOneBy({ + id: chatflowid + }) + if (!chatflow) return `Chatflow ${chatflowid} not found` + + const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent'] + const uploadProcessingNodes = ['chatOpenAI'] + + const flowObj = JSON.parse(chatflow.flowData) + const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] + + let isSpeechToTextEnabled = false + if (chatflow.speechToText) { + const speechToTextProviders = JSON.parse(chatflow.speechToText) + for (const provider in speechToTextProviders) { + if (provider !== 'none') { + const providerObj = speechToTextProviders[provider] + if (providerObj.status) { + isSpeechToTextEnabled = true + break + } + } + } + } + + let isImageUploadAllowed = false + const nodes: IReactFlowNode[] = flowObj.nodes + + /* + * Condition for isImageUploadAllowed + * 1.) one of the uploadAllowedNodes exists + * 2.) one of the uploadProcessingNodes exists + allowImageUploads is ON + */ + if (!nodes.some((node) => uploadAllowedNodes.includes(node.data.name))) { + return { + isSpeechToTextEnabled, + isImageUploadAllowed: false, + imgUploadSizeAndTypes + } + } + + nodes.forEach((node: IReactFlowNode) => { + if (uploadProcessingNodes.indexOf(node.data.name) > -1) { + // TODO: for now the maxUploadSize is hardcoded to 5MB, we need to add it to the node properties + node.data.inputParams.map((param: INodeParams) => { + if (param.name === 'allowImageUploads' && node.data.inputs?.['allowImageUploads']) { + imgUploadSizeAndTypes.push({ + fileTypes: 'image/gif;image/jpeg;image/png;image/webp;'.split(';'), + maxUploadSize: 5 + }) + isImageUploadAllowed = true + } + }) + } + }) + + return { + isSpeechToTextEnabled, + isImageUploadAllowed, + imgUploadSizeAndTypes + } + } + /** * Method that get chat messages. * @param {string} chatflowid @@ -1637,6 +1797,57 @@ export class App { if (!isKeyValidated) return res.status(401).send('Unauthorized') } + let fileUploads: IFileUpload[] = [] + if (incomingInput.uploads) { + fileUploads = incomingInput.uploads + for (let i = 0; i < fileUploads.length; i += 1) { + const upload = fileUploads[i] + if ((upload.type === 'file' || upload.type === 'audio') && upload.data) { + const filename = upload.name + const dir = path.join(getStoragePath(), chatflowid, chatId) + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }) + } + const filePath = path.join(dir, filename) + const splitDataURI = upload.data.split(',') + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + fs.writeFileSync(filePath, bf) + + // Omit upload.data since we don't store the content in database + upload.type = 'stored-file' + fileUploads[i] = omit(upload, ['data']) + } + + // Run Speech to Text conversion + if (upload.mime === 'audio/webm') { + let speechToTextConfig: ICommonObject = {} + if (chatflow.speechToText) { + const speechToTextProviders = JSON.parse(chatflow.speechToText) + for (const provider in speechToTextProviders) { + const providerObj = speechToTextProviders[provider] + if (providerObj.status) { + speechToTextConfig = providerObj + speechToTextConfig['name'] = provider + break + } + } + } + if (speechToTextConfig) { + const options: ICommonObject = { + chatId, + chatflowid, + appDataSource: this.AppDataSource, + databaseEntities: databaseEntities + } + const speechToTextResult = await convertSpeechToText(upload, speechToTextConfig, options) + if (speechToTextResult) { + incomingInput.question = speechToTextResult + } + } + } + } + } + let isStreamValid = false const files = (req.files as any[]) || [] @@ -1802,7 +2013,10 @@ export class App { chatflowid, this.AppDataSource, incomingInput?.overrideConfig, - this.cachePool + this.cachePool, + false, + undefined, + incomingInput.uploads ) const nodeToExecute = @@ -1841,6 +2055,7 @@ export class App { appDataSource: this.AppDataSource, databaseEntities, analytic: chatflow.analytic, + uploads: incomingInput.uploads, socketIO, socketIOClientId: incomingInput.socketIOClientId }) @@ -1851,7 +2066,8 @@ export class App { logger, appDataSource: this.AppDataSource, databaseEntities, - analytic: chatflow.analytic + analytic: chatflow.analytic, + uploads: incomingInput.uploads }) result = typeof result === 'string' ? { text: result } : result @@ -1869,7 +2085,8 @@ export class App { chatId, memoryType, sessionId, - createdDate: userMessageDateTime + createdDate: userMessageDateTime, + fileUploads: incomingInput.uploads ? JSON.stringify(fileUploads) : undefined } await this.addChatMessage(userMessage) @@ -1891,7 +2108,6 @@ export class App { if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools) if (result?.fileAnnotations) apiMessage.fileAnnotations = JSON.stringify(result.fileAnnotations) const chatMessage = await this.addChatMessage(apiMessage) - result.chatMessageId = chatMessage.id logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`) await this.telemetry.sendTelemetry('prediction_sent', { @@ -1903,7 +2119,11 @@ export class App { }) // Prepare response + // return the question in the response + // this is used when input text is empty but question is in audio format + result.question = incomingInput.question result.chatId = chatId + result.chatMessageId = chatMessage.id if (sessionId) result.sessionId = sessionId if (memoryType) result.memoryType = memoryType diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 2903d426..8092fbd8 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -27,7 +27,8 @@ import { ICommonObject, IDatabaseEntity, IMessage, - FlowiseMemory + FlowiseMemory, + IFileUpload } from 'flowise-components' import { randomBytes } from 'crypto' import { AES, enc } from 'crypto-js' @@ -279,7 +280,8 @@ export const buildFlow = async ( overrideConfig?: ICommonObject, cachePool?: CachePool, isUpsert?: boolean, - stopNodeId?: string + stopNodeId?: string, + uploads?: IFileUpload[] ) => { const flowNodes = cloneDeep(reactFlowNodes) @@ -325,7 +327,8 @@ export const buildFlow = async ( appDataSource, databaseEntities, cachePool, - dynamicVariables + dynamicVariables, + uploads }) logger.debug(`[server]: Finished upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`) break @@ -340,7 +343,8 @@ export const buildFlow = async ( appDataSource, databaseEntities, cachePool, - dynamicVariables + dynamicVariables, + uploads }) // Save dynamic variables @@ -596,7 +600,6 @@ export const resolveVariables = ( } const paramsObj = flowNodeData[types] ?? {} - getParamValues(paramsObj) return flowNodeData @@ -1128,6 +1131,34 @@ export const getAllValuesFromJson = (obj: any): any[] => { return values } +/** + * Delete file & folder recursively + * @param {string} directory + */ +export const deleteFolderRecursive = (directory: string) => { + if (fs.existsSync(directory)) { + fs.readdir(directory, (error, files) => { + if (error) throw new Error('Could not read directory') + + files.forEach((file) => { + const file_path = path.join(directory, file) + + fs.stat(file_path, (error, stat) => { + if (error) throw new Error('File do not exist') + + if (!stat.isDirectory()) { + fs.unlink(file_path, (error) => { + if (error) throw new Error('Could not delete file') + }) + } else { + deleteFolderRecursive(file_path) + } + }) + }) + }) + } +} + /** * Get only essential flow data items for telemetry * @param {IReactFlowNode[]} nodes diff --git a/packages/ui/src/api/chatflows.js b/packages/ui/src/api/chatflows.js index 8810b5a5..586fe183 100644 --- a/packages/ui/src/api/chatflows.js +++ b/packages/ui/src/api/chatflows.js @@ -14,6 +14,8 @@ const deleteChatflow = (id) => client.delete(`/chatflows/${id}`) const getIsChatflowStreaming = (id) => client.get(`/chatflows-streaming/${id}`) +const getAllowChatflowUploads = (id) => client.get(`/chatflows-uploads/${id}`) + export default { getAllChatflows, getSpecificChatflow, @@ -21,5 +23,6 @@ export default { createNewChatflow, updateChatflow, deleteChatflow, - getIsChatflowStreaming + getIsChatflowStreaming, + getAllowChatflowUploads } diff --git a/packages/ui/src/api/chatmessage.js b/packages/ui/src/api/chatmessage.js index 5f1a4bad..f1651247 100644 --- a/packages/ui/src/api/chatmessage.js +++ b/packages/ui/src/api/chatmessage.js @@ -4,10 +4,12 @@ const getInternalChatmessageFromChatflow = (id) => client.get(`/internal-chatmes const getAllChatmessageFromChatflow = (id, params = {}) => client.get(`/chatmessage/${id}`, { params: { order: 'DESC', ...params } }) const getChatmessageFromPK = (id, params = {}) => client.get(`/chatmessage/${id}`, { params: { order: 'ASC', ...params } }) const deleteChatmessage = (id, params = {}) => client.delete(`/chatmessage/${id}`, { params: { ...params } }) +const getStoragePath = () => client.get(`/get-upload-path`) export default { getInternalChatmessageFromChatflow, getAllChatmessageFromChatflow, getChatmessageFromPK, - deleteChatmessage + deleteChatmessage, + getStoragePath } diff --git a/packages/ui/src/assets/images/assemblyai.png b/packages/ui/src/assets/images/assemblyai.png new file mode 100644 index 00000000..8919cb18 Binary files /dev/null and b/packages/ui/src/assets/images/assemblyai.png differ diff --git a/packages/ui/src/assets/images/openai.svg b/packages/ui/src/assets/images/openai.svg new file mode 100644 index 00000000..5c20398a --- /dev/null +++ b/packages/ui/src/assets/images/openai.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/packages/ui/src/assets/images/wave-sound.jpg b/packages/ui/src/assets/images/wave-sound.jpg new file mode 100644 index 00000000..9f56d67d Binary files /dev/null and b/packages/ui/src/assets/images/wave-sound.jpg differ diff --git a/packages/ui/src/menu-items/settings.js b/packages/ui/src/menu-items/settings.js index 1e0f58dd..af45fba7 100644 --- a/packages/ui/src/menu-items/settings.js +++ b/packages/ui/src/menu-items/settings.js @@ -1,8 +1,17 @@ // assets -import { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff } from '@tabler/icons' +import { + IconTrash, + IconFileUpload, + IconFileExport, + IconCopy, + IconSearch, + IconMessage, + IconPictureInPictureOff, + IconMicrophone +} from '@tabler/icons' // constant -const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff } +const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage, IconPictureInPictureOff, IconMicrophone } // ==============================|| SETTINGS MENU ITEMS ||============================== // @@ -25,6 +34,13 @@ const settings = { url: '', icon: icons.IconMessage }, + { + id: 'enableSpeechToText', + title: 'Speech to Text', + type: 'item', + url: '', + icon: icons.IconMicrophone + }, { id: 'duplicateChatflow', title: 'Duplicate Chatflow', diff --git a/packages/ui/src/ui-component/button/ImageButton.js b/packages/ui/src/ui-component/button/ImageButton.js new file mode 100644 index 00000000..7a10b966 --- /dev/null +++ b/packages/ui/src/ui-component/button/ImageButton.js @@ -0,0 +1,57 @@ +import { styled } from '@mui/material/styles' +import ButtonBase from '@mui/material/ButtonBase' + +export const ImageButton = styled(ButtonBase)(({ theme }) => ({ + position: 'relative', + height: 200, + borderRadius: '10px', + [theme.breakpoints.down('sm')]: { + width: '100% !important', // Overrides inline-style + height: 100 + }, + '&:hover, &.Mui-focusVisible': { + zIndex: 1, + '& .MuiImageBackdrop-root': { + opacity: 0.4 + }, + '& .MuiImageMarked-root': { + opacity: 1 + }, + '& .MuiTypography-root': { + border: '4px solid currentColor' + } + } +})) + +export const ImageSrc = styled('span')({ + position: 'absolute', + borderRadius: '10px', + left: 0, + right: 0, + top: 0, + bottom: 0, + backgroundSize: 'cover', + backgroundPosition: 'center 40%' +}) + +export const ImageBackdrop = styled('span')(({ theme }) => ({ + position: 'absolute', + borderRadius: '10px', + left: 0, + right: 0, + top: 0, + bottom: 0, + backgroundColor: theme.palette.common.black, + opacity: 0.1, + transition: theme.transitions.create('opacity') +})) + +export const ImageMarked = styled('span')(() => ({ + height: 25, + width: 25, + backgroundColor: 'transparent', + position: 'absolute', + top: 'auto', + left: 'auto', + opacity: 0 +})) diff --git a/packages/ui/src/ui-component/cards/StarterPromptsCard.css b/packages/ui/src/ui-component/cards/StarterPromptsCard.css index 85c2d415..8fc6c07c 100644 --- a/packages/ui/src/ui-component/cards/StarterPromptsCard.css +++ b/packages/ui/src/ui-component/cards/StarterPromptsCard.css @@ -1,7 +1,4 @@ .button-container { - position: absolute; - bottom: 0; - z-index: 1000; display: flex; overflow-x: auto; -webkit-overflow-scrolling: touch; /* For momentum scroll on mobile devices */ @@ -10,5 +7,4 @@ .button { flex: 0 0 auto; /* Don't grow, don't shrink, base width on content */ - margin: 5px; /* Adjust as needed for spacing between buttons */ } diff --git a/packages/ui/src/ui-component/cards/StarterPromptsCard.js b/packages/ui/src/ui-component/cards/StarterPromptsCard.js index 3abd8378..bb3fbdfa 100644 --- a/packages/ui/src/ui-component/cards/StarterPromptsCard.js +++ b/packages/ui/src/ui-component/cards/StarterPromptsCard.js @@ -3,9 +3,12 @@ import PropTypes from 'prop-types' import { Chip } from '@mui/material' import './StarterPromptsCard.css' -const StarterPromptsCard = ({ isGrid, starterPrompts, onPromptClick }) => { +const StarterPromptsCard = ({ isGrid, starterPrompts, sx, onPromptClick }) => { return ( - + {starterPrompts.map((sp, index) => ( onPromptClick(sp.prompt, e)} /> ))} @@ -15,7 +18,8 @@ const StarterPromptsCard = ({ isGrid, starterPrompts, onPromptClick }) => { StarterPromptsCard.propTypes = { isGrid: PropTypes.bool, - starterPrompts: PropTypes.arrayOf(PropTypes.string), + starterPrompts: PropTypes.array, + sx: PropTypes.object, onPromptClick: PropTypes.func } diff --git a/packages/ui/src/ui-component/dialog/SpeechToTextDialog.js b/packages/ui/src/ui-component/dialog/SpeechToTextDialog.js new file mode 100644 index 00000000..489d4335 --- /dev/null +++ b/packages/ui/src/ui-component/dialog/SpeechToTextDialog.js @@ -0,0 +1,348 @@ +import { createPortal } from 'react-dom' +import { useDispatch } from 'react-redux' +import { useState, useEffect } from 'react' +import PropTypes from 'prop-types' +import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction, SET_CHATFLOW } from 'store/actions' + +// material-ui +import { + Typography, + Box, + Button, + Dialog, + DialogContent, + DialogTitle, + DialogActions, + FormControl, + ListItem, + ListItemAvatar, + ListItemText, + MenuItem, + Select +} from '@mui/material' +import { IconX } from '@tabler/icons' + +// Project import +import CredentialInputHandler from 'views/canvas/CredentialInputHandler' +import { TooltipWithParser } from 'ui-component/tooltip/TooltipWithParser' +import { SwitchInput } from 'ui-component/switch/Switch' +import { Input } from 'ui-component/input/Input' +import { StyledButton } from 'ui-component/button/StyledButton' +import { Dropdown } from 'ui-component/dropdown/Dropdown' +import openAISVG from 'assets/images/openai.svg' +import assemblyAIPng from 'assets/images/assemblyai.png' + +// store +import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions' +import useNotifier from 'utils/useNotifier' + +// API +import chatflowsApi from 'api/chatflows' + +const speechToTextProviders = { + openAIWhisper: { + label: 'OpenAI Whisper', + name: 'openAIWhisper', + icon: openAISVG, + url: 'https://platform.openai.com/docs/guides/speech-to-text', + inputs: [ + { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['openAIApi'] + }, + { + label: 'Language', + name: 'language', + type: 'string', + description: + 'The language of the input audio. Supplying the input language in ISO-639-1 format will improve accuracy and latency.', + placeholder: 'en', + optional: true + }, + { + label: 'Prompt', + name: 'prompt', + type: 'string', + rows: 4, + description: `An optional text to guide the model's style or continue a previous audio segment. The prompt should match the audio language.`, + optional: true + }, + { + label: 'Temperature', + name: 'temperature', + type: 'number', + step: 0.1, + description: `The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.`, + optional: true + } + ] + }, + assemblyAiTranscribe: { + label: 'Assembly AI', + name: 'assemblyAiTranscribe', + icon: assemblyAIPng, + url: 'https://www.assemblyai.com/', + inputs: [ + { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['assemblyAIApi'] + } + ] + } +} + +const SpeechToTextDialog = ({ show, dialogProps, onCancel }) => { + const portalElement = document.getElementById('portal') + const dispatch = useDispatch() + + useNotifier() + + const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) + const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args)) + + const [speechToText, setSpeechToText] = useState({}) + const [selectedProvider, setSelectedProvider] = useState('none') + + const onSave = async () => { + const speechToText = setValue(true, selectedProvider, 'status') + try { + const saveResp = await chatflowsApi.updateChatflow(dialogProps.chatflow.id, { + speechToText: JSON.stringify(speechToText) + }) + if (saveResp.data) { + enqueueSnackbar({ + message: 'Speech To Text Configuration Saved', + options: { + key: new Date().getTime() + Math.random(), + variant: 'success', + action: (key) => ( + + ) + } + }) + dispatch({ type: SET_CHATFLOW, chatflow: saveResp.data }) + } + onCancel() + } catch (error) { + const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}` + enqueueSnackbar({ + message: `Failed to save Speech To Text Configuration: ${errorData}`, + options: { + key: new Date().getTime() + Math.random(), + variant: 'error', + persist: true, + action: (key) => ( + + ) + } + }) + } + } + + const setValue = (value, providerName, inputParamName) => { + let newVal = {} + if (!Object.prototype.hasOwnProperty.call(speechToText, providerName)) { + newVal = { ...speechToText, [providerName]: {} } + } else { + newVal = { ...speechToText } + } + + newVal[providerName][inputParamName] = value + if (inputParamName === 'status' && value === true) { + // ensure that the others are turned off + Object.keys(speechToTextProviders).forEach((key) => { + const provider = speechToTextProviders[key] + if (provider.name !== providerName) { + newVal[provider.name] = { ...speechToText[provider.name], status: false } + } + }) + } + setSpeechToText(newVal) + return newVal + } + + const handleProviderChange = (event) => { + setSelectedProvider(event.target.value) + } + + useEffect(() => { + if (dialogProps.chatflow && dialogProps.chatflow.speechToText) { + try { + const speechToText = JSON.parse(dialogProps.chatflow.speechToText) + let selectedProvider = 'none' + Object.keys(speechToTextProviders).forEach((key) => { + const providerConfig = speechToText[key] + if (providerConfig && providerConfig.status) { + selectedProvider = key + } + }) + setSelectedProvider(selectedProvider) + setSpeechToText(speechToText) + } catch (e) { + setSpeechToText({}) + setSelectedProvider('none') + console.error(e) + } + } + + return () => { + setSpeechToText({}) + setSelectedProvider('none') + } + }, [dialogProps]) + + useEffect(() => { + if (show) dispatch({ type: SHOW_CANVAS_DIALOG }) + else dispatch({ type: HIDE_CANVAS_DIALOG }) + return () => dispatch({ type: HIDE_CANVAS_DIALOG }) + }, [show, dispatch]) + + const component = ( + + + Speech To Text Configuration + + + + Speech To Text Providers + + + + + {selectedProvider !== 'none' && ( + <> + + +
+ AI +
+ + + {speechToTextProviders[selectedProvider].url} + + } + /> + + {speechToTextProviders[selectedProvider].inputs.map((inputParam, index) => ( + +
+ + {inputParam.label} + {!inputParam.optional &&  *} + {inputParam.description && ( + + )} + +
+ {inputParam.type === 'credential' && ( + setValue(newValue, selectedProvider, 'credentialId')} + /> + )} + {inputParam.type === 'boolean' && ( + setValue(newValue, selectedProvider, inputParam.name)} + value={ + speechToText[selectedProvider] + ? speechToText[selectedProvider][inputParam.name] + : inputParam.default ?? false + } + /> + )} + {(inputParam.type === 'string' || inputParam.type === 'password' || inputParam.type === 'number') && ( + setValue(newValue, selectedProvider, inputParam.name)} + value={ + speechToText[selectedProvider] + ? speechToText[selectedProvider][inputParam.name] + : inputParam.default ?? '' + } + /> + )} + + {inputParam.type === 'options' && ( + setValue(newValue, selectedProvider, inputParam.name)} + value={ + speechToText[selectedProvider] + ? speechToText[selectedProvider][inputParam.name] + : inputParam.default ?? 'choose an option' + } + /> + )} + + ))} + + )} + + + + Save + + + + ) + + return createPortal(component, portalElement) +} + +SpeechToTextDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func +} + +export default SpeechToTextDialog diff --git a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.js b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.js index cadd4abd..7456aa81 100644 --- a/packages/ui/src/ui-component/dialog/ViewMessagesDialog.js +++ b/packages/ui/src/ui-component/dialog/ViewMessagesDialog.js @@ -21,7 +21,9 @@ import { DialogTitle, ListItem, ListItemText, - Chip + Chip, + Card, + CardMedia } from '@mui/material' import { useTheme } from '@mui/material/styles' import DatePicker from 'react-datepicker' @@ -47,7 +49,7 @@ import useApi from 'hooks/useApi' import useConfirm from 'hooks/useConfirm' // Utils -import { isValidURL, removeDuplicateURL } from 'utils/genericHelper' +import { getOS, isValidURL, removeDuplicateURL } from 'utils/genericHelper' import useNotifier from 'utils/useNotifier' import { baseURL } from 'store/constant' @@ -69,6 +71,12 @@ DatePickerCustomInput.propTypes = { onClick: PropTypes.func } +const messageImageStyle = { + width: '128px', + height: '128px', + objectFit: 'cover' +} + const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { const portalElement = document.getElementById('portal') const dispatch = useDispatch() @@ -92,6 +100,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { const getChatmessageApi = useApi(chatmessageApi.getAllChatmessageFromChatflow) const getChatmessageFromPKApi = useApi(chatmessageApi.getChatmessageFromPK) + const getStoragePathFromServer = useApi(chatmessageApi.getStoragePath) + let storagePath = '' const onStartDateSelected = (date) => { setStartDate(date) @@ -120,16 +130,35 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { }) } - const exportMessages = () => { + const exportMessages = async () => { + if (!storagePath && getStoragePathFromServer.data) { + storagePath = getStoragePathFromServer.data.storagePath + } const obj = {} + let fileSeparator = '/' + if ('windows' === getOS()) { + fileSeparator = '\\' + } for (let i = 0; i < allChatlogs.length; i += 1) { const chatmsg = allChatlogs[i] const chatPK = getChatPK(chatmsg) + let filePaths = [] + if (chatmsg.fileUploads) { + chatmsg.fileUploads = JSON.parse(chatmsg.fileUploads) + chatmsg.fileUploads.forEach((file) => { + if (file.type === 'stored-file') { + filePaths.push( + `${storagePath}${fileSeparator}${chatmsg.chatflowid}${fileSeparator}${chatmsg.chatId}${fileSeparator}${file.name}` + ) + } + }) + } const msg = { content: chatmsg.content, role: chatmsg.role === 'apiMessage' ? 'bot' : 'user', time: chatmsg.createdDate } + if (filePaths.length) msg.filePaths = filePaths if (chatmsg.sourceDocuments) msg.sourceDocuments = JSON.parse(chatmsg.sourceDocuments) if (chatmsg.usedTools) msg.usedTools = JSON.parse(chatmsg.usedTools) if (chatmsg.fileAnnotations) msg.fileAnnotations = JSON.parse(chatmsg.fileAnnotations) @@ -249,6 +278,14 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { }) } } + if (chatmsg.fileUploads) { + chatmsg.fileUploads = JSON.parse(chatmsg.fileUploads) + chatmsg.fileUploads.forEach((file) => { + if (file.type === 'stored-file') { + file.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatmsg.chatflowid}&chatId=${chatmsg.chatId}&fileName=${file.name}` + } + }) + } const obj = { ...chatmsg, message: chatmsg.content, @@ -357,6 +394,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { useEffect(() => { if (getChatmessageApi.data) { + getStoragePathFromServer.request() + setAllChatLogs(getChatmessageApi.data) const chatPK = processChatLogs(getChatmessageApi.data) setSelectedMessageIndex(0) @@ -593,8 +632,8 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { sx={{ background: message.type === 'apiMessage' ? theme.palette.asyncSelect.main : '', - pl: 1, - pr: 1 + py: '1rem', + px: '1.5rem' }} key={index} style={{ display: 'flex', justifyContent: 'center', alignContent: 'center' }} @@ -644,6 +683,51 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => { })} )} + {message.fileUploads && message.fileUploads.length > 0 && ( +
+ {message.fileUploads.map((item, index) => { + return ( + <> + {item.mime.startsWith('image/') ? ( + + + + ) : ( + // eslint-disable-next-line jsx-a11y/media-has-caption + + )} + + ) + })} +
+ )}
{/* Messages are being rendered in Markdown format */} { + let userAgent = window.navigator.userAgent.toLowerCase(), + macosPlatforms = /(macintosh|macintel|macppc|mac68k|macos)/i, + windowsPlatforms = /(win32|win64|windows|wince)/i, + iosPlatforms = /(iphone|ipad|ipod)/i, + os = null + + if (macosPlatforms.test(userAgent)) { + os = 'macos' + } else if (iosPlatforms.test(userAgent)) { + os = 'ios' + } else if (windowsPlatforms.test(userAgent)) { + os = 'windows' + } else if (/android/.test(userAgent)) { + os = 'android' + } else if (!os && /linux/.test(userAgent)) { + os = 'linux' + } + + return os +} diff --git a/packages/ui/src/views/canvas/CanvasHeader.js b/packages/ui/src/views/canvas/CanvasHeader.js index 85408cd8..a8589f48 100644 --- a/packages/ui/src/views/canvas/CanvasHeader.js +++ b/packages/ui/src/views/canvas/CanvasHeader.js @@ -28,6 +28,7 @@ import useApi from 'hooks/useApi' import { generateExportFlowData } from 'utils/genericHelper' import { uiBaseURL } from 'store/constant' import { SET_CHATFLOW } from 'store/actions' +import SpeechToTextDialog from '../../ui-component/dialog/SpeechToTextDialog' // ==============================|| CANVAS HEADER ||============================== // @@ -46,6 +47,8 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl const [apiDialogProps, setAPIDialogProps] = useState({}) const [analyseDialogOpen, setAnalyseDialogOpen] = useState(false) const [analyseDialogProps, setAnalyseDialogProps] = useState({}) + const [speechToAudioDialogOpen, setSpeechToAudioDialogOpen] = useState(false) + const [speechToAudioDialogProps, setSpeechToAudioialogProps] = useState({}) const [conversationStartersDialogOpen, setConversationStartersDialogOpen] = useState(false) const [conversationStartersDialogProps, setConversationStartersDialogProps] = useState({}) const [viewMessagesDialogOpen, setViewMessagesDialogOpen] = useState(false) @@ -71,6 +74,12 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl chatflow: chatflow }) setAnalyseDialogOpen(true) + } else if (setting === 'enableSpeechToText') { + setSpeechToAudioialogProps({ + title: 'Speech to Text', + chatflow: chatflow + }) + setSpeechToAudioDialogOpen(true) } else if (setting === 'viewMessages') { setViewMessagesDialogProps({ title: 'View Messages', @@ -385,6 +394,11 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl /> setAPIDialogOpen(false)} /> setAnalyseDialogOpen(false)} /> + setSpeechToAudioDialogOpen(false)} + /> { + setCredentialId(data?.credential ?? '') + }, [data]) + return (
{inputParam && ( diff --git a/packages/ui/src/views/chatmessage/ChatExpandDialog.js b/packages/ui/src/views/chatmessage/ChatExpandDialog.js index 1b2037a8..9b526e56 100644 --- a/packages/ui/src/views/chatmessage/ChatExpandDialog.js +++ b/packages/ui/src/views/chatmessage/ChatExpandDialog.js @@ -7,7 +7,7 @@ import { ChatMessage } from './ChatMessage' import { StyledButton } from 'ui-component/button/StyledButton' import { IconEraser } from '@tabler/icons' -const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel }) => { +const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel, previews, setPreviews }) => { const portalElement = document.getElementById('portal') const customization = useSelector((state) => state.customization) @@ -21,7 +21,7 @@ const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel }) => { aria-describedby='alert-dialog-description' sx={{ overflow: 'visible' }} > - +
{dialogProps.title}
@@ -43,8 +43,17 @@ const ChatExpandDialog = ({ show, dialogProps, onClear, onCancel }) => { )}
- - + + ) : null @@ -56,7 +65,9 @@ ChatExpandDialog.propTypes = { show: PropTypes.bool, dialogProps: PropTypes.object, onClear: PropTypes.func, - onCancel: PropTypes.func + onCancel: PropTypes.func, + previews: PropTypes.array, + setPreviews: PropTypes.func } export default ChatExpandDialog diff --git a/packages/ui/src/views/chatmessage/ChatMessage.css b/packages/ui/src/views/chatmessage/ChatMessage.css index 2298fee6..e1646c80 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.css +++ b/packages/ui/src/views/chatmessage/ChatMessage.css @@ -1,8 +1,6 @@ .messagelist { width: 100%; - height: 100%; - overflow-y: scroll; - overflow-x: hidden; + height: auto; border-radius: 0.5rem; } @@ -108,31 +106,57 @@ } .center { + width: 100%; display: flex; justify-content: center; align-items: center; position: relative; flex-direction: column; - padding: 10px; + padding: 12px; } -.cloud { +.cloud-wrapper { width: 400px; - height: calc(100vh - 260px); - border-radius: 0.5rem; - display: flex; - justify-content: center; - align-items: center; + height: calc(100vh - 180px); } +.cloud-dialog-wrapper { + width: 100%; + height: calc(100vh - 120px); +} + +.cloud-wrapper > div, +.cloud-dialog-wrapper > div { + width: 100%; + height: 100%; + display: flex; + align-items: flex-start; + justify-content: flex-start; + flex-direction: column; + position: relative; +} + +.image-dropzone { + position: absolute; + width: 100%; + height: 100%; + top: 0; + left: 0; + bottom: 0; + right: 0; + z-index: 2001; /* Ensure it's above other content */ +} + +.cloud, .cloud-dialog { width: 100%; - height: 100vh; + height: auto; + max-height: calc(100% - 54px); overflow-y: scroll; - border-radius: 0.5rem; display: flex; justify-content: center; - align-items: center; + align-items: flex-start; + flex-grow: 1; } .cloud-message { @@ -144,3 +168,38 @@ justify-content: center; align-items: center; } + +.preview { + position: absolute; + bottom: 0; + z-index: 1000; + display: flex; + overflow-x: auto; + -webkit-overflow-scrolling: touch; /* For momentum scroll on mobile devices */ + scrollbar-width: none; /* For Firefox */ +} + +.file-drop-field { + position: relative; /* Needed to position the icon correctly */ + /* Other styling for the field */ +} + +.drop-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: rgba(137, 134, 134, 0.83); /* Semi-transparent white */ + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + z-index: 2000; /* Ensure it's above other content */ + border: 2px dashed #0094ff; /* Example style */ +} + +.center audio { + height: 100%; + border-radius: 0; +} diff --git a/packages/ui/src/views/chatmessage/ChatMessage.js b/packages/ui/src/views/chatmessage/ChatMessage.js index b52ff2da..75a466d3 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.js +++ b/packages/ui/src/views/chatmessage/ChatMessage.js @@ -1,4 +1,4 @@ -import { useState, useRef, useEffect, useCallback } from 'react' +import { useState, useRef, useEffect, useCallback, Fragment } from 'react' import { useSelector } from 'react-redux' import PropTypes from 'prop-types' import socketIOClient from 'socket.io-client' @@ -9,15 +9,34 @@ import remarkGfm from 'remark-gfm' import remarkMath from 'remark-math' import axios from 'axios' -import { CircularProgress, OutlinedInput, Divider, InputAdornment, IconButton, Box, Chip, Button } from '@mui/material' +import { + Box, + Button, + Card, + CardMedia, + Chip, + CircularProgress, + Divider, + IconButton, + InputAdornment, + OutlinedInput, + Typography +} from '@mui/material' import { useTheme } from '@mui/material/styles' -import { IconSend, IconDownload } from '@tabler/icons' +import { IconCircleDot, IconDownload, IconSend, IconMicrophone, IconPhotoPlus, IconTrash, IconX } from '@tabler/icons' +import robotPNG from 'assets/images/robot.png' +import userPNG from 'assets/images/account.png' +import audioUploadSVG from 'assets/images/wave-sound.jpg' // project import import { CodeBlock } from 'ui-component/markdown/CodeBlock' import { MemoizedReactMarkdown } from 'ui-component/markdown/MemoizedReactMarkdown' import SourceDocDialog from 'ui-component/dialog/SourceDocDialog' +import StarterPromptsCard from 'ui-component/cards/StarterPromptsCard' +import { cancelAudioRecording, startAudioRecording, stopAudioRecording } from './audio-recording' +import { ImageButton, ImageSrc, ImageBackdrop, ImageMarked } from 'ui-component/button/ImageButton' import './ChatMessage.css' +import './audio-recording.css' // api import chatmessageApi from 'api/chatmessage' @@ -30,12 +49,16 @@ import useApi from 'hooks/useApi' // Const import { baseURL, maxScroll } from 'store/constant' -import robotPNG from 'assets/images/robot.png' -import userPNG from 'assets/images/account.png' -import StarterPromptsCard from '../../ui-component/cards/StarterPromptsCard' +// Utils import { isValidURL, removeDuplicateURL, setLocalStorageChatflow } from 'utils/genericHelper' -export const ChatMessage = ({ open, chatflowid, isDialog }) => { +const messageImageStyle = { + width: '128px', + height: '128px', + objectFit: 'cover' +} + +export const ChatMessage = ({ open, chatflowid, isDialog, previews, setPreviews }) => { const theme = useTheme() const customization = useSelector((state) => state.customization) @@ -51,6 +74,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { ]) const [socketIOClientId, setSocketIOClientId] = useState('') const [isChatFlowAvailableToStream, setIsChatFlowAvailableToStream] = useState(false) + const [isChatFlowAvailableForSpeech, setIsChatFlowAvailableForSpeech] = useState(false) const [sourceDialogOpen, setSourceDialogOpen] = useState(false) const [sourceDialogProps, setSourceDialogProps] = useState({}) const [chatId, setChatId] = useState(undefined) @@ -58,10 +82,220 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { const inputRef = useRef(null) const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow) const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming) + const getAllowChatFlowUploads = useApi(chatflowsApi.getAllowChatflowUploads) const getChatflowConfig = useApi(chatflowsApi.getSpecificChatflow) const [starterPrompts, setStarterPrompts] = useState([]) + // drag & drop and file input + const fileUploadRef = useRef(null) + const [isChatFlowAvailableForUploads, setIsChatFlowAvailableForUploads] = useState(false) + const [isDragActive, setIsDragActive] = useState(false) + + // recording + const [isRecording, setIsRecording] = useState(false) + const [recordingNotSupported, setRecordingNotSupported] = useState(false) + const [isLoadingRecording, setIsLoadingRecording] = useState(false) + + const isFileAllowedForUpload = (file) => { + const constraints = getAllowChatFlowUploads.data + /** + * {isImageUploadAllowed: boolean, imgUploadSizeAndTypes: Array<{ fileTypes: string[], maxUploadSize: number }>} + */ + let acceptFile = false + if (constraints.isImageUploadAllowed) { + const fileType = file.type + const sizeInMB = file.size / 1024 / 1024 + constraints.imgUploadSizeAndTypes.map((allowed) => { + if (allowed.fileTypes.includes(fileType) && sizeInMB <= allowed.maxUploadSize) { + acceptFile = true + } + }) + } + if (!acceptFile) { + alert(`Cannot upload file. Kindly check the allowed file types and maximum allowed size.`) + } + return acceptFile + } + + const handleDrop = async (e) => { + if (!isChatFlowAvailableForUploads) { + return + } + e.preventDefault() + setIsDragActive(false) + let files = [] + if (e.dataTransfer.files.length > 0) { + for (const file of e.dataTransfer.files) { + if (isFileAllowedForUpload(file) === false) { + return + } + const reader = new FileReader() + const { name } = file + files.push( + new Promise((resolve) => { + reader.onload = (evt) => { + if (!evt?.target?.result) { + return + } + const { result } = evt.target + let previewUrl + if (file.type.startsWith('audio/')) { + previewUrl = audioUploadSVG + } else if (file.type.startsWith('image/')) { + previewUrl = URL.createObjectURL(file) + } + resolve({ + data: result, + preview: previewUrl, + type: 'file', + name: name, + mime: file.type + }) + } + reader.readAsDataURL(file) + }) + ) + } + + const newFiles = await Promise.all(files) + setPreviews((prevPreviews) => [...prevPreviews, ...newFiles]) + } + + if (e.dataTransfer.items) { + for (const item of e.dataTransfer.items) { + if (item.kind === 'string' && item.type.match('^text/uri-list')) { + item.getAsString((s) => { + let upload = { + data: s, + preview: s, + type: 'url', + name: s.substring(s.lastIndexOf('/') + 1) + } + setPreviews((prevPreviews) => [...prevPreviews, upload]) + }) + } else if (item.kind === 'string' && item.type.match('^text/html')) { + item.getAsString((s) => { + if (s.indexOf('href') === -1) return + //extract href + let start = s.substring(s.indexOf('href') + 6) + let hrefStr = start.substring(0, start.indexOf('"')) + + let upload = { + data: hrefStr, + preview: hrefStr, + type: 'url', + name: hrefStr.substring(hrefStr.lastIndexOf('/') + 1) + } + setPreviews((prevPreviews) => [...prevPreviews, upload]) + }) + } + } + } + } + + const handleFileChange = async (event) => { + const fileObj = event.target.files && event.target.files[0] + if (!fileObj) { + return + } + let files = [] + for (const file of event.target.files) { + if (isFileAllowedForUpload(file) === false) { + return + } + const reader = new FileReader() + const { name } = file + files.push( + new Promise((resolve) => { + reader.onload = (evt) => { + if (!evt?.target?.result) { + return + } + const { result } = evt.target + resolve({ + data: result, + preview: URL.createObjectURL(file), + type: 'file', + name: name, + mime: file.type + }) + } + reader.readAsDataURL(file) + }) + ) + } + + const newFiles = await Promise.all(files) + setPreviews((prevPreviews) => [...prevPreviews, ...newFiles]) + // 👇️ reset file input + event.target.value = null + } + + const addRecordingToPreviews = (blob) => { + const mimeType = blob.type.substring(0, blob.type.indexOf(';')) + // read blob and add to previews + const reader = new FileReader() + reader.readAsDataURL(blob) + reader.onloadend = () => { + const base64data = reader.result + const upload = { + data: base64data, + preview: audioUploadSVG, + type: 'audio', + name: 'audio.wav', + mime: mimeType + } + setPreviews((prevPreviews) => [...prevPreviews, upload]) + } + } + + const handleDrag = (e) => { + if (isChatFlowAvailableForUploads) { + e.preventDefault() + e.stopPropagation() + if (e.type === 'dragenter' || e.type === 'dragover') { + setIsDragActive(true) + } else if (e.type === 'dragleave') { + setIsDragActive(false) + } + } + } + + const handleDeletePreview = (itemToDelete) => { + if (itemToDelete.type === 'file') { + URL.revokeObjectURL(itemToDelete.preview) // Clean up for file + } + setPreviews(previews.filter((item) => item !== itemToDelete)) + } + + const handleUploadClick = () => { + // 👇️ open file input box on click of another element + fileUploadRef.current.click() + } + + const clearPreviews = () => { + // Revoke the data uris to avoid memory leaks + previews.forEach((file) => URL.revokeObjectURL(file.preview)) + setPreviews([]) + } + + const onMicrophonePressed = () => { + setIsRecording(true) + startAudioRecording(setIsRecording, setRecordingNotSupported) + } + + const onRecordingCancelled = () => { + if (!recordingNotSupported) cancelAudioRecording() + setIsRecording(false) + setRecordingNotSupported(false) + } + + const onRecordingStopped = async () => { + setIsLoadingRecording(true) + stopAudioRecording(addRecordingToPreviews) + } + const onSourceDialogClick = (data, title) => { setSourceDialogProps({ data, title }) setSourceDialogOpen(true) @@ -118,7 +352,10 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { if (e) e.preventDefault() if (!promptStarterInput && userInput.trim() === '') { - return + const containsAudio = previews.filter((item) => item.type === 'audio').length > 0 + if (!(previews.length >= 1 && containsAudio)) { + return + } } let input = userInput @@ -126,7 +363,16 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { if (promptStarterInput !== undefined && promptStarterInput.trim() !== '') input = promptStarterInput setLoading(true) - setMessages((prevMessages) => [...prevMessages, { message: input, type: 'userMessage' }]) + const urls = previews.map((item) => { + return { + data: item.data, + type: item.type, + name: item.name, + mime: item.mime + } + }) + clearPreviews() + setMessages((prevMessages) => [...prevMessages, { message: input, type: 'userMessage', fileUploads: urls }]) // Send user question and history to API try { @@ -135,6 +381,7 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { history: messages.filter((msg) => msg.message !== 'Hi there! How can I help?'), chatId } + if (urls && urls.length > 0) params.uploads = urls if (isChatFlowAvailableToStream) params.socketIOClientId = socketIOClientId const response = await predictionApi.sendMessageAndGetPrediction(chatflowid, params) @@ -144,6 +391,17 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { if (!chatId) setChatId(data.chatId) + if (input === '' && data.question) { + // the response contains the question even if it was in an audio format + // so if input is empty but the response contains the question, update the user message to show the question + setMessages((prevMessages) => { + let allMessages = [...cloneDeep(prevMessages)] + if (allMessages[allMessages.length - 2].type === 'apiMessage') return allMessages + allMessages[allMessages.length - 2].message = data.question + return allMessages + }) + } + if (!isChatFlowAvailableToStream) { let text = '' if (data.text) text = data.text @@ -222,6 +480,14 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments) if (message.usedTools) obj.usedTools = JSON.parse(message.usedTools) if (message.fileAnnotations) obj.fileAnnotations = JSON.parse(message.fileAnnotations) + if (message.fileUploads) { + obj.fileUploads = JSON.parse(message.fileUploads) + obj.fileUploads.forEach((file) => { + if (file.type === 'stored-file') { + file.data = `${baseURL}/api/v1/get-upload-file?chatflowId=${chatflowid}&chatId=${chatId}&fileName=${file.name}` + } + }) + } return obj }) setMessages((prevMessages) => [...prevMessages, ...loadedMessages]) @@ -239,6 +505,15 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [getIsChatflowStreamingApi.data]) + // Get chatflow uploads capability + useEffect(() => { + if (getAllowChatFlowUploads.data) { + setIsChatFlowAvailableForUploads(getAllowChatFlowUploads.data?.isImageUploadAllowed ?? false) + setIsChatFlowAvailableForSpeech(getAllowChatFlowUploads.data?.isSpeechToTextEnabled ?? false) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getAllowChatFlowUploads.data]) + useEffect(() => { if (getChatflowConfig.data) { if (getChatflowConfig.data?.chatbotConfig && JSON.parse(getChatflowConfig.data?.chatbotConfig)) { @@ -273,11 +548,18 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { useEffect(() => { let socket if (open && chatflowid) { + // API request getChatmessageApi.request(chatflowid) getIsChatflowStreamingApi.request(chatflowid) + getAllowChatFlowUploads.request(chatflowid) getChatflowConfig.request(chatflowid) + + // Scroll to bottom scrollToBottom() + setIsRecording(false) + + // SocketIO socket = socketIOClient(baseURL) socket.on('connect', () => { @@ -311,147 +593,330 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [open, chatflowid]) + useEffect(() => { + // wait for audio recording to load and then send + const containsAudio = previews.filter((item) => item.type === 'audio').length > 0 + if (previews.length >= 1 && containsAudio) { + setIsRecording(false) + setRecordingNotSupported(false) + handlePromptClick('') + } + // eslint-disable-next-line + }, [previews]) + return ( - <> -
-
+
+ {isDragActive && ( +
+ )} + {isDragActive && getAllowChatFlowUploads.data?.isImageUploadAllowed && ( + + Drop here to upload + {getAllowChatFlowUploads.data.imgUploadSizeAndTypes.map((allowed) => { + return ( + <> + {allowed.fileTypes?.join(', ')} + Max Allowed Size: {allowed.maxUploadSize} MB + + ) + })} + + )} +
+
{messages && messages.map((message, index) => { return ( // The latest message sent by the user will be animated while waiting for a response - <> - - {/* Display the correct icon depending on the message type */} - {message.type === 'apiMessage' ? ( - AI - ) : ( - Me - )} -
- {message.usedTools && ( -
- {message.usedTools.map((tool, index) => { - return ( - onSourceDialogClick(tool, 'Used Tools')} - /> - ) - })} -
- )} -
- {/* Messages are being rendered in Markdown format */} - - ) : ( - - {children} - - ) - } - }} - > - {message.message} - + + {/* Display the correct icon depending on the message type */} + {message.type === 'apiMessage' ? ( + AI + ) : ( + Me + )} +
+ {message.usedTools && ( +
+ {message.usedTools.map((tool, index) => { + return ( + onSourceDialogClick(tool, 'Used Tools')} + /> + ) + })}
- {message.fileAnnotations && ( -
- {message.fileAnnotations.map((fileAnnotation, index) => { - return ( - - ) - })} -
- )} - {message.sourceDocuments && ( -
- {removeDuplicateURL(message).map((source, index) => { - const URL = - source.metadata && source.metadata.source - ? isValidURL(source.metadata.source) - : undefined - return ( - - URL ? onURLClick(source.metadata.source) : onSourceDialogClick(source) - } + )} + {message.fileUploads && message.fileUploads.length > 0 && ( +
+ {message.fileUploads.map((item, index) => { + return ( + <> + {item.mime.startsWith('image/') ? ( + + + + ) : ( + // eslint-disable-next-line jsx-a11y/media-has-caption + + )} + + ) + })} +
+ )} +
+ {/* Messages are being rendered in Markdown format */} + + ) : ( + + {children} + ) - })} -
- )} + } + }} + > + {message.message} +
- - + {message.fileAnnotations && ( +
+ {message.fileAnnotations.map((fileAnnotation, index) => { + return ( + + ) + })} +
+ )} + {message.sourceDocuments && ( +
+ {removeDuplicateURL(message).map((source, index) => { + const URL = + source.metadata && source.metadata.source + ? isValidURL(source.metadata.source) + : undefined + return ( + + URL ? onURLClick(source.metadata.source) : onSourceDialogClick(source) + } + /> + ) + })} +
+ )} +
+ ) })}
-
- {messages && messages.length === 1 && ( - - )} - -
+ {messages && messages.length === 1 && starterPrompts.length > 0 && ( +
+ 0 ? 70 : 0 }} + starterPrompts={starterPrompts || []} + onPromptClick={handlePromptClick} + isGrid={isDialog} + /> +
+ )} + + +
-
+ {previews && previews.length > 0 && ( + + {previews.map((item, index) => ( + + {item.mime.startsWith('image/') ? ( + handleDeletePreview(item)} + > + + + + + + + ) : ( + + + handleDeletePreview(item)} size='small'> + + + + )} + + ))} + + )} + {isRecording ? ( + <> + {recordingNotSupported ? ( +
+
+ + To record audio, use modern browsers like Chrome or Firefox that support audio recording. + + +
+
+ ) : ( + +
+ + + + 00:00 + {isLoadingRecording && Sending...} +
+
+ + + + + + +
+ + )} + + ) : (
{ onChange={onChange} multiline={true} maxRows={isDialog ? 7 : 2} - endAdornment={ - - - {loading ? ( -
- -
- ) : ( - // Send icon SVG in input field - + + - )} - - + + + ) + } + endAdornment={ + <> + {isChatFlowAvailableForSpeech && ( + + onMicrophonePressed()} + type='button' + disabled={loading || !chatflowid} + edge='end' + > + + + + )} + + + {loading ? ( +
+ +
+ ) : ( + // Send icon SVG in input field + + )} + + + } /> + {isChatFlowAvailableForUploads && ( + + )} -
+ )}
setSourceDialogOpen(false)} /> - +
) } ChatMessage.propTypes = { open: PropTypes.bool, chatflowid: PropTypes.string, - isDialog: PropTypes.bool + isDialog: PropTypes.bool, + previews: PropTypes.array, + setPreviews: PropTypes.func } diff --git a/packages/ui/src/views/chatmessage/ChatPopUp.js b/packages/ui/src/views/chatmessage/ChatPopUp.js index 670fb00f..74d4b908 100644 --- a/packages/ui/src/views/chatmessage/ChatPopUp.js +++ b/packages/ui/src/views/chatmessage/ChatPopUp.js @@ -35,6 +35,7 @@ export const ChatPopUp = ({ chatflowid }) => { const [open, setOpen] = useState(false) const [showExpandDialog, setShowExpandDialog] = useState(false) const [expandDialogProps, setExpandDialogProps] = useState({}) + const [previews, setPreviews] = useState([]) const anchorRef = useRef(null) const prevOpen = useRef(open) @@ -191,8 +192,15 @@ export const ChatPopUp = ({ chatflowid }) => { - - + + @@ -204,6 +212,8 @@ export const ChatPopUp = ({ chatflowid }) => { dialogProps={expandDialogProps} onClear={clearChat} onCancel={() => setShowExpandDialog(false)} + previews={previews} + setPreviews={setPreviews} > ) diff --git a/packages/ui/src/views/chatmessage/audio-recording.css b/packages/ui/src/views/chatmessage/audio-recording.css new file mode 100644 index 00000000..c5d9fac9 --- /dev/null +++ b/packages/ui/src/views/chatmessage/audio-recording.css @@ -0,0 +1,249 @@ +/* style.css*/ + +/* Media Queries */ + +/* Small Devices*/ + +@media (min-width: 0px) { + * { + box-sizing: border-box; + } + .start-recording-button { + font-size: 70px; + color: #435f7a; + cursor: pointer; + } + .start-recording-button:hover { + opacity: 1; + } + .recording-control-buttons-container { + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + justify-content: center; + /*horizontal centering*/ + align-items: center; + gap: 12px; + } + .recording-elapsed-time { + font-size: 16px; + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + justify-content: center; + /*horizontal centering*/ + align-items: center; + } + .recording-elapsed-time #elapsed-time { + margin: 0; + } + .recording-indicator-wrapper { + position: relative; + display: flex; + width: 16px; + height: 16px; + } + .red-recording-dot { + font-size: 25px; + color: red; + margin-right: 12px; + /*transitions with Firefox, IE and Opera Support browser support*/ + animation-name: flashing-recording-dot; + -webkit-animation-name: flashing-recording-dot; + -moz-animation-name: flashing-recording-dot; + -o-animation-name: flashing-recording-dot; + animation-duration: 2s; + -webkit-animation-duration: 2s; + -moz-animation-duration: 2s; + -o-animation-duration: 2s; + animation-iteration-count: infinite; + -webkit-animation-iteration-count: infinite; + -moz-animation-iteration-count: infinite; + -o-animation-iteration-count: infinite; + } + /* The animation code */ + @keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + @-webkit-keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + @-moz-keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + @-o-keyframes flashing-recording-dot { + 0% { + opacity: 1; + } + 50% { + opacity: 0; + } + 100% { + opacity: 1; + } + } + .recording-control-buttons-container.hide { + display: none; + } + .overlay { + width: 100%; + height: '54px'; + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + justify-content: center; + /*horizontal centering*/ + align-items: center; + } + .overlay.hide { + display: none; + } + .browser-not-supporting-audio-recording-box { + /*targeting Chrome & Safari*/ + display: -webkit-flex; + /*targeting IE10*/ + display: -ms-flex; + display: flex; + justify-content: space-between; + /*horizontal centering*/ + align-items: center; + width: 100%; + font-size: 16px; + gap: 12px; + } + .browser-not-supporting-audio-recording-box > p { + margin: 0; + } + .close-browser-not-supported-box { + cursor: pointer; + background-color: #abc1c05c; + border-radius: 10px; + font-size: 16px; + border: none; + } + .close-browser-not-supported-box:hover { + background-color: #92a5a45c; + } + .close-browser-not-supported-box:focus { + outline: none; + border: none; + } + .audio-element.hide { + display: none; + } + .text-indication-of-audio-playing-container { + height: 20px; + } + .text-indication-of-audio-playing { + font-size: 20px; + } + .text-indication-of-audio-playing.hide { + display: none; + } + /* 3 Dots animation*/ + .text-indication-of-audio-playing span { + /*transitions with Firefox, IE and Opera Support browser support*/ + animation-name: blinking-dot; + -webkit-animation-name: blinking-dot; + -moz-animation-name: blinking-dot; + -o-animation-name: blinking-dot; + animation-duration: 2s; + -webkit-animation-duration: 2s; + -moz-animation-duration: 2s; + -o-animation-duration: 2s; + animation-iteration-count: infinite; + -webkit-animation-iteration-count: infinite; + -moz-animation-iteration-count: infinite; + -o-animation-iteration-count: infinite; + } + .text-indication-of-audio-playing span:nth-child(2) { + animation-delay: 0.4s; + -webkit-animation-delay: 0.4s; + -moz-animation-delay: 0.4s; + -o-animation-delay: 0.4s; + } + .text-indication-of-audio-playing span:nth-child(3) { + animation-delay: 0.8s; + -webkit-animation-delay: 0.8s; + -moz-animation-delay: 0.8s; + -o-animation-delay: 0.8s; + } + /* The animation code */ + @keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } + /* The animation code */ + @-webkit-keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } + /* The animation code */ + @-moz-keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } + /* The animation code */ + @-o-keyframes blinking-dot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } +} diff --git a/packages/ui/src/views/chatmessage/audio-recording.js b/packages/ui/src/views/chatmessage/audio-recording.js new file mode 100644 index 00000000..1fbaddc1 --- /dev/null +++ b/packages/ui/src/views/chatmessage/audio-recording.js @@ -0,0 +1,337 @@ +/** + * @fileoverview This file contains the API to handle audio recording. + * Originally from 'https://ralzohairi.medium.com/audio-recording-in-javascript-96eed45b75ee' + */ + +// audio-recording.js --------------- +let microphoneButton, elapsedTimeTag + +/** Initialize controls */ +function initializeControls() { + microphoneButton = document.getElementsByClassName('start-recording-button')[0] +} + +/** Displays recording control buttons */ +function handleDisplayingRecordingControlButtons() { + //Hide the microphone button that starts audio recording + microphoneButton.style.display = 'none' + + //Handle the displaying of the elapsed recording time + handleElapsedRecordingTime() +} + +/** Hide the displayed recording control buttons */ +function handleHidingRecordingControlButtons() { + //Display the microphone button that starts audio recording + microphoneButton.style.display = 'block' + + //stop interval that handles both time elapsed and the red dot + clearInterval(elapsedTimeTimer) +} + +/** Stores the actual start time when an audio recording begins to take place to ensure elapsed time start time is accurate*/ +let audioRecordStartTime + +/** Stores the maximum recording time in hours to stop recording once maximum recording hour has been reached */ +let maximumRecordingTimeInHours = 1 + +/** Stores the reference of the setInterval function that controls the timer in audio recording*/ +let elapsedTimeTimer + +/** Starts the audio recording*/ +export function startAudioRecording(onRecordingStart, onUnsupportedBrowser) { + initializeControls() + + //start recording using the audio recording API + audioRecorder + .start() + .then(() => { + //on success show the controls to stop and cancel the recording + if (onRecordingStart) { + onRecordingStart(true) + } + //store the recording start time to display the elapsed time according to it + audioRecordStartTime = new Date() + + //display control buttons to offer the functionality of stop and cancel + handleDisplayingRecordingControlButtons() + }) + .catch((error) => { + //on error + //No Browser Support Error + if (error.message.includes('mediaDevices API or getUserMedia method is not supported in this browser.')) { + if (onUnsupportedBrowser) { + onUnsupportedBrowser(true) + } + } + + //Error handling structure + switch (error.name) { + case 'AbortError': //error from navigator.mediaDevices.getUserMedia + // eslint-disable-next-line no-console + console.log('An AbortError has occurred.') + break + case 'NotAllowedError': //error from navigator.mediaDevices.getUserMedia + // eslint-disable-next-line no-console + console.log('A NotAllowedError has occurred. User might have denied permission.') + break + case 'NotFoundError': //error from navigator.mediaDevices.getUserMedia + // eslint-disable-next-line no-console + console.log('A NotFoundError has occurred.') + break + case 'NotReadableError': //error from navigator.mediaDevices.getUserMedia + // eslint-disable-next-line no-console + console.log('A NotReadableError has occurred.') + break + case 'SecurityError': //error from navigator.mediaDevices.getUserMedia or from the MediaRecorder.start + // eslint-disable-next-line no-console + console.log('A SecurityError has occurred.') + break + case 'TypeError': //error from navigator.mediaDevices.getUserMedia + // eslint-disable-next-line no-console + console.log('A TypeError has occurred.') + break + case 'InvalidStateError': //error from the MediaRecorder.start + // eslint-disable-next-line no-console + console.log('An InvalidStateError has occurred.') + break + case 'UnknownError': //error from the MediaRecorder.start + // eslint-disable-next-line no-console + console.log('An UnknownError has occurred.') + break + default: + // eslint-disable-next-line no-console + console.log('An error occurred with the error name ' + error.name) + } + }) +} +/** Stop the currently started audio recording & sends it + */ +export function stopAudioRecording(addRecordingToPreviews) { + //stop the recording using the audio recording API + audioRecorder + .stop() + .then((audioBlob) => { + //hide recording control button & return record icon + handleHidingRecordingControlButtons() + if (addRecordingToPreviews) { + addRecordingToPreviews(audioBlob) + } + }) + .catch((error) => { + //Error handling structure + switch (error.name) { + case 'InvalidStateError': //error from the MediaRecorder.stop + // eslint-disable-next-line no-console + console.log('An InvalidStateError has occurred.') + break + default: + // eslint-disable-next-line no-console + console.log('An error occurred with the error name ' + error.name) + } + }) +} + +/** Cancel the currently started audio recording */ +export function cancelAudioRecording() { + //cancel the recording using the audio recording API + audioRecorder.cancel() + + //hide recording control button & return record icon + handleHidingRecordingControlButtons() +} + +/** Computes the elapsed recording time since the moment the function is called in the format h:m:s*/ +function handleElapsedRecordingTime() { + elapsedTimeTag = document.getElementById('elapsed-time') + //display initial time when recording begins + displayElapsedTimeDuringAudioRecording('00:00') + + //create an interval that compute & displays elapsed time, as well as, animate red dot - every second + elapsedTimeTimer = setInterval(() => { + //compute the elapsed time every second + let elapsedTime = computeElapsedTime(audioRecordStartTime) //pass the actual record start time + //display the elapsed time + displayElapsedTimeDuringAudioRecording(elapsedTime) + }, 1000) //every second +} + +/** Display elapsed time during audio recording + * @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss + */ +function displayElapsedTimeDuringAudioRecording(elapsedTime) { + //1. display the passed elapsed time as the elapsed time in the elapsedTime HTML element + elapsedTimeTag.innerHTML = elapsedTime + //2. Stop the recording when the max number of hours is reached + if (elapsedTimeReachedMaximumNumberOfHours(elapsedTime)) { + stopAudioRecording() + } +} + +/** + * @param {String} elapsedTime - elapsed time in the format mm:ss or hh:mm:ss + * @returns {Boolean} whether the elapsed time reached the maximum number of hours or not + */ +function elapsedTimeReachedMaximumNumberOfHours(elapsedTime) { + //Split the elapsed time by the symbol that separates the hours, minutes and seconds : + let elapsedTimeSplit = elapsedTime.split(':') + + //Turn the maximum recording time in hours to a string and pad it with zero if less than 10 + let maximumRecordingTimeInHoursAsString = + maximumRecordingTimeInHours < 10 ? '0' + maximumRecordingTimeInHours : maximumRecordingTimeInHours.toString() + + //if the elapsed time reach hours and also reach the maximum recording time in hours return true + return elapsedTimeSplit.length === 3 && elapsedTimeSplit[0] === maximumRecordingTimeInHoursAsString +} + +/** Computes the elapsedTime since the moment the function is called in the format mm:ss or hh:mm:ss + * @param {String} startTime - start time to compute the elapsed time since + * @returns {String} elapsed time in mm:ss format or hh:mm:ss format, if elapsed hours are 0. + */ +function computeElapsedTime(startTime) { + //record end time + let endTime = new Date() + + //time difference in ms + let timeDiff = endTime - startTime + + //convert time difference from ms to seconds + timeDiff = timeDiff / 1000 + + //extract integer seconds that don't form a minute using % + let seconds = Math.floor(timeDiff % 60) //ignoring incomplete seconds (floor) + + //pad seconds with a zero if necessary + seconds = seconds < 10 ? '0' + seconds : seconds + + //convert time difference from seconds to minutes using % + timeDiff = Math.floor(timeDiff / 60) + + //extract integer minutes that don't form an hour using % + let minutes = timeDiff % 60 //no need to floor possible incomplete minutes, because they've been handled as seconds + minutes = minutes < 10 ? '0' + minutes : minutes + + //convert time difference from minutes to hours + timeDiff = Math.floor(timeDiff / 60) + + //extract integer hours that don't form a day using % + let hours = timeDiff % 24 //no need to floor possible incomplete hours, because they've been handled as seconds + + //convert time difference from hours to days + timeDiff = Math.floor(timeDiff / 24) + + // the rest of timeDiff is number of days + let days = timeDiff //add days to hours + + let totalHours = hours + days * 24 + totalHours = totalHours < 10 ? '0' + totalHours : totalHours + + if (totalHours === '00') { + return minutes + ':' + seconds + } else { + return totalHours + ':' + minutes + ':' + seconds + } +} + +//API to handle audio recording + +export const audioRecorder = { + /** Stores the recorded audio as Blob objects of audio data as the recording continues*/ + audioBlobs: [] /*of type Blob[]*/, + /** Stores the reference of the MediaRecorder instance that handles the MediaStream when recording starts*/ + mediaRecorder: null /*of type MediaRecorder*/, + /** Stores the reference to the stream currently capturing the audio*/ + streamBeingCaptured: null /*of type MediaStream*/, + /** Start recording the audio + * @returns {Promise} - returns a promise that resolves if audio recording successfully started + */ + start: function () { + //Feature Detection + if (!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia)) { + //Feature is not supported in browser + //return a custom error + return Promise.reject(new Error('mediaDevices API or getUserMedia method is not supported in this browser.')) + } else { + //Feature is supported in browser + + //create an audio stream + return ( + navigator.mediaDevices + .getUserMedia({ audio: true } /*of type MediaStreamConstraints*/) + //returns a promise that resolves to the audio stream + .then((stream) /*of type MediaStream*/ => { + //save the reference of the stream to be able to stop it when necessary + audioRecorder.streamBeingCaptured = stream + + //create a media recorder instance by passing that stream into the MediaRecorder constructor + audioRecorder.mediaRecorder = new MediaRecorder(stream) + /*the MediaRecorder interface of the MediaStream Recording API provides functionality to easily record media*/ + + //clear previously saved audio Blobs, if any + audioRecorder.audioBlobs = [] + + //add a dataavailable event listener in order to store the audio data Blobs when recording + audioRecorder.mediaRecorder.addEventListener('dataavailable', (event) => { + //store audio Blob object + audioRecorder.audioBlobs.push(event.data) + }) + + //start the recording by calling the start method on the media recorder + audioRecorder.mediaRecorder.start() + }) + ) + + /* errors are not handled in the API because if its handled and the promise is chained, the .then after the catch will be executed*/ + } + }, + /** Stop the started audio recording + * @returns {Promise} - returns a promise that resolves to the audio as a blob file + */ + stop: function () { + //return a promise that would return the blob or URL of the recording + return new Promise((resolve) => { + //save audio type to pass to set the Blob type + let mimeType = audioRecorder.mediaRecorder.mimeType + + //listen to the stop event in order to create & return a single Blob object + audioRecorder.mediaRecorder.addEventListener('stop', () => { + //create a single blob object, as we might have gathered a few Blob objects that needs to be joined as one + let audioBlob = new Blob(audioRecorder.audioBlobs, { type: mimeType }) + + //resolve promise with the single audio blob representing the recorded audio + resolve(audioBlob) + }) + audioRecorder.cancel() + }) + }, + /** Cancel audio recording*/ + cancel: function () { + //stop the recording feature + audioRecorder.mediaRecorder.stop() + + //stop all the tracks on the active stream in order to stop the stream + audioRecorder.stopStream() + + //reset API properties for next recording + audioRecorder.resetRecordingProperties() + }, + /** Stop all the tracks on the active stream in order to stop the stream and remove + * the red flashing dot showing in the tab + */ + stopStream: function () { + //stopping the capturing request by stopping all the tracks on the active stream + audioRecorder.streamBeingCaptured + .getTracks() //get all tracks from the stream + .forEach((track) /*of type MediaStreamTrack*/ => track.stop()) //stop each one + }, + /** Reset all the recording properties including the media recorder and stream being captured*/ + resetRecordingProperties: function () { + audioRecorder.mediaRecorder = null + audioRecorder.streamBeingCaptured = null + + /*No need to remove event listeners attached to mediaRecorder as + If a DOM element which is removed is reference-free (no references pointing to it), the element itself is picked + up by the garbage collector as well as any event handlers/listeners associated with it. + getEventListeners(audioRecorder.mediaRecorder) will return an empty array of events.*/ + } +} diff --git a/packages/ui/src/views/settings/index.js b/packages/ui/src/views/settings/index.js index 8d76cc0d..f7019b68 100644 --- a/packages/ui/src/views/settings/index.js +++ b/packages/ui/src/views/settings/index.js @@ -1,9 +1,11 @@ -import { useState, useEffect } from 'react' +import { useState, useEffect, useRef } from 'react' import PropTypes from 'prop-types' +import { useSelector } from 'react-redux' // material-ui import { useTheme } from '@mui/material/styles' -import { Box, List, Paper, Popper, ClickAwayListener } from '@mui/material' +import { ListItemButton, ListItemIcon, ListItemText, Typography, Box, List, Paper, Popper, ClickAwayListener } from '@mui/material' +import FiberManualRecordIcon from '@mui/icons-material/FiberManualRecord' // third-party import PerfectScrollbar from 'react-perfect-scrollbar' @@ -11,8 +13,6 @@ import PerfectScrollbar from 'react-perfect-scrollbar' // project imports import MainCard from 'ui-component/cards/MainCard' import Transitions from 'ui-component/extended/Transitions' -import NavItem from 'layout/MainLayout/Sidebar/MenuList/NavItem' - import settings from 'menu-items/settings' // ==============================|| SETTINGS ||============================== // @@ -20,9 +20,26 @@ import settings from 'menu-items/settings' const Settings = ({ chatflow, isSettingsOpen, anchorEl, onSettingsItemClick, onUploadFile, onClose }) => { const theme = useTheme() const [settingsMenu, setSettingsMenu] = useState([]) - + const customization = useSelector((state) => state.customization) + const inputFile = useRef(null) const [open, setOpen] = useState(false) + const handleFileUpload = (e) => { + if (!e.target.files) return + + const file = e.target.files[0] + + const reader = new FileReader() + reader.onload = (evt) => { + if (!evt?.target?.result) { + return + } + const { result } = evt.target + onUploadFile(result) + } + reader.readAsText(file) + } + useEffect(() => { if (chatflow && !chatflow.id) { const settingsMenu = settings.children.filter((menu) => menu.id === 'loadChatflow') @@ -39,16 +56,40 @@ const Settings = ({ chatflow, isSettingsOpen, anchorEl, onSettingsItemClick, onU // settings list items const items = settingsMenu.map((menu) => { - return ( - onSettingsItemClick(id)} - onUploadFile={onUploadFile} + const Icon = menu.icon + const itemIcon = menu?.icon ? ( + + ) : ( + id === menu?.id) > -1 ? 8 : 6, + height: customization.isOpen.findIndex((id) => id === menu?.id) > -1 ? 8 : 6 + }} + fontSize={level > 0 ? 'inherit' : 'medium'} /> ) + return ( + { + if (menu.id === 'loadChatflow' && inputFile) { + inputFile?.current.click() + } else { + onSettingsItemClick(menu.id) + } + }} + > + {itemIcon} + {menu.title}} /> + + ) }) return ( @@ -82,6 +123,14 @@ const Settings = ({ chatflow, isSettingsOpen, anchorEl, onSettingsItemClick, onU {items} + handleFileUpload(e)} + />