From 23c62bdc0b0a7b7465737c851b3d4681d969eaea Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sun, 22 Oct 2023 11:52:06 +0530 Subject: [PATCH] Redis Vector Store - addition of similaritySearchVectorWithScore method and other updates --- .../vectorstores/Redis/RedisSearchBase.ts | 104 +++++++++++++++++- .../vectorstores/Redis/Redis_Existing.ts | 12 +- .../nodes/vectorstores/Redis/Redis_Upsert.ts | 19 +++- .../nodes/vectorstores/Redis/utils.ts | 25 +++++ 4 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 packages/components/nodes/vectorstores/Redis/utils.ts diff --git a/packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts b/packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts index c47c1e51..e7fb5f11 100644 --- a/packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts +++ b/packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts @@ -11,8 +11,9 @@ import { import { Embeddings } from 'langchain/embeddings/base' import { VectorStore } from 'langchain/vectorstores/base' import { Document } from 'langchain/document' -import { createClient } from 'redis' +import { createClient, SearchOptions } from 'redis' import { RedisVectorStore } from 'langchain/vectorstores/redis' +import { escapeSpecialChars, unEscapeSpecialChars } from './utils' export abstract class RedisSearchBase { label: string @@ -51,6 +52,40 @@ export abstract class RedisSearchBase { placeholder: '', type: 'string' }, + { + label: 'Delete and Recreate the Index (will remove all contents as well) ?', + name: 'deleteIndex', + description: 'Delete the index if it already exists', + default: false, + type: 'boolean' + }, + { + label: 'Content Field', + name: 'contentKey', + description: 'Name of the field (column) that contains the actual content', + type: 'string', + default: 'content', + additionalParams: true, + optional: true + }, + { + label: 'Metadata Field', + name: 'metadataKey', + description: 'Name of the field (column) that contains the metadata of the document', + type: 'string', + default: 'metadata', + additionalParams: true, + optional: true + }, + { + label: 'Vector Field', + name: 'vectorKey', + description: 'Name of the field (column) that contains the vector', + type: 'string', + default: 'content_vector', + additionalParams: true, + optional: true + }, { label: 'Top K', name: 'topK', @@ -78,14 +113,19 @@ export abstract class RedisSearchBase { abstract constructVectorStore( embeddings: Embeddings, indexName: string, + deleteIndex: boolean, docs: Document>[] | undefined ): Promise async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document>[] | undefined): Promise { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const indexName = nodeData.inputs?.indexName as string + let contentKey = nodeData.inputs?.contentKey as string + let metadataKey = nodeData.inputs?.metadataKey as string + let vectorKey = nodeData.inputs?.vectorKey as string const embeddings = nodeData.inputs?.embeddings as Embeddings const topK = nodeData.inputs?.topK as string + const deleteIndex = nodeData.inputs?.deleteIndex as boolean const k = topK ? parseFloat(topK) : 4 const output = nodeData.outputs?.output as string @@ -102,7 +142,67 @@ export abstract class RedisSearchBase { this.redisClient = createClient({ url: redisUrl }) await this.redisClient.connect() - const vectorStore = await this.constructVectorStore(embeddings, indexName, docs) + const vectorStore = await this.constructVectorStore(embeddings, indexName, deleteIndex, docs) + if (!contentKey || contentKey === '') contentKey = 'content' + if (!metadataKey || metadataKey === '') metadataKey = 'metadata' + if (!vectorKey || vectorKey === '') vectorKey = 'content_vector' + + const buildQuery = (query: number[], k: number, filter?: string[]): [string, SearchOptions] => { + const vectorScoreField = 'vector_score' + + let hybridFields = '*' + // if a filter is set, modify the hybrid query + if (filter && filter.length) { + // `filter` is a list of strings, then it's applied using the OR operator in the metadata key + hybridFields = `@${metadataKey}:(${filter.map(escapeSpecialChars).join('|')})` + } + + const baseQuery = `${hybridFields} => [KNN ${k} @${vectorKey} $vector AS ${vectorScoreField}]` + const returnFields = [metadataKey, contentKey, vectorScoreField] + + const options: SearchOptions = { + PARAMS: { + vector: Buffer.from(new Float32Array(query).buffer) + }, + RETURN: returnFields, + SORTBY: vectorScoreField, + DIALECT: 2, + LIMIT: { + from: 0, + size: k + } + } + + return [baseQuery, options] + } + + vectorStore.similaritySearchVectorWithScore = async ( + query: number[], + k: number, + filter?: string[] + ): Promise<[Document, number][]> => { + const results = await this.redisClient.ft.search(indexName, ...buildQuery(query, k, filter)) + const result: [Document, number][] = [] + + if (results.total) { + for (const res of results.documents) { + if (res.value) { + const document = res.value + if (document.vector_score) { + const metadataString = unEscapeSpecialChars(document[metadataKey] as string) + result.push([ + new Document({ + pageContent: document[contentKey] as string, + metadata: JSON.parse(metadataString) + }), + Number(document.vector_score) + ]) + } + } + } + } + return result + } if (output === 'retriever') { return vectorStore.asRetriever(k) diff --git a/packages/components/nodes/vectorstores/Redis/Redis_Existing.ts b/packages/components/nodes/vectorstores/Redis/Redis_Existing.ts index 88166659..754e46c8 100644 --- a/packages/components/nodes/vectorstores/Redis/Redis_Existing.ts +++ b/packages/components/nodes/vectorstores/Redis/Redis_Existing.ts @@ -13,9 +13,19 @@ class RedisExisting_VectorStores extends RedisSearchBase implements INode { this.name = 'RedisIndex' this.version = 1.0 this.description = 'Load existing index from Redis (i.e: Document has been upserted)' + + // Remove deleteIndex from inputs as it is not applicable while fetching data from Redis + let input = this.inputs.find((i) => i.name === 'deleteIndex') + if (input) this.inputs.splice(this.inputs.indexOf(input), 1) } - async constructVectorStore(embeddings: Embeddings, indexName: string, _: Document>[]): Promise { + async constructVectorStore( + embeddings: Embeddings, + indexName: string, + // eslint-disable-next-line unused-imports/no-unused-vars + deleteIndex: boolean, + _: Document>[] + ): Promise { const storeConfig: RedisVectorStoreConfig = { redisClient: this.redisClient, indexName: indexName diff --git a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts index 2af445eb..6637e703 100644 --- a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts +++ b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts @@ -6,6 +6,7 @@ import { flatten } from 'lodash' import { RedisSearchBase } from './RedisSearchBase' import { VectorStore } from 'langchain/vectorstores/base' import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis' +import { escapeAllStrings } from './utils' class RedisUpsert_VectorStores extends RedisSearchBase implements INode { constructor() { @@ -22,11 +23,23 @@ class RedisUpsert_VectorStores extends RedisSearchBase implements INode { }) } - async constructVectorStore(embeddings: Embeddings, indexName: string, docs: Document>[]): Promise { + async constructVectorStore( + embeddings: Embeddings, + indexName: string, + deleteIndex: boolean, + docs: Document>[] + ): Promise { const storeConfig: RedisVectorStoreConfig = { redisClient: this.redisClient, indexName: indexName } + if (deleteIndex) { + let response = await this.redisClient.ft.dropIndex(indexName) + if (process.env.DEBUG === 'true') { + // eslint-disable-next-line no-console + console.log(`Redis Vector Store :: Dropping index [${indexName}], Received Response [${response}]`) + } + } return await RedisVectorStore.fromDocuments(docs, embeddings, storeConfig) } @@ -36,7 +49,9 @@ class RedisUpsert_VectorStores extends RedisSearchBase implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + const document = new Document(flattenDocs[i]) + escapeAllStrings(document.metadata) + finalDocs.push(document) } return super.init(nodeData, _, options, flattenDocs) diff --git a/packages/components/nodes/vectorstores/Redis/utils.ts b/packages/components/nodes/vectorstores/Redis/utils.ts new file mode 100644 index 00000000..acef03ef --- /dev/null +++ b/packages/components/nodes/vectorstores/Redis/utils.ts @@ -0,0 +1,25 @@ +/* + * Escapes all '-' characters. + * Redis Search considers '-' as a negative operator, hence we need + * to escape it + */ +export const escapeSpecialChars = (str: string) => { + return str.replaceAll('-', '\\-') +} + +export const escapeAllStrings = (obj: object) => { + Object.keys(obj).forEach((key: string) => { + // @ts-ignore + let item = obj[key] + if (typeof item === 'object') { + escapeAllStrings(item) + } else if (typeof item === 'string') { + // @ts-ignore + obj[key] = escapeSpecialChars(item) + } + }) +} + +export const unEscapeSpecialChars = (str: string) => { + return str.replaceAll('\\-', '-') +}