seg_yinzy
/
Flowise
mirror of https://github.com/FlowiseAI/Flowise.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bugfix/Files not removed when doc store loader is deleted (#2502) 

fix files not removed when doc store loader is deleted
pull/2507/head
Henry Heng 2024-05-28 22:36:12 +01:00 committed by GitHub
parent 22f39692e5
commit 97386bc3b2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 24 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
packages/server/src/services/documentstore/index.ts
View File

@ -71,9 +71,12 @@ const deleteLoaderFromDocumentStore = async (storeId: string, loaderId: string)
const existingLoaders = JSON.parse(entity.loaders)
const found = existingLoaders.find((uFile: IDocumentStoreLoader) => uFile.id === loaderId)
if (found) {
if (found.path) {
//remove the existing files, if any of the file loaders were used.
await removeSpecificFileFromStorage(DOCUMENT_STORE_BASE_FOLDER, entity.id, found.path)
if (found.files?.length) {
for (const file of found.files) {
if (file.name) {
await removeSpecificFileFromStorage(DOCUMENT_STORE_BASE_FOLDER, storeId, file.name)
}
}
}
const index = existingLoaders.indexOf(found)
if (index > -1) {
@ -536,8 +539,23 @@ const _saveChunksToStorage = async (data: IDocumentStoreLoaderForPreview, entity
await _normalizeFilePaths(data, entity)
//step 2: split the file into chunks
previewChunks(data).then(async (response) => {
//{ chunks: docs, totalChunks: totalChunks, previewChunkCount: data.previewChunkCount }
//step 3: remove base64 files and save them to storage, this needs to be rewritten
//step 3: remove all files associated with the loader
const existingLoaders = JSON.parse(entity.loaders)
const loader = existingLoaders.find((ldr: IDocumentStoreLoader) => ldr.id === newLoaderId)
if (data.id) {
const index = existingLoaders.indexOf(loader)
if (index > -1) {
existingLoaders.splice(index, 1)
if (!data.rehydrated) {
if (loader.files) {
loader.files.map(async (file: IDocumentStoreLoaderFile) => {
await removeSpecificFileFromStorage(DOCUMENT_STORE_BASE_FOLDER, entity.id, file.name)
})
}
}
}
}
//step 4: save new file to storage
let filesWithMetadata = []
const keys = Object.getOwnPropertyNames(data.loaderConfig)
for (let i = 0; i < keys.length; i++) {
@ -569,23 +587,7 @@ const _saveChunksToStorage = async (data: IDocumentStoreLoaderForPreview, entity
break
}
}
const existingLoaders = JSON.parse(entity.loaders)
const loader = existingLoaders.find((ldr: IDocumentStoreLoader) => ldr.id === newLoaderId)
if (data.id) {
//step 4: remove all files and chunks associated with the previous loader
const index = existingLoaders.indexOf(loader)
if (index > -1) {
existingLoaders.splice(index, 1)
if (!data.rehydrated) {
if (loader.files) {
loader.files.map(async (file: IDocumentStoreLoaderFile) => {
await removeSpecificFileFromStorage(DOCUMENT_STORE_BASE_FOLDER, entity.id, file.name)
})
}
}
}
}
//step 5: upload with the new files and loaderConfig
//step 5: update with the new files and loaderConfig
if (filesWithMetadata.length > 0) {
loader.loaderConfig = data.loaderConfig
loader.files = filesWithMetadata