From 3fb800190778a67deb46928708ad8e8773154fd3 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sat, 30 Dec 2023 14:13:39 -0500 Subject: [PATCH 01/21] Added support to exclude specific Airtable Field Ids --- .../documentloaders/Airtable/Airtable.ts | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index a2c1eef3..dfa05983 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -64,6 +64,16 @@ class Airtable_DocumentLoaders implements INode { 'If your view URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, viw9UrP77Id0CE4ee is the view id', optional: true }, + { + label: 'Exclude Field Ids', + name: 'excludeFieldIds', + type: 'string', + placeholder: 'fld1u0qUz0SoOQ9Gg, fldAMOvPfwxr12VrK', + optional: true, + additionalParams: true, + description: + 'Comma-separated list of field ids to exclude' + }, { label: 'Return All', name: 'returnAll', @@ -93,6 +103,7 @@ class Airtable_DocumentLoaders implements INode { const baseId = nodeData.inputs?.baseId as string const tableId = nodeData.inputs?.tableId as string const viewId = nodeData.inputs?.viewId as string + const excludeFieldIds = nodeData.inputs?.excludeFieldIds as string const returnAll = nodeData.inputs?.returnAll as boolean const limit = nodeData.inputs?.limit as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter @@ -105,6 +116,7 @@ class Airtable_DocumentLoaders implements INode { baseId, tableId, viewId, + excludeFieldIds: excludeFieldIds ? excludeFieldIds.split(',').map(id => id.trim()) : [], returnAll, accessToken, limit: limit ? parseInt(limit, 10) : 100 @@ -145,6 +157,7 @@ interface AirtableLoaderParams { tableId: string accessToken: string viewId?: string + excludeFieldIds?: string[] limit?: number returnAll?: boolean } @@ -167,17 +180,20 @@ class AirtableLoader extends BaseDocumentLoader { public readonly viewId?: string + public readonly excludeFieldIds: string[] + public readonly accessToken: string public readonly limit: number public readonly returnAll: boolean - constructor({ baseId, tableId, viewId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { + constructor({ baseId, tableId, viewId, excludeFieldIds = [], accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { super() this.baseId = baseId this.tableId = tableId this.viewId = viewId + this.excludeFieldIds = excludeFieldIds this.accessToken = accessToken this.limit = limit this.returnAll = returnAll @@ -207,10 +223,16 @@ class AirtableLoader extends BaseDocumentLoader { private createDocumentFromPage(page: AirtableLoaderPage): Document { // Generate the URL const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}` + const fields = { ...page.fields }; + + // Exclude any specified fields + this.excludeFieldIds.forEach(id => { + delete fields[id]; + }); // Return a langchain document return new Document({ - pageContent: JSON.stringify(page.fields, null, 2), + pageContent: JSON.stringify(fields, null, 2), metadata: { url: pageUrl } From 6006157958f21347c6feff68ca56a313e813712b Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sat, 30 Dec 2023 15:17:33 -0500 Subject: [PATCH 02/21] Updated Airtable field exclusion support to use field names instead of field ids --- .../documentloaders/Airtable/Airtable.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index dfa05983..d7ae2667 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -65,14 +65,14 @@ class Airtable_DocumentLoaders implements INode { optional: true }, { - label: 'Exclude Field Ids', - name: 'excludeFieldIds', + label: 'Exclude Field Names', + name: 'excludeFieldNames', type: 'string', - placeholder: 'fld1u0qUz0SoOQ9Gg, fldAMOvPfwxr12VrK', + placeholder: 'Name, Assignee', optional: true, additionalParams: true, description: - 'Comma-separated list of field ids to exclude' + 'Comma-separated list of field names to exclude' }, { label: 'Return All', @@ -103,7 +103,7 @@ class Airtable_DocumentLoaders implements INode { const baseId = nodeData.inputs?.baseId as string const tableId = nodeData.inputs?.tableId as string const viewId = nodeData.inputs?.viewId as string - const excludeFieldIds = nodeData.inputs?.excludeFieldIds as string + const excludeFieldNames = nodeData.inputs?.excludeFieldNames as string const returnAll = nodeData.inputs?.returnAll as boolean const limit = nodeData.inputs?.limit as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter @@ -116,7 +116,7 @@ class Airtable_DocumentLoaders implements INode { baseId, tableId, viewId, - excludeFieldIds: excludeFieldIds ? excludeFieldIds.split(',').map(id => id.trim()) : [], + excludeFieldNames: excludeFieldNames ? excludeFieldNames.split(',').map(id => id.trim()) : [], returnAll, accessToken, limit: limit ? parseInt(limit, 10) : 100 @@ -157,7 +157,7 @@ interface AirtableLoaderParams { tableId: string accessToken: string viewId?: string - excludeFieldIds?: string[] + excludeFieldNames?: string[] limit?: number returnAll?: boolean } @@ -180,7 +180,7 @@ class AirtableLoader extends BaseDocumentLoader { public readonly viewId?: string - public readonly excludeFieldIds: string[] + public readonly excludeFieldNames: string[] public readonly accessToken: string @@ -188,12 +188,12 @@ class AirtableLoader extends BaseDocumentLoader { public readonly returnAll: boolean - constructor({ baseId, tableId, viewId, excludeFieldIds = [], accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { + constructor({ baseId, tableId, viewId, excludeFieldNames = [], accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { super() this.baseId = baseId this.tableId = tableId this.viewId = viewId - this.excludeFieldIds = excludeFieldIds + this.excludeFieldNames = excludeFieldNames this.accessToken = accessToken this.limit = limit this.returnAll = returnAll @@ -226,7 +226,7 @@ class AirtableLoader extends BaseDocumentLoader { const fields = { ...page.fields }; // Exclude any specified fields - this.excludeFieldIds.forEach(id => { + this.excludeFieldNames.forEach(id => { delete fields[id]; }); From e88859f5d4326a44743e0a87e871e31ca9e00c7d Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Mon, 1 Jan 2024 13:33:09 -0500 Subject: [PATCH 03/21] Added support for gpt-4 and gpt-4-32k models --- .../components/nodes/llms/Azure OpenAI/AzureOpenAI.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts index f50e3d95..a8ac8830 100644 --- a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts +++ b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts @@ -18,7 +18,7 @@ class AzureOpenAI_LLMs implements INode { constructor() { this.label = 'Azure OpenAI' this.name = 'azureOpenAI' - this.version = 2.0 + this.version = 2.1 this.type = 'AzureOpenAI' this.icon = 'Azure.svg' this.category = 'LLMs' @@ -89,6 +89,14 @@ class AzureOpenAI_LLMs implements INode { { label: 'gpt-35-turbo', name: 'gpt-35-turbo' + }, + { + label: 'gpt-4', + name: 'gpt-4' + }, + { + label: 'gpt-4-32k', + name: 'gpt-4-32k' } ], default: 'text-davinci-003', From 66701cec8a8def398813f584663ab83b3cb13a26 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Wed, 3 Jan 2024 13:35:25 -0500 Subject: [PATCH 04/21] Fixing linting issues using 'yarn lint-fix' --- .../documentloaders/Airtable/Airtable.ts | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index d7ae2667..04d7d735 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -55,24 +55,23 @@ class Airtable_DocumentLoaders implements INode { description: 'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id' }, - { - label: 'View Id', - name: 'viewId', - type: 'string', - placeholder: 'viw9UrP77Id0CE4ee', - description: - 'If your view URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, viw9UrP77Id0CE4ee is the view id', - optional: true + { + label: 'View Id', + name: 'viewId', + type: 'string', + placeholder: 'viw9UrP77Id0CE4ee', + description: + 'If your view URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, viw9UrP77Id0CE4ee is the view id', + optional: true }, { - label: 'Exclude Field Names', - name: 'excludeFieldNames', - type: 'string', - placeholder: 'Name, Assignee', - optional: true, + label: 'Exclude Field Names', + name: 'excludeFieldNames', + type: 'string', + placeholder: 'Name, Assignee', + optional: true, additionalParams: true, - description: - 'Comma-separated list of field names to exclude' + description: 'Comma-separated list of field names to exclude' }, { label: 'Return All', @@ -116,7 +115,7 @@ class Airtable_DocumentLoaders implements INode { baseId, tableId, viewId, - excludeFieldNames: excludeFieldNames ? excludeFieldNames.split(',').map(id => id.trim()) : [], + excludeFieldNames: excludeFieldNames ? excludeFieldNames.split(',').map((id) => id.trim()) : [], returnAll, accessToken, limit: limit ? parseInt(limit, 10) : 100 @@ -223,12 +222,12 @@ class AirtableLoader extends BaseDocumentLoader { private createDocumentFromPage(page: AirtableLoaderPage): Document { // Generate the URL const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}` - const fields = { ...page.fields }; + const fields = { ...page.fields } // Exclude any specified fields - this.excludeFieldNames.forEach(id => { - delete fields[id]; - }); + this.excludeFieldNames.forEach((id) => { + delete fields[id] + }) // Return a langchain document return new Document({ From e8c85035f238cdacd3f23e148914d49502aff753 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Wed, 3 Jan 2024 15:25:00 -0500 Subject: [PATCH 05/21] Made streaming support a configurable option within the AzureChatOpenAI node --- .../chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 9b7b724a..33f6b76e 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -19,7 +19,7 @@ class AzureChatOpenAI_ChatModels implements INode { constructor() { this.label = 'Azure ChatOpenAI' this.name = 'azureChatOpenAI' - this.version = 2.0 + this.version = 2.1 this.type = 'AzureChatOpenAI' this.icon = 'Azure.svg' this.category = 'Chat Models' @@ -102,6 +102,14 @@ class AzureChatOpenAI_ChatModels implements INode { step: 1, optional: true, additionalParams: true + }, + { + label: 'Streaming', + name: 'streaming', + type: 'boolean', + default: true, + optional: true, + additionalParams: true } ] } From e3982476b0064bbc4bfe42af0cd06500a1095751 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Thu, 4 Jan 2024 12:07:25 -0500 Subject: [PATCH 06/21] Bumping version --- packages/components/nodes/documentloaders/Airtable/Airtable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 04d7d735..0f212a0a 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -20,7 +20,7 @@ class Airtable_DocumentLoaders implements INode { constructor() { this.label = 'Airtable' this.name = 'airtable' - this.version = 1.0 + this.version = 2.0 this.type = 'Document' this.icon = 'airtable.svg' this.category = 'Document Loaders' From 3d2b4077cfd5231e908bc279e0bba5fd9e995056 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Thu, 4 Jan 2024 12:09:08 -0500 Subject: [PATCH 07/21] Removed streaming feature since it broke chatflows --- .../nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 33f6b76e..a459a8ec 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -102,14 +102,6 @@ class AzureChatOpenAI_ChatModels implements INode { step: 1, optional: true, additionalParams: true - }, - { - label: 'Streaming', - name: 'streaming', - type: 'boolean', - default: true, - optional: true, - additionalParams: true } ] } From 71f456af90ab0d9855f9a1f3207e6589decc034b Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Thu, 25 Jan 2024 11:00:58 -0500 Subject: [PATCH 08/21] Switched to specifying Airtable fields to include rather than exclude - this helps reduce the amount of data fetched by the DocumentLoader when there are massive numbers of fields in an Airtable table. --- .../documentloaders/Airtable/Airtable.ts | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 0f212a0a..78ad7a66 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -65,13 +65,13 @@ class Airtable_DocumentLoaders implements INode { optional: true }, { - label: 'Exclude Field Names', - name: 'excludeFieldNames', + label: 'Fields', + name: 'fields', type: 'string', - placeholder: 'Name, Assignee', + placeholder: 'Name, Assignee, fld1u0qUz0SoOQ9Gg, fldew39v6LBN5CjUl', optional: true, additionalParams: true, - description: 'Comma-separated list of field names to exclude' + description: 'Comma-separated list of field names or IDs to include. Use field IDs if field names contain commas.' }, { label: 'Return All', @@ -102,7 +102,8 @@ class Airtable_DocumentLoaders implements INode { const baseId = nodeData.inputs?.baseId as string const tableId = nodeData.inputs?.tableId as string const viewId = nodeData.inputs?.viewId as string - const excludeFieldNames = nodeData.inputs?.excludeFieldNames as string + const fieldsInput = nodeData.inputs?.fields as string + const fields = fieldsInput ? fieldsInput.split(',').map((field) => field.trim()) : [] const returnAll = nodeData.inputs?.returnAll as boolean const limit = nodeData.inputs?.limit as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter @@ -115,7 +116,7 @@ class Airtable_DocumentLoaders implements INode { baseId, tableId, viewId, - excludeFieldNames: excludeFieldNames ? excludeFieldNames.split(',').map((id) => id.trim()) : [], + fields, returnAll, accessToken, limit: limit ? parseInt(limit, 10) : 100 @@ -156,7 +157,7 @@ interface AirtableLoaderParams { tableId: string accessToken: string viewId?: string - excludeFieldNames?: string[] + fields?: string[] limit?: number returnAll?: boolean } @@ -179,7 +180,7 @@ class AirtableLoader extends BaseDocumentLoader { public readonly viewId?: string - public readonly excludeFieldNames: string[] + public readonly fields: string[] public readonly accessToken: string @@ -187,12 +188,12 @@ class AirtableLoader extends BaseDocumentLoader { public readonly returnAll: boolean - constructor({ baseId, tableId, viewId, excludeFieldNames = [], accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { + constructor({ baseId, tableId, viewId, fields = [], accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { super() this.baseId = baseId this.tableId = tableId this.viewId = viewId - this.excludeFieldNames = excludeFieldNames + this.fields = fields this.accessToken = accessToken this.limit = limit this.returnAll = returnAll @@ -205,14 +206,14 @@ class AirtableLoader extends BaseDocumentLoader { return this.loadLimit() } - protected async fetchAirtableData(url: string, params: ICommonObject): Promise { + protected async fetchAirtableData(url: string, data: any): Promise { try { const headers = { Authorization: `Bearer ${this.accessToken}`, 'Content-Type': 'application/json', Accept: 'application/json' } - const response = await axios.get(url, { params, headers }) + const response = await axios.get(url, data, { headers }) return response.data } catch (error) { throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) @@ -222,12 +223,6 @@ class AirtableLoader extends BaseDocumentLoader { private createDocumentFromPage(page: AirtableLoaderPage): Document { // Generate the URL const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}` - const fields = { ...page.fields } - - // Exclude any specified fields - this.excludeFieldNames.forEach((id) => { - delete fields[id] - }) // Return a langchain document return new Document({ @@ -239,24 +234,40 @@ class AirtableLoader extends BaseDocumentLoader { } private async loadLimit(): Promise { - const params = { maxRecords: this.limit, view: this.viewId } - const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params) - if (data.records.length === 0) { + const data = { + maxRecords: this.limit, + view: this.viewId + } + + if (this.fields.length > 0) { + data.fields = this.fields + } + + const response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, data) + if (response.records.length === 0) { return [] } - return data.records.map((page) => this.createDocumentFromPage(page)) + return response.records.map((page) => this.createDocumentFromPage(page)) } private async loadAll(): Promise { - const params: ICommonObject = { pageSize: 100, view: this.viewId } - let data: AirtableLoaderResponse + const data = { + pageSize: 100, + view: this.viewId + } + + if (this.fields.length > 0) { + data.fields = this.fields + } + + let response: AirtableLoaderResponse let returnPages: AirtableLoaderPage[] = [] do { - data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params) - returnPages.push.apply(returnPages, data.records) - params.offset = data.offset - } while (data.offset !== undefined) + response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, data) + returnPages.push.apply(returnPages, response.records) + params.offset = response.offset + } while (response.offset !== undefined) return returnPages.map((page) => this.createDocumentFromPage(page)) } } From ae64854baedca9c6f224136578bb87a1df93afa4 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Thu, 25 Jan 2024 11:29:06 -0500 Subject: [PATCH 09/21] Fixing a bunch of build errors --- .../nodes/documentloaders/Airtable/Airtable.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 78ad7a66..6fdb070a 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -162,6 +162,12 @@ interface AirtableLoaderParams { returnAll?: boolean } +interface AirtableLoaderRequest { + maxRecords: number + view: string | undefined + fields?: string[] +} + interface AirtableLoaderResponse { records: AirtableLoaderPage[] offset?: string @@ -213,7 +219,7 @@ class AirtableLoader extends BaseDocumentLoader { 'Content-Type': 'application/json', Accept: 'application/json' } - const response = await axios.get(url, data, { headers }) + const response = await axios.post(url, data, { headers }) return response.data } catch (error) { throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) @@ -226,7 +232,7 @@ class AirtableLoader extends BaseDocumentLoader { // Return a langchain document return new Document({ - pageContent: JSON.stringify(fields, null, 2), + pageContent: JSON.stringify(page.fields, null, 2), metadata: { url: pageUrl } @@ -234,7 +240,7 @@ class AirtableLoader extends BaseDocumentLoader { } private async loadLimit(): Promise { - const data = { + let data: AirtableLoaderRequest = { maxRecords: this.limit, view: this.viewId } @@ -251,7 +257,7 @@ class AirtableLoader extends BaseDocumentLoader { } private async loadAll(): Promise { - const data = { + let data: AirtableLoaderRequest = { pageSize: 100, view: this.viewId } From 1a7cb5a010039f714fbe8e117a9875985866f2f4 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Thu, 25 Jan 2024 11:43:56 -0500 Subject: [PATCH 10/21] Fixing more build errors --- .../components/nodes/documentloaders/Airtable/Airtable.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 6fdb070a..de913b90 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -166,6 +166,7 @@ interface AirtableLoaderRequest { maxRecords: number view: string | undefined fields?: string[] + offset?: string } interface AirtableLoaderResponse { @@ -258,7 +259,7 @@ class AirtableLoader extends BaseDocumentLoader { private async loadAll(): Promise { let data: AirtableLoaderRequest = { - pageSize: 100, + pageSize: this.limit, view: this.viewId } @@ -272,7 +273,7 @@ class AirtableLoader extends BaseDocumentLoader { do { response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, data) returnPages.push.apply(returnPages, response.records) - params.offset = response.offset + data.offset = response.offset } while (response.offset !== undefined) return returnPages.map((page) => this.createDocumentFromPage(page)) } From 72ec7878b609e013f3a7b17b3875beab2f936d0e Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Thu, 25 Jan 2024 12:08:52 -0500 Subject: [PATCH 11/21] Added more error checking and also fixed yet more build errors --- .../nodes/documentloaders/Airtable/Airtable.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index de913b90..4558edb5 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -124,6 +124,10 @@ class Airtable_DocumentLoaders implements INode { const loader = new AirtableLoader(airtableOptions) + if (!baseId || !tableId) { + throw new Error('Base ID and Table ID must be provided.') + } + let docs = [] if (textSplitter) { @@ -213,7 +217,7 @@ class AirtableLoader extends BaseDocumentLoader { return this.loadLimit() } - protected async fetchAirtableData(url: string, data: any): Promise { + protected async fetchAirtableData(url: string, data: AirtableLoaderRequest): Promise { try { const headers = { Authorization: `Bearer ${this.accessToken}`, @@ -223,7 +227,11 @@ class AirtableLoader extends BaseDocumentLoader { const response = await axios.post(url, data, { headers }) return response.data } catch (error) { - throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) + if (axios.isAxiosError(error)) { + throw new Error(`Failed to fetch ${url} from Airtable: ${error.message}, status: ${error.response?.status}`) + } else { + throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) + } } } @@ -259,7 +267,7 @@ class AirtableLoader extends BaseDocumentLoader { private async loadAll(): Promise { let data: AirtableLoaderRequest = { - pageSize: this.limit, + maxRecords: this.limit, view: this.viewId } @@ -272,7 +280,7 @@ class AirtableLoader extends BaseDocumentLoader { do { response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, data) - returnPages.push.apply(returnPages, response.records) + returnPages.push(...response.records) data.offset = response.offset } while (response.offset !== undefined) return returnPages.map((page) => this.createDocumentFromPage(page)) From 8ae848110eb98726e30994f4442fa9e4c5544e8a Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Thu, 25 Jan 2024 21:36:42 -0500 Subject: [PATCH 12/21] Clarifying the description for the optional fields param --- .../components/nodes/documentloaders/Airtable/Airtable.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 4558edb5..ed157264 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -65,13 +65,13 @@ class Airtable_DocumentLoaders implements INode { optional: true }, { - label: 'Fields', + label: 'Include Only Fields', name: 'fields', type: 'string', placeholder: 'Name, Assignee, fld1u0qUz0SoOQ9Gg, fldew39v6LBN5CjUl', optional: true, additionalParams: true, - description: 'Comma-separated list of field names or IDs to include. Use field IDs if field names contain commas.' + description: 'Comma-separated list of field names or IDs to include. If empty, then ALL fields are used. Use field IDs if field names contain commas.' }, { label: 'Return All', From 456dfabc6febea31ab11ce1bab22a4b51ac70c8a Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Fri, 26 Jan 2024 13:48:09 -0500 Subject: [PATCH 13/21] For some reason, Airtable doesn't like the POST operations, so I need to add logging to figure out why this happens --- .../components/nodes/documentloaders/Airtable/Airtable.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index ed157264..d0ed3b44 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -71,7 +71,8 @@ class Airtable_DocumentLoaders implements INode { placeholder: 'Name, Assignee, fld1u0qUz0SoOQ9Gg, fldew39v6LBN5CjUl', optional: true, additionalParams: true, - description: 'Comma-separated list of field names or IDs to include. If empty, then ALL fields are used. Use field IDs if field names contain commas.' + description: + 'Comma-separated list of field names or IDs to include. If empty, then ALL fields are used. Use field IDs if field names contain commas.' }, { label: 'Return All', @@ -224,12 +225,15 @@ class AirtableLoader extends BaseDocumentLoader { 'Content-Type': 'application/json', Accept: 'application/json' } + console.log('Sending request to Airtable with data: ', data) const response = await axios.post(url, data, { headers }) return response.data } catch (error) { if (axios.isAxiosError(error)) { + console.error('Error response from Airtable:', error.response?.data) throw new Error(`Failed to fetch ${url} from Airtable: ${error.message}, status: ${error.response?.status}`) } else { + console.error('An unexpected error occurred:', error) throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) } } From 3b788e42e1520c193a909001faa8793a35420caa Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Fri, 26 Jan 2024 15:03:30 -0500 Subject: [PATCH 14/21] When you switch from GET to POST, you're supposed to adjust the Airtable URL to use the /listRecords endpoint. I didn't RTFM clearly. This is currently documented here: https://support.airtable.com/docs/enforcement-of-url-length-limit-for-web-api-requests --- .../components/nodes/documentloaders/Airtable/Airtable.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index d0ed3b44..7d5e8ad0 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -262,7 +262,7 @@ class AirtableLoader extends BaseDocumentLoader { data.fields = this.fields } - const response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, data) + const response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data) if (response.records.length === 0) { return [] } @@ -283,7 +283,7 @@ class AirtableLoader extends BaseDocumentLoader { let returnPages: AirtableLoaderPage[] = [] do { - response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, data) + response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data) returnPages.push(...response.records) data.offset = response.offset } while (response.offset !== undefined) From 2237b1ab165d6ff0acd6252b0be7c1e31591a8ec Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Fri, 26 Jan 2024 17:15:39 -0500 Subject: [PATCH 15/21] Fix worked, removing debug logging, and bumped node version. --- .../components/nodes/documentloaders/Airtable/Airtable.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 7d5e8ad0..76d6e349 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -20,7 +20,7 @@ class Airtable_DocumentLoaders implements INode { constructor() { this.label = 'Airtable' this.name = 'airtable' - this.version = 2.0 + this.version = 3.0 this.type = 'Document' this.icon = 'airtable.svg' this.category = 'Document Loaders' @@ -225,15 +225,12 @@ class AirtableLoader extends BaseDocumentLoader { 'Content-Type': 'application/json', Accept: 'application/json' } - console.log('Sending request to Airtable with data: ', data) const response = await axios.post(url, data, { headers }) return response.data } catch (error) { if (axios.isAxiosError(error)) { - console.error('Error response from Airtable:', error.response?.data) throw new Error(`Failed to fetch ${url} from Airtable: ${error.message}, status: ${error.response?.status}`) } else { - console.error('An unexpected error occurred:', error) throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) } } From 9b71f683fffe916baced5693cbe0f070dd043d12 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sat, 27 Jan 2024 13:58:20 -0500 Subject: [PATCH 16/21] Support pagination even in loadLimit(), so that if a user wants to load more than 100 records but not all of them, they can. Currently, there's a bug where the document loader doesn't work on loading more than 100 records because of internal Airtable API limitations. The Airtable API can only fetch up to 100 records per call - anything more than that requires pagination. --- .../documentloaders/Airtable/Airtable.ts | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 76d6e349..011975a7 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -251,7 +251,7 @@ class AirtableLoader extends BaseDocumentLoader { private async loadLimit(): Promise { let data: AirtableLoaderRequest = { - maxRecords: this.limit, + maxRecords: Math.min(this.limit, 100), // Airtable only returns up to 100 records per request view: this.viewId } @@ -259,11 +259,25 @@ class AirtableLoader extends BaseDocumentLoader { data.fields = this.fields } - const response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data) - if (response.records.length === 0) { - return [] + let response: AirtableLoaderResponse + let returnPages: AirtableLoaderPage[] = [] + + // Paginate if the user specifies a limit > 100 (like 200) but not return all. + do { + response = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}/listRecords`, data) + returnPages.push(...response.records) + data.offset = response.offset + + // Stop if we have fetched enough records + if (returnPages.length >= this.limit) break + } while (response.offset !== undefined) + + // Truncate array to the limit if necessary + if (returnPages.length > this.limit) { + returnPages.length = this.limit } - return response.records.map((page) => this.createDocumentFromPage(page)) + + return returnPages.map((page) => this.createDocumentFromPage(page)) } private async loadAll(): Promise { From dc39d7e2bec837d863bde0c8b8b52fcdb1b37078 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sat, 27 Jan 2024 16:33:16 -0500 Subject: [PATCH 17/21] So Airtable API expects a maxRecords value to be the total set of records you want across multiple API calls. If the maxRecords is greater than 100, then it will provide pagination hints accordingly. --- packages/components/nodes/documentloaders/Airtable/Airtable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 011975a7..90a7ba03 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -251,7 +251,7 @@ class AirtableLoader extends BaseDocumentLoader { private async loadLimit(): Promise { let data: AirtableLoaderRequest = { - maxRecords: Math.min(this.limit, 100), // Airtable only returns up to 100 records per request + maxRecords: this.limit, view: this.viewId } From 37945fc998cd1e2818756a7877ffb4613be46094 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sat, 27 Jan 2024 21:25:43 -0800 Subject: [PATCH 18/21] The loadAll() function should ignore any maxRecords specified, because the intention is the load *all* of the records. Also, marking both the Return All and Limit params as optional, so as to not confuse the user. Making them both required adds a lot of confusion that doesn't make sense. Ideally, the user either specifies Return All OR specifies the Limit value but not BOTH. It seems there's no way to define "conditional requirements" in Flowise params, so it's better to make both params optional. --- packages/components/nodes/documentloaders/Airtable/Airtable.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 90a7ba03..8845bbe0 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -78,6 +78,7 @@ class Airtable_DocumentLoaders implements INode { label: 'Return All', name: 'returnAll', type: 'boolean', + optional: true, default: true, additionalParams: true, description: 'If all results should be returned or only up to a given limit' @@ -86,6 +87,7 @@ class Airtable_DocumentLoaders implements INode { label: 'Limit', name: 'limit', type: 'number', + optional: true, default: 100, additionalParams: true, description: 'Number of results to return' @@ -282,7 +284,6 @@ class AirtableLoader extends BaseDocumentLoader { private async loadAll(): Promise { let data: AirtableLoaderRequest = { - maxRecords: this.limit, view: this.viewId } From 66eef8463315a5f045b428478abf71b1b53b9996 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sat, 27 Jan 2024 21:40:05 -0800 Subject: [PATCH 19/21] Forgot to make maxRecords optional now --- packages/components/nodes/documentloaders/Airtable/Airtable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 8845bbe0..b0478100 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -170,7 +170,7 @@ interface AirtableLoaderParams { } interface AirtableLoaderRequest { - maxRecords: number + maxRecords?: number view: string | undefined fields?: string[] offset?: string From b960f061ebbedaa2cbc0b1cdc802999ab9a948df Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sun, 28 Jan 2024 08:21:02 -0800 Subject: [PATCH 20/21] Clarifying that the Limit value is ignored when Return All is set to true. --- packages/components/nodes/documentloaders/Airtable/Airtable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index b0478100..14815297 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -90,7 +90,7 @@ class Airtable_DocumentLoaders implements INode { optional: true, default: 100, additionalParams: true, - description: 'Number of results to return' + description: 'Number of results to return. Ignored when Return All is enabled.' }, { label: 'Metadata', From 905c9fc2bed208c7cf5327b077c941533eac8157 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Sun, 28 Jan 2024 20:48:08 -0800 Subject: [PATCH 21/21] Reverting version bump --- .../nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index a459a8ec..9b7b724a 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -19,7 +19,7 @@ class AzureChatOpenAI_ChatModels implements INode { constructor() { this.label = 'Azure ChatOpenAI' this.name = 'azureChatOpenAI' - this.version = 2.1 + this.version = 2.0 this.type = 'AzureChatOpenAI' this.icon = 'Azure.svg' this.category = 'Chat Models'