diff --git a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts index 4aa6aead037..875a8e0f048 100644 --- a/packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/__tests__/request-helper-functions.test.ts @@ -9,7 +9,6 @@ import type { INode, IRequestOptions, IWorkflowExecuteAdditionalData, - PaginationOptions, Workflow, } from 'n8n-workflow'; import { UserError } from 'n8n-workflow'; @@ -21,7 +20,6 @@ import type { SsrfBridge } from '@/execution-engine'; import type { ExecutionLifecycleHooks } from '@/execution-engine/execution-lifecycle-hooks'; import { - applyPaginationRequestData, convertN8nRequestToAxios, httpRequest, invokeAxios, @@ -703,150 +701,6 @@ describe('Request Helper Functions', () => { }); }); - describe('applyPaginationRequestData', () => { - test('should merge pagination request data with original request options', () => { - const originalRequestOptions: IRequestOptions = { - uri: 'https://original.com/api', - method: 'GET', - qs: { page: 1 }, - headers: { 'X-Original-Header': 'original' }, - }; - - const paginationRequestData: PaginationOptions['request'] = { - url: 'https://pagination.com/api', - body: { key: 'value' }, - headers: { 'X-Pagination-Header': 'pagination' }, - }; - - const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); - - expect(result).toEqual({ - uri: 'https://pagination.com/api', - url: 'https://pagination.com/api', - method: 'GET', - qs: { page: 1 }, - headers: { - 'X-Original-Header': 'original', - 'X-Pagination-Header': 'pagination', - }, - body: { key: 'value' }, - }); - }); - - test('should handle formData correctly', () => { - const originalRequestOptions: IRequestOptions = { - uri: 'https://original.com/api', - method: 'POST', - formData: { original: 'data' }, - }; - - const paginationRequestData: PaginationOptions['request'] = { - url: 'https://pagination.com/api', - body: { key: 'value' }, - }; - - const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); - - expect(result).toEqual({ - uri: 'https://pagination.com/api', - url: 'https://pagination.com/api', - method: 'POST', - formData: { key: 'value', original: 'data' }, - }); - }); - - test('should handle form data correctly', () => { - const originalRequestOptions: IRequestOptions = { - uri: 'https://original.com/api', - method: 'POST', - form: { original: 'data' }, - }; - - const paginationRequestData: PaginationOptions['request'] = { - url: 'https://pagination.com/api', - body: { key: 'value' }, - }; - - const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); - - expect(result).toEqual({ - uri: 'https://pagination.com/api', - url: 'https://pagination.com/api', - method: 'POST', - form: { key: 'value', original: 'data' }, - }); - }); - - test('should prefer pagination body over original body', () => { - const originalRequestOptions: IRequestOptions = { - uri: 'https://original.com/api', - method: 'POST', - body: { original: 'data' }, - }; - - const paginationRequestData: PaginationOptions['request'] = { - url: 'https://pagination.com/api', - body: { key: 'value' }, - }; - - const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); - - expect(result).toEqual({ - uri: 'https://pagination.com/api', - url: 'https://pagination.com/api', - method: 'POST', - body: { key: 'value', original: 'data' }, - }); - }); - - test('should merge complex request options', () => { - const originalRequestOptions: IRequestOptions = { - uri: 'https://original.com/api', - method: 'GET', - qs: { page: 1, limit: 10 }, - headers: { 'X-Original-Header': 'original' }, - body: { filter: 'active' }, - }; - - const paginationRequestData: PaginationOptions['request'] = { - url: 'https://pagination.com/api', - body: { key: 'value' }, - headers: { 'X-Pagination-Header': 'pagination' }, - qs: { offset: 20 }, - }; - - const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); - - expect(result).toEqual({ - uri: 'https://pagination.com/api', - url: 'https://pagination.com/api', - method: 'GET', - qs: { offset: 20, limit: 10, page: 1 }, - headers: { - 'X-Original-Header': 'original', - 'X-Pagination-Header': 'pagination', - }, - body: { key: 'value', filter: 'active' }, - }); - }); - - test('should handle edge cases with empty pagination data', () => { - const originalRequestOptions: IRequestOptions = { - uri: 'https://original.com/api', - method: 'GET', - }; - - const paginationRequestData: PaginationOptions['request'] = {}; - - const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); - - expect(result).toEqual({ - uri: 'https://original.com/api', - method: 'GET', - }); - }); - }); - describe('httpRequest', () => { const baseUrl = 'https://example.com'; diff --git a/packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts b/packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts index ff2fd21dd27..a88f4899650 100644 --- a/packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/request-helper-functions.ts @@ -24,7 +24,6 @@ import { IncomingMessage } from 'http'; import { type AgentOptions } from 'https'; import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; -import merge from 'lodash/merge'; import pick from 'lodash/pick'; import { NodeApiError, @@ -34,7 +33,6 @@ import { ExecutionBaseError, jsonParse, ApplicationError, - sleep, } from 'n8n-workflow'; import type { GenericValue, @@ -56,7 +54,6 @@ import type { IWorkflowExecuteAdditionalData, Logger as WorkflowLogger, NodeParameterValueType, - PaginationOptions, RequestHelperFunctions, Workflow, WorkflowExecuteMode, @@ -89,6 +86,7 @@ import { tryParseUrl, } from './request-helpers'; import { throwIfDomainNotAllowed } from './request-helpers/axios-utils'; +import { requestWithAuthenticationPaginated } from './request-helpers/pagination'; export async function invokeAxios( axiosConfig: AxiosRequestConfig, @@ -692,26 +690,6 @@ export async function httpRequest( return result.data; } -export function applyPaginationRequestData( - requestData: IRequestOptions, - paginationRequestData: PaginationOptions['request'], -): IRequestOptions { - const preparedPaginationData: Partial = { - ...paginationRequestData, - uri: paginationRequestData.url, - }; - - if ('formData' in requestData) { - preparedPaginationData.formData = paginationRequestData.body; - delete preparedPaginationData.body; - } else if ('form' in requestData) { - preparedPaginationData.form = paginationRequestData.body; - delete preparedPaginationData.body; - } - - return merge({}, requestData, preparedPaginationData); -} - function createOAuth2Client(credentials: OAuth2CredentialData): ClientOAuth2 { // Split and trim scopes; empty scope tokens are not RFC 6749-compliant and may be rejected by authorization servers const scopes = credentials.scope @@ -1435,219 +1413,6 @@ export const getRequestHelperFunctions = ( return parameterValue; }; - // eslint-disable-next-line complexity - async function requestWithAuthenticationPaginated( - this: IExecuteFunctions, - requestOptions: IRequestOptions, - itemIndex: number, - paginationOptions: PaginationOptions, - credentialsType?: string, - additionalCredentialOptions?: IAdditionalCredentialOptions, - ): Promise { - const responseData = []; - if (!requestOptions.qs) { - requestOptions.qs = {}; - } - requestOptions.resolveWithFullResponse = true; - requestOptions.simple = false; - - let tempResponseData: IN8nHttpFullResponse; - let makeAdditionalRequest: boolean; - let paginateRequestData: PaginationOptions['request']; - - const runIndex = 0; - - const additionalKeys: IWorkflowDataProxyAdditionalKeys = { - $request: requestOptions, - $response: {} as IN8nHttpFullResponse, - $version: node.typeVersion, - $pageCount: 0, - }; - - const executeData: IExecuteData = { - data: {}, - node, - source: null, - }; - - const hashData = { - identicalCount: 0, - previousLength: 0, - previousHash: '', - }; - do { - paginateRequestData = getResolvedValue( - paginationOptions.request as unknown as NodeParameterValueType, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as object as PaginationOptions['request']; - - const tempRequestOptions = applyPaginationRequestData(requestOptions, paginateRequestData); - - if (!tryParseUrl(tempRequestOptions.uri as string)) { - throw new NodeOperationError(node, `'${paginateRequestData.url}' is not a valid URL.`, { - itemIndex, - runIndex, - type: 'invalid_url', - }); - } - - if (credentialsType) { - tempResponseData = await this.helpers.requestWithAuthentication.call( - this, - credentialsType, - tempRequestOptions, - additionalCredentialOptions, - ); - } else { - tempResponseData = await this.helpers.request(tempRequestOptions); - } - - const newResponse: IN8nHttpFullResponse = Object.assign( - { - body: {}, - headers: {}, - statusCode: 0, - }, - pick(tempResponseData, ['body', 'headers', 'statusCode']), - ); - - let contentBody: Exclude; - - if (newResponse.body instanceof Readable && paginationOptions.binaryResult !== true) { - // Keep the original string version that we can use it to hash if needed - contentBody = await binaryToString(newResponse.body as Buffer | Readable); - - const responseContentType = newResponse.headers['content-type']?.toString() ?? ''; - if (responseContentType.includes('application/json')) { - newResponse.body = jsonParse(contentBody, { fallbackValue: {} }); - } else { - newResponse.body = contentBody; - } - tempResponseData.__bodyResolved = true; - tempResponseData.body = newResponse.body; - } else { - contentBody = newResponse.body; - } - - if (paginationOptions.binaryResult !== true || tempResponseData.headers.etag) { - // If the data is not binary (and so not a stream), or an etag is present, - // we check via etag or hash if identical data is received - - let contentLength = 0; - if ('content-length' in tempResponseData.headers) { - contentLength = parseInt(tempResponseData.headers['content-length'] as string) || 0; - } - - if (hashData.previousLength === contentLength) { - let hash: string; - if (tempResponseData.headers.etag) { - // If an etag is provided, we use it as "hash" - hash = tempResponseData.headers.etag as string; - } else { - // If there is no etag, we calculate a hash from the data in the body - if (typeof contentBody !== 'string') { - contentBody = JSON.stringify(contentBody); - } - hash = crypto.createHash('md5').update(contentBody).digest('base64'); - } - - if (hashData.previousHash === hash) { - hashData.identicalCount += 1; - if (hashData.identicalCount > 2) { - // Length was identical 5x and hash 3x - throw new NodeOperationError( - node, - 'The returned response was identical 5x, so requests got stopped', - { - itemIndex, - description: - 'Check if "Pagination Completed When" has been configured correctly.', - }, - ); - } - } else { - hashData.identicalCount = 0; - } - hashData.previousHash = hash; - } else { - hashData.identicalCount = 0; - } - hashData.previousLength = contentLength; - } - - responseData.push(tempResponseData); - - additionalKeys.$response = newResponse; - additionalKeys.$pageCount = (additionalKeys.$pageCount ?? 0) + 1; - - const maxRequests = getResolvedValue( - paginationOptions.maxRequests, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as number; - - if (maxRequests && additionalKeys.$pageCount >= maxRequests) { - break; - } - - makeAdditionalRequest = getResolvedValue( - paginationOptions.continue, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as boolean; - - if (makeAdditionalRequest) { - if (paginationOptions.requestInterval) { - const requestInterval = getResolvedValue( - paginationOptions.requestInterval, - itemIndex, - runIndex, - executeData, - additionalKeys, - false, - ) as number; - - await sleep(requestInterval); - } - if (tempResponseData.statusCode < 200 || tempResponseData.statusCode >= 300) { - // We have it configured to let all requests pass no matter the response code - // via "requestOptions.simple = false" to not by default fail if it is for example - // configured to stop on 404 response codes. For that reason we have to throw here - // now an error manually if the response code is not a success one. - let data = tempResponseData.body; - if (data instanceof Readable && paginationOptions.binaryResult !== true) { - data = await binaryToString(data as Buffer | Readable); - } else if (typeof data === 'object') { - data = JSON.stringify(data); - } - - throw Object.assign(new Error(`${tempResponseData.statusCode} - "${data?.toString()}"`), { - statusCode: tempResponseData.statusCode, - error: data, - isAxiosError: true, - response: { - headers: tempResponseData.headers, - status: tempResponseData.statusCode, - statusText: tempResponseData.statusMessage, - }, - }); - } - } - } while (makeAdditionalRequest); - - return responseData; - } - // Eval LLM mock handler: extract once for use in direct helpers below const evalLlmMock = additionalData.evalLlmMockHandler; @@ -1672,7 +1437,25 @@ export const getRequestHelperFunctions = ( } return await httpRequest(requestOptions, additionalData.ssrfBridge); }, - requestWithAuthenticationPaginated, + async requestWithAuthenticationPaginated( + this: IExecuteFunctions, + requestOptions, + itemIndex, + paginationOptions, + credentialsType, + additionalCredentialOptions, + ): Promise { + return await requestWithAuthenticationPaginated.call( + this, + requestOptions, + itemIndex, + paginationOptions, + getResolvedValue, + node, + credentialsType, + additionalCredentialOptions, + ); + }, async httpRequestWithAuthentication( this, credentialsType, diff --git a/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/__tests__/pagination.test.ts b/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/__tests__/pagination.test.ts new file mode 100644 index 00000000000..4e886d9c648 --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/__tests__/pagination.test.ts @@ -0,0 +1,149 @@ +import type { IRequestOptions, PaginationOptions } from 'n8n-workflow'; + +import { applyPaginationRequestData } from '../pagination'; + +describe('pagination', () => { + describe('applyPaginationRequestData', () => { + test('should merge pagination request data with original request options', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'GET', + qs: { page: 1 }, + headers: { 'X-Original-Header': 'original' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + headers: { 'X-Pagination-Header': 'pagination' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'GET', + qs: { page: 1 }, + headers: { + 'X-Original-Header': 'original', + 'X-Pagination-Header': 'pagination', + }, + body: { key: 'value' }, + }); + }); + + test('should handle formData correctly', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'POST', + formData: { original: 'data' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'POST', + formData: { key: 'value', original: 'data' }, + }); + }); + + test('should handle form data correctly', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'POST', + form: { original: 'data' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'POST', + form: { key: 'value', original: 'data' }, + }); + }); + + test('should prefer pagination body over original body', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'POST', + body: { original: 'data' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'POST', + body: { key: 'value', original: 'data' }, + }); + }); + + test('should merge complex request options', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'GET', + qs: { page: 1, limit: 10 }, + headers: { 'X-Original-Header': 'original' }, + body: { filter: 'active' }, + }; + + const paginationRequestData: PaginationOptions['request'] = { + url: 'https://pagination.com/api', + body: { key: 'value' }, + headers: { 'X-Pagination-Header': 'pagination' }, + qs: { offset: 20 }, + }; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://pagination.com/api', + url: 'https://pagination.com/api', + method: 'GET', + qs: { offset: 20, limit: 10, page: 1 }, + headers: { + 'X-Original-Header': 'original', + 'X-Pagination-Header': 'pagination', + }, + body: { key: 'value', filter: 'active' }, + }); + }); + + test('should handle edge cases with empty pagination data', () => { + const originalRequestOptions: IRequestOptions = { + uri: 'https://original.com/api', + method: 'GET', + }; + + const paginationRequestData: PaginationOptions['request'] = {}; + + const result = applyPaginationRequestData(originalRequestOptions, paginationRequestData); + + expect(result).toEqual({ + uri: 'https://original.com/api', + method: 'GET', + }); + }); + }); +}); diff --git a/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/index.ts b/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/index.ts index 716fc635f12..898c1ab40b7 100644 --- a/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/index.ts +++ b/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/index.ts @@ -12,3 +12,8 @@ export { getUrlFromProxyConfig, setAxiosAgents, } from './axios-utils'; +export { + applyPaginationRequestData, + requestWithAuthenticationPaginated, + type ResolveValueFn, +} from './pagination'; diff --git a/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/pagination.ts b/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/pagination.ts new file mode 100644 index 00000000000..da5631c7c0d --- /dev/null +++ b/packages/core/src/execution-engine/node-execution-context/utils/request-helpers/pagination.ts @@ -0,0 +1,272 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/prefer-nullish-coalescing */ + +import crypto from 'crypto'; +import merge from 'lodash/merge'; +import pick from 'lodash/pick'; +import { NodeOperationError, jsonParse, sleep } from 'n8n-workflow'; +import type { + IAdditionalCredentialOptions, + IExecuteData, + IExecuteFunctions, + IN8nHttpFullResponse, + IN8nHttpResponse, + INode, + IRequestOptions, + IWorkflowDataProxyAdditionalKeys, + NodeParameterValueType, + PaginationOptions, +} from 'n8n-workflow'; +import { Readable } from 'stream'; + +import { binaryToString } from '../binary-helper-functions'; +import { tryParseUrl } from './axios-utils'; + +/** + * Resolves a (possibly expression) parameter value within the current execution + * context. Matches the signature of the `getResolvedValue` closure defined in + * `getRequestHelperFunctions`. + */ +export type ResolveValueFn = ( + parameterValue: NodeParameterValueType, + itemIndex: number, + runIndex: number, + executeData: IExecuteData, + additionalKeys?: IWorkflowDataProxyAdditionalKeys, + returnObjectAsString?: boolean, +) => NodeParameterValueType; + +export function applyPaginationRequestData( + requestData: IRequestOptions, + paginationRequestData: PaginationOptions['request'], +): IRequestOptions { + const preparedPaginationData: Partial = { + ...paginationRequestData, + uri: paginationRequestData.url, + }; + + if ('formData' in requestData) { + preparedPaginationData.formData = paginationRequestData.body; + delete preparedPaginationData.body; + } else if ('form' in requestData) { + preparedPaginationData.form = paginationRequestData.body; + delete preparedPaginationData.body; + } + + return merge({}, requestData, preparedPaginationData); +} + +// eslint-disable-next-line complexity +export async function requestWithAuthenticationPaginated( + this: IExecuteFunctions, + requestOptions: IRequestOptions, + itemIndex: number, + paginationOptions: PaginationOptions, + resolveValue: ResolveValueFn, + node: INode, + credentialsType?: string, + additionalCredentialOptions?: IAdditionalCredentialOptions, +): Promise { + const responseData = []; + if (!requestOptions.qs) { + requestOptions.qs = {}; + } + requestOptions.resolveWithFullResponse = true; + requestOptions.simple = false; + + let tempResponseData: IN8nHttpFullResponse; + let makeAdditionalRequest: boolean; + let paginateRequestData: PaginationOptions['request']; + + const runIndex = 0; + + const additionalKeys: IWorkflowDataProxyAdditionalKeys = { + $request: requestOptions, + $response: {} as IN8nHttpFullResponse, + $version: node.typeVersion, + $pageCount: 0, + }; + + const executeData: IExecuteData = { + data: {}, + node, + source: null, + }; + + const hashData = { + identicalCount: 0, + previousLength: 0, + previousHash: '', + }; + do { + paginateRequestData = resolveValue( + paginationOptions.request as unknown as NodeParameterValueType, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as object as PaginationOptions['request']; + + const tempRequestOptions = applyPaginationRequestData(requestOptions, paginateRequestData); + + if (!tryParseUrl(tempRequestOptions.uri as string)) { + throw new NodeOperationError(node, `'${paginateRequestData.url}' is not a valid URL.`, { + itemIndex, + runIndex, + type: 'invalid_url', + }); + } + + if (credentialsType) { + tempResponseData = await this.helpers.requestWithAuthentication.call( + this, + credentialsType, + tempRequestOptions, + additionalCredentialOptions, + ); + } else { + tempResponseData = await this.helpers.request(tempRequestOptions); + } + + const newResponse: IN8nHttpFullResponse = Object.assign( + { + body: {}, + headers: {}, + statusCode: 0, + }, + pick(tempResponseData, ['body', 'headers', 'statusCode']), + ); + + let contentBody: Exclude; + + if (newResponse.body instanceof Readable && paginationOptions.binaryResult !== true) { + // Keep the original string version that we can use it to hash if needed + contentBody = await binaryToString(newResponse.body as Buffer | Readable); + + const responseContentType = newResponse.headers['content-type']?.toString() ?? ''; + if (responseContentType.includes('application/json')) { + newResponse.body = jsonParse(contentBody, { fallbackValue: {} }); + } else { + newResponse.body = contentBody; + } + tempResponseData.__bodyResolved = true; + tempResponseData.body = newResponse.body; + } else { + contentBody = newResponse.body; + } + + if (paginationOptions.binaryResult !== true || tempResponseData.headers.etag) { + // If the data is not binary (and so not a stream), or an etag is present, + // we check via etag or hash if identical data is received + + let contentLength = 0; + if ('content-length' in tempResponseData.headers) { + contentLength = parseInt(tempResponseData.headers['content-length'] as string) || 0; + } + + if (hashData.previousLength === contentLength) { + let hash: string; + if (tempResponseData.headers.etag) { + // If an etag is provided, we use it as "hash" + hash = tempResponseData.headers.etag as string; + } else { + // If there is no etag, we calculate a hash from the data in the body + if (typeof contentBody !== 'string') { + contentBody = JSON.stringify(contentBody); + } + hash = crypto.createHash('md5').update(contentBody).digest('base64'); + } + + if (hashData.previousHash === hash) { + hashData.identicalCount += 1; + if (hashData.identicalCount > 2) { + // Length was identical 5x and hash 3x + throw new NodeOperationError( + node, + 'The returned response was identical 5x, so requests got stopped', + { + itemIndex, + description: 'Check if "Pagination Completed When" has been configured correctly.', + }, + ); + } + } else { + hashData.identicalCount = 0; + } + hashData.previousHash = hash; + } else { + hashData.identicalCount = 0; + } + hashData.previousLength = contentLength; + } + + responseData.push(tempResponseData); + + additionalKeys.$response = newResponse; + additionalKeys.$pageCount = (additionalKeys.$pageCount ?? 0) + 1; + + const maxRequests = resolveValue( + paginationOptions.maxRequests, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as number; + + if (maxRequests && additionalKeys.$pageCount >= maxRequests) { + break; + } + + makeAdditionalRequest = resolveValue( + paginationOptions.continue, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as boolean; + + if (makeAdditionalRequest) { + if (paginationOptions.requestInterval) { + const requestInterval = resolveValue( + paginationOptions.requestInterval, + itemIndex, + runIndex, + executeData, + additionalKeys, + false, + ) as number; + + await sleep(requestInterval); + } + if (tempResponseData.statusCode < 200 || tempResponseData.statusCode >= 300) { + // We have it configured to let all requests pass no matter the response code + // via "requestOptions.simple = false" to not by default fail if it is for example + // configured to stop on 404 response codes. For that reason we have to throw here + // now an error manually if the response code is not a success one. + let data = tempResponseData.body; + if (data instanceof Readable && paginationOptions.binaryResult !== true) { + data = await binaryToString(data as Buffer | Readable); + } else if (typeof data === 'object') { + data = JSON.stringify(data); + } + + throw Object.assign(new Error(`${tempResponseData.statusCode} - "${data?.toString()}"`), { + statusCode: tempResponseData.statusCode, + error: data, + isAxiosError: true, + response: { + headers: tempResponseData.headers, + status: tempResponseData.statusCode, + statusText: tempResponseData.statusMessage, + }, + }); + } + } + } while (makeAdditionalRequest); + + return responseData; +}