From b3f3ce25ddd2a8e91f0216b80d5ee96f290fd6c7 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 24 Dec 2019 18:25:44 -0600 Subject: [PATCH 01/22] :construction_worker: Automatically run tests --- .github/workflows/tests.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000000..adb4812df21 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,27 @@ +name: Node CI + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [10.x, 12.x] + + steps: + - uses: actions/checkout@v1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: npm install, build, and test + run: | + npm install + npm run bootstrap + npm run build --if-present + npm test + env: + CI: true From 04cdb0890617418706e4a50d30da93d090da613c Mon Sep 17 00:00:00 2001 From: Ram Yalamanchili Date: Thu, 26 Dec 2019 18:49:48 -0800 Subject: [PATCH 02/22] Enable external module imports in Function-node Enable whitelist support to pick external modules which are allowed in Function-node. The environment variable NODE_FUNCTION_ALLOW_EXTERNAL specifies the list of modules to whitelist. --- docs/configuration.md | 13 +++++++++---- packages/nodes-base/nodes/Function.node.ts | 10 +++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fb612d31fc8..65dd36f472c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -100,11 +100,13 @@ export N8N_CUSTOM_EXTENSIONS="/home/jim/n8n/custom-nodes;/data/n8n/nodes" ``` -## Use built-in modules in Function-Nodes +## Use built-in and external modules in Function-Nodes -By default is it for security reasons not allowed to import modules in Function-Nodes. -It is, however, possible to lift that restriction for built-in modules by setting the -environment variable `NODE_FUNCTION_ALLOW_BUILTIN`. +For security reasons, importing modules is restricted by default in Function-Nodes. +It is, however, possible to lift that restriction for built-in and external modules by +setting the following environment variables: +`NODE_FUNCTION_ALLOW_BUILTIN`: For builtin modules +`NODE_FUNCTION_ALLOW_EXTERNAL`: For external modules sourced from n8n/node_modules directory. External module support is disabled when env variable is not set. ```bash # Allows usage of all builtin modules @@ -115,6 +117,9 @@ export NODE_FUNCTION_ALLOW_BUILTIN=crypto # Allows usage of only crypto and fs export NODE_FUNCTION_ALLOW_BUILTIN=crypto,fs + +# Allow usage of external npm modules. Wildcard matching is not supported. +export NODE_FUNCTION_ALLOW_EXTERNAL=moment,lodash ``` diff --git a/packages/nodes-base/nodes/Function.node.ts b/packages/nodes-base/nodes/Function.node.ts index 558bb50ed36..7892a0a5c09 100644 --- a/packages/nodes-base/nodes/Function.node.ts +++ b/packages/nodes-base/nodes/Function.node.ts @@ -63,9 +63,8 @@ export class Function implements INodeType { console: 'inherit', sandbox, require: { - external: false, + external: false as boolean | { modules: string[] }, builtin: [] as string[], - root: './', } }; @@ -73,6 +72,11 @@ export class Function implements INodeType { options.require.builtin = process.env.NODE_FUNCTION_ALLOW_BUILTIN.split(','); } + if (process.env.NODE_FUNCTION_ALLOW_EXTERNAL) { + options.require.external = { modules: process.env.NODE_FUNCTION_ALLOW_EXTERNAL.split(',') }; + } + + const vm = new NodeVM(options); // Get the code to execute @@ -80,7 +84,7 @@ export class Function implements INodeType { try { // Execute the function code - items = (await vm.run(`module.exports = async function() {${functionCode}}()`)); + items = (await vm.run(`module.exports = async function() {${functionCode}}()`, './')); } catch (e) { return Promise.reject(e); } From 973dde5d9dd7b172c27a152e42988aea85b70e6d Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 26 Dec 2019 21:18:02 -0600 Subject: [PATCH 03/22] :bookmark: Release n8n-nodes-base@0.36.0 --- packages/nodes-base/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 8d89334a2aa..fa5665b66e4 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -1,6 +1,6 @@ { "name": "n8n-nodes-base", - "version": "0.35.0", + "version": "0.36.0", "description": "Base nodes of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From 12734199fbf033ec5f2ab179a418e95ca3e91606 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 26 Dec 2019 21:20:09 -0600 Subject: [PATCH 04/22] :arrow_up: Set n8n-nodes-base@0.36.0 on n8n --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index c7053147741..32c0a921ac0 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -94,7 +94,7 @@ "mongodb": "^3.2.3", "n8n-core": "~0.18.0", "n8n-editor-ui": "~0.29.0", - "n8n-nodes-base": "~0.35.0", + "n8n-nodes-base": "~0.36.0", "n8n-workflow": "~0.18.0", "open": "^6.1.0", "pg": "^7.11.0", From a75b844f0e67dabf37fe4e38dabf19e308232eff Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 26 Dec 2019 21:20:43 -0600 Subject: [PATCH 05/22] :bookmark: Release n8n@0.41.0 --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 32c0a921ac0..2d037e15659 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "0.40.0", + "version": "0.41.0", "description": "n8n Workflow Automation Tool", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From a3ad579f89edcd90a9a00bd5f23d5379ba17258f Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Fri, 27 Dec 2019 08:50:22 -0600 Subject: [PATCH 06/22] :zap: Make it possible to read RAW data from Spreadsheets --- .../nodes-base/nodes/SpreadsheetFile.node.ts | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/packages/nodes-base/nodes/SpreadsheetFile.node.ts b/packages/nodes-base/nodes/SpreadsheetFile.node.ts index 9f82a2b9504..c4026f86e24 100644 --- a/packages/nodes-base/nodes/SpreadsheetFile.node.ts +++ b/packages/nodes-base/nodes/SpreadsheetFile.node.ts @@ -166,22 +166,36 @@ export class SpreadsheetFile implements INodeType { name: 'options', type: 'collection', placeholder: 'Add Option', - displayOptions: { - show: { - operation: [ - 'toFile', - ], - }, - }, default: {}, options: [ { displayName: 'File Name', name: 'fileName', type: 'string', + displayOptions: { + show: { + '/operation': [ + 'toFile', + ], + }, + }, default: '', description: 'File name to set in binary data. By default will "spreadsheet." be used.', }, + { + displayName: 'RAW Data', + name: 'rawData', + type: 'boolean', + displayOptions: { + show: { + '/operation': [ + 'fromFile' + ], + }, + }, + default: false, + description: 'If the data should be returned RAW instead of parsed.', + }, ], }, ] @@ -203,7 +217,8 @@ export class SpreadsheetFile implements INodeType { for (let i = 0; i < items.length; i++) { item = items[i]; - const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string; + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i) as string; + const options = this.getNodeParameter('options', i, {}) as IDataObject; if (item.binary === undefined || item.binary[binaryPropertyName] === undefined) { // Property did not get found on item @@ -212,7 +227,7 @@ export class SpreadsheetFile implements INodeType { // Read the binary spreadsheet data const binaryData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING); - const workbook = xlsxRead(binaryData); + const workbook = xlsxRead(binaryData, { raw: options.rawData as boolean }); if (workbook.SheetNames.length === 0) { throw new Error('File does not have any sheets!'); @@ -235,9 +250,9 @@ export class SpreadsheetFile implements INodeType { return this.prepareOutputData(newItems); } else if (operation === 'toFile') { // Write the workflow data to spreadsheet file - const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string; - const fileFormat = this.getNodeParameter('fileFormat', 0) as string; - const options = this.getNodeParameter('options', 0, {}) as IDataObject; + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i) as string; + const fileFormat = this.getNodeParameter('fileFormat', i) as string; + const options = this.getNodeParameter('options', i, {}) as IDataObject; // Get the json data of the items and flatten it let item: INodeExecutionData; From 72cf3252476ed7e7317c60f7cea2fe84094f71fa Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Fri, 27 Dec 2019 09:32:04 -0600 Subject: [PATCH 07/22] :bug: Fix introduced bug --- packages/nodes-base/nodes/SpreadsheetFile.node.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nodes-base/nodes/SpreadsheetFile.node.ts b/packages/nodes-base/nodes/SpreadsheetFile.node.ts index c4026f86e24..18544091edb 100644 --- a/packages/nodes-base/nodes/SpreadsheetFile.node.ts +++ b/packages/nodes-base/nodes/SpreadsheetFile.node.ts @@ -250,9 +250,9 @@ export class SpreadsheetFile implements INodeType { return this.prepareOutputData(newItems); } else if (operation === 'toFile') { // Write the workflow data to spreadsheet file - const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i) as string; - const fileFormat = this.getNodeParameter('fileFormat', i) as string; - const options = this.getNodeParameter('options', i, {}) as IDataObject; + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string; + const fileFormat = this.getNodeParameter('fileFormat', 0) as string; + const options = this.getNodeParameter('options', 0, {}) as IDataObject; // Get the json data of the items and flatten it let item: INodeExecutionData; From 4a4373882940f2c5fe0c07b8391805ceacf62cdd Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Fri, 27 Dec 2019 10:33:16 -0600 Subject: [PATCH 08/22] :zap: Allow to set custom sheet name --- .../nodes-base/nodes/SpreadsheetFile.node.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/SpreadsheetFile.node.ts b/packages/nodes-base/nodes/SpreadsheetFile.node.ts index 18544091edb..2328b0a004f 100644 --- a/packages/nodes-base/nodes/SpreadsheetFile.node.ts +++ b/packages/nodes-base/nodes/SpreadsheetFile.node.ts @@ -196,6 +196,24 @@ export class SpreadsheetFile implements INodeType { default: false, description: 'If the data should be returned RAW instead of parsed.', }, + { + displayName: 'Sheet Name', + name: 'sheetName', + type: 'string', + displayOptions: { + show: { + '/operation': [ + 'toFile', + ], + '/fileFormat': [ + 'ods', + 'xls', + ], + }, + }, + default: 'Sheet', + description: 'Name of the Sheet in the Spreadsheet.', + }, ], }, ] @@ -282,7 +300,7 @@ export class SpreadsheetFile implements INodeType { } // Convert the data in the correct format - const sheetName = 'Sheet'; + const sheetName = options.sheetName as string || 'Sheet'; const wb: WorkBook = { SheetNames: [sheetName], Sheets: { From d943a8ca03cf454e1bc6381474bb12088402fc83 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Fri, 27 Dec 2019 10:33:42 -0600 Subject: [PATCH 09/22] :zap: Improve display name of fileFormats --- .../nodes-base/nodes/SpreadsheetFile.node.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/nodes-base/nodes/SpreadsheetFile.node.ts b/packages/nodes-base/nodes/SpreadsheetFile.node.ts index 2328b0a004f..e43b4b2d5e2 100644 --- a/packages/nodes-base/nodes/SpreadsheetFile.node.ts +++ b/packages/nodes-base/nodes/SpreadsheetFile.node.ts @@ -109,27 +109,27 @@ export class SpreadsheetFile implements INodeType { type: 'options', options: [ { - name: 'csv', + name: 'CSV', value: 'csv', description: 'Comma-separated values', }, { - name: 'ods', - value: 'ods', - description: 'OpenDocument Spreadsheet', - }, - { - name: 'rtf', - value: 'rtf', - description: 'Rich Text Format', - }, - { - name: 'html', + name: 'HTML', value: 'html', description: 'HTML Table', }, { - name: 'xls', + name: 'ODS', + value: 'ods', + description: 'OpenDocument Spreadsheet', + }, + { + name: 'RTF', + value: 'rtf', + description: 'Rich Text Format', + }, + { + name: 'XLS', value: 'xls', description: 'Excel', }, From ec30f10091b5172535fc7be69e8fe5136d91e063 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Fri, 27 Dec 2019 10:54:51 -0600 Subject: [PATCH 10/22] :zap: Make it possible to read from specific sheet --- .../nodes-base/nodes/SpreadsheetFile.node.ts | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/nodes-base/nodes/SpreadsheetFile.node.ts b/packages/nodes-base/nodes/SpreadsheetFile.node.ts index e43b4b2d5e2..37fb564e09a 100644 --- a/packages/nodes-base/nodes/SpreadsheetFile.node.ts +++ b/packages/nodes-base/nodes/SpreadsheetFile.node.ts @@ -196,6 +196,20 @@ export class SpreadsheetFile implements INodeType { default: false, description: 'If the data should be returned RAW instead of parsed.', }, + { + displayName: 'Sheet Name', + name: 'sheetName', + type: 'string', + displayOptions: { + show: { + '/operation': [ + 'fromFile', + ], + }, + }, + default: 'Sheet', + description: 'Name of the sheet to read from in the spreadsheet (if supported). If not set, the first one gets chosen.', + }, { displayName: 'Sheet Name', name: 'sheetName', @@ -212,7 +226,7 @@ export class SpreadsheetFile implements INodeType { }, }, default: 'Sheet', - description: 'Name of the Sheet in the Spreadsheet.', + description: 'Name of the sheet to create in the spreadsheet.', }, ], }, @@ -248,11 +262,19 @@ export class SpreadsheetFile implements INodeType { const workbook = xlsxRead(binaryData, { raw: options.rawData as boolean }); if (workbook.SheetNames.length === 0) { - throw new Error('File does not have any sheets!'); + throw new Error('Spreadsheet does not have any sheets!'); + } + + let sheetName = workbook.SheetNames[0]; + if (options.sheetName) { + if (!workbook.SheetNames.includes(options.sheetName as string)) { + throw new Error(`Spreadsheet does not contain sheet called "${options.sheetName}"!`); + } + sheetName = options.sheetName as string; } // Convert it to json - const sheetJson = xlsxUtils.sheet_to_json(workbook.Sheets[workbook.SheetNames[0]]); + const sheetJson = xlsxUtils.sheet_to_json(workbook.Sheets[sheetName]); // Check if data could be found in file if (sheetJson.length === 0) { From fe9b37375b386c0f8ac88765308af6d4e8964c4d Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2019 20:11:32 +0000 Subject: [PATCH 11/22] Update @types/compression to version 1.0.1 --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 2d037e15659..0a489c6a697 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -52,7 +52,7 @@ "devDependencies": { "@oclif/dev-cli": "^1.22.2", "@types/basic-auth": "^1.1.2", - "@types/compression": "0.0.36", + "@types/compression": "1.0.1", "@types/connect-history-api-fallback": "^1.3.1", "@types/convict": "^4.2.1", "@types/dotenv": "^6.1.1", From e1be29194916792c33ce758677af125b6559b02c Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Sat, 28 Dec 2019 21:24:14 -0600 Subject: [PATCH 12/22] :zap: Add subtitle to HTTP Request Node --- packages/nodes-base/nodes/HttpRequest.node.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nodes-base/nodes/HttpRequest.node.ts b/packages/nodes-base/nodes/HttpRequest.node.ts index 3795c2772fe..f62409983f5 100644 --- a/packages/nodes-base/nodes/HttpRequest.node.ts +++ b/packages/nodes-base/nodes/HttpRequest.node.ts @@ -25,6 +25,7 @@ export class HttpRequest implements INodeType { icon: 'fa:at', group: ['input'], version: 1, + subtitle: '={{$parameter["requestMethod"] + ": " + $parameter["url"]}}', description: 'Makes a HTTP request and returns the received data', defaults: { name: 'HTTP Request', From 50f3d9bb5a3b949001ba6fee6b0444ead5018ff5 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Sat, 28 Dec 2019 21:28:15 -0600 Subject: [PATCH 13/22] :sparkles: Add HTML Extract-Node --- packages/editor-ui/src/main.ts | 2 + .../nodes/HtmlExtract/HtmlExtract.node.ts | 277 ++++++++++++++++++ packages/nodes-base/package.json | 3 + 3 files changed, 282 insertions(+) create mode 100644 packages/nodes-base/nodes/HtmlExtract/HtmlExtract.node.ts diff --git a/packages/editor-ui/src/main.ts b/packages/editor-ui/src/main.ts index 9cfa172e336..048a718d96e 100644 --- a/packages/editor-ui/src/main.ts +++ b/packages/editor-ui/src/main.ts @@ -34,6 +34,7 @@ import { faClone, faCloud, faCopy, + faCut, faDotCircle, faEdit, faEnvelope, @@ -106,6 +107,7 @@ library.add(faCogs); library.add(faClone); library.add(faCloud); library.add(faCopy); +library.add(faCut); library.add(faDotCircle); library.add(faEdit); library.add(faEnvelope); diff --git a/packages/nodes-base/nodes/HtmlExtract/HtmlExtract.node.ts b/packages/nodes-base/nodes/HtmlExtract/HtmlExtract.node.ts new file mode 100644 index 00000000000..ddedbc08235 --- /dev/null +++ b/packages/nodes-base/nodes/HtmlExtract/HtmlExtract.node.ts @@ -0,0 +1,277 @@ +import * as cheerio from 'cheerio'; +import { IExecuteFunctions } from 'n8n-core'; +import { + INodeExecutionData, + INodeType, + INodeTypeDescription, + IDataObject, +} from 'n8n-workflow'; + +interface IValueData { + attribute?: string; + cssSelector: string; + returnValue: string; + key: string; + returnArray: boolean; +} + + +// The extraction functions +const extractFunctions: { + [key: string]: ($: Cheerio, valueData: IValueData) => string | undefined; +} = { + attribute: ($: Cheerio, valueData: IValueData): string | undefined => $.attr(valueData.attribute!), + html: ($: Cheerio, valueData: IValueData): string | undefined => $.html() || undefined, + text: ($: Cheerio, valueData: IValueData): string | undefined => $.text(), + value: ($: Cheerio, valueData: IValueData): string | undefined => $.val(), +}; + + +/** + * Simple helper function which applies options + */ +function getValue($: Cheerio, valueData: IValueData, options: IDataObject) { + const value = extractFunctions[valueData.returnValue]($, valueData); + if (options.trimValues === false || value === undefined) { + return value; + } + + return value.trim(); +} + + +export class HtmlExtract implements INodeType { + description: INodeTypeDescription = { + displayName: 'HTML Extract', + name: 'htmlExtract', + icon: 'fa:cut', + group: ['transform'], + version: 1, + subtitle: '={{$parameter["sourceData"] + ": " + $parameter["dataPropertyName"]}}', + description: 'Extracts data from HTML', + defaults: { + name: 'HTML Extract', + color: '#333377', + }, + inputs: ['main'], + outputs: ['main'], + properties: [ + { + displayName: 'Source Data', + name: 'sourceData', + type: 'options', + options: [ + { + name: 'Binary', + value: 'binary', + }, + { + name: 'JSON', + value: 'json', + }, + ], + default: 'json', + description: 'If HTML should be read from binary or json data.', + }, + { + displayName: 'Binary Property', + name: 'dataPropertyName', + type: 'string', + displayOptions: { + show: { + sourceData: [ + 'binary', + ], + }, + }, + default: 'data', + required: true, + description: 'Name of the binary property in which the HTML to extract the data from can be found.', + }, + { + displayName: 'JSON Property', + name: 'dataPropertyName', + type: 'string', + displayOptions: { + show: { + sourceData: [ + 'json', + ], + }, + }, + default: 'data', + required: true, + description: 'Name of the json property in which the HTML to extract the data from can be found.
The property can either contain a string or an array of strings.', + }, + { + displayName: 'Extraction Values', + name: 'extractionValues', + placeholder: 'Add Value', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + description: 'The extraction values.', + default: {}, + options: [ + { + name: 'values', + displayName: 'Values', + values: [ + { + displayName: 'Key', + name: 'key', + type: 'string', + default: '', + description: 'The key under which the extracted value should be saved.', + }, + { + displayName: 'CSS Selector', + name: 'cssSelector', + type: 'string', + default: '', + placeholder: '.price', + description: 'The CSS selector to use.', + }, + { + displayName: 'Return Value', + name: 'returnValue', + type: 'options', + options: [ + { + name: 'Attribute', + value: 'attribute', + description: 'Get an attribute value like "class" from an element.', + }, + { + name: 'HTML', + value: 'html', + description: 'Get the HTML the element contains.', + }, + { + name: 'Text', + value: 'text', + description: 'Get only the text content of the element.', + }, + { + name: 'Value', + value: 'value', + description: 'Get value of an input, select or textarea.', + }, + ], + default: 'text', + description: 'What kind of data should be returned.', + }, + { + displayName: 'Attribute', + name: 'attribute', + type: 'string', + displayOptions: { + show: { + returnValue: [ + 'attribute', + ], + }, + }, + default: '', + placeholder: 'class', + description: 'The name of the attribute to return the value off.', + }, + { + displayName: 'Return Array', + name: 'returnArray', + type: 'boolean', + default: false, + description: 'Returns the values as an array so if multiple ones get found they also get
returned separately.If not set all will be returned as a single string.', + }, + ], + }, + ], + }, + + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + { + displayName: 'Trim Values', + name: 'trimValues', + type: 'boolean', + default: true, + description: 'Removes automatically all spaces and newlines from
the beginning and end of the values.', + }, + ], + } + ] + }; + + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + + const returnData: INodeExecutionData[] = []; + + let item: INodeExecutionData; + for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { + const dataPropertyName = this.getNodeParameter('dataPropertyName', itemIndex) as string; + const extractionValues = this.getNodeParameter('extractionValues', itemIndex) as IDataObject; + const options = this.getNodeParameter('options', itemIndex, {}) as IDataObject; + const sourceData = this.getNodeParameter('sourceData', itemIndex) as string; + + item = items[itemIndex]; + + let htmlArray: string[] | string = []; + if (sourceData === 'json') { + if (item.json[dataPropertyName] === undefined) { + throw new Error(`No property named "${dataPropertyName}" exists!`); + } + htmlArray = item.json[dataPropertyName] as string; + } else { + if (item.binary === undefined) { + throw new Error(`No item does not contain binary data!`); + } + if (item.binary[dataPropertyName] === undefined) { + throw new Error(`No property named "${dataPropertyName}" exists!`); + } + htmlArray = Buffer.from(item.binary[dataPropertyName].data, 'base64').toString('utf8'); + } + + // Convert it always to array that it works with a string or an array of strings + if (!Array.isArray(htmlArray)) { + htmlArray = [htmlArray]; + } + + for (const html of htmlArray as string[]) { + const $ = cheerio.load(html); + + const newItem: INodeExecutionData = { + json: {}, + }; + + // Itterate over all the defined values which should be extracted + let htmlElement; + for (const valueData of extractionValues.values as IValueData[]) { + htmlElement = $(valueData.cssSelector); + + if (valueData.returnArray === true) { + // An array should be returned so itterate over one + // value at a time + newItem.json[valueData.key as string] = []; + htmlElement.each((i, el) => { + (newItem.json[valueData.key as string] as Array).push(getValue($(el), valueData, options)); + }); + } else { + // One single value should be returned + newItem.json[valueData.key as string] = getValue(htmlElement, valueData, options); + } + } + returnData.push(newItem); + } + } + + return this.prepareOutputData(returnData); + } +} diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index fa5665b66e4..ebd64cab02e 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -109,6 +109,7 @@ "dist/nodes/Google/GoogleDrive.node.js", "dist/nodes/Google/GoogleSheets.node.js", "dist/nodes/GraphQL/GraphQL.node.js", + "dist/nodes/HtmlExtract/HtmlExtract.node.js", "dist/nodes/HttpRequest.node.js", "dist/nodes/Hubspot/Hubspot.node.js", "dist/nodes/If.node.js", @@ -167,6 +168,7 @@ "devDependencies": { "@types/aws4": "^1.5.1", "@types/basic-auth": "^1.1.2", + "@types/cheerio": "^0.22.15", "@types/cron": "^1.6.1", "@types/express": "^4.16.1", "@types/gm": "^1.18.2", @@ -189,6 +191,7 @@ "dependencies": { "aws4": "^1.8.0", "basic-auth": "^2.0.1", + "cheerio": "^1.0.0-rc.3", "cron": "^1.6.0", "glob-promise": "^3.4.0", "gm": "^1.23.1", From bf0fb1b5fed46542062d2cbcaf072f6ec1542231 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2019 15:07:42 +0000 Subject: [PATCH 14/22] Update @types/dotenv to version 8.2.0 --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 0a489c6a697..2b899ef8c65 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -55,7 +55,7 @@ "@types/compression": "1.0.1", "@types/connect-history-api-fallback": "^1.3.1", "@types/convict": "^4.2.1", - "@types/dotenv": "^6.1.1", + "@types/dotenv": "^8.2.0", "@types/express": "^4.16.1", "@types/jest": "^24.0.18", "@types/localtunnel": "^1.9.0", From 1043e81472ec3a1b676e66e1252cb79d223a5d94 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2019 16:20:19 +0000 Subject: [PATCH 15/22] Update inquirer to version 7.0.1 --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 0a489c6a697..7cd60ec9eb6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -86,7 +86,7 @@ "flatted": "^2.0.0", "glob-promise": "^3.4.0", "google-timezones-json": "^1.0.2", - "inquirer": "^6.5.1", + "inquirer": "^7.0.1", "jsonwebtoken": "^8.5.1", "jwks-rsa": "^1.6.0", "localtunnel": "^1.9.1", From da425ff8575af206f3b2e938eeb2b1830118a753 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Sun, 29 Dec 2019 12:22:37 -0600 Subject: [PATCH 16/22] :arrow_up: Set localtunnel@2.0.0 on n8n --- packages/cli/commands/start.ts | 6 ++---- packages/cli/package.json | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/cli/commands/start.ts b/packages/cli/commands/start.ts index 62e6615897f..5494b5c5c30 100644 --- a/packages/cli/commands/start.ts +++ b/packages/cli/commands/start.ts @@ -5,8 +5,7 @@ import { } from "n8n-core"; import { Command, flags } from '@oclif/command'; const open = require('open'); -import { promisify } from 'util'; -import { dirname } from 'path'; +// import { dirname } from 'path'; import * as config from '../config'; import { @@ -20,7 +19,6 @@ import { TestWebhooks, } from "../src"; -const tunnel = promisify(localtunnel); // // Add support for internationalization // const fullIcuPath = require.resolve('full-icu'); @@ -151,7 +149,7 @@ export class Start extends Command { const port = config.get('port') as number; // @ts-ignore - const webhookTunnel = await tunnel(port, tunnelSettings); + const webhookTunnel = await localtunnel(port, tunnelSettings); process.env.WEBHOOK_TUNNEL_URL = webhookTunnel.url + '/'; this.log(`Tunnel URL: ${process.env.WEBHOOK_TUNNEL_URL}\n`); diff --git a/packages/cli/package.json b/packages/cli/package.json index 6dd6f69711e..41302c17433 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -89,7 +89,7 @@ "inquirer": "^7.0.1", "jsonwebtoken": "^8.5.1", "jwks-rsa": "^1.6.0", - "localtunnel": "^1.9.1", + "localtunnel": "^2.0.0", "lodash.get": "^4.4.2", "mongodb": "^3.2.3", "n8n-core": "~0.18.0", From 0ca4362d66b0c9abdbf5c9e8cb2c9ae9db992863 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2019 18:45:16 +0000 Subject: [PATCH 17/22] Update nodemon to version 2.0.2 --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 6dd6f69711e..dc6224d018a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -65,7 +65,7 @@ "@types/parseurl": "^1.3.1", "@types/request-promise-native": "^1.0.15", "jest": "^24.9.0", - "nodemon": "^1.19.1", + "nodemon": "^2.0.2", "run-script-os": "^1.0.7", "ts-jest": "^24.0.2", "tslint": "^5.17.0", From e2b74abe62f45e4e507d64101f5bd2cea60c666b Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2019 19:55:17 +0000 Subject: [PATCH 18/22] Update open to version 7.0.0 --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 41302c17433..22656fd2dfc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -96,7 +96,7 @@ "n8n-editor-ui": "~0.29.0", "n8n-nodes-base": "~0.36.0", "n8n-workflow": "~0.18.0", - "open": "^6.1.0", + "open": "^7.0.0", "pg": "^7.11.0", "request-promise-native": "^1.0.7", "sqlite3": "^4.0.6", From 49e22855733a23e832465bcc0a0431a01a62578d Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Sun, 29 Dec 2019 15:02:21 -0600 Subject: [PATCH 19/22] :arrow_up: Update packages to latest version on n8n-editor-ui --- packages/editor-ui/.eslintrc.js | 3 +- packages/editor-ui/package.json | 38 ++++++++++--------- .../src/components/BinaryDataDisplay.vue | 4 -- .../src/components/CredentialsList.vue | 7 +--- .../editor-ui/src/components/MainSidebar.vue | 5 ++- .../src/components/ParameterInputList.vue | 2 +- .../src/components/VariableSelector.vue | 28 +++++++------- .../src/components/WorkflowActivator.vue | 2 +- .../editor-ui/src/components/WorkflowOpen.vue | 4 +- .../src/components/WorkflowSettings.vue | 4 +- .../src/components/mixins/genericHelpers.ts | 2 +- .../src/components/mixins/restApi.ts | 4 +- .../src/components/mixins/workflowHelpers.ts | 4 +- .../src/components/mixins/workflowSave.ts | 4 +- packages/editor-ui/src/views/NodeView.vue | 11 +++--- packages/editor-ui/vue.config.js | 2 +- 16 files changed, 61 insertions(+), 63 deletions(-) diff --git a/packages/editor-ui/.eslintrc.js b/packages/editor-ui/.eslintrc.js index 25493bb780e..b2f7cd19817 100644 --- a/packages/editor-ui/.eslintrc.js +++ b/packages/editor-ui/.eslintrc.js @@ -5,7 +5,6 @@ module.exports = { }, 'extends': [ 'plugin:vue/essential', - '@vue/standard', '@vue/typescript', ], rules: { @@ -18,6 +17,6 @@ module.exports = { 'no-labels': 0, }, parserOptions: { - parser: 'typescript-eslint-parser', + parser: '@typescript-eslint/parser', }, }; diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index 2cbd05db4d2..ec0310dfcc0 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -34,23 +34,27 @@ "@types/jest": "^24.0.18", "@types/lodash.get": "^4.4.5", "@types/lodash.set": "^4.3.6", + "@types/node": "12.12.22", "@types/quill": "^2.0.1", - "@vue/cli-plugin-babel": "^3.8.0", - "@vue/cli-plugin-e2e-cypress": "^3.8.0", - "@vue/cli-plugin-eslint": "^3.8.0", - "@vue/cli-plugin-typescript": "~3.8.1", - "@vue/cli-plugin-unit-jest": "^3.8.0", - "@vue/cli-service": "^3.8.0", - "@vue/eslint-config-standard": "^4.0.0", - "@vue/eslint-config-typescript": "~3.2.0", - "@vue/test-utils": "^1.0.0-beta.20", - "axios": "^0.18.1", + "@typescript-eslint/eslint-plugin": "^2.13.0", + "@typescript-eslint/parser": "^2.13.0", + "@vue/cli-plugin-babel": "^4.1.2", + "@vue/cli-plugin-e2e-cypress": "^4.1.2", + "@vue/cli-plugin-eslint": "^4.1.2", + "@vue/cli-plugin-typescript": "~4.1.2", + "@vue/cli-plugin-unit-jest": "^4.1.2", + "@vue/cli-service": "^4.1.2", + "@vue/eslint-config-standard": "^5.0.1", + "@vue/eslint-config-typescript": "~5.0.1", + "@vue/test-utils": "^1.0.0-beta.24", + "axios": "^0.19.0", "babel-core": "7.0.0-bridge.0", "babel-eslint": "^10.0.1", "dateformat": "^3.0.3", - "element-ui": "~2.4.11", - "eslint": "^5.8.0", - "eslint-plugin-vue": "^5.0.0-0", + "element-ui": "~2.13.0", + "eslint": "^6.8.0", + "eslint-plugin-import": "^2.19.1", + "eslint-plugin-vue": "^6.1.1", "file-saver": "^2.0.2", "flatted": "^2.0.0", "jquery": "^3.4.1", @@ -64,18 +68,18 @@ "prismjs": "^1.17.1", "quill": "^2.0.0-dev.3", "quill-autoformat": "^0.1.1", - "sass-loader": "^7.0.1", + "sass-loader": "^8.0.0", "string-template-parser": "^1.2.6", "ts-jest": "^24.0.2", "tslint": "^5.17.0", - "typescript": "~3.5.2", + "typescript": "~3.7.4", "vue": "^2.6.9", - "vue-cli-plugin-webpack-bundle-analyzer": "^1.3.0", + "vue-cli-plugin-webpack-bundle-analyzer": "^2.0.0", "vue-json-tree": "^0.4.1", "vue-prism-editor": "^0.3.0", "vue-router": "^3.0.6", "vue-template-compiler": "^2.5.17", - "vue-typed-mixins": "^0.1.0", + "vue-typed-mixins": "^0.2.0", "vuex": "^3.1.1" } } diff --git a/packages/editor-ui/src/components/BinaryDataDisplay.vue b/packages/editor-ui/src/components/BinaryDataDisplay.vue index b66fdd02eab..c9aca4856d5 100644 --- a/packages/editor-ui/src/components/BinaryDataDisplay.vue +++ b/packages/editor-ui/src/components/BinaryDataDisplay.vue @@ -21,14 +21,10 @@ diff --git a/packages/editor-ui/src/components/WorkflowOpen.vue b/packages/editor-ui/src/components/WorkflowOpen.vue index 6fe447cfb9a..9592ecb4d55 100644 --- a/packages/editor-ui/src/components/WorkflowOpen.vue +++ b/packages/editor-ui/src/components/WorkflowOpen.vue @@ -104,13 +104,13 @@ export default mixins( workflowData.updatedAt = this.convertToDisplayDate(workflowData.updatedAt as number); }); this.isDataLoading = false; - } + }, ) .catch( (error: Error) => { this.$showError(error, 'Problem loading workflows', 'There was a problem loading the workflows:'); this.isDataLoading = false; - } + }, ); }, workflowActiveChanged (data: { id: string, active: boolean }) { diff --git a/packages/editor-ui/src/components/WorkflowSettings.vue b/packages/editor-ui/src/components/WorkflowSettings.vue index 9b804e6dc20..178d51a8135 100644 --- a/packages/editor-ui/src/components/WorkflowSettings.vue +++ b/packages/editor-ui/src/components/WorkflowSettings.vue @@ -185,7 +185,7 @@ export default mixins( key: 'none', value: 'Do not save', }, - ] + ], ); }, async loadSaveDataSuccessExecutionOptions () { @@ -204,7 +204,7 @@ export default mixins( key: 'none', value: 'Do not save', }, - ] + ], ); }, async loadSaveManualOptions () { diff --git a/packages/editor-ui/src/components/mixins/genericHelpers.ts b/packages/editor-ui/src/components/mixins/genericHelpers.ts index 4b77564a545..a51eecd5479 100644 --- a/packages/editor-ui/src/components/mixins/genericHelpers.ts +++ b/packages/editor-ui/src/components/mixins/genericHelpers.ts @@ -63,7 +63,7 @@ export const genericHelpers = mixins(showMessage).extend({ text: 'Loading', spinner: 'el-icon-loading', background: 'rgba(255, 255, 255, 0.8)', - } + }, ); }, stopLoading () { diff --git a/packages/editor-ui/src/components/mixins/restApi.ts b/packages/editor-ui/src/components/mixins/restApi.ts index 875e3661530..a72520718e4 100644 --- a/packages/editor-ui/src/components/mixins/restApi.ts +++ b/packages/editor-ui/src/components/mixins/restApi.ts @@ -1,7 +1,7 @@ import Vue from 'vue'; import { parse } from 'flatted'; -import axios, { AxiosRequestConfig } from 'axios'; +import axios, { AxiosRequestConfig, Method } from 'axios'; import { IActivationError, ICredentialsDecryptedResponse, @@ -94,7 +94,7 @@ export const restApi = Vue.extend({ restApi (): IRestApi { const self = this; return { - async makeRestApiRequest (method: string, endpoint: string, data?: IDataObject): Promise { // tslint:disable-line:no-any + async makeRestApiRequest (method: Method, endpoint: string, data?: IDataObject): Promise { // tslint:disable-line:no-any try { const options: AxiosRequestConfig = { method, diff --git a/packages/editor-ui/src/components/mixins/workflowHelpers.ts b/packages/editor-ui/src/components/mixins/workflowHelpers.ts index 6c39c61ce49..e12eeb21a04 100644 --- a/packages/editor-ui/src/components/mixins/workflowHelpers.ts +++ b/packages/editor-ui/src/components/mixins/workflowHelpers.ts @@ -370,7 +370,7 @@ export const workflowHelpers = mixins( { confirmButtonText: 'Save', cancelButtonText: 'Cancel', - } + }, ) .then((data) => { // @ts-ignore @@ -403,7 +403,7 @@ export const workflowHelpers = mixins( if (currentWorkflow === undefined || withNewName === true) { // Workflow is new or is supposed to get saved under a new name // so create a new entry in database - workflowData.name = workflowName.trim() as string; + workflowData.name = workflowName!.trim() as string; if (withNewName === true) { // If an existing workflow gets resaved with a new name diff --git a/packages/editor-ui/src/components/mixins/workflowSave.ts b/packages/editor-ui/src/components/mixins/workflowSave.ts index 25963290326..1584d9a7c8e 100644 --- a/packages/editor-ui/src/components/mixins/workflowSave.ts +++ b/packages/editor-ui/src/components/mixins/workflowSave.ts @@ -29,7 +29,7 @@ export const workflowSave = mixins( { confirmButtonText: 'Save', cancelButtonText: 'Cancel', - } + }, ) .then((data) => { // @ts-ignore @@ -62,7 +62,7 @@ export const workflowSave = mixins( if (currentWorkflow === undefined || withNewName === true) { // Workflow is new or is supposed to get saved under a new name // so create a new entry in database - workflowData.name = workflowName.trim() as string; + workflowData.name = workflowName!.trim() as string; if (withNewName === true) { // If an existing workflow gets resaved with a new name diff --git a/packages/editor-ui/src/views/NodeView.vue b/packages/editor-ui/src/views/NodeView.vue index ba5abdbb40a..b9db49e03fe 100644 --- a/packages/editor-ui/src/views/NodeView.vue +++ b/packages/editor-ui/src/views/NodeView.vue @@ -102,6 +102,7 @@