From f06db123e4e6d8a0bb2ed2ddeda102d8d3404d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ir=C3=A9n=C3=A9e?= Date: Fri, 7 Aug 2026 12:50:41 +0100 Subject: [PATCH] feat: spike multi-repository source control and project isolation Add multi-repo source control connections with per-project scoping (LIGO-923): connection entities/repositories/services, sync service, REST controller, DB migration, schema docs, and editor-ui for managing connections, push/pull, and project-level source control. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/generated/postgres-schema/README.md | 27 + .../postgres-schema/public.project.md | 11 +- .../public.source_control_connection.md | 78 ++ .../public.source_control_scope.md | 81 ++ docs/generated/sqlite-schema/README.md | 27 + docs/generated/sqlite-schema/project.md | 11 +- .../source_control_connection.md | 81 ++ .../sqlite-schema/source_control_scope.md | 88 ++ ...000-CreateSourceControlConnectionTables.ts | 66 ++ .../db/src/migrations/postgresdb/index.ts | 2 + .../@n8n/db/src/migrations/sqlite/index.ts | 2 + .../n8n-packages/engine/import-telemetry.ts | 4 +- .../engine/project-package-importer.ts | 8 +- .../engine/workflow-package-importer.ts | 4 +- .../credential/credential.exporter.ts | 6 +- .../data-table/data-table.exporter.ts | 6 +- .../entities/folder/folder.exporter.ts | 7 +- .../entities/project/project.exporter.ts | 8 +- .../n8n-packages/entities/tag/tag.exporter.ts | 6 +- .../entities/variable/variable.exporter.ts | 4 +- .../entities/variable/variable.types.ts | 3 +- .../auto-included-workflow.exporter.ts | 32 +- .../entities/workflow/workflow.exporter.ts | 7 +- .../n8n-packages/io/fs/fs-package-reader.ts | 50 ++ .../n8n-packages/io/fs/fs-package-writer.ts | 29 + .../io/unique-filename-allocator.ts | 10 +- .../n8n-packages/n8n-packages.service.ts | 31 +- .../n8n-packages/n8n-packages.types.ts | 13 + .../multi-repo/connection-git.ts | 179 ++++ .../multi-repo/multi-repo.types.ts | 68 ++ .../source-control-connection.entity.ts | 48 ++ .../source-control-connection.repository.ts | 19 + .../source-control-connection.service.ts | 259 ++++++ .../source-control-connections.controller.ts | 123 +++ .../multi-repo/source-control-scope.entity.ts | 32 + .../source-control-scope.repository.ts | 31 + .../multi-repo/source-control-sync.service.ts | 129 +++ .../source-control.module.ts | 10 + .../ownership-transfer.manifest.json | 5 + .../frontend/@n8n/i18n/src/locales/en.json | 44 + .../components/MainSidebarSourceControl.vue | 80 +- .../editor-ui/src/app/components/Modals.vue | 16 + .../editor-ui/src/app/stores/ui.store.ts | 4 + .../components/ProjectSourceControl.vue | 172 ++++ .../projects/views/ProjectSettings.vue | 3 + .../SourceControlConnectionForm.vue | 318 +++++++ .../SourceControlConnectionPullModal.vue | 137 +++ .../SourceControlConnectionPushModal.vue | 151 ++++ .../sourceControl.constants.ts | 2 + .../sourceControlConnections.api.ts | 120 +++ .../sourceControlConnections.store.ts | 162 ++++ .../sourceControlConnections.types.ts | 58 ++ .../views/SettingsSourceControl.vue | 780 +++++------------- 53 files changed, 3018 insertions(+), 634 deletions(-) create mode 100644 docs/generated/postgres-schema/public.source_control_connection.md create mode 100644 docs/generated/postgres-schema/public.source_control_scope.md create mode 100644 docs/generated/sqlite-schema/source_control_connection.md create mode 100644 docs/generated/sqlite-schema/source_control_scope.md create mode 100644 packages/@n8n/db/src/migrations/common/1786000000000-CreateSourceControlConnectionTables.ts create mode 100644 packages/cli/src/modules/n8n-packages/io/fs/fs-package-reader.ts create mode 100644 packages/cli/src/modules/n8n-packages/io/fs/fs-package-writer.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/connection-git.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/multi-repo.types.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.entity.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.repository.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.service.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/source-control-connections.controller.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.entity.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.repository.ts create mode 100644 packages/cli/src/modules/source-control.ee/multi-repo/source-control-sync.service.ts create mode 100644 packages/frontend/editor-ui/src/features/collaboration/projects/components/ProjectSourceControl.vue create mode 100644 packages/frontend/editor-ui/src/features/integrations/sourceControl.ee/components/SourceControlConnectionForm.vue create mode 100644 packages/frontend/editor-ui/src/features/integrations/sourceControl.ee/components/SourceControlConnectionPullModal.vue create mode 100644 packages/frontend/editor-ui/src/features/integrations/sourceControl.ee/components/SourceControlConnectionPushModal.vue create mode 100644 packages/frontend/editor-ui/src/features/integrations/sourceControl.ee/sourceControlConnections.api.ts create mode 100644 packages/frontend/editor-ui/src/features/integrations/sourceControl.ee/sourceControlConnections.store.ts create mode 100644 packages/frontend/editor-ui/src/features/integrations/sourceControl.ee/sourceControlConnections.types.ts diff --git a/docs/generated/postgres-schema/README.md b/docs/generated/postgres-schema/README.md index 13aa56c7d7c..09627e7ac22 100644 --- a/docs/generated/postgres-schema/README.md +++ b/docs/generated/postgres-schema/README.md @@ -106,6 +106,8 @@ Auto-generated from the PostgreSQL migrations in @n8n/db. Do not edit by hand. | [public.settings](public.settings.md) | 3 | | BASE TABLE | | [public.shared_credentials](public.shared_credentials.md) | 5 | | BASE TABLE | | [public.shared_workflow](public.shared_workflow.md) | 5 | | BASE TABLE | +| [public.source_control_connection](public.source_control_connection.md) | 13 | | BASE TABLE | +| [public.source_control_scope](public.source_control_scope.md) | 6 | | BASE TABLE | | [public.tag_entity](public.tag_entity.md) | 4 | | BASE TABLE | | [public.test_case_execution](public.test_case_execution.md) | 14 | | BASE TABLE | | [public.test_run](public.test_run.md) | 16 | | BASE TABLE | @@ -293,6 +295,8 @@ erDiagram "public.shared_credentials" }o--|| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" "public.shared_workflow" }o--|| "public.workflow_entity" : "FOREIGN KEY (#quot;workflowId#quot;) REFERENCES workflow_entity(id) ON DELETE CASCADE" "public.shared_workflow" }o--|| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" +"public.source_control_scope" }o--o| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" +"public.source_control_scope" }o--|| "public.source_control_connection" : "FOREIGN KEY (#quot;connectionId#quot;) REFERENCES source_control_connection(id) ON DELETE CASCADE" "public.test_case_execution" }o--o| "public.execution_entity" : "FOREIGN KEY (#quot;executionId#quot;) REFERENCES execution_entity(id) ON DELETE SET NULL" "public.test_case_execution" }o--|| "public.test_run" : "FOREIGN KEY (#quot;testRunId#quot;) REFERENCES test_run(id) ON DELETE CASCADE" "public.test_run" }o--|| "public.workflow_entity" : "FOREIGN KEY (#quot;workflowId#quot;) REFERENCES workflow_entity(id) ON DELETE CASCADE" @@ -1298,6 +1302,29 @@ erDiagram timestamp_3__with_time_zone updatedAt varchar_36_ workflowId FK } +"public.source_control_connection" { + varchar_16_ branchColor + varchar_255_ branchName + boolean branchReadOnly + boolean connected + varchar_16_ connectionType + timestamp_3__with_time_zone createdAt + text encryptedPassword + text encryptedPrivateKey + text encryptedUsername + varchar_36_ id + text publicKey + text repositoryUrl + timestamp_3__with_time_zone updatedAt +} +"public.source_control_scope" { + varchar_36_ connectionId FK + timestamp_3__with_time_zone createdAt + varchar_36_ id + varchar_36_ projectId FK + varchar_16_ scopeType + timestamp_3__with_time_zone updatedAt +} "public.tag_entity" { timestamp_3__with_time_zone createdAt varchar_36_ id diff --git a/docs/generated/postgres-schema/public.project.md b/docs/generated/postgres-schema/public.project.md index 373cd0e8ea2..b7a200e318e 100644 --- a/docs/generated/postgres-schema/public.project.md +++ b/docs/generated/postgres-schema/public.project.md @@ -9,7 +9,7 @@ | customTelemetryTags | json | '[]'::json | false | | | | | description | varchar(512) | | true | | | | | icon | json | | true | | | | -| id | varchar(36) | | false | [public.agent_chat_attachments](public.agent_chat_attachments.md) [public.agent_execution_threads](public.agent_execution_threads.md) [public.agents](public.agents.md) [public.data_table](public.data_table.md) [public.folder](public.folder.md) [public.insights_metadata](public.insights_metadata.md) [public.instance_ai_threads](public.instance_ai_threads.md) [public.project_relation](public.project_relation.md) [public.project_secrets_provider_access](public.project_secrets_provider_access.md) [public.role_mapping_rule_project](public.role_mapping_rule_project.md) [public.shared_credentials](public.shared_credentials.md) [public.shared_workflow](public.shared_workflow.md) [public.variables](public.variables.md) [public.workflow_review_request](public.workflow_review_request.md) | | | +| id | varchar(36) | | false | [public.agent_chat_attachments](public.agent_chat_attachments.md) [public.agent_execution_threads](public.agent_execution_threads.md) [public.agents](public.agents.md) [public.data_table](public.data_table.md) [public.folder](public.folder.md) [public.insights_metadata](public.insights_metadata.md) [public.instance_ai_threads](public.instance_ai_threads.md) [public.project_relation](public.project_relation.md) [public.project_secrets_provider_access](public.project_secrets_provider_access.md) [public.role_mapping_rule_project](public.role_mapping_rule_project.md) [public.shared_credentials](public.shared_credentials.md) [public.shared_workflow](public.shared_workflow.md) [public.source_control_scope](public.source_control_scope.md) [public.variables](public.variables.md) [public.workflow_review_request](public.workflow_review_request.md) | | | | name | varchar(255) | | false | | | | | type | varchar(36) | | false | | | | | updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | | @@ -51,6 +51,7 @@ erDiagram "public.role_mapping_rule_project" }o--|| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" "public.shared_credentials" }o--|| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" "public.shared_workflow" }o--|| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" +"public.source_control_scope" }o--o| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" "public.variables" }o--o| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" "public.workflow_review_request" }o--|| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" @@ -193,6 +194,14 @@ erDiagram timestamp_3__with_time_zone updatedAt varchar_36_ workflowId FK } +"public.source_control_scope" { + varchar_36_ connectionId FK + timestamp_3__with_time_zone createdAt + varchar_36_ id + varchar_36_ projectId FK + varchar_16_ scopeType + timestamp_3__with_time_zone updatedAt +} "public.variables" { varchar_36_ id varchar_50_ key diff --git a/docs/generated/postgres-schema/public.source_control_connection.md b/docs/generated/postgres-schema/public.source_control_connection.md new file mode 100644 index 00000000000..b86c0080f16 --- /dev/null +++ b/docs/generated/postgres-schema/public.source_control_connection.md @@ -0,0 +1,78 @@ +# public.source_control_connection + +## Columns + +| Name | Type | Default | Nullable | Children | Parents | Comment | +| ---- | ---- | ------- | -------- | -------- | ------- | ------- | +| branchColor | varchar(16) | '#5296D6'::character varying | false | | | | +| branchName | varchar(255) | 'main'::character varying | false | | | | +| branchReadOnly | boolean | false | false | | | | +| connected | boolean | false | false | | | | +| connectionType | varchar(16) | | false | | | | +| createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | | +| encryptedPassword | text | | true | | | | +| encryptedPrivateKey | text | | true | | | | +| encryptedUsername | text | | true | | | | +| id | varchar(36) | | false | [public.source_control_scope](public.source_control_scope.md) | | | +| publicKey | text | | true | | | | +| repositoryUrl | text | | false | | | | +| updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | | + +## Constraints + +| Name | Type | Definition | +| ---- | ---- | ---------- | +| CHK_source_control_connection_connectionType | CHECK | CHECK ((("connectionType")::text = ANY ((ARRAY['ssh'::character varying, 'https'::character varying])::text[]))) | +| PK_62b13b8ae2c3016f25728202252 | PRIMARY KEY | PRIMARY KEY (id) | +| source_control_connection_branchColor_not_null | n | NOT NULL "branchColor" | +| source_control_connection_branchName_not_null | n | NOT NULL "branchName" | +| source_control_connection_branchReadOnly_not_null | n | NOT NULL "branchReadOnly" | +| source_control_connection_connected_not_null | n | NOT NULL connected | +| source_control_connection_connectionType_not_null | n | NOT NULL "connectionType" | +| source_control_connection_createdAt_not_null | n | NOT NULL "createdAt" | +| source_control_connection_id_not_null | n | NOT NULL id | +| source_control_connection_repositoryUrl_not_null | n | NOT NULL "repositoryUrl" | +| source_control_connection_updatedAt_not_null | n | NOT NULL "updatedAt" | + +## Indexes + +| Name | Definition | +| ---- | ---------- | +| IDX_source_control_connection_repo_branch | CREATE UNIQUE INDEX "IDX_source_control_connection_repo_branch" ON public.source_control_connection USING btree ("repositoryUrl", "branchName") | +| PK_62b13b8ae2c3016f25728202252 | CREATE UNIQUE INDEX "PK_62b13b8ae2c3016f25728202252" ON public.source_control_connection USING btree (id) | + +## Relations + +```mermaid +erDiagram + +"public.source_control_scope" }o--|| "public.source_control_connection" : "FOREIGN KEY (#quot;connectionId#quot;) REFERENCES source_control_connection(id) ON DELETE CASCADE" + +"public.source_control_connection" { + varchar_16_ branchColor + varchar_255_ branchName + boolean branchReadOnly + boolean connected + varchar_16_ connectionType + timestamp_3__with_time_zone createdAt + text encryptedPassword + text encryptedPrivateKey + text encryptedUsername + varchar_36_ id + text publicKey + text repositoryUrl + timestamp_3__with_time_zone updatedAt +} +"public.source_control_scope" { + varchar_36_ connectionId FK + timestamp_3__with_time_zone createdAt + varchar_36_ id + varchar_36_ projectId FK + varchar_16_ scopeType + timestamp_3__with_time_zone updatedAt +} +``` + +--- + +> Generated by [tbls](https://github.com/k1LoW/tbls) diff --git a/docs/generated/postgres-schema/public.source_control_scope.md b/docs/generated/postgres-schema/public.source_control_scope.md new file mode 100644 index 00000000000..b2c30a49289 --- /dev/null +++ b/docs/generated/postgres-schema/public.source_control_scope.md @@ -0,0 +1,81 @@ +# public.source_control_scope + +## Columns + +| Name | Type | Default | Nullable | Children | Parents | Comment | +| ---- | ---- | ------- | -------- | -------- | ------- | ------- | +| connectionId | varchar(36) | | false | | [public.source_control_connection](public.source_control_connection.md) | | +| createdAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | | +| id | varchar(36) | | false | | | | +| projectId | varchar(36) | | true | | [public.project](public.project.md) | | +| scopeType | varchar(16) | | false | | | | +| updatedAt | timestamp(3) with time zone | CURRENT_TIMESTAMP(3) | false | | | | + +## Constraints + +| Name | Type | Definition | +| ---- | ---- | ---------- | +| CHK_source_control_scope_scopeType | CHECK | CHECK ((("scopeType")::text = ANY ((ARRAY['project'::character varying, 'instance'::character varying])::text[]))) | +| FK_72b7e38e545688d47bce4534384 | FOREIGN KEY | FOREIGN KEY ("connectionId") REFERENCES source_control_connection(id) ON DELETE CASCADE | +| FK_97d8e9baacd10e8ea11db44ed35 | FOREIGN KEY | FOREIGN KEY ("projectId") REFERENCES project(id) ON DELETE CASCADE | +| PK_5466e741188f23e3a7b29ed0c10 | PRIMARY KEY | PRIMARY KEY (id) | +| source_control_scope_connectionId_not_null | n | NOT NULL "connectionId" | +| source_control_scope_createdAt_not_null | n | NOT NULL "createdAt" | +| source_control_scope_id_not_null | n | NOT NULL id | +| source_control_scope_scopeType_not_null | n | NOT NULL "scopeType" | +| source_control_scope_updatedAt_not_null | n | NOT NULL "updatedAt" | + +## Indexes + +| Name | Definition | +| ---- | ---------- | +| IDX_source_control_scope_project | CREATE UNIQUE INDEX "IDX_source_control_scope_project" ON public.source_control_scope USING btree ("projectId") | +| PK_5466e741188f23e3a7b29ed0c10 | CREATE UNIQUE INDEX "PK_5466e741188f23e3a7b29ed0c10" ON public.source_control_scope USING btree (id) | + +## Relations + +```mermaid +erDiagram + +"public.source_control_scope" }o--|| "public.source_control_connection" : "FOREIGN KEY (#quot;connectionId#quot;) REFERENCES source_control_connection(id) ON DELETE CASCADE" +"public.source_control_scope" }o--o| "public.project" : "FOREIGN KEY (#quot;projectId#quot;) REFERENCES project(id) ON DELETE CASCADE" + +"public.source_control_scope" { + varchar_36_ connectionId FK + timestamp_3__with_time_zone createdAt + varchar_36_ id + varchar_36_ projectId FK + varchar_16_ scopeType + timestamp_3__with_time_zone updatedAt +} +"public.source_control_connection" { + varchar_16_ branchColor + varchar_255_ branchName + boolean branchReadOnly + boolean connected + varchar_16_ connectionType + timestamp_3__with_time_zone createdAt + text encryptedPassword + text encryptedPrivateKey + text encryptedUsername + varchar_36_ id + text publicKey + text repositoryUrl + timestamp_3__with_time_zone updatedAt +} +"public.project" { + timestamp_3__with_time_zone createdAt + uuid creatorId FK + json customTelemetryTags + varchar_512_ description + json icon + varchar_36_ id + varchar_255_ name + varchar_36_ type + timestamp_3__with_time_zone updatedAt +} +``` + +--- + +> Generated by [tbls](https://github.com/k1LoW/tbls) diff --git a/docs/generated/sqlite-schema/README.md b/docs/generated/sqlite-schema/README.md index 7182604a07c..9121004c96f 100644 --- a/docs/generated/sqlite-schema/README.md +++ b/docs/generated/sqlite-schema/README.md @@ -106,6 +106,8 @@ Auto-generated from the SQLite migrations in @n8n/db. Do not edit by hand. | [settings](settings.md) | 3 | | table | | [shared_credentials](shared_credentials.md) | 5 | | table | | [shared_workflow](shared_workflow.md) | 5 | | table | +| [source_control_connection](source_control_connection.md) | 13 | | table | +| [source_control_scope](source_control_scope.md) | 6 | | table | | [tag_entity](tag_entity.md) | 4 | | table | | [test_case_execution](test_case_execution.md) | 16 | | table | | [test_run](test_run.md) | 16 | | table | @@ -276,6 +278,8 @@ erDiagram "shared_credentials" |o--|| "credentials_entity" : "FOREIGN KEY (credentialsId) REFERENCES credentials_entity (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "shared_workflow" |o--|| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "shared_workflow" |o--|| "workflow_entity" : "FOREIGN KEY (workflowId) REFERENCES workflow_entity (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" +"source_control_scope" }o--o| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" +"source_control_scope" }o--|| "source_control_connection" : "FOREIGN KEY (connectionId) REFERENCES source_control_connection (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "test_case_execution" }o--|| "test_run" : "FOREIGN KEY (testRunId) REFERENCES test_run (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "test_case_execution" }o--o| "execution_entity" : "FOREIGN KEY (pastExecutionId) REFERENCES execution_entity (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE" "test_case_execution" }o--o| "execution_entity" : "FOREIGN KEY (executionId) REFERENCES execution_entity (id) ON UPDATE NO ACTION ON DELETE SET NULL MATCH NONE" @@ -1285,6 +1289,29 @@ erDiagram datetime_3_ updatedAt varchar_36_ workflowId PK } +"source_control_connection" { + varchar_16_ branchColor + varchar_255_ branchName + boolean branchReadOnly + boolean connected + varchar_16_ connectionType + datetime_3_ createdAt + TEXT encryptedPassword + TEXT encryptedPrivateKey + TEXT encryptedUsername + varchar_36_ id PK + TEXT publicKey + TEXT repositoryUrl + datetime_3_ updatedAt +} +"source_control_scope" { + varchar_36_ connectionId FK + datetime_3_ createdAt + varchar_36_ id PK + varchar_36_ projectId FK + varchar_16_ scopeType + datetime_3_ updatedAt +} "tag_entity" { datetime_3_ createdAt varchar_36_ id PK diff --git a/docs/generated/sqlite-schema/project.md b/docs/generated/sqlite-schema/project.md index ee58c8dc61e..c0b3d8ebd2b 100644 --- a/docs/generated/sqlite-schema/project.md +++ b/docs/generated/sqlite-schema/project.md @@ -20,7 +20,7 @@ CREATE TABLE "project" ("id" varchar(36) PRIMARY KEY NOT NULL, "name" varchar(25 | customTelemetryTags | TEXT | '[]' | false | | | | | description | varchar(512) | | true | | | | | icon | TEXT | | true | | | | -| id | varchar(36) | | false | [agent_chat_attachments](agent_chat_attachments.md) [agent_execution_threads](agent_execution_threads.md) [agents](agents.md) [data_table](data_table.md) [folder](folder.md) [insights_metadata](insights_metadata.md) [instance_ai_threads](instance_ai_threads.md) [project_relation](project_relation.md) [project_secrets_provider_access](project_secrets_provider_access.md) [role_mapping_rule_project](role_mapping_rule_project.md) [shared_credentials](shared_credentials.md) [shared_workflow](shared_workflow.md) [variables](variables.md) [workflow_review_request](workflow_review_request.md) | | | +| id | varchar(36) | | false | [agent_chat_attachments](agent_chat_attachments.md) [agent_execution_threads](agent_execution_threads.md) [agents](agents.md) [data_table](data_table.md) [folder](folder.md) [insights_metadata](insights_metadata.md) [instance_ai_threads](instance_ai_threads.md) [project_relation](project_relation.md) [project_secrets_provider_access](project_secrets_provider_access.md) [role_mapping_rule_project](role_mapping_rule_project.md) [shared_credentials](shared_credentials.md) [shared_workflow](shared_workflow.md) [source_control_scope](source_control_scope.md) [variables](variables.md) [workflow_review_request](workflow_review_request.md) | | | | name | varchar(255) | | false | | | | | type | varchar(36) | | false | | | | | updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | | @@ -57,6 +57,7 @@ erDiagram "role_mapping_rule_project" |o--|| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "shared_credentials" |o--|| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "shared_workflow" |o--|| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" +"source_control_scope" }o--o| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "variables" }o--o| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" "workflow_review_request" }o--|| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" @@ -199,6 +200,14 @@ erDiagram datetime_3_ updatedAt varchar_36_ workflowId PK } +"source_control_scope" { + varchar_36_ connectionId FK + datetime_3_ createdAt + varchar_36_ id PK + varchar_36_ projectId FK + varchar_16_ scopeType + datetime_3_ updatedAt +} "variables" { varchar_36_ id PK TEXT key diff --git a/docs/generated/sqlite-schema/source_control_connection.md b/docs/generated/sqlite-schema/source_control_connection.md new file mode 100644 index 00000000000..491b7a39905 --- /dev/null +++ b/docs/generated/sqlite-schema/source_control_connection.md @@ -0,0 +1,81 @@ +# source_control_connection + +## Description + +
+Table Definition + +```sql +CREATE TABLE "source_control_connection" ("id" varchar(36) PRIMARY KEY NOT NULL, "repositoryUrl" text NOT NULL, "branchName" varchar(255) NOT NULL DEFAULT ('main'), "branchReadOnly" boolean NOT NULL DEFAULT (false), "branchColor" varchar(16) NOT NULL DEFAULT ('#5296D6'), "connectionType" varchar(16) NOT NULL, "connected" boolean NOT NULL DEFAULT (false), "publicKey" text, "encryptedPrivateKey" text, "encryptedUsername" text, "encryptedPassword" text, "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), CONSTRAINT "CHK_source_control_connection_connectionType" CHECK ("connectionType" IN ('ssh', 'https'))) +``` + +
+ +## Columns + +| Name | Type | Default | Nullable | Children | Parents | Comment | +| ---- | ---- | ------- | -------- | -------- | ------- | ------- | +| branchColor | varchar(16) | '#5296D6' | false | | | | +| branchName | varchar(255) | 'main' | false | | | | +| branchReadOnly | boolean | false | false | | | | +| connected | boolean | false | false | | | | +| connectionType | varchar(16) | | false | | | | +| createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | | +| encryptedPassword | TEXT | | true | | | | +| encryptedPrivateKey | TEXT | | true | | | | +| encryptedUsername | TEXT | | true | | | | +| id | varchar(36) | | false | [source_control_scope](source_control_scope.md) | | | +| publicKey | TEXT | | true | | | | +| repositoryUrl | TEXT | | false | | | | +| updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | | + +## Constraints + +| Name | Type | Definition | +| ---- | ---- | ---------- | +| - | CHECK | CHECK ("connectionType" IN ('ssh', 'https')) | +| id | PRIMARY KEY | PRIMARY KEY (id) | +| sqlite_autoindex_source_control_connection_1 | PRIMARY KEY | PRIMARY KEY (id) | + +## Indexes + +| Name | Definition | +| ---- | ---------- | +| IDX_source_control_connection_repo_branch | CREATE UNIQUE INDEX "IDX_source_control_connection_repo_branch"
ON "source_control_connection"("repositoryUrl", "branchName") | +| sqlite_autoindex_source_control_connection_1 | PRIMARY KEY (id) | + +## Relations + +```mermaid +erDiagram + +"source_control_scope" }o--|| "source_control_connection" : "FOREIGN KEY (connectionId) REFERENCES source_control_connection (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" + +"source_control_connection" { + varchar_16_ branchColor + varchar_255_ branchName + boolean branchReadOnly + boolean connected + varchar_16_ connectionType + datetime_3_ createdAt + TEXT encryptedPassword + TEXT encryptedPrivateKey + TEXT encryptedUsername + varchar_36_ id PK + TEXT publicKey + TEXT repositoryUrl + datetime_3_ updatedAt +} +"source_control_scope" { + varchar_36_ connectionId FK + datetime_3_ createdAt + varchar_36_ id PK + varchar_36_ projectId FK + varchar_16_ scopeType + datetime_3_ updatedAt +} +``` + +--- + +> Generated by [tbls](https://github.com/k1LoW/tbls) diff --git a/docs/generated/sqlite-schema/source_control_scope.md b/docs/generated/sqlite-schema/source_control_scope.md new file mode 100644 index 00000000000..725a0ace72b --- /dev/null +++ b/docs/generated/sqlite-schema/source_control_scope.md @@ -0,0 +1,88 @@ +# source_control_scope + +## Description + +
+Table Definition + +```sql +CREATE TABLE "source_control_scope" ("id" varchar(36) PRIMARY KEY NOT NULL, "connectionId" varchar(36) NOT NULL, "scopeType" varchar(16) NOT NULL, "projectId" varchar(36), "createdAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), "updatedAt" datetime(3) NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), CONSTRAINT "CHK_source_control_scope_scopeType" CHECK ("scopeType" IN ('project', 'instance')), CONSTRAINT "FK_72b7e38e545688d47bce4534384" FOREIGN KEY ("connectionId") REFERENCES "source_control_connection" ("id") ON DELETE CASCADE, CONSTRAINT "FK_97d8e9baacd10e8ea11db44ed35" FOREIGN KEY ("projectId") REFERENCES "project" ("id") ON DELETE CASCADE) +``` + +
+ +## Columns + +| Name | Type | Default | Nullable | Children | Parents | Comment | +| ---- | ---- | ------- | -------- | -------- | ------- | ------- | +| connectionId | varchar(36) | | false | | [source_control_connection](source_control_connection.md) | | +| createdAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | | +| id | varchar(36) | | false | | | | +| projectId | varchar(36) | | true | | [project](project.md) | | +| scopeType | varchar(16) | | false | | | | +| updatedAt | datetime(3) | STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') | false | | | | + +## Constraints + +| Name | Type | Definition | +| ---- | ---- | ---------- | +| - | CHECK | CHECK ("scopeType" IN ('project', 'instance')) | +| - (Foreign key ID: 0) | FOREIGN KEY | FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE | +| - (Foreign key ID: 1) | FOREIGN KEY | FOREIGN KEY (connectionId) REFERENCES source_control_connection (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE | +| id | PRIMARY KEY | PRIMARY KEY (id) | +| sqlite_autoindex_source_control_scope_1 | PRIMARY KEY | PRIMARY KEY (id) | + +## Indexes + +| Name | Definition | +| ---- | ---------- | +| IDX_source_control_scope_project | CREATE UNIQUE INDEX "IDX_source_control_scope_project"
ON "source_control_scope"("projectId") | +| sqlite_autoindex_source_control_scope_1 | PRIMARY KEY (id) | + +## Relations + +```mermaid +erDiagram + +"source_control_scope" }o--|| "source_control_connection" : "FOREIGN KEY (connectionId) REFERENCES source_control_connection (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" +"source_control_scope" }o--o| "project" : "FOREIGN KEY (projectId) REFERENCES project (id) ON UPDATE NO ACTION ON DELETE CASCADE MATCH NONE" + +"source_control_scope" { + varchar_36_ connectionId FK + datetime_3_ createdAt + varchar_36_ id PK + varchar_36_ projectId FK + varchar_16_ scopeType + datetime_3_ updatedAt +} +"source_control_connection" { + varchar_16_ branchColor + varchar_255_ branchName + boolean branchReadOnly + boolean connected + varchar_16_ connectionType + datetime_3_ createdAt + TEXT encryptedPassword + TEXT encryptedPrivateKey + TEXT encryptedUsername + varchar_36_ id PK + TEXT publicKey + TEXT repositoryUrl + datetime_3_ updatedAt +} +"project" { + datetime_3_ createdAt + varchar creatorId FK + TEXT customTelemetryTags + varchar_512_ description + TEXT icon + varchar_36_ id PK + varchar_255_ name + varchar_36_ type + datetime_3_ updatedAt +} +``` + +--- + +> Generated by [tbls](https://github.com/k1LoW/tbls) diff --git a/packages/@n8n/db/src/migrations/common/1786000000000-CreateSourceControlConnectionTables.ts b/packages/@n8n/db/src/migrations/common/1786000000000-CreateSourceControlConnectionTables.ts new file mode 100644 index 00000000000..fef72ac3ca8 --- /dev/null +++ b/packages/@n8n/db/src/migrations/common/1786000000000-CreateSourceControlConnectionTables.ts @@ -0,0 +1,66 @@ +import type { MigrationContext, ReversibleMigration } from '../migration-types'; + +const CONNECTION_TABLE = 'source_control_connection'; +const SCOPE_TABLE = 'source_control_scope'; +const PROJECT_TABLE = 'project'; + +const UQ_CONNECTION_REPO_BRANCH = 'source_control_connection_repo_branch'; +const UQ_SCOPE_PROJECT = 'source_control_scope_project'; + +export class CreateSourceControlConnectionTables1786000000000 implements ReversibleMigration { + async up({ schemaBuilder: { createTable, column }, runQuery, escape }: MigrationContext) { + await createTable(CONNECTION_TABLE).withColumns( + column('id').varchar(36).primary, + column('repositoryUrl').text.notNull, + column('branchName').varchar(255).notNull.default("'main'"), + column('branchReadOnly').bool.notNull.default(false), + column('branchColor').varchar(16).notNull.default("'#5296D6'"), + column('connectionType').varchar(16).notNull.withEnumCheck(['ssh', 'https']), + column('connected').bool.notNull.default(false), + column('publicKey').text, + column('encryptedPrivateKey').text, + column('encryptedUsername').text, + column('encryptedPassword').text, + ).withTimestamps; + + await createTable(SCOPE_TABLE) + .withColumns( + column('id').varchar(36).primary, + column('connectionId').varchar(36).notNull, + column('scopeType').varchar(16).notNull.withEnumCheck(['project', 'instance']), + column('projectId').varchar(36), + ) + .withTimestamps.withForeignKey('connectionId', { + tableName: CONNECTION_TABLE, + columnName: 'id', + onDelete: 'CASCADE', + }) + .withForeignKey('projectId', { + tableName: PROJECT_TABLE, + columnName: 'id', + onDelete: 'CASCADE', + }); + + const connectionTable = escape.tableName(CONNECTION_TABLE); + const scopeTable = escape.tableName(SCOPE_TABLE); + // repositoryUrl is text; index its column pair via an expression-safe varchar cast is + // unnecessary here — both sqlite and postgres index text columns directly. + await runQuery( + `CREATE UNIQUE INDEX IF NOT EXISTS ${escape.indexName(UQ_CONNECTION_REPO_BRANCH)} + ON ${connectionTable}(${escape.columnName('repositoryUrl')}, ${escape.columnName('branchName')})`, + ); + // NULL projectIds (instance scopes) are distinct in both sqlite and postgres, + // so this enforces "a project is claimed by at most one connection" only. + await runQuery( + `CREATE UNIQUE INDEX IF NOT EXISTS ${escape.indexName(UQ_SCOPE_PROJECT)} + ON ${scopeTable}(${escape.columnName('projectId')})`, + ); + } + + async down({ schemaBuilder: { dropTable }, runQuery, escape }: MigrationContext) { + await runQuery(`DROP INDEX IF EXISTS ${escape.indexName(UQ_SCOPE_PROJECT)}`); + await runQuery(`DROP INDEX IF EXISTS ${escape.indexName(UQ_CONNECTION_REPO_BRANCH)}`); + await dropTable(SCOPE_TABLE); + await dropTable(CONNECTION_TABLE); + } +} diff --git a/packages/@n8n/db/src/migrations/postgresdb/index.ts b/packages/@n8n/db/src/migrations/postgresdb/index.ts index 9fcafac14c1..be7164af31b 100644 --- a/packages/@n8n/db/src/migrations/postgresdb/index.ts +++ b/packages/@n8n/db/src/migrations/postgresdb/index.ts @@ -238,6 +238,7 @@ import { AddMisfirePolicyToScheduler1785247194307 } from '../common/178524719430 import { CreateAgentChatAttachmentsTable1785255306000 } from '../common/1785255306000-CreateAgentChatAttachmentsTable'; import { AddSetupCompletedAtToAgents1785500832626 } from '../common/1785500832626-AddSetupCompletedAtToAgents'; import { AddAgentExecutionRuntimeState1785828155091 } from '../common/1785828155091-AddAgentExecutionRuntimeState'; +import { CreateSourceControlConnectionTables1786000000000 } from '../common/1786000000000-CreateSourceControlConnectionTables'; import type { Migration } from '../migration-types'; export const postgresMigrations: Migration[] = [ @@ -481,4 +482,5 @@ export const postgresMigrations: Migration[] = [ CreateAgentChatAttachmentsTable1785255306000, AddSetupCompletedAtToAgents1785500832626, AddAgentExecutionRuntimeState1785828155091, + CreateSourceControlConnectionTables1786000000000, ]; diff --git a/packages/@n8n/db/src/migrations/sqlite/index.ts b/packages/@n8n/db/src/migrations/sqlite/index.ts index e8fffaa8759..e52dcd64dc5 100644 --- a/packages/@n8n/db/src/migrations/sqlite/index.ts +++ b/packages/@n8n/db/src/migrations/sqlite/index.ts @@ -230,6 +230,7 @@ import { AddAgentFileStorageColumns1785186578138 } from '../common/1785186578138 import { CrashStaleEnqueuedExecutions1785247194306 } from '../common/1785247194306-CrashStaleEnqueuedExecutions'; import { CreateAgentChatAttachmentsTable1785255306000 } from '../common/1785255306000-CreateAgentChatAttachmentsTable'; import { AddAgentExecutionRuntimeState1785828155091 } from '../common/1785828155091-AddAgentExecutionRuntimeState'; +import { CreateSourceControlConnectionTables1786000000000 } from '../common/1786000000000-CreateSourceControlConnectionTables'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -463,6 +464,7 @@ const sqliteMigrations: Migration[] = [ CreateAgentChatAttachmentsTable1785255306000, AddSetupCompletedAtToAgents1785500832626, AddAgentExecutionRuntimeState1785828155091, + CreateSourceControlConnectionTables1786000000000, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts b/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts index 0c63be9bf73..3c55eee40b8 100644 --- a/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts +++ b/packages/cli/src/modules/n8n-packages/engine/import-telemetry.ts @@ -6,7 +6,7 @@ import type { TagImportPlan, TagImportRequest } from '../entities/tag/tag.types' import type { VariableImportRequest } from '../entities/variable/variable.types'; import type { PersistedWorkflowOutcome } from '../entities/workflow/workflow-import.types'; import { VariableParentPolicy } from '../n8n-packages.types'; -import type { ImportContext, ImportPackageRequest } from '../n8n-packages.types'; +import type { ImportContext, ImportRequestOptions } from '../n8n-packages.types'; import type { ImportContentResult } from './import-orchestrator'; import { reconcileVariableSummary } from './import-result'; import type { PackageManifest } from '../spec/manifest.schema'; @@ -24,7 +24,7 @@ export interface PackageImportScope { export function emitPackageImportedEvent( eventService: EventService, params: { - request: ImportPackageRequest; + request: ImportRequestOptions; manifest: PackageManifest; scopes: PackageImportScope[]; }, diff --git a/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts b/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts index 36e014bddc4..d024c5eee87 100644 --- a/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts +++ b/packages/cli/src/modules/n8n-packages/engine/project-package-importer.ts @@ -17,7 +17,7 @@ import type { ImportBindingMap, ImportedFolderSummary, ImportedWorkflowSummary, - ImportPackageRequest, + ImportRequestOptions, ImportResult, ImportTagSummary, PackageImportBindings, @@ -58,7 +58,7 @@ export class ProjectPackageImporter { ) {} async import( - request: ImportPackageRequest, + request: ImportRequestOptions, reader: PackageReader, manifest: PackageManifest, ): Promise { @@ -193,7 +193,7 @@ export class ProjectPackageImporter { } private async buildImportContextForProject( - request: ImportPackageRequest, + request: ImportRequestOptions, reader: PackageReader, manifest: PackageManifest, project: ManifestEntry, @@ -261,7 +261,7 @@ export class ProjectPackageImporter { } private assertAdequatePermissions( - request: ImportPackageRequest, + request: ImportRequestOptions, manifest: PackageManifest, ): void { // A project package can create new projects or update matched ones (by source id), so require both — diff --git a/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts b/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts index c7573836885..c4fad70245d 100644 --- a/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts +++ b/packages/cli/src/modules/n8n-packages/engine/workflow-package-importer.ts @@ -18,7 +18,7 @@ import type { VariableImportRequest } from '../entities/variable/variable.types' import { WorkflowPublisher } from '../entities/workflow/workflow-publisher'; import type { PackageReader } from '../io/package-reader'; import { VariableParentPolicy } from '../n8n-packages.types'; -import type { ImportContext, ImportPackageRequest, ImportResult } from '../n8n-packages.types'; +import type { ImportContext, ImportRequestOptions, ImportResult } from '../n8n-packages.types'; import { assertPackageImportApiKeyScopes, assertTagWritesAllowed } from './import-gates'; import { ImportOrchestrator } from './import-orchestrator'; import { @@ -51,7 +51,7 @@ export class WorkflowPackageImporter { ) {} async import( - request: ImportPackageRequest, + request: ImportRequestOptions, reader: PackageReader, manifest: PackageManifest, ): Promise { diff --git a/packages/cli/src/modules/n8n-packages/entities/credential/credential.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/credential/credential.exporter.ts index fbe959ff44e..103471dc360 100644 --- a/packages/cli/src/modules/n8n-packages/entities/credential/credential.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/credential/credential.exporter.ts @@ -7,6 +7,7 @@ import { CredentialSerializer } from './credential.serializer'; import type { WorkflowCredentialRequirement } from './credential.types'; import type { PackageWriter } from '../../io/package-writer'; import { UniqueFilenameAllocator } from '../../io/unique-filename-allocator'; +import type { PathStyle } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import type { PackageCredentialRequirement } from '../../spec/requirements.schema'; @@ -23,6 +24,7 @@ export interface CredentialExportRequest { // Contains a map of projectId to export location // p123 -> /project/p123/ projectTargetsById?: Map; + pathStyle?: PathStyle; } export interface CredentialExportResult { @@ -44,7 +46,7 @@ export class CredentialExporter { const allocatorFor = (baseDir: string) => { const existing = allocators.get(baseDir); if (existing) return existing; - const created = new UniqueFilenameAllocator(baseDir, 'credential'); + const created = new UniqueFilenameAllocator(baseDir, 'credential', request.pathStyle); allocators.set(baseDir, created); return created; }; @@ -69,7 +71,7 @@ export class CredentialExporter { if (credential) { const baseDir = this.resolveBaseDir(credential, request.projectTargetsById); - const target = allocatorFor(baseDir).allocate(name); + const target = allocatorFor(baseDir).allocate(name, id); request.writer.writeDirectory(target); request.writer.writeFile( `${target}/credential.json`, diff --git a/packages/cli/src/modules/n8n-packages/entities/data-table/data-table.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/data-table/data-table.exporter.ts index 83d1f4b9c2e..795103257b1 100644 --- a/packages/cli/src/modules/n8n-packages/entities/data-table/data-table.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/data-table/data-table.exporter.ts @@ -10,6 +10,7 @@ import { DataTableSerializer } from './data-table.serializer'; import type { WorkflowDataTableRequirement } from './data-table.types'; import type { PackageWriter } from '../../io/package-writer'; import { UniqueFilenameAllocator } from '../../io/unique-filename-allocator'; +import type { PathStyle } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import type { PackageDataTableRequirement } from '../../spec/requirements.schema'; @@ -18,6 +19,7 @@ export interface DataTableExportRequest { requirements: WorkflowDataTableRequirement[]; writer: PackageWriter; projectTargetsById?: Map; + pathStyle?: PathStyle; } export interface DataTableExportResult { @@ -61,7 +63,7 @@ export class DataTableExporter { const allocatorFor = (baseDir: string) => { const existing = allocators.get(baseDir); if (existing) return existing; - const created = new UniqueFilenameAllocator(baseDir, 'data-table'); + const created = new UniqueFilenameAllocator(baseDir, 'data-table', request.pathStyle); allocators.set(baseDir, created); return created; }; @@ -72,7 +74,7 @@ export class DataTableExporter { for (const dataTable of dataTables) { const usedByWorkflows = usedByWorkflowsById.get(dataTable.id) ?? []; const baseDir = this.resolveBaseDir(dataTable, request.projectTargetsById); - const target = allocatorFor(baseDir).allocate(dataTable.name); + const target = allocatorFor(baseDir).allocate(dataTable.name, dataTable.id); request.writer.writeDirectory(target); request.writer.writeFile( diff --git a/packages/cli/src/modules/n8n-packages/entities/folder/folder.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/folder/folder.exporter.ts index aaaa343f194..9de520a86d0 100644 --- a/packages/cli/src/modules/n8n-packages/entities/folder/folder.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/folder/folder.exporter.ts @@ -7,6 +7,7 @@ import { WorkflowFinderService } from '@/workflows/workflow-finder.service'; import { FolderSerializer } from './folder.serializer'; import type { PackageWriter } from '../../io/package-writer'; import { UniqueFilenameAllocator } from '../../io/unique-filename-allocator'; +import type { PathStyle } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import { assertEveryRequestedEntityAccessible } from '../package-export.errors'; import { mergeRequirements } from '../requirements.types'; @@ -25,6 +26,7 @@ export interface FolderExportRequest { * same walk nests under `projects//folders/...`. */ basePrefix?: string; + pathStyle?: PathStyle; } export interface FolderExportResult { @@ -115,12 +117,12 @@ export class FolderExporter { reservedName?: string, ): Promise { // File names need to be unique within a folder only. - const allocator = new UniqueFilenameAllocator(parentDir, 'folder'); + const allocator = new UniqueFilenameAllocator(parentDir, 'folder', context.request.pathStyle); if (reservedName) allocator.reserve(reservedName); const results: FolderExportResult[] = []; for (const folder of this.orderedByCreation(siblings)) { - const target = allocator.allocate(folder.name); + const target = allocator.allocate(folder.name, folder.id); results.push(await this.exportFolder(folder, target, effectiveParentId, context)); } @@ -184,6 +186,7 @@ export class FolderExporter { workflowIds, includeTags: request.includeTags, basePrefix, + pathStyle: request.pathStyle, }); } diff --git a/packages/cli/src/modules/n8n-packages/entities/project/project.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/project/project.exporter.ts index ccd7c91a4e4..08d80566516 100644 --- a/packages/cli/src/modules/n8n-packages/entities/project/project.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/project/project.exporter.ts @@ -8,6 +8,7 @@ import { WorkflowFinderService } from '@/workflows/workflow-finder.service'; import { ProjectSerializer } from './project.serializer'; import type { PackageWriter } from '../../io/package-writer'; import { UniqueFilenameAllocator } from '../../io/unique-filename-allocator'; +import type { PathStyle } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import { FolderExporter } from '../folder/folder.exporter'; import type { FolderExportResult } from '../folder/folder.exporter'; @@ -22,6 +23,7 @@ export interface ProjectExportRequest { projectIds: string[]; writer: PackageWriter; includeTags: boolean; + pathStyle?: PathStyle; } interface ProjectExportResult { @@ -57,11 +59,11 @@ export class ProjectExporter { async (ids) => await this.projectService.findExistingProjectIds(ids), ); - const allocator = new UniqueFilenameAllocator('projects', 'project'); + const allocator = new UniqueFilenameAllocator('projects', 'project', request.pathStyle); const results: ProjectExportResult[] = []; for (const project of projects) { - const target = allocator.allocate(project.name); + const target = allocator.allocate(project.name, project.id); results.push(await this.exportProject(project, target, request)); } @@ -112,6 +114,7 @@ export class ProjectExporter { writer: request.writer, includeTags: request.includeTags, basePrefix: target, + pathStyle: request.pathStyle, }); } @@ -131,6 +134,7 @@ export class ProjectExporter { writer: request.writer, includeTags: request.includeTags, basePrefix: target, + pathStyle: request.pathStyle, }); } diff --git a/packages/cli/src/modules/n8n-packages/entities/tag/tag.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/tag/tag.exporter.ts index f564c545471..178d68304f3 100644 --- a/packages/cli/src/modules/n8n-packages/entities/tag/tag.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/tag/tag.exporter.ts @@ -2,6 +2,7 @@ import { Service } from '@n8n/di'; import type { PackageWriter } from '../../io/package-writer'; import { UniqueFilenameAllocator } from '../../io/unique-filename-allocator'; +import type { PathStyle } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import type { PackageTagRequirement } from '../../spec/requirements.schema'; import { serializedTagSchema } from '../../spec/serialized/tag.schema'; @@ -10,6 +11,7 @@ import { compareTagsByName, type WorkflowTagUsage } from './tag.types'; export interface TagExportRequest { usages: WorkflowTagUsage[]; writer: PackageWriter; + pathStyle?: PathStyle; } export interface TagExportResult { @@ -36,11 +38,11 @@ export class TagExporter { const requirements = [...requirementsByTagId.values()].sort(compareTagsByName); - const allocator = new UniqueFilenameAllocator('tags', 'tag'); + const allocator = new UniqueFilenameAllocator('tags', 'tag', request.pathStyle); const entries: ManifestEntry[] = []; for (const { id, name } of requirements) { - const tagDirectory = allocator.allocate(name); + const tagDirectory = allocator.allocate(name, id); const serializedTag = serializedTagSchema.parse({ id, name }); request.writer.writeDirectory(tagDirectory); request.writer.writeFile( diff --git a/packages/cli/src/modules/n8n-packages/entities/variable/variable.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/variable/variable.exporter.ts index 6b0de68038b..3a9a76d3af7 100644 --- a/packages/cli/src/modules/n8n-packages/entities/variable/variable.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/variable/variable.exporter.ts @@ -73,7 +73,7 @@ export class VariableExporter { const allocatorFor = (baseDir: string) => { const existing = allocators.get(baseDir); if (existing) return existing; - const created = new UniqueFilenameAllocator(baseDir, 'variable'); + const created = new UniqueFilenameAllocator(baseDir, 'variable', request.pathStyle); allocators.set(baseDir, created); return created; }; @@ -90,7 +90,7 @@ export class VariableExporter { bundledVariableIds.add(variable.id); const baseDir = this.resolveBaseDir(variable, request.projectTargetsById); - const target = allocatorFor(baseDir).allocate(variable.key); + const target = allocatorFor(baseDir).allocate(variable.key, variable.id); request.writer.writeDirectory(target); request.writer.writeFile( `${target}/variable.json`, diff --git a/packages/cli/src/modules/n8n-packages/entities/variable/variable.types.ts b/packages/cli/src/modules/n8n-packages/entities/variable/variable.types.ts index c605cd1e563..a7a37725db9 100644 --- a/packages/cli/src/modules/n8n-packages/entities/variable/variable.types.ts +++ b/packages/cli/src/modules/n8n-packages/entities/variable/variable.types.ts @@ -1,7 +1,7 @@ import type { User } from '@n8n/db'; import type { PackageWriter } from '../../io/package-writer'; -import type { VariableMissingMode } from '../../n8n-packages.types'; +import type { PathStyle, VariableMissingMode } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import type { PackageVariableRequirement } from '../../spec/requirements.schema'; @@ -16,6 +16,7 @@ export interface VariableExportRequest { writer: PackageWriter; includeVariableValues: boolean; projectTargetsById?: Map; + pathStyle?: PathStyle; } export interface VariableExportResult { diff --git a/packages/cli/src/modules/n8n-packages/entities/workflow/auto-included-workflow.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/workflow/auto-included-workflow.exporter.ts index 0a4d07848c4..d07e60a2708 100644 --- a/packages/cli/src/modules/n8n-packages/entities/workflow/auto-included-workflow.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/workflow/auto-included-workflow.exporter.ts @@ -2,11 +2,12 @@ import type { Folder, Project, WorkflowEntity } from '@n8n/db'; import { Service } from '@n8n/di'; import { UnexpectedError } from 'n8n-workflow'; -import type { WorkflowNodeTypeSource } from './node-type-usage'; import type { AutoIncludedWorkflow } from './auto-included-workflow-resolver'; +import type { WorkflowNodeTypeSource } from './node-type-usage'; import { WorkflowSerializer } from './workflow.serializer'; import type { PackageWriter } from '../../io/package-writer'; import { UniqueFilenameAllocator } from '../../io/unique-filename-allocator'; +import type { PathStyle } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import { CredentialRequirementsExtractor } from '../credential/credential-requirements.extractor'; import type { WorkflowCredentialRequirement } from '../credential/credential.types'; @@ -28,6 +29,7 @@ export interface AutoIncludedWorkflowExportRequest { existingProjectEntries: ManifestEntry[]; includeTags: boolean; projectTargetsById?: Map; + pathStyle?: PathStyle; } export interface AutoIncludedWorkflowExportResult { @@ -42,6 +44,7 @@ interface ExportAllocators { workflows: Map; folders: Map; project: UniqueFilenameAllocator; + pathStyle?: PathStyle; } /** @@ -64,7 +67,8 @@ export class AutoIncludedWorkflowExporter { const allocators: ExportAllocators = { workflows: new Map(), folders: new Map(), - project: new UniqueFilenameAllocator('projects', 'project'), + project: new UniqueFilenameAllocator('projects', 'project', request.pathStyle), + pathStyle: request.pathStyle, }; const workflowEntriesById = new Map( @@ -119,6 +123,7 @@ export class AutoIncludedWorkflowExporter { request.writer, allocators.workflows, request.includeTags, + request.pathStyle, ); workflowEntries.push(entry); workflowEntriesById.set(entry.id, entry); @@ -173,6 +178,7 @@ export class AutoIncludedWorkflowExporter { folderEntriesById: options.folderEntriesById, folderEntries: options.folderEntries, allocators: options.allocators.folders, + pathStyle: options.allocators.pathStyle, }); return `${folderTarget}/workflows`; } @@ -185,6 +191,7 @@ export class AutoIncludedWorkflowExporter { folderEntriesById: options.folderEntriesById, folderEntries: options.folderEntries, allocators: options.allocators.folders, + pathStyle: options.allocators.pathStyle, }); return `${folderTarget}/workflows`; } @@ -203,7 +210,7 @@ export class AutoIncludedWorkflowExporter { const existing = options.projectEntriesById.get(options.project.id); if (existing) return existing.target; - const target = options.allocator.allocate(options.project.name); + const target = options.allocator.allocate(options.project.name, options.project.id); const serialized = this.projectSerializer.serialize(options.project); options.writer.writeDirectory(target); options.writer.writeFile(`${target}/project.json`, JSON.stringify(serialized, null, '\t')); @@ -222,6 +229,7 @@ export class AutoIncludedWorkflowExporter { folderEntriesById: Map; folderEntries: ManifestEntry[]; allocators: Map; + pathStyle?: PathStyle; }): string { let parentTarget: string | undefined; let effectiveParentId: string | null = null; @@ -234,12 +242,17 @@ export class AutoIncludedWorkflowExporter { continue; } - const allocator = allocatorFor(options.allocators, parentTarget ?? options.baseDir, 'folder'); + const allocator = allocatorFor( + options.allocators, + parentTarget ?? options.baseDir, + 'folder', + options.pathStyle, + ); // A folder's directly-contained workflows are written under `/workflows`, // so reserve that segment before placing child folders — otherwise a child folder // named "workflows" would collide with its parent's workflow directory. if (parentTarget) allocator.reserve('workflows'); - const target = allocator.allocate(folder.name); + const target = allocator.allocate(folder.name, folder.id); const serialized = this.folderSerializer.serialize(folder, effectiveParentId); options.writer.writeDirectory(target); options.writer.writeFile(`${target}/folder.json`, JSON.stringify(serialized, null, '\t')); @@ -269,8 +282,12 @@ export class AutoIncludedWorkflowExporter { writer: PackageWriter, allocators: Map, includeTags: boolean, + pathStyle?: PathStyle, ): ManifestEntry { - const target = allocatorFor(allocators, baseDir, 'workflow').allocate(workflow.name); + const target = allocatorFor(allocators, baseDir, 'workflow', pathStyle).allocate( + workflow.name, + workflow.id, + ); const serialized = this.workflowSerializer.serialize(workflow, { includeTags }); writer.writeDirectory(target); writer.writeFile(`${target}/workflow.json`, JSON.stringify(serialized, null, '\t')); @@ -286,11 +303,12 @@ function allocatorFor( allocators: Map, baseDir: string, fallback: string, + pathStyle?: PathStyle, ): UniqueFilenameAllocator { const existing = allocators.get(baseDir); if (existing) return existing; - const allocator = new UniqueFilenameAllocator(baseDir, fallback); + const allocator = new UniqueFilenameAllocator(baseDir, fallback, pathStyle); allocators.set(baseDir, allocator); return allocator; } diff --git a/packages/cli/src/modules/n8n-packages/entities/workflow/workflow.exporter.ts b/packages/cli/src/modules/n8n-packages/entities/workflow/workflow.exporter.ts index 478c4fa8584..47bd758f2ee 100644 --- a/packages/cli/src/modules/n8n-packages/entities/workflow/workflow.exporter.ts +++ b/packages/cli/src/modules/n8n-packages/entities/workflow/workflow.exporter.ts @@ -3,15 +3,16 @@ import { Service } from '@n8n/di'; import { WorkflowFinderService } from '@/workflows/workflow-finder.service'; +import type { WorkflowNodeTypeSource } from './node-type-usage'; import { WorkflowSerializer } from './workflow.serializer'; import type { PackageWriter } from '../../io/package-writer'; import { UniqueFilenameAllocator } from '../../io/unique-filename-allocator'; +import type { PathStyle } from '../../n8n-packages.types'; import type { ManifestEntry } from '../../spec/manifest.schema'; import { CredentialRequirementsExtractor } from '../credential/credential-requirements.extractor'; import type { WorkflowCredentialRequirement } from '../credential/credential.types'; import { DataTableRequirementsExtractor } from '../data-table/data-table-requirements.extractor'; import type { WorkflowDataTableRequirement } from '../data-table/data-table.types'; -import type { WorkflowNodeTypeSource } from './node-type-usage'; import { assertEveryRequestedEntityAccessible } from '../package-export.errors'; import type { WorkflowExportRequirements } from '../requirements.types'; import { TagRequirementsExtractor } from '../tag/tag-requirements.extractor'; @@ -27,6 +28,7 @@ export interface WorkflowExportRequest { // Directory the workflow is written under. e.g. folders/{folderId}/ basePrefix?: string; + pathStyle?: PathStyle; } export interface WorkflowExportResult { @@ -70,10 +72,11 @@ export class WorkflowExporter { const fileNames = new UniqueFilenameAllocator( request.basePrefix ? `${request.basePrefix}/workflows` : 'workflows', 'workflow', + request.pathStyle, ); for (const workflow of workflowsForExport) { - const target = fileNames.allocate(workflow.name); + const target = fileNames.allocate(workflow.name, workflow.id); const serialized = this.workflowSerializer.serialize(workflow, { includeTags: request.includeTags, }); diff --git a/packages/cli/src/modules/n8n-packages/io/fs/fs-package-reader.ts b/packages/cli/src/modules/n8n-packages/io/fs/fs-package-reader.ts new file mode 100644 index 00000000000..37fde118568 --- /dev/null +++ b/packages/cli/src/modules/n8n-packages/io/fs/fs-package-reader.ts @@ -0,0 +1,50 @@ +import { jsonParse } from 'n8n-workflow'; +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; + +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; + +import type { PackageManifest } from '../../spec/manifest.schema'; +import type { PackageReader } from '../package-reader'; + +const MANIFEST_PATH = 'manifest.json'; + +/** + * Reads a package from a directory tree (a connection's git working directory) + * instead of a tar archive — counterpart of FsPackageWriter (LIGO-923 POC). + */ +export class FsPackageReader implements PackageReader { + constructor(private readonly rootDir: string) {} + + async readManifest(): Promise { + const manifest = await this.readFile(MANIFEST_PATH); + try { + return jsonParse(manifest.toString('utf-8')); + } catch { + throw new BadRequestError('Package manifest is not valid JSON'); + } + } + + async readFile(entryPath: string): Promise { + try { + return readFileSync(path.join(this.rootDir, entryPath)); + } catch { + throw new BadRequestError(`Package does not contain entry: ${entryPath}`); + } + } + + async listEntries(): Promise { + const entries: string[] = []; + const walk = (relativeDir: string) => { + const absoluteDir = path.join(this.rootDir, relativeDir); + for (const dirent of readdirSync(absoluteDir, { withFileTypes: true })) { + if (dirent.name === '.git') continue; + const relativePath = relativeDir ? `${relativeDir}/${dirent.name}` : dirent.name; + if (dirent.isDirectory()) walk(relativePath); + else if (dirent.isFile()) entries.push(relativePath); + } + }; + walk(''); + return entries; + } +} diff --git a/packages/cli/src/modules/n8n-packages/io/fs/fs-package-writer.ts b/packages/cli/src/modules/n8n-packages/io/fs/fs-package-writer.ts new file mode 100644 index 00000000000..909f4ee3e1b --- /dev/null +++ b/packages/cli/src/modules/n8n-packages/io/fs/fs-package-writer.ts @@ -0,0 +1,29 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +import type { PackageWriter } from '../package-writer'; + +/** + * Materializes a package export directly into a directory tree instead of a tar + * archive — used by the multi-repo source control sync (LIGO-923 POC) to write + * into a connection's git working directory. + */ +export class FsPackageWriter implements PackageWriter { + constructor(private readonly rootDir: string) {} + + writeFile(entryPath: string, content: string | Buffer): void { + const absolute = path.join(this.rootDir, entryPath); + mkdirSync(path.dirname(absolute), { recursive: true }); + writeFileSync(absolute, content); + } + + writeDirectory(entryPath: string): void { + mkdirSync(path.join(this.rootDir, entryPath), { recursive: true }); + } + + finalize(): Readable { + // Files are already on disk; the stream contract is satisfied with an empty one. + return Readable.from([]); + } +} diff --git a/packages/cli/src/modules/n8n-packages/io/unique-filename-allocator.ts b/packages/cli/src/modules/n8n-packages/io/unique-filename-allocator.ts index 9cee5dc3ede..d9162afcb71 100644 --- a/packages/cli/src/modules/n8n-packages/io/unique-filename-allocator.ts +++ b/packages/cli/src/modules/n8n-packages/io/unique-filename-allocator.ts @@ -1,6 +1,7 @@ import { UnexpectedError } from 'n8n-workflow'; import { generateSlug } from './slug.utils'; +import type { PathStyle } from '../n8n-packages.types'; export class UniqueFilenameAllocator { private readonly used = new Set(); @@ -8,6 +9,7 @@ export class UniqueFilenameAllocator { constructor( private readonly baseDir: string, private readonly fallback: string, + private readonly pathStyle: PathStyle = 'slug', ) {} reserve(segment: string): void { @@ -26,8 +28,12 @@ export class UniqueFilenameAllocator { this.used.add(path); } - allocate(name: string): string { - const base = `${this.baseDir}/${generateSlug(name, this.fallback)}`; + allocate(name: string, id?: string): string { + // 'id' paths stay stable across renames and allocation order — required for + // long-lived git trees (source control sync), where slug churn is destructive. + const segment = + this.pathStyle === 'id' && id !== undefined ? id : generateSlug(name, this.fallback); + const base = `${this.baseDir}/${segment}`; if (!this.used.has(base)) { this.used.add(base); diff --git a/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts b/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts index ade94045f97..e932a393955 100644 --- a/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts +++ b/packages/cli/src/modules/n8n-packages/n8n-packages.service.ts @@ -17,16 +17,18 @@ import { ProjectExporter } from './entities/project/project.exporter'; import { mergeRequirements } from './entities/requirements.types'; import { TagExporter } from './entities/tag/tag.exporter'; import { VariableExporter } from './entities/variable/variable.exporter'; -import { collectNodeTypeUsage } from './entities/workflow/node-type-usage'; -import { assertStaticSubWorkflowsIncluded } from './entities/workflow/static-sub-workflow-requirements'; import { AutoIncludedWorkflowResolver } from './entities/workflow/auto-included-workflow-resolver'; import { AutoIncludedWorkflowExporter, type AutoIncludedWorkflowExportResult, } from './entities/workflow/auto-included-workflow.exporter'; +import { collectNodeTypeUsage } from './entities/workflow/node-type-usage'; +import { assertStaticSubWorkflowsIncluded } from './entities/workflow/static-sub-workflow-requirements'; import { WorkflowDependencyResolver } from './entities/workflow/workflow-dependency-resolver'; import { WorkflowRequirementExporter } from './entities/workflow/workflow-requirement.exporter'; import { WorkflowExporter } from './entities/workflow/workflow.exporter'; +import type { PackageReader } from './io/package-reader'; +import type { PackageWriter } from './io/package-writer'; import { TarPackageReader } from './io/tar/tar-package-reader'; import { TarPackageWriter } from './io/tar/tar-package-writer'; import { PackageImportConfig } from './n8n-packages.config'; @@ -36,6 +38,7 @@ import { type ExportPackageRequest, type ExportPackageResult, type ImportPackageRequest, + type ImportRequestOptions, type ImportResult, } from './n8n-packages.types'; import { FORMAT_VERSION } from './spec/constants'; @@ -69,12 +72,13 @@ export class N8nPackagesService { private readonly autoIncludedWorkflowExporter: AutoIncludedWorkflowExporter, ) {} - async exportPackage(request: ExportPackageRequest): Promise { - const { missingWorkflowDependencyPolicy } = request; + async exportPackage( + request: ExportPackageRequest, + writer: PackageWriter = new TarPackageWriter(), + ): Promise { + const { missingWorkflowDependencyPolicy, pathStyle } = request; const isReferenceOnly = missingWorkflowDependencyPolicy === MissingWorkflowDependencyPolicy.ReferenceOnly; - - const writer = new TarPackageWriter(); const workflowIds = request.workflowIds ?? []; const folderIds = request.folderIds ?? []; const projectIds = request.projectIds ?? []; @@ -87,6 +91,7 @@ export class N8nPackagesService { folderIds, writer, includeTags, + pathStyle, }) : undefined; @@ -102,6 +107,7 @@ export class N8nPackagesService { workflowIds: workflowsForExport, writer, includeTags, + pathStyle, }) : undefined; @@ -112,6 +118,7 @@ export class N8nPackagesService { projectIds, writer, includeTags, + pathStyle, }) : undefined; @@ -155,6 +162,7 @@ export class N8nPackagesService { existingProjectEntries: allProjectsBeforeAutoInclude, includeTags, projectTargetsById: projectExportResult?.projectTargetsById, + pathStyle, }); } @@ -208,6 +216,7 @@ export class N8nPackagesService { writer, // Routes project-owned credentials into their project namespace; others stay top-level. projectTargetsById, + pathStyle, }); const dataTableExportResult = await this.dataTableExporter.export({ @@ -216,6 +225,7 @@ export class N8nPackagesService { writer, // Routes project-owned data tables into their project namespace; others stay top-level. projectTargetsById, + pathStyle, }); const workflowRequirementExportResult = await this.workflowRequirementExporter.export({ @@ -230,11 +240,13 @@ export class N8nPackagesService { writer, includeVariableValues, projectTargetsById, + pathStyle, }); const tagExportResult = this.tagExporter.export({ usages: requirements.tags, writer, + pathStyle, }); const manifestRequirements = this.buildManifestRequirements({ @@ -295,6 +307,13 @@ export class N8nPackagesService { async importPackage(request: ImportPackageRequest): Promise { const reader = new TarPackageReader(request.packageBuffer, this.packageImportConfig); + return await this.importFromReader(request, reader); + } + + async importFromReader( + request: ImportRequestOptions, + reader: PackageReader, + ): Promise { const manifest = await this.packageParser.getManifest(reader); if (isProjectPackage(manifest)) { if (request.variableParentPolicy !== undefined) { diff --git a/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts b/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts index 11bb493b075..d96a037f101 100644 --- a/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts +++ b/packages/cli/src/modules/n8n-packages/n8n-packages.types.ts @@ -148,6 +148,15 @@ export type TagMissingMode = (typeof TagMissingMode)[keyof typeof TagMissingMode export type TagConflictPolicy = (typeof TagConflictPolicy)[keyof typeof TagConflictPolicy]; +/** + * How exported entity paths are derived: 'slug' (readable, name-derived — the + * `.n8np` archive default) or 'id' (rename-stable — used for git-backed sync). + * + * POC (LIGO-923): internal-only option, deliberately not propagated through the + * DTO / public API / CLI layers described in this module's CLAUDE.md. + */ +export type PathStyle = 'slug' | 'id'; + export interface ExportPackageRequest { user: User; workflowIds?: string[]; @@ -157,6 +166,7 @@ export interface ExportPackageRequest { canExportVariableValues?: boolean; includeTags?: boolean; missingWorkflowDependencyPolicy?: MissingWorkflowDependencyPolicy; + pathStyle?: PathStyle; } export type ImportPackageRequest = { @@ -173,6 +183,9 @@ export type ImportPackageRequest = { ImportVariableProperties & ImportTagProperties; +/** Import request without the archive buffer — for callers that supply their own PackageReader. */ +export type ImportRequestOptions = Omit; + export type ImportCredentialProperties = { credentialMatchingMode: CredentialMatchingMode; credentialMissingMode: CredentialMissingMode; diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/connection-git.ts b/packages/cli/src/modules/source-control.ee/multi-repo/connection-git.ts new file mode 100644 index 00000000000..b2a4ba056c2 --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/connection-git.ts @@ -0,0 +1,179 @@ +import type { Logger } from '@n8n/backend-common'; +import { resolveProxyUrl } from '@n8n/backend-network'; +import type { User } from '@n8n/db'; +import * as path from 'path'; +import type { SimpleGit, SimpleGitOptions } from 'simple-git'; + +import { + SOURCE_CONTROL_DEFAULT_EMAIL, + SOURCE_CONTROL_DEFAULT_NAME, + SOURCE_CONTROL_ORIGIN, +} from '../constants'; +import { sourceControlFoldersExistCheck } from '../source-control-helper.ee'; + +export interface ConnectionGitOptions { + /** Per-connection git working directory. */ + workDir: string; + /** Per-connection SSH folder (key + known_hosts). */ + sshFolder: string; + connectionType: 'ssh' | 'https'; + repositoryUrl: string; + /** Decrypted credentials, https connections only. */ + httpsCredentials?: { username: string; password: string }; + /** Path to the decrypted private key on disk, ssh connections only. */ + privateKeyPath?: string; + logger: Logger; +} + +export interface ConnectionGitFileStatus { + path: string; + status: 'A' | 'M' | 'D'; +} + +/** + * Git operations bound to one source control connection. Unlike the legacy + * singleton SourceControlGitService (one instance-wide SimpleGit configured from + * the preferences blob), this is a plain class constructed per operation from a + * connection row, so any number of repos can coexist. + */ +export class ConnectionGit { + private git: SimpleGit | null = null; + + constructor(private readonly opts: ConnectionGitOptions) {} + + private async getGit(): Promise { + if (this.git) return this.git; + + sourceControlFoldersExistCheck([this.opts.workDir, this.opts.sshFolder]); + + const baseOptions: Partial = { + baseDir: this.opts.workDir, + binary: 'git', + maxConcurrentProcesses: 6, + trimmed: false, + }; + + const { simpleGit } = await import('simple-git'); + + if (this.opts.connectionType === 'https') { + const credentials = this.opts.httpsCredentials ?? { username: '', password: '' }; + const escapeShellArg = (arg: string) => `'${arg.replace(/'/g, "'\"'\"'")}'`; + const credentialScript = `!f() { echo username=${escapeShellArg(credentials.username)}; echo password=${escapeShellArg(credentials.password)}; }; f`; + + const config = [ + 'credential.helper=' + credentialScript, + // ensures that the credentials are only used for this connection's repositoryUrl + 'credential.useHttpPath=true', + ]; + const proxyUrl = resolveProxyUrl(this.opts.repositoryUrl); + if (proxyUrl) config.push(`http.proxy=${proxyUrl}`); + + this.git = simpleGit({ + ...baseOptions, + config, + unsafe: { allowUnsafeCredentialHelper: true }, + }).env('GIT_TERMINAL_PROMPT', '0'); + } else { + const knownHostsPath = path.join(this.opts.sshFolder, 'known_hosts'); + const normalize = (p: string) => p.split(/[/\\]/).join('/').replace(/"/g, '\\"'); + const sshCommand = `ssh -o UserKnownHostsFile="${normalize(knownHostsPath)}" -o StrictHostKeyChecking=accept-new -i "${normalize(this.opts.privateKeyPath ?? '')}"`; + + this.git = simpleGit({ + ...baseOptions, + unsafe: { allowUnsafeSshCommand: true }, + }) + .env('GIT_SSH_COMMAND', sshCommand) + .env('GIT_TERMINAL_PROMPT', '0'); + } + + return this.git; + } + + /** Initialize the working dir, point origin at the repo and track the branch. */ + async initAndTrack(branchName: string, user: User): Promise { + const git = await this.getGit(); + + if (!(await git.checkIsRepo())) { + await git.init(); + } + + try { + await git.addRemote(SOURCE_CONTROL_ORIGIN, this.opts.repositoryUrl); + } catch (error) { + if (!(error as Error).message.includes('remote origin already exists')) throw error; + } + + await git.addConfig( + 'user.name', + user.firstName && user.lastName + ? `${user.firstName} ${user.lastName}` + : SOURCE_CONTROL_DEFAULT_NAME, + ); + await git.addConfig('user.email', user.email ?? SOURCE_CONTROL_DEFAULT_EMAIL); + + await git.fetch(); + const { branches, currentBranch } = await this.getBranches(); + if (!currentBranch && branches.includes(branchName)) { + await git.checkout(branchName); + await git.branch([`--set-upstream-to=${SOURCE_CONTROL_ORIGIN}/${branchName}`, branchName]); + } else if (!branches.includes(branchName)) { + // Empty remote: name the local branch; `push -u` creates it remotely later. + await git.raw(['branch', '-M', branchName]); + } + } + + async fetch(): Promise { + const git = await this.getGit(); + await git.fetch(); + } + + async getBranches(): Promise<{ branches: string[]; currentBranch: string }> { + const git = await this.getGit(); + const { branches } = await git.branch(['-r']); + const remoteBranches = Object.keys(branches) + .map((name) => name.split('/').slice(1).join('/')) + .filter((name) => name !== 'HEAD'); + const { current } = await git.branch(); + return { branches: remoteBranches, currentBranch: current }; + } + + async resetToRemote(branch: string): Promise { + const git = await this.getGit(); + await git.raw(['reset', '--hard', `${SOURCE_CONTROL_ORIGIN}/${branch}`]); + await git.clean('f', ['-d']); + } + + /** Working tree vs remote branch — one symmetric preview for push and pull. */ + async diffVsRemote(branch: string): Promise { + const git = await this.getGit(); + try { + const raw = await git.raw(['diff', `${SOURCE_CONTROL_ORIGIN}/${branch}`, '--name-status']); + return raw + .split('\n') + .filter((line) => line.trim().length > 0) + .map((line) => { + const [status, ...pathParts] = line.split('\t'); + return { path: pathParts.join('\t'), status: status.charAt(0) as 'A' | 'M' | 'D' }; + }); + } catch { + // No remote branch yet (first push to an empty repo): everything is an addition. + const status = await git.status(); + return [...status.not_added, ...status.created, ...status.modified].map((filePath) => ({ + path: filePath, + status: 'A' as const, + })); + } + } + + async addAllCommitPush(message: string, branch: string): Promise<{ commitHash: string }> { + const git = await this.getGit(); + await git.add(['-A']); + const commit = await git.commit(message); + await git.push(SOURCE_CONTROL_ORIGIN, branch, ['-u']); + this.opts.logger.info('Pushed source control connection changes', { + branch, + commit: commit.commit, + }); + return { commitHash: commit.commit }; + } +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/multi-repo.types.ts b/packages/cli/src/modules/source-control.ee/multi-repo/multi-repo.types.ts new file mode 100644 index 00000000000..94508a461c8 --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/multi-repo.types.ts @@ -0,0 +1,68 @@ +import type { SourceControlConnectionType } from './source-control-connection.entity'; +import type { SourceControlScopeType } from './source-control-scope.entity'; +import type { KeyPairType } from '../types/key-pair-type'; + +// POC (LIGO-923): plain module types instead of zod DTOs in @n8n/api-types. +// The frontend re-declares these shapes; promote to @n8n/api-types when this +// graduates from POC. + +export interface ConnectionScopeResponse { + id: string; + scopeType: SourceControlScopeType; + projectId: string | null; +} + +export interface ConnectionResponse { + id: string; + repositoryUrl: string; + branchName: string; + branchReadOnly: boolean; + branchColor: string; + connectionType: SourceControlConnectionType; + connected: boolean; + publicKey: string | null; + scopes: ConnectionScopeResponse[]; +} + +export interface CreateConnectionPayload { + repositoryUrl: string; + connectionType: SourceControlConnectionType; + branchName?: string; + branchColor?: string; + branchReadOnly?: boolean; + httpsUsername?: string; + httpsPassword?: string; + keyGeneratorType?: KeyPairType; +} + +export interface UpdateConnectionPayload { + branchName?: string; + branchColor?: string; + branchReadOnly?: boolean; + httpsUsername?: string; + httpsPassword?: string; +} + +export interface ClaimScopePayload { + scopeType: SourceControlScopeType; + projectId?: string; +} + +export interface ConnectionStatusFile { + path: string; + status: 'A' | 'M' | 'D'; +} + +export interface ConnectionStatus { + branchName: string; + files: ConnectionStatusFile[]; +} + +export interface ConnectionPushRequest { + commitMessage?: string; +} + +export interface ConnectionPushResult { + commitHash: string; + pushedFiles: ConnectionStatusFile[]; +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.entity.ts b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.entity.ts new file mode 100644 index 00000000000..74ce356c52c --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.entity.ts @@ -0,0 +1,48 @@ +import { WithTimestampsAndStringId } from '@n8n/db'; +import { Column, Entity, Index, OneToMany } from '@n8n/typeorm'; + +import type { SourceControlScope } from './source-control-scope.entity'; + +export type SourceControlConnectionType = 'ssh' | 'https'; + +/** + * A source control connection = one (repository, branch) with its own auth, + * SSH key material and git working directory. Multi-repo counterpart of the + * legacy singleton preferences blob (LIGO-923 POC). + */ +@Entity() +@Index(['repositoryUrl', 'branchName'], { unique: true }) +export class SourceControlConnection extends WithTimestampsAndStringId { + @Column() + repositoryUrl: string; + + @Column({ default: 'main' }) + branchName: string; + + @Column({ default: false }) + branchReadOnly: boolean; + + @Column({ default: '#5296D6' }) + branchColor: string; + + @Column({ type: 'varchar', length: 16 }) + connectionType: SourceControlConnectionType; + + @Column({ default: false }) + connected: boolean; + + @Column({ type: 'text', nullable: true }) + publicKey: string | null; + + @Column({ type: 'text', nullable: true }) + encryptedPrivateKey: string | null; + + @Column({ type: 'text', nullable: true }) + encryptedUsername: string | null; + + @Column({ type: 'text', nullable: true }) + encryptedPassword: string | null; + + @OneToMany('SourceControlScope', 'connection') + scopes: SourceControlScope[]; +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.repository.ts b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.repository.ts new file mode 100644 index 00000000000..d88bd086df7 --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.repository.ts @@ -0,0 +1,19 @@ +import { Service } from '@n8n/di'; +import { DataSource, Repository } from '@n8n/typeorm'; + +import { SourceControlConnection } from './source-control-connection.entity'; + +@Service() +export class SourceControlConnectionRepository extends Repository { + constructor(dataSource: DataSource) { + super(SourceControlConnection, dataSource.manager); + } + + async findAllWithScopes() { + return await this.find({ relations: { scopes: true }, order: { createdAt: 'ASC' } }); + } + + async findWithScopes(id: string) { + return await this.findOne({ where: { id }, relations: { scopes: true } }); + } +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.service.ts b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.service.ts new file mode 100644 index 00000000000..83812707821 --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connection.service.ts @@ -0,0 +1,259 @@ +import { Logger } from '@n8n/backend-common'; +import { ProjectRepository, type User } from '@n8n/db'; +import { Service } from '@n8n/di'; +import { Cipher, InstanceSettings } from 'n8n-core'; +import { UserError } from 'n8n-workflow'; +import { rmSync, writeFileSync } from 'node:fs'; +import * as path from 'node:path'; + +import { SOURCE_CONTROL_DEFAULT_BRANCH, SOURCE_CONTROL_DEFAULT_BRANCH_COLOR } from '../constants'; +import { generateSshKeyPair, sourceControlFoldersExistCheck } from '../source-control-helper.ee'; +import { SourceControlConfig } from '../source-control.config'; +import { ConnectionGit } from './connection-git'; +import type { + ClaimScopePayload, + ConnectionResponse, + CreateConnectionPayload, + UpdateConnectionPayload, +} from './multi-repo.types'; +import type { SourceControlConnection } from './source-control-connection.entity'; +import { SourceControlConnectionRepository } from './source-control-connection.repository'; +import { SourceControlScopeRepository } from './source-control-scope.repository'; +import type { KeyPairType } from '../types/key-pair-type'; + +const CONNECTIONS_GIT_FOLDER = 'git-connections'; +const CONNECTIONS_SSH_FOLDER = 'ssh-connections'; +const SSH_KEY_NAME = 'key'; + +@Service() +export class SourceControlConnectionService { + constructor( + private readonly connectionRepository: SourceControlConnectionRepository, + private readonly scopeRepository: SourceControlScopeRepository, + private readonly projectRepository: ProjectRepository, + private readonly cipher: Cipher, + private readonly instanceSettings: InstanceSettings, + private readonly sourceControlConfig: SourceControlConfig, + private readonly logger: Logger, + ) {} + + workDirFor(connectionId: string): string { + return path.join(this.instanceSettings.n8nFolder, CONNECTIONS_GIT_FOLDER, connectionId); + } + + sshFolderFor(connectionId: string): string { + return path.join(this.instanceSettings.n8nFolder, CONNECTIONS_SSH_FOLDER, connectionId); + } + + async list(): Promise { + const connections = await this.connectionRepository.findAllWithScopes(); + return connections.map((connection) => this.toResponse(connection)); + } + + async get(connectionId: string): Promise { + const connection = await this.connectionRepository.findWithScopes(connectionId); + if (!connection) throw new UserError(`Source control connection ${connectionId} not found`); + return connection; + } + + async create(payload: CreateConnectionPayload): Promise { + const connection = this.connectionRepository.create({ + repositoryUrl: payload.repositoryUrl, + branchName: payload.branchName ?? SOURCE_CONTROL_DEFAULT_BRANCH, + branchColor: payload.branchColor ?? SOURCE_CONTROL_DEFAULT_BRANCH_COLOR, + branchReadOnly: payload.branchReadOnly ?? false, + connectionType: payload.connectionType, + connected: false, + publicKey: null, + encryptedPrivateKey: null, + encryptedUsername: null, + encryptedPassword: null, + scopes: [], + }); + + if (payload.connectionType === 'https') { + connection.encryptedUsername = await this.cipher.encryptV2(payload.httpsUsername ?? ''); + connection.encryptedPassword = await this.cipher.encryptV2(payload.httpsPassword ?? ''); + } else { + const keyPair = await generateSshKeyPair( + payload.keyGeneratorType ?? this.sourceControlConfig.defaultKeyPairType, + ); + connection.publicKey = keyPair.publicKey; + connection.encryptedPrivateKey = await this.cipher.encryptV2(keyPair.privateKey); + } + + const saved = await this.connectionRepository.save(connection); + return this.toResponse({ ...saved, scopes: [] }); + } + + async update( + connectionId: string, + payload: UpdateConnectionPayload, + ): Promise { + const connection = await this.get(connectionId); + + if (payload.branchName !== undefined) connection.branchName = payload.branchName; + if (payload.branchColor !== undefined) connection.branchColor = payload.branchColor; + if (payload.branchReadOnly !== undefined) connection.branchReadOnly = payload.branchReadOnly; + if (payload.httpsUsername !== undefined) { + connection.encryptedUsername = await this.cipher.encryptV2(payload.httpsUsername); + } + if (payload.httpsPassword !== undefined) { + connection.encryptedPassword = await this.cipher.encryptV2(payload.httpsPassword); + } + + const saved = await this.connectionRepository.save(connection); + return this.toResponse(saved); + } + + async delete(connectionId: string): Promise { + await this.connectionRepository.delete({ id: connectionId }); + rmSync(this.workDirFor(connectionId), { recursive: true, force: true }); + rmSync(this.sshFolderFor(connectionId), { recursive: true, force: true }); + } + + async regenerateKeyPair( + connectionId: string, + keyGeneratorType?: KeyPairType, + ): Promise { + const connection = await this.get(connectionId); + const keyPair = await generateSshKeyPair( + keyGeneratorType ?? this.sourceControlConfig.defaultKeyPairType, + ); + connection.publicKey = keyPair.publicKey; + connection.encryptedPrivateKey = await this.cipher.encryptV2(keyPair.privateKey); + const saved = await this.connectionRepository.save(connection); + return this.toResponse(saved); + } + + async connect(connectionId: string, user: User): Promise { + const connection = await this.get(connectionId); + const git = await this.gitFor(connection); + await git.initAndTrack(connection.branchName, user); + connection.connected = true; + const saved = await this.connectionRepository.save(connection); + return this.toResponse(saved); + } + + async disconnect(connectionId: string): Promise { + const connection = await this.get(connectionId); + connection.connected = false; + const saved = await this.connectionRepository.save(connection); + return this.toResponse(saved); + } + + async getBranches(connectionId: string): Promise<{ branches: string[]; currentBranch: string }> { + const connection = await this.get(connectionId); + const git = await this.gitFor(connection); + await git.fetch(); + return await git.getBranches(); + } + + async claimScope(connectionId: string, payload: ClaimScopePayload): Promise { + if (payload.scopeType === 'instance') { + const existing = await this.scopeRepository.findInstanceScope(); + if (existing && existing.connectionId !== connectionId) { + throw new UserError('Another connection already owns the instance scope'); + } + if (existing) return; + await this.scopeRepository.save( + this.scopeRepository.create({ connectionId, scopeType: 'instance', projectId: null }), + ); + return; + } + + // The unique index on projectId surfaces double-claims; message it as a domain error. + const claimed = await this.scopeRepository.findScopeForProject(payload.projectId!); + if (claimed) { + if (claimed.connectionId === connectionId) return; + throw new UserError('This project is already owned by another connection'); + } + await this.scopeRepository.save( + this.scopeRepository.create({ + connectionId, + scopeType: 'project', + projectId: payload.projectId, + }), + ); + } + + async unclaimProject(projectId: string): Promise { + await this.scopeRepository.delete({ scopeType: 'project', projectId }); + } + + async removeInstanceScope(connectionId: string): Promise { + await this.scopeRepository.delete({ scopeType: 'instance', connectionId }); + } + + /** + * Projects a connection pushes/pulls. A `project` scope owns exactly that + * project; the `instance` scope owns the derived complement — every team + * project no other connection has claimed. Personal projects are out of + * scope for the POC. + */ + async resolveOwnedProjectIds(connectionId: string): Promise { + const connection = await this.get(connectionId); + const projectScopes = connection.scopes.filter((scope) => scope.scopeType === 'project'); + if (projectScopes.length > 0 || !connection.scopes.some((s) => s.scopeType === 'instance')) { + return projectScopes + .map((scope) => scope.projectId) + .filter((id): id is string => id !== null); + } + + const teamProjects = await this.projectRepository.find({ + select: ['id'], + where: { type: 'team' }, + }); + const claimedIds = new Set(await this.scopeRepository.findClaimedProjectIds()); + return teamProjects.map((project) => project.id).filter((id) => !claimedIds.has(id)); + } + + async gitFor(connection: SourceControlConnection): Promise { + const sshFolder = this.sshFolderFor(connection.id); + let privateKeyPath: string | undefined; + let httpsCredentials: { username: string; password: string } | undefined; + + if (connection.connectionType === 'ssh') { + sourceControlFoldersExistCheck([sshFolder]); + privateKeyPath = path.join(sshFolder, SSH_KEY_NAME); + writeFileSync( + privateKeyPath, + (await this.cipher.decryptV2(connection.encryptedPrivateKey ?? '')) + '\n', + { mode: 0o600 }, + ); + } else { + httpsCredentials = { + username: await this.cipher.decryptV2(connection.encryptedUsername ?? ''), + password: await this.cipher.decryptV2(connection.encryptedPassword ?? ''), + }; + } + + return new ConnectionGit({ + workDir: this.workDirFor(connection.id), + sshFolder, + connectionType: connection.connectionType, + repositoryUrl: connection.repositoryUrl, + httpsCredentials, + privateKeyPath, + logger: this.logger, + }); + } + + private toResponse(connection: SourceControlConnection): ConnectionResponse { + return { + id: connection.id, + repositoryUrl: connection.repositoryUrl, + branchName: connection.branchName, + branchReadOnly: connection.branchReadOnly, + branchColor: connection.branchColor, + connectionType: connection.connectionType, + connected: connection.connected, + publicKey: connection.publicKey, + scopes: (connection.scopes ?? []).map((scope) => ({ + id: scope.id, + scopeType: scope.scopeType, + projectId: scope.projectId, + })), + }; + } +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connections.controller.ts b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connections.controller.ts new file mode 100644 index 00000000000..ede5744fe8a --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-connections.controller.ts @@ -0,0 +1,123 @@ +import { AuthenticatedRequest } from '@n8n/db'; +import { Delete, Get, GlobalScope, Patch, Post, RestController } from '@n8n/decorators'; + +import type { + ClaimScopePayload, + ConnectionPushRequest, + ConnectionResponse, + CreateConnectionPayload, + UpdateConnectionPayload, +} from './multi-repo.types'; +import { SourceControlConnectionService } from './source-control-connection.service'; +import { SourceControlSyncService } from './source-control-sync.service'; +import type { KeyPairType } from '../types/key-pair-type'; + +type ConnectionRequest = AuthenticatedRequest< + { connectionId: string; projectId?: string }, + unknown, + Body +>; + +/** + * Multi-repo source control connections (LIGO-923 POC). Lives beside the legacy + * singleton controller under a distinct prefix; request bodies are plain module + * types instead of zod DTOs — promote to @n8n/api-types when this graduates. + */ +@RestController('/source-control/connections') +export class SourceControlConnectionsController { + constructor( + private readonly connectionService: SourceControlConnectionService, + private readonly syncService: SourceControlSyncService, + ) {} + + @Get('/') + @GlobalScope('sourceControl:manage') + async listConnections(): Promise { + return await this.connectionService.list(); + } + + @Post('/') + @GlobalScope('sourceControl:manage') + async createConnection(req: ConnectionRequest) { + return await this.connectionService.create(req.body); + } + + @Patch('/:connectionId') + @GlobalScope('sourceControl:manage') + async updateConnection(req: ConnectionRequest) { + return await this.connectionService.update(req.params.connectionId, req.body); + } + + @Delete('/:connectionId') + @GlobalScope('sourceControl:manage') + async deleteConnection(req: ConnectionRequest) { + await this.connectionService.delete(req.params.connectionId); + return { success: true }; + } + + @Post('/:connectionId/connect') + @GlobalScope('sourceControl:manage') + async connect(req: ConnectionRequest) { + return await this.connectionService.connect(req.params.connectionId, req.user); + } + + @Post('/:connectionId/disconnect') + @GlobalScope('sourceControl:manage') + async disconnect(req: ConnectionRequest) { + return await this.connectionService.disconnect(req.params.connectionId); + } + + @Post('/:connectionId/generate-key-pair') + @GlobalScope('sourceControl:manage') + async generateKeyPair(req: ConnectionRequest<{ keyGeneratorType?: KeyPairType }>) { + return await this.connectionService.regenerateKeyPair( + req.params.connectionId, + req.body.keyGeneratorType, + ); + } + + @Get('/:connectionId/branches') + @GlobalScope('sourceControl:manage') + async getBranches(req: ConnectionRequest) { + return await this.connectionService.getBranches(req.params.connectionId); + } + + @Post('/:connectionId/scopes') + @GlobalScope('sourceControl:manage') + async claimScope(req: ConnectionRequest) { + await this.connectionService.claimScope(req.params.connectionId, req.body); + return { success: true }; + } + + @Delete('/:connectionId/scopes/project/:projectId') + @GlobalScope('sourceControl:manage') + async unclaimProject(req: ConnectionRequest) { + await this.connectionService.unclaimProject(req.params.projectId!); + return { success: true }; + } + + @Delete('/:connectionId/scopes/instance') + @GlobalScope('sourceControl:manage') + async removeInstanceScope(req: ConnectionRequest) { + await this.connectionService.removeInstanceScope(req.params.connectionId); + return { success: true }; + } + + @Get('/:connectionId/status') + @GlobalScope('sourceControl:push') + async status(req: ConnectionRequest) { + return await this.syncService.status(req.params.connectionId, req.user); + } + + @Post('/:connectionId/push') + @GlobalScope('sourceControl:push') + async push(req: ConnectionRequest) { + return await this.syncService.push(req.params.connectionId, req.user, req.body.commitMessage); + } + + @Post('/:connectionId/pull') + @GlobalScope('sourceControl:pull') + async pull(req: ConnectionRequest) { + return await this.syncService.pull(req.params.connectionId, req.user); + } +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.entity.ts b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.entity.ts new file mode 100644 index 00000000000..adcd3d87573 --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.entity.ts @@ -0,0 +1,32 @@ +import { WithTimestampsAndStringId } from '@n8n/db'; +import { Column, Entity, Index, JoinColumn, ManyToOne } from '@n8n/typeorm'; + +import { SourceControlConnection } from './source-control-connection.entity'; + +export type SourceControlScopeType = 'project' | 'instance'; + +/** + * What a connection owns. `project` claims exactly one project (unique index on + * projectId = no-overlap). `instance` is the catch-all: global entities plus every + * unclaimed project, derived by complement — never stored per project. + */ +@Entity() +export class SourceControlScope extends WithTimestampsAndStringId { + @ManyToOne( + () => SourceControlConnection, + (connection) => connection.scopes, + { onDelete: 'CASCADE' }, + ) + @JoinColumn({ name: 'connectionId' }) + connection: SourceControlConnection; + + @Column() + connectionId: string; + + @Column({ type: 'varchar', length: 16 }) + scopeType: SourceControlScopeType; + + @Index({ unique: true }) + @Column({ type: 'varchar', length: 36, nullable: true }) + projectId: string | null; +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.repository.ts b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.repository.ts new file mode 100644 index 00000000000..f5e058199dd --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.repository.ts @@ -0,0 +1,31 @@ +import { Service } from '@n8n/di'; +import { DataSource, Not, IsNull, Repository } from '@n8n/typeorm'; + +import { SourceControlScope } from './source-control-scope.entity'; + +@Service() +export class SourceControlScopeRepository extends Repository { + constructor(dataSource: DataSource) { + super(SourceControlScope, dataSource.manager); + } + + async findInstanceScope() { + return await this.findOne({ where: { scopeType: 'instance' } }); + } + + async findScopeForProject(projectId: string) { + return await this.findOne({ where: { scopeType: 'project', projectId } }); + } + + async findProjectScopesForConnection(connectionId: string) { + return await this.find({ where: { scopeType: 'project', connectionId } }); + } + + async findClaimedProjectIds(): Promise { + const scopes = await this.find({ + select: ['projectId'], + where: { scopeType: 'project', projectId: Not(IsNull()) }, + }); + return scopes.map((scope) => scope.projectId).filter((id): id is string => id !== null); + } +} diff --git a/packages/cli/src/modules/source-control.ee/multi-repo/source-control-sync.service.ts b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-sync.service.ts new file mode 100644 index 00000000000..45dcbef00a2 --- /dev/null +++ b/packages/cli/src/modules/source-control.ee/multi-repo/source-control-sync.service.ts @@ -0,0 +1,129 @@ +import { Logger } from '@n8n/backend-common'; +import type { User } from '@n8n/db'; +import { Service } from '@n8n/di'; +import { existsSync, readdirSync, rmSync } from 'node:fs'; +import * as path from 'node:path'; + +import { FsPackageReader } from '@/modules/n8n-packages/io/fs/fs-package-reader'; +import { FsPackageWriter } from '@/modules/n8n-packages/io/fs/fs-package-writer'; +import { N8nPackagesService } from '@/modules/n8n-packages/n8n-packages.service'; +import type { ImportRequestOptions } from '@/modules/n8n-packages/n8n-packages.types'; + +import type { ConnectionPushResult, ConnectionStatus } from './multi-repo.types'; +import type { SourceControlConnection } from './source-control-connection.entity'; +import { SourceControlConnectionService } from './source-control-connection.service'; + +/** + * Sync-shaped import profile: id-preserving upserts that converge the instance + * to the repo state, mirroring v1 source control semantics (stubs for + * credentials, activation state preserved). + */ +const SYNC_IMPORT_PROFILE: Omit = { + workflowIdPolicy: 'source', + workflowConflictPolicy: 'new-version', + workflowPublishingPolicy: 'preserve-published-state', + missingNodeTypeMode: 'import-anyway', + credentialMatchingMode: 'id-only', + credentialMissingMode: 'create-stub', + folderConflictPolicy: 'merge', + dataTableMatchingMode: 'by-id', + dataTableMissingMode: 'create', + dataTableSchemaConflictPolicy: 'keep-existing', + variableMissingMode: 'create-with-value', + tagMissingMode: 'create', + tagConflictPolicy: 'rename', +}; + +@Service() +export class SourceControlSyncService { + constructor( + private readonly connectionService: SourceControlConnectionService, + private readonly n8nPackagesService: N8nPackagesService, + private readonly logger: Logger, + ) {} + + async push( + connectionId: string, + user: User, + commitMessage?: string, + ): Promise { + const connection = await this.connectionService.get(connectionId); + const git = await this.connectionService.gitFor(connection); + + await git.fetch(); + await this.exportToWorkDir(connection, user); + + const pushedFiles = await git.diffVsRemote(connection.branchName); + const { commitHash } = await git.addAllCommitPush( + commitMessage ?? 'Updated workfolder', + connection.branchName, + ); + + return { commitHash, pushedFiles }; + } + + async pull(connectionId: string, user: User) { + const connection = await this.connectionService.get(connectionId); + const git = await this.connectionService.gitFor(connection); + const workDir = this.connectionService.workDirFor(connection.id); + + await git.fetch(); + await git.resetToRemote(connection.branchName); + + if (!existsSync(path.join(workDir, 'manifest.json'))) { + this.logger.info('Pull skipped: repository has no package manifest yet', { connectionId }); + return null; + } + + const reader = new FsPackageReader(workDir); + const manifest = await reader.readManifest(); + if (!manifest.projects?.length) { + this.logger.info('Pull skipped: package contains no projects', { connectionId }); + return null; + } + + return await this.n8nPackagesService.importFromReader({ user, ...SYNC_IMPORT_PROFILE }, reader); + } + + async status(connectionId: string, user: User): Promise { + const connection = await this.connectionService.get(connectionId); + const git = await this.connectionService.gitFor(connection); + + await git.fetch(); + await this.exportToWorkDir(connection, user); + + const files = await git.diffVsRemote(connection.branchName); + return { branchName: connection.branchName, files }; + } + + /** Wipe-and-re-export so renames and deletions surface as plain git diffs. */ + private async exportToWorkDir(connection: SourceControlConnection, user: User): Promise { + const workDir = this.connectionService.workDirFor(connection.id); + if (existsSync(workDir)) { + for (const entry of readdirSync(workDir)) { + if (entry === '.git') continue; + rmSync(path.join(workDir, entry), { recursive: true, force: true }); + } + } + + const projectIds = await this.connectionService.resolveOwnedProjectIds(connection.id); + if (projectIds.length === 0) { + this.logger.info('Nothing to export: connection owns no projects', { + connectionId: connection.id, + }); + return; + } + + await this.n8nPackagesService.exportPackage( + { + user, + projectIds, + includeVariableValues: true, + canExportVariableValues: true, + missingWorkflowDependencyPolicy: 'reference-only', + pathStyle: 'id', + }, + new FsPackageWriter(workDir), + ); + } +} diff --git a/packages/cli/src/modules/source-control.ee/source-control.module.ts b/packages/cli/src/modules/source-control.ee/source-control.module.ts index 61abe1d5a01..833d4bfaaec 100644 --- a/packages/cli/src/modules/source-control.ee/source-control.module.ts +++ b/packages/cli/src/modules/source-control.ee/source-control.module.ts @@ -10,8 +10,18 @@ import { Container } from '@n8n/di'; export class SourceControlModule implements ModuleInterface { async init() { await import('./source-control.controller.ee.js'); + await import('./multi-repo/source-control-connections.controller.js'); const { SourceControlService } = await import('./source-control.service.ee.js'); await Container.get(SourceControlService).start(); } + + async entities() { + const { SourceControlConnection } = await import( + './multi-repo/source-control-connection.entity.js' + ); + const { SourceControlScope } = await import('./multi-repo/source-control-scope.entity.js'); + + return [SourceControlConnection, SourceControlScope]; + } } diff --git a/packages/cli/src/services/ownership-transfer/ownership-transfer.manifest.json b/packages/cli/src/services/ownership-transfer/ownership-transfer.manifest.json index d1fef090970..8a59e0a7bc9 100644 --- a/packages/cli/src/services/ownership-transfer/ownership-transfer.manifest.json +++ b/packages/cli/src/services/ownership-transfer/ownership-transfer.manifest.json @@ -68,6 +68,11 @@ "name": "WorkflowReviewRequest", "path": "packages/@n8n/db/src/entities/workflow-review-request.ee.ts", "reason": "Not transferred with the project. Open reviews should be closed when their workflows move to another project; that close-on-move behaviour is deferred." + }, + { + "name": "SourceControlScope", + "path": "packages/cli/src/modules/source-control.ee/multi-repo/source-control-scope.entity.ts", + "reason": "Source-control ownership claim for the deleted project itself; dropped via FK cascade, never moved to another project (LIGO-923 POC)." } ] } diff --git a/packages/frontend/@n8n/i18n/src/locales/en.json b/packages/frontend/@n8n/i18n/src/locales/en.json index a054716bfab..2afe1fb229f 100644 --- a/packages/frontend/@n8n/i18n/src/locales/en.json +++ b/packages/frontend/@n8n/i18n/src/locales/en.json @@ -4105,6 +4105,40 @@ "settings.sourceControl.refreshBranches.tooltip": "Reload branches list", "settings.sourceControl.refreshBranches.success": "Branches successfully refreshed", "settings.sourceControl.refreshBranches.error": "Error refreshing branches", + "settings.sourceControl.connections.description": "Connect one or more Git repositories. Each connection owns either specific projects or the whole instance (everything not claimed by another connection).", + "settings.sourceControl.connections.heading": "Repository connections", + "settings.sourceControl.connections.add": "Add connection", + "settings.sourceControl.connections.empty.heading": "No connections yet", + "settings.sourceControl.connections.empty.description": "Add a repository connection to start syncing your instance or individual projects.", + "settings.sourceControl.connections.instanceBadge": "Instance", + "settings.sourceControl.connections.status.connected": "Connected", + "settings.sourceControl.connections.status.notConnected": "Not connected", + "settings.sourceControl.connections.scope.instance": "Owns the instance: all global entities and every unclaimed project", + "settings.sourceControl.connections.scope.none": "No scopes claimed yet", + "settings.sourceControl.connections.scope.projects": "Owns projects: {projects}", + "settings.sourceControl.connections.makeInstance": "Make instance connection", + "settings.sourceControl.connections.removeInstance": "Remove instance scope", + "settings.sourceControl.connections.button.edit": "Edit", + "settings.sourceControl.connections.button.delete": "Delete", + "settings.sourceControl.connections.button.cancel": "Cancel", + "settings.sourceControl.connections.button.create": "Create connection", + "settings.sourceControl.connections.modals.delete.title": "Delete this connection?", + "settings.sourceControl.connections.modals.delete.message": "This removes the connection, its SSH key and its local repository clone. The remote repository is not touched.", + "settings.sourceControl.connections.toast.saved": "Connection saved", + "settings.sourceControl.connections.toast.saveError": "Error saving connection", + "settings.sourceControl.connections.toast.deleted": "Connection deleted", + "settings.sourceControl.connections.sshKeyNotice": "Grant this key read-write access to your repository before connecting.", + "settings.sourceControl.connections.protected": "Protected: block pushes to this branch from this instance", + "settings.sourceControl.connections.modal.status.error": "Error loading connection status", + "settings.sourceControl.connections.modal.noChanges": "Everything is up to date — no differences with the remote branch.", + "settings.sourceControl.connections.modal.commitMessage.placeholder": "Commit message", + "settings.sourceControl.connections.modal.push.title": "Push changes", + "settings.sourceControl.connections.modal.push.success": "Changes pushed to repository", + "settings.sourceControl.connections.modal.push.error": "Error pushing to repository", + "settings.sourceControl.connections.modal.pull.title": "Pull changes", + "settings.sourceControl.connections.modal.pull.description": "Differences between this instance and the remote branch. Pulling overwrites the instance with the repository state.", + "settings.sourceControl.connections.modal.pull.success": "Changes pulled from repository", + "settings.sourceControl.connections.modal.pull.error": "Error pulling from repository", "settings.sourceControl.docs.url": "https://docs.n8n.io/source-control-environments/", "settings.sourceControl.docs.setup.url": "https://docs.n8n.io/source-control-environments/setup/", "settings.sourceControl.docs.setup.ssh.url": "https://docs.n8n.io/source-control-environments/setup/#step-3-set-up-a-deploy-key", @@ -5698,6 +5732,16 @@ "projects.settings.member.removed.title": "Member removed successfully", "projects.settings.member.added.title": "Member added successfully", "projects.settings.users.search.error": "An error occurred while searching for users", + "projects.settings.sourceControl": "Source control", + "projects.settings.sourceControl.claimed": "This project is owned by a dedicated connection:", + "projects.settings.sourceControl.coveredByInstance": "This project is covered by the instance connection:", + "projects.settings.sourceControl.notSynced": "This project is not synced to any repository.", + "projects.settings.sourceControl.select.placeholder": "Select a connection...", + "projects.settings.sourceControl.button.claim": "Claim project", + "projects.settings.sourceControl.button.unclaim": "Unclaim", + "projects.settings.sourceControl.toast.claimed": "Project claimed by connection", + "projects.settings.sourceControl.toast.unclaimed": "Project unclaimed", + "projects.settings.sourceControl.toast.error": "Source control action failed", "projects.settings.externalSecrets": "External secret vaults", "projects.settings.externalSecrets.emptyState.heading": "No external secrets available yet", "projects.settings.externalSecrets.emptyState.instanceAdmin.noProjectProviders.description": "Share secrets vault with this project to make external secrets available here. Once shared, secrets will appear automatically and can be referenced through aliases in credentials.", diff --git a/packages/frontend/editor-ui/src/app/components/MainSidebarSourceControl.vue b/packages/frontend/editor-ui/src/app/components/MainSidebarSourceControl.vue index 36a76be3d9a..a2fdd17e9d2 100644 --- a/packages/frontend/editor-ui/src/app/components/MainSidebarSourceControl.vue +++ b/packages/frontend/editor-ui/src/app/components/MainSidebarSourceControl.vue @@ -1,27 +1,35 @@