From aaaef76ed1fe7dcfecf031e5c3bc4fc83c8c16e2 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 02:46:38 +0800 Subject: [PATCH 001/103] generic multi step auth form --- .../sources/modal/GCSMultiStepForm.svelte | 177 +++++++----------- .../sources/modal/MultiStepAuthForm.svelte | 101 ++++++++++ 2 files changed, 164 insertions(+), 114 deletions(-) create mode 100644 web-common/src/features/sources/modal/MultiStepAuthForm.svelte diff --git a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte index e007489218a..77cadc5735c 100644 --- a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte +++ b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte @@ -1,11 +1,8 @@ - -
-
-
Authentication method
- - - {#if option.value === "credentials"} - - {:else if option.value === "hmac"} -
- - -
- {/if} - - -
- - - {#each filteredParamsProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {#if propertyKey !== "path" && propertyKey !== "google_application_credentials" && propertyKey !== "key_id" && propertyKey !== "secret"} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} + + + {#if option.value === "credentials"} + + {:else if option.value === "hmac"} +
+ +
{/if} - {/each} -
+ + diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte new file mode 100644 index 00000000000..a88defcfe26 --- /dev/null +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -0,0 +1,101 @@ + + + +
+
Authentication method
+ + + + + +
+ + +{#each properties as property (property.key)} + {@const propertyKey = property.key ?? ""} + {#if !excluded.has(propertyKey)} +
+ {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} + onStringInputChange(e)} + alwaysShowError + /> + {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} + + {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} + + {/if} +
+ {/if} +{/each} From 0ab65306f90635ec21b026d5ef66d64b510512ad Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 02:57:12 +0800 Subject: [PATCH 002/103] inital s3 and azure --- .../features/sources/modal/AddDataForm.svelte | 46 ++++++-- .../sources/modal/AzureMultiStepForm.svelte | 108 ++++++++++++++++++ .../sources/modal/S3MultiStepForm.svelte | 102 +++++++++++++++++ .../src/features/sources/modal/constants.ts | 47 +++++++- 4 files changed, 293 insertions(+), 10 deletions(-) create mode 100644 web-common/src/features/sources/modal/AzureMultiStepForm.svelte create mode 100644 web-common/src/features/sources/modal/S3MultiStepForm.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index ee081dc8f34..61b4daa1d7b 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -23,6 +23,8 @@ import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; + import S3MultiStepForm from "./S3MultiStepForm.svelte"; + import AzureMultiStepForm from "./AzureMultiStepForm.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; @@ -380,22 +382,48 @@ {:else if isMultiStepConnector} {#if stepState.step === "connector"} - + - + {#if connector.name === "gcs"} + + {:else if connector.name === "s3"} + + {:else if connector.name === "azure"} + + {:else} + + {/if} {:else} - + + import Input from "@rilldata/web-common/components/forms/Input.svelte"; + import MultiStepAuthForm from "./MultiStepAuthForm.svelte"; + import { normalizeErrors } from "./utils"; + import { AZURE_AUTH_OPTIONS, type AzureAuthMethod } from "./constants"; + + export let properties: any[] = []; + export let paramsForm: any; + export let paramsErrors: Record; + export let onStringInputChange: (e: Event) => void; + export let handleFileUpload: (file: File) => Promise; + + const filteredParamsProperties = properties; + + const AZURE_CLEAR_FIELDS: Record = { + account_key: ["azure_storage_connection_string", "azure_storage_sas_token"], + sas_token: ["azure_storage_connection_string", "azure_storage_key"], + connection_string: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + ], + }; + + const AZURE_EXCLUDED_KEYS = [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + ]; + + + + + {#if option.value === "account_key"} +
+ + +
+ {:else if option.value === "sas_token"} +
+ + +
+ {:else if option.value === "connection_string"} + + {/if} + + diff --git a/web-common/src/features/sources/modal/S3MultiStepForm.svelte b/web-common/src/features/sources/modal/S3MultiStepForm.svelte new file mode 100644 index 00000000000..09c9ab73061 --- /dev/null +++ b/web-common/src/features/sources/modal/S3MultiStepForm.svelte @@ -0,0 +1,102 @@ + + + + + {#if option.value === "access_keys"} +
+ + +
+ {:else if option.value === "role"} +
+ + + +
+ {/if} + + diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 4bffc52825d..34b052b2dcb 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -39,6 +39,51 @@ export const GCS_AUTH_OPTIONS: { }, ]; +export type S3AuthMethod = "access_keys" | "role"; + +export const S3_AUTH_OPTIONS: { + value: S3AuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "access_keys", + label: "Access keys", + description: "Use AWS access key ID and secret access key.", + }, + { + value: "role", + label: "Assume role", + description: "Assume an AWS IAM role using your local or provided credentials.", + }, +]; + +export type AzureAuthMethod = "account_key" | "sas_token" | "connection_string"; + +export const AZURE_AUTH_OPTIONS: { + value: AzureAuthMethod; + label: string; + description: string; + hint?: string; +}[] = [ + { + value: "account_key", + label: "Access key", + description: "Authenticate with storage account name and access key.", + }, + { + value: "sas_token", + label: "SAS token", + description: "Authenticate with storage account name and SAS token.", + }, + { + value: "connection_string", + label: "Connection string", + description: "Authenticate with a full Azure storage connection string.", + }, +]; + // pre-defined order for sources export const SOURCES = [ "athena", @@ -67,7 +112,7 @@ export const OLAP_ENGINES = [ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) -export const MULTI_STEP_CONNECTORS = ["gcs"]; +export const MULTI_STEP_CONNECTORS = ["gcs", "s3", "azure"]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; From 968f49a9550bbbfd7af14041458c700142f72bb7 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:20:45 +0800 Subject: [PATCH 003/103] multi step form renderer --- .../features/sources/modal/AddDataForm.svelte | 33 ++-- .../modal/MultiStepFormRenderer.svelte | 65 +++++++ .../src/features/sources/modal/constants.ts | 183 +++++++++++++++++- .../src/features/sources/modal/types.ts | 33 ++++ .../src/features/sources/modal/yupSchemas.ts | 16 +- 5 files changed, 300 insertions(+), 30 deletions(-) create mode 100644 web-common/src/features/sources/modal/MultiStepFormRenderer.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 61b4daa1d7b..458a6553595 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -15,6 +15,7 @@ import { isEmpty } from "./utils"; import { CONNECTION_TAB_OPTIONS, + multiStepFormConfigs, type ClickHouseConnectorType, } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; @@ -22,9 +23,8 @@ import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; - import S3MultiStepForm from "./S3MultiStepForm.svelte"; - import AzureMultiStepForm from "./AzureMultiStepForm.svelte"; + import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; + import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; @@ -180,6 +180,12 @@ } })(); + $: activeMultiStepConfig = isMultiStepConnector + ? multiStepFormConfigs[ + connector.name as keyof typeof multiStepFormConfigs + ] || null + : null; + $: isSubmitting = submitting; // Reset errors when form is modified @@ -388,24 +394,9 @@ enhance={paramsEnhance} onSubmit={paramsSubmit} > - {#if connector.name === "gcs"} - - {:else if connector.name === "s3"} - - {:else if connector.name === "azure"} - + import CredentialsInput from "@rilldata/web-common/components/forms/CredentialsInput.svelte"; + import Input from "@rilldata/web-common/components/forms/Input.svelte"; + import MultiStepAuthForm from "./MultiStepAuthForm.svelte"; + import { normalizeErrors } from "./utils"; + import { type MultiStepFormConfig } from "./types"; + + export let config: MultiStepFormConfig | null = null; + export let properties: any[] = []; + export let paramsForm: any; + export let paramsErrors: Record; + export let onStringInputChange: (e: Event) => void; + export let handleFileUpload: (file: File) => Promise; + + +{#if config} + + + {#if config.authFieldGroups?.[option.value]} +
+ {#each config.authFieldGroups[option.value] as field (field.id)} + {#if field.type === "credentials"} + + {:else} + + {/if} + {/each} +
+ {/if} + + +{/if} diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 34b052b2dcb..571f4ce310b 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -1,3 +1,5 @@ +import type { MultiStepFormConfig } from "./types"; + export type ClickHouseConnectorType = | "rill-managed" | "self-hosted" @@ -55,7 +57,8 @@ export const S3_AUTH_OPTIONS: { { value: "role", label: "Assume role", - description: "Assume an AWS IAM role using your local or provided credentials.", + description: + "Assume an AWS IAM role using your local or provided credentials.", }, ]; @@ -121,3 +124,181 @@ export const TALL_FORM_CONNECTORS = new Set([ "snowflake", "salesforce", ]); + +export const multiStepFormConfigs: Record = { + gcs: { + authOptions: GCS_AUTH_OPTIONS, + defaultAuthMethod: "credentials", + clearFieldsByMethod: { + credentials: ["key_id", "secret"], + hmac: ["google_application_credentials"], + }, + excludedKeys: ["google_application_credentials", "key_id", "secret"], + authFieldGroups: { + credentials: [ + { + type: "credentials", + id: "google_application_credentials", + optional: false, + hint: "Upload a JSON key file for a service account with GCS access.", + accept: ".json", + }, + ], + hmac: [ + { + type: "input", + id: "key_id", + label: "Access Key ID", + placeholder: "Enter your HMAC access key ID", + optional: false, + secret: true, + hint: "HMAC access key ID for S3-compatible authentication", + }, + { + type: "input", + id: "secret", + label: "Secret Access Key", + placeholder: "Enter your HMAC secret access key", + optional: false, + secret: true, + hint: "HMAC secret access key for S3-compatible authentication", + }, + ], + }, + }, + s3: { + authOptions: S3_AUTH_OPTIONS, + defaultAuthMethod: "access_keys", + clearFieldsByMethod: { + access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], + role: ["aws_access_key_id", "aws_secret_access_key"], + }, + excludedKeys: [ + "aws_access_key_id", + "aws_secret_access_key", + "aws_role_arn", + "aws_role_session_name", + "aws_external_id", + ], + authFieldGroups: { + access_keys: [ + { + type: "input", + id: "aws_access_key_id", + label: "Access Key ID", + placeholder: "Enter AWS access key ID", + optional: false, + secret: true, + hint: "AWS access key ID for the bucket", + }, + { + type: "input", + id: "aws_secret_access_key", + label: "Secret Access Key", + placeholder: "Enter AWS secret access key", + optional: false, + secret: true, + hint: "AWS secret access key for the bucket", + }, + ], + role: [ + { + type: "input", + id: "aws_role_arn", + label: "Role ARN", + placeholder: "Enter AWS IAM role ARN", + optional: false, + secret: true, + hint: "Role ARN to assume for accessing the bucket", + }, + { + type: "input", + id: "aws_role_session_name", + label: "Role session name", + placeholder: "Optional session name (defaults to rill-session)", + optional: true, + }, + { + type: "input", + id: "aws_external_id", + label: "External ID", + placeholder: "Optional external ID for cross-account access", + optional: true, + secret: true, + }, + ], + }, + }, + azure: { + authOptions: AZURE_AUTH_OPTIONS, + defaultAuthMethod: "account_key", + clearFieldsByMethod: { + account_key: [ + "azure_storage_connection_string", + "azure_storage_sas_token", + ], + sas_token: ["azure_storage_connection_string", "azure_storage_key"], + connection_string: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + ], + }, + excludedKeys: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + ], + authFieldGroups: { + account_key: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + hint: "The name of the Azure storage account", + }, + { + type: "input", + id: "azure_storage_key", + label: "Access key", + placeholder: "Enter Azure storage access key", + optional: false, + secret: true, + hint: "Primary or secondary access key for the storage account", + }, + ], + sas_token: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + }, + { + type: "input", + id: "azure_storage_sas_token", + label: "SAS token", + placeholder: "Enter Azure SAS token", + optional: false, + secret: true, + hint: "Shared Access Signature token for the storage account", + }, + ], + connection_string: [ + { + type: "input", + id: "azure_storage_connection_string", + label: "Connection string", + placeholder: "Enter Azure storage connection string", + optional: false, + secret: true, + hint: "Full connection string for the storage account", + }, + ], + }, + }, +}; diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index 5cb0785a5a5..b9cd39a8b1a 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -1,3 +1,36 @@ export type AddDataFormType = "source" | "connector"; export type ConnectorType = "parameters" | "dsn"; + +export type AuthOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export type AuthField = + | { + type: "credentials"; + id: string; + hint?: string; + optional?: boolean; + accept?: string; + } + | { + type: "input"; + id: string; + label: string; + placeholder?: string; + optional?: boolean; + secret?: boolean; + hint?: string; + }; + +export type MultiStepFormConfig = { + authOptions: AuthOption[]; + clearFieldsByMethod: Record; + excludedKeys: string[]; + authFieldGroups: Record; + defaultAuthMethod?: string; +}; diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index b12ec332eae..45e0ad40974 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -11,10 +11,10 @@ export const getYupSchema = { .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") .required("S3 URI is required"), aws_region: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + // name: yup + // .string() + // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + // .required("Source name is required"), }), gcs: yup.object().shape({ @@ -74,10 +74,10 @@ export const getYupSchema = { ) .required("Path is required"), azure_storage_account: yup.string(), - name: yup - .string() - .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + // name: yup + // .string() + // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + // .required("Source name is required"), }), postgres: yup.object().shape({ From 781fb5b7e072f5d8967132bd70fee97235d6b478 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:32:23 +0800 Subject: [PATCH 004/103] exclude name from connector form --- web-common/src/features/sources/modal/constants.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 571f4ce310b..53d6b7574ee 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -133,7 +133,12 @@ export const multiStepFormConfigs: Record = { credentials: ["key_id", "secret"], hmac: ["google_application_credentials"], }, - excludedKeys: ["google_application_credentials", "key_id", "secret"], + excludedKeys: [ + "google_application_credentials", + "key_id", + "secret", + "name", + ], authFieldGroups: { credentials: [ { @@ -179,6 +184,7 @@ export const multiStepFormConfigs: Record = { "aws_role_arn", "aws_role_session_name", "aws_external_id", + "name", ], authFieldGroups: { access_keys: [ @@ -249,6 +255,7 @@ export const multiStepFormConfigs: Record = { "azure_storage_key", "azure_storage_sas_token", "azure_storage_connection_string", + "name", ], authFieldGroups: { account_key: [ From 6ac31f16d3fe0c4983dbe38d5c24510a4134d964 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:39:05 +0800 Subject: [PATCH 005/103] reset --- .../src/features/sources/modal/yupSchemas.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 45e0ad40974..b12ec332eae 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -11,10 +11,10 @@ export const getYupSchema = { .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") .required("S3 URI is required"), aws_region: yup.string(), - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Source name is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Source name is required"), }), gcs: yup.object().shape({ @@ -74,10 +74,10 @@ export const getYupSchema = { ) .required("Path is required"), azure_storage_account: yup.string(), - // name: yup - // .string() - // .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - // .required("Source name is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Source name is required"), }), postgres: yup.object().shape({ From 00fc8ee4dcc73a45a904b89d24e7e4836032dc90 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 03:41:56 +0800 Subject: [PATCH 006/103] bump specs --- runtime/drivers/azure/azure.go | 10 +++++---- runtime/drivers/s3/s3.go | 38 +++++++++++++--------------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/runtime/drivers/azure/azure.go b/runtime/drivers/azure/azure.go index dbfd51ffd90..9bcd743bef8 100644 --- a/runtime/drivers/azure/azure.go +++ b/runtime/drivers/azure/azure.go @@ -39,12 +39,14 @@ var spec = drivers.Spec{ Secret: true, }, { - Key: "azure_storage_connection_string", - Type: drivers.StringPropertyType, - Secret: true, + Key: "azure_storage_connection_string", + Type: drivers.StringPropertyType, + DisplayName: "Azure Connection String", + Description: "Azure connection string for storage account", + Placeholder: "Paste your Azure connection string here", + Secret: true, }, }, - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ { Key: "path", diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 90e84dfd0ca..81bd18aaf56 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -26,14 +26,22 @@ var spec = drivers.Spec{ DocsURL: "https://docs.rilldata.com/build/connectors/data-source/s3", ConfigProperties: []*drivers.PropertySpec{ { - Key: "aws_access_key_id", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_access_key_id", + Type: drivers.StringPropertyType, + DisplayName: "AWS access key ID", + Description: "AWS access key ID for explicit credentials", + Placeholder: "Enter your AWS access key ID", + Secret: true, + Required: true, }, { - Key: "aws_secret_access_key", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_secret_access_key", + Type: drivers.StringPropertyType, + DisplayName: "AWS secret access key", + Description: "AWS secret access key for explicit credentials", + Placeholder: "Enter your AWS secret access key", + Secret: true, + Required: true, }, { Key: "region", @@ -53,24 +61,6 @@ var spec = drivers.Spec{ Required: false, Hint: "Overrides the S3 endpoint to connect to. This should only be used to connect to S3 compatible services, such as Cloudflare R2 or MinIO.", }, - { - Key: "aws_role_arn", - Type: drivers.StringPropertyType, - Secret: true, - Description: "AWS Role ARN to assume", - }, - { - Key: "aws_role_session_name", - Type: drivers.StringPropertyType, - Secret: true, - Description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", - }, - { - Key: "aws_external_id", - Type: drivers.StringPropertyType, - Secret: true, - Description: "Optional external ID to use when assuming an AWS role for cross-account access.", - }, }, SourceProperties: []*drivers.PropertySpec{ { From b96446280d7de7e32396f2af2e271af9a5e703dc Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 04:06:15 +0800 Subject: [PATCH 007/103] gate multi step configs buttno --- .../features/sources/modal/AddDataForm.svelte | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 458a6553595..79b5d4fda08 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -134,6 +134,30 @@ let clickhouseShowSaveAnyway: boolean = false; $: isSubmitDisabled = (() => { + // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) + if (isMultiStepConnector && stepState.step === "connector") { + const config = + multiStepFormConfigs[ + connector.name as keyof typeof multiStepFormConfigs + ]; + if (!config) return true; + const groups = Object.values(config.authFieldGroups || {}); + if (!groups.length) return false; + const hasError = (fieldId: string) => + Boolean(($paramsErrors[fieldId] as any)?.length); + const groupSatisfied = groups.some((fields) => + fields.every((field: any) => { + const required = !(field.optional ?? false); + if (!required) return true; + const value = $paramsForm[field.id]; + if (isEmpty(value)) return false; + if (hasError(field.id)) return false; + return true; + }), + ); + return !groupSatisfied; + } + if (onlyDsn || connectionTab === "dsn") { // DSN form: check required DSN properties for (const property of dsnProperties) { From e0f66b9ee7178af4ab1c5d2b62f63618518a9c17 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 04:09:44 +0800 Subject: [PATCH 008/103] clean up --- .../sources/modal/AddDataFormManager.ts | 2 ++ .../features/sources/modal/FormValidation.ts | 18 +++++++++++++++++- .../src/features/sources/modal/yupSchemas.ts | 14 ++++++++++---- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index e2d0f9c4c10..d183eab6265 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -127,6 +127,8 @@ export class AddDataFormManager { // Superforms: params const paramsSchemaDef = getValidationSchemaForConnector( connector.name as string, + formType, + { isMultiStepConnector: this.isMultiStepConnector }, ); const paramsAdapter = yup(paramsSchemaDef); type ParamsOut = YupInfer; diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 92c0f87c338..a3e784ac27c 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,23 @@ import { dsnSchema, getYupSchema } from "./yupSchemas"; +import type { AddDataFormType } from "./types"; export { dsnSchema }; -export function getValidationSchemaForConnector(name: string) { +export function getValidationSchemaForConnector( + name: string, + formType: AddDataFormType, + opts?: { isMultiStepConnector?: boolean }, +) { + const { isMultiStepConnector } = opts || {}; + + // For multi-step source flows, prefer the connector-specific schema when present + // so step 1 (connector) validation doesn't require source-only fields. + if (isMultiStepConnector && formType === "source") { + const connectorKey = `${name}_connector`; + if (connectorKey in getYupSchema) { + return getYupSchema[connectorKey as keyof typeof getYupSchema]; + } + } + return getYupSchema[name as keyof typeof getYupSchema]; } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index b12ec332eae..86d15fb30bc 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,16 +5,22 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { - s3: yup.object().shape({ + s3_connector: yup.object().shape({ + aws_access_key_id: yup.string().required("AWS access key ID is required"), + aws_secret_access_key: yup + .string() + .required("AWS secret access key is required"), + }), + + s3_source: yup.object().shape({ path: yup .string() .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") - .required("S3 URI is required"), - aws_region: yup.string(), + .required(), name: yup .string() .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + .required(), }), gcs: yup.object().shape({ From d8ed85dfb5973ce567eb5aa92cc0e45ac52fa4bf Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 04:13:02 +0800 Subject: [PATCH 009/103] remove unused --- .../sources/modal/AzureMultiStepForm.svelte | 108 ------------------ .../sources/modal/GCSMultiStepForm.svelte | 78 ------------- .../sources/modal/S3MultiStepForm.svelte | 102 ----------------- 3 files changed, 288 deletions(-) delete mode 100644 web-common/src/features/sources/modal/AzureMultiStepForm.svelte delete mode 100644 web-common/src/features/sources/modal/GCSMultiStepForm.svelte delete mode 100644 web-common/src/features/sources/modal/S3MultiStepForm.svelte diff --git a/web-common/src/features/sources/modal/AzureMultiStepForm.svelte b/web-common/src/features/sources/modal/AzureMultiStepForm.svelte deleted file mode 100644 index d877a939304..00000000000 --- a/web-common/src/features/sources/modal/AzureMultiStepForm.svelte +++ /dev/null @@ -1,108 +0,0 @@ - - - - - {#if option.value === "account_key"} -
- - -
- {:else if option.value === "sas_token"} -
- - -
- {:else if option.value === "connection_string"} - - {/if} - - diff --git a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte deleted file mode 100644 index 77cadc5735c..00000000000 --- a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte +++ /dev/null @@ -1,78 +0,0 @@ - - - - - {#if option.value === "credentials"} - - {:else if option.value === "hmac"} -
- - -
- {/if} - - diff --git a/web-common/src/features/sources/modal/S3MultiStepForm.svelte b/web-common/src/features/sources/modal/S3MultiStepForm.svelte deleted file mode 100644 index 09c9ab73061..00000000000 --- a/web-common/src/features/sources/modal/S3MultiStepForm.svelte +++ /dev/null @@ -1,102 +0,0 @@ - - - - - {#if option.value === "access_keys"} -
- - -
- {:else if option.value === "role"} -
- - - -
- {/if} - - From 931889be1d78c8420e0a32c35f92b9de3579c220 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 11:47:00 +0800 Subject: [PATCH 010/103] lint, prettier --- .../features/sources/modal/MultiStepAuthForm.svelte | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte index a88defcfe26..15a70f11a18 100644 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -5,13 +5,7 @@ import InformationalField from "@rilldata/web-common/components/forms/InformationalField.svelte"; import { ConnectorDriverPropertyType } from "@rilldata/web-common/runtime-client"; import { normalizeErrors } from "./utils"; - - export type AuthOption = { - value: string; - label: string; - description: string; - hint?: string; - }; + import type { AuthOption } from "./types"; export let properties: any[] = []; export let paramsForm: any; @@ -56,7 +50,7 @@ name="auth-fields" {option} paramsFormStore={paramsForm} - paramsErrors={paramsErrors} + {paramsErrors} {handleFileUpload} /> From a55e8469f9dc357640c43536a8f9117b044bd9ad Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 11:55:44 +0800 Subject: [PATCH 011/103] separate the long constant --- .../features/sources/modal/AddDataForm.svelte | 2 +- .../src/features/sources/modal/constants.ts | 187 ----------------- .../sources/modal/multi-step-auth-configs.ts | 191 ++++++++++++++++++ 3 files changed, 192 insertions(+), 188 deletions(-) create mode 100644 web-common/src/features/sources/modal/multi-step-auth-configs.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 79b5d4fda08..553be8ac27c 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -15,7 +15,6 @@ import { isEmpty } from "./utils"; import { CONNECTION_TAB_OPTIONS, - multiStepFormConfigs, type ClickHouseConnectorType, } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; @@ -28,6 +27,7 @@ import { AddDataFormManager } from "./AddDataFormManager"; import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; + import { multiStepFormConfigs } from "./multi-step-auth-configs"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 53d6b7574ee..384abf7777c 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -1,5 +1,3 @@ -import type { MultiStepFormConfig } from "./types"; - export type ClickHouseConnectorType = | "rill-managed" | "self-hosted" @@ -124,188 +122,3 @@ export const TALL_FORM_CONNECTORS = new Set([ "snowflake", "salesforce", ]); - -export const multiStepFormConfigs: Record = { - gcs: { - authOptions: GCS_AUTH_OPTIONS, - defaultAuthMethod: "credentials", - clearFieldsByMethod: { - credentials: ["key_id", "secret"], - hmac: ["google_application_credentials"], - }, - excludedKeys: [ - "google_application_credentials", - "key_id", - "secret", - "name", - ], - authFieldGroups: { - credentials: [ - { - type: "credentials", - id: "google_application_credentials", - optional: false, - hint: "Upload a JSON key file for a service account with GCS access.", - accept: ".json", - }, - ], - hmac: [ - { - type: "input", - id: "key_id", - label: "Access Key ID", - placeholder: "Enter your HMAC access key ID", - optional: false, - secret: true, - hint: "HMAC access key ID for S3-compatible authentication", - }, - { - type: "input", - id: "secret", - label: "Secret Access Key", - placeholder: "Enter your HMAC secret access key", - optional: false, - secret: true, - hint: "HMAC secret access key for S3-compatible authentication", - }, - ], - }, - }, - s3: { - authOptions: S3_AUTH_OPTIONS, - defaultAuthMethod: "access_keys", - clearFieldsByMethod: { - access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], - role: ["aws_access_key_id", "aws_secret_access_key"], - }, - excludedKeys: [ - "aws_access_key_id", - "aws_secret_access_key", - "aws_role_arn", - "aws_role_session_name", - "aws_external_id", - "name", - ], - authFieldGroups: { - access_keys: [ - { - type: "input", - id: "aws_access_key_id", - label: "Access Key ID", - placeholder: "Enter AWS access key ID", - optional: false, - secret: true, - hint: "AWS access key ID for the bucket", - }, - { - type: "input", - id: "aws_secret_access_key", - label: "Secret Access Key", - placeholder: "Enter AWS secret access key", - optional: false, - secret: true, - hint: "AWS secret access key for the bucket", - }, - ], - role: [ - { - type: "input", - id: "aws_role_arn", - label: "Role ARN", - placeholder: "Enter AWS IAM role ARN", - optional: false, - secret: true, - hint: "Role ARN to assume for accessing the bucket", - }, - { - type: "input", - id: "aws_role_session_name", - label: "Role session name", - placeholder: "Optional session name (defaults to rill-session)", - optional: true, - }, - { - type: "input", - id: "aws_external_id", - label: "External ID", - placeholder: "Optional external ID for cross-account access", - optional: true, - secret: true, - }, - ], - }, - }, - azure: { - authOptions: AZURE_AUTH_OPTIONS, - defaultAuthMethod: "account_key", - clearFieldsByMethod: { - account_key: [ - "azure_storage_connection_string", - "azure_storage_sas_token", - ], - sas_token: ["azure_storage_connection_string", "azure_storage_key"], - connection_string: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - ], - }, - excludedKeys: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - "azure_storage_connection_string", - "name", - ], - authFieldGroups: { - account_key: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, - hint: "The name of the Azure storage account", - }, - { - type: "input", - id: "azure_storage_key", - label: "Access key", - placeholder: "Enter Azure storage access key", - optional: false, - secret: true, - hint: "Primary or secondary access key for the storage account", - }, - ], - sas_token: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, - }, - { - type: "input", - id: "azure_storage_sas_token", - label: "SAS token", - placeholder: "Enter Azure SAS token", - optional: false, - secret: true, - hint: "Shared Access Signature token for the storage account", - }, - ], - connection_string: [ - { - type: "input", - id: "azure_storage_connection_string", - label: "Connection string", - placeholder: "Enter Azure storage connection string", - optional: false, - secret: true, - hint: "Full connection string for the storage account", - }, - ], - }, - }, -}; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts new file mode 100644 index 00000000000..361dcb2f580 --- /dev/null +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -0,0 +1,191 @@ +import { + AZURE_AUTH_OPTIONS, + GCS_AUTH_OPTIONS, + S3_AUTH_OPTIONS, +} from "./constants"; +import type { MultiStepFormConfig } from "./types"; + +export const multiStepFormConfigs: Record = { + gcs: { + authOptions: GCS_AUTH_OPTIONS, + defaultAuthMethod: "credentials", + clearFieldsByMethod: { + credentials: ["key_id", "secret"], + hmac: ["google_application_credentials"], + }, + excludedKeys: [ + "google_application_credentials", + "key_id", + "secret", + "name", + ], + authFieldGroups: { + credentials: [ + { + type: "credentials", + id: "google_application_credentials", + optional: false, + hint: "Upload a JSON key file for a service account with GCS access.", + accept: ".json", + }, + ], + hmac: [ + { + type: "input", + id: "key_id", + label: "Access Key ID", + placeholder: "Enter your HMAC access key ID", + optional: false, + secret: true, + hint: "HMAC access key ID for S3-compatible authentication", + }, + { + type: "input", + id: "secret", + label: "Secret Access Key", + placeholder: "Enter your HMAC secret access key", + optional: false, + secret: true, + hint: "HMAC secret access key for S3-compatible authentication", + }, + ], + }, + }, + s3: { + authOptions: S3_AUTH_OPTIONS, + defaultAuthMethod: "access_keys", + clearFieldsByMethod: { + access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], + role: ["aws_access_key_id", "aws_secret_access_key"], + }, + excludedKeys: [ + "aws_access_key_id", + "aws_secret_access_key", + "aws_role_arn", + "aws_role_session_name", + "aws_external_id", + "name", + ], + authFieldGroups: { + access_keys: [ + { + type: "input", + id: "aws_access_key_id", + label: "Access Key ID", + placeholder: "Enter AWS access key ID", + optional: false, + secret: true, + hint: "AWS access key ID for the bucket", + }, + { + type: "input", + id: "aws_secret_access_key", + label: "Secret Access Key", + placeholder: "Enter AWS secret access key", + optional: false, + secret: true, + hint: "AWS secret access key for the bucket", + }, + ], + role: [ + { + type: "input", + id: "aws_role_arn", + label: "Role ARN", + placeholder: "Enter AWS IAM role ARN", + optional: false, + secret: true, + hint: "Role ARN to assume for accessing the bucket", + }, + { + type: "input", + id: "aws_role_session_name", + label: "Role session name", + placeholder: "Optional session name (defaults to rill-session)", + optional: true, + }, + { + type: "input", + id: "aws_external_id", + label: "External ID", + placeholder: "Optional external ID for cross-account access", + optional: true, + secret: true, + }, + ], + }, + }, + azure: { + authOptions: AZURE_AUTH_OPTIONS, + defaultAuthMethod: "account_key", + clearFieldsByMethod: { + account_key: [ + "azure_storage_connection_string", + "azure_storage_sas_token", + ], + sas_token: ["azure_storage_connection_string", "azure_storage_key"], + connection_string: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + ], + }, + excludedKeys: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + "name", + ], + authFieldGroups: { + connection_string: [ + { + type: "input", + id: "azure_storage_connection_string", + label: "Connection string", + placeholder: "Enter Azure storage connection string", + optional: false, + secret: true, + hint: "Full connection string for the storage account", + }, + ], + account_key: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + hint: "The name of the Azure storage account", + }, + { + type: "input", + id: "azure_storage_key", + label: "Access key", + placeholder: "Enter Azure storage access key", + optional: false, + secret: true, + hint: "Primary or secondary access key for the storage account", + }, + ], + sas_token: [ + { + type: "input", + id: "azure_storage_account", + label: "Storage account", + placeholder: "Enter Azure storage account", + optional: false, + }, + { + type: "input", + id: "azure_storage_sas_token", + label: "SAS token", + placeholder: "Enter Azure SAS token", + optional: false, + secret: true, + hint: "Shared Access Signature token for the storage account", + }, + ], + }, + }, +}; From 818ad79c574aa9159e5a4452a089162f635fbe4c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:02:52 +0800 Subject: [PATCH 012/103] copy changes --- .../src/features/sources/modal/constants.ts | 18 +++++++++--------- .../sources/modal/multi-step-auth-configs.ts | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 384abf7777c..1b5e70f1bd9 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -69,19 +69,19 @@ export const AZURE_AUTH_OPTIONS: { hint?: string; }[] = [ { - value: "account_key", - label: "Access key", - description: "Authenticate with storage account name and access key.", + value: "connection_string", + label: "Connection String", + description: "Alternative for cloud deployment", }, { - value: "sas_token", - label: "SAS token", - description: "Authenticate with storage account name and SAS token.", + value: "account_key", + label: "Storage Account Key", + description: "Recommended for cloud deployment", }, { - value: "connection_string", - label: "Connection string", - description: "Authenticate with a full Azure storage connection string.", + value: "sas_token", + label: "Shared Access Signature (SAS) Token", + description: "Most secure, fine-grained control", }, ]; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 361dcb2f580..9ef4b7f8dc5 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -146,7 +146,7 @@ export const multiStepFormConfigs: Record = { placeholder: "Enter Azure storage connection string", optional: false, secret: true, - hint: "Full connection string for the storage account", + hint: "Paste an Azure Storage connection string", }, ], account_key: [ From 2479c85cfad776b8256c67addebdc581a9f1cd40 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:28:20 +0800 Subject: [PATCH 013/103] clean up s3 connector form --- .../sources/modal/MultiStepAuthForm.svelte | 48 +++++++++++++------ .../src/features/sources/modal/constants.ts | 8 +--- .../sources/modal/multi-step-auth-configs.ts | 38 ++++----------- 3 files changed, 45 insertions(+), 49 deletions(-) diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte index 15a70f11a18..27fd21a34b3 100644 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -19,6 +19,10 @@ // Keep auth method local to this component; default to provided value or first option. let authMethod: string = defaultAuthMethod || authOptions?.[0]?.value || ""; + $: hasSingleAuthOption = authOptions?.length === 1; + $: if (hasSingleAuthOption && authOptions?.[0]?.value) { + authMethod = authOptions[0].value; + } // Reactive clearing of fields not relevant to the selected auth method. $: if (authMethod && clearFieldsByMethod[authMethod]?.length) { @@ -42,20 +46,36 @@ -
-
Authentication method
- - - - - -
+{#if !hasSingleAuthOption} +
+
Authentication method
+ + + + + +
+{:else if authOptions?.[0]} +
+ +
+{/if} {#each properties as property (property.key)} diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 1b5e70f1bd9..ec7af49839e 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -39,7 +39,7 @@ export const GCS_AUTH_OPTIONS: { }, ]; -export type S3AuthMethod = "access_keys" | "role"; +export type S3AuthMethod = "access_keys"; export const S3_AUTH_OPTIONS: { value: S3AuthMethod; @@ -52,12 +52,6 @@ export const S3_AUTH_OPTIONS: { label: "Access keys", description: "Use AWS access key ID and secret access key.", }, - { - value: "role", - label: "Assume role", - description: - "Assume an AWS IAM role using your local or provided credentials.", - }, ]; export type AzureAuthMethod = "account_key" | "sas_token" | "connection_string"; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 9ef4b7f8dc5..ab0e3f66687 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -55,17 +55,9 @@ export const multiStepFormConfigs: Record = { authOptions: S3_AUTH_OPTIONS, defaultAuthMethod: "access_keys", clearFieldsByMethod: { - access_keys: ["aws_role_arn", "aws_role_session_name", "aws_external_id"], - role: ["aws_access_key_id", "aws_secret_access_key"], + access_keys: [], }, - excludedKeys: [ - "aws_access_key_id", - "aws_secret_access_key", - "aws_role_arn", - "aws_role_session_name", - "aws_external_id", - "name", - ], + excludedKeys: ["aws_access_key_id", "aws_secret_access_key", "name"], authFieldGroups: { access_keys: [ { @@ -86,31 +78,21 @@ export const multiStepFormConfigs: Record = { secret: true, hint: "AWS secret access key for the bucket", }, - ], - role: [ - { - type: "input", - id: "aws_role_arn", - label: "Role ARN", - placeholder: "Enter AWS IAM role ARN", - optional: false, - secret: true, - hint: "Role ARN to assume for accessing the bucket", - }, { type: "input", - id: "aws_role_session_name", - label: "Role session name", - placeholder: "Optional session name (defaults to rill-session)", + id: "aws_region", + label: "Region", + placeholder: "us-east-1", optional: true, + hint: "AWS region for the bucket", }, { type: "input", - id: "aws_external_id", - label: "External ID", - placeholder: "Optional external ID for cross-account access", + id: "aws_endpoint", + label: "Endpoint", + placeholder: "https://s3.example.com", optional: true, - secret: true, + hint: "AWS endpoint for the bucket", }, ], }, From 61bd19bc470c5dcae609f01116ada9e9a0ae51a5 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:45:41 +0800 Subject: [PATCH 014/103] fix multi step connector preview --- .../features/sources/modal/AddDataFormManager.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index d183eab6265..005d2e098e3 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -394,16 +394,22 @@ export class AddDataFormManager { clickhouseDsnValues, } = ctx; + const connectorPropertiesForPreview = + isMultiStepConnector && stepState?.step === "connector" + ? (connector.configProperties ?? []) + : filteredParamsProperties; + const getConnectorYamlPreview = (values: Record) => { + const orderedProperties = + onlyDsn || connectionTab === "dsn" + ? filteredDsnProperties + : connectorPropertiesForPreview; return compileConnectorYAML(connector, values, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; return !property.noPrompt; }, - orderedProperties: - onlyDsn || connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, + orderedProperties, }); }; From 09079fcebd12b88054197f2aaf8e20831ba2749b Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 12:46:50 +0800 Subject: [PATCH 015/103] fix excluded keys of s3 --- .../sources/modal/multi-step-auth-configs.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index ab0e3f66687..aac8fab3900 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -57,7 +57,13 @@ export const multiStepFormConfigs: Record = { clearFieldsByMethod: { access_keys: [], }, - excludedKeys: ["aws_access_key_id", "aws_secret_access_key", "name"], + excludedKeys: [ + "aws_access_key_id", + "aws_secret_access_key", + "region", + "endpoint", + "name", + ], authFieldGroups: { access_keys: [ { @@ -80,19 +86,19 @@ export const multiStepFormConfigs: Record = { }, { type: "input", - id: "aws_region", + id: "region", label: "Region", placeholder: "us-east-1", optional: true, - hint: "AWS region for the bucket", + hint: "Rill uses your default AWS region unless you set it explicitly.", }, { type: "input", - id: "aws_endpoint", + id: "endpoint", label: "Endpoint", placeholder: "https://s3.example.com", optional: true, - hint: "AWS endpoint for the bucket", + hint: "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", }, ], }, From 9ac8a4bcd300646ed6cc88ce256fd55cad5feba0 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 15:07:48 +0800 Subject: [PATCH 016/103] spacing between preview and help --- web-common/src/features/sources/modal/AddDataForm.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 553be8ac27c..42cf07beaa1 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -525,7 +525,7 @@
{#if dsnError || paramsError || clickhouseError} Date: Tue, 9 Dec 2025 15:31:14 +0800 Subject: [PATCH 017/103] reorg --- .../sources/modal/submitAddDataForm.ts | 182 +++++++++--------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index 06b6425f83c..331d8f16af9 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -45,6 +45,15 @@ interface AddDataFormValues { // in-flight Test-and-Connect submissions don't roll them back. const savedAnywayPaths = new Set(); +const connectorSubmissions = new Map< + string, + { + promise: Promise; + connectorName: string; + completed: boolean; + } +>(); + async function beforeSubmitForm( instanceId: string, connector?: V1ConnectorDriver, @@ -145,97 +154,6 @@ async function getOriginalEnvBlob( } } -export async function submitAddSourceForm( - queryClient: QueryClient, - connector: V1ConnectorDriver, - formValues: AddDataFormValues, -): Promise { - const instanceId = get(runtime).instanceId; - await beforeSubmitForm(instanceId, connector); - - const newSourceName = formValues.name as string; - - const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( - connector, - formValues, - ); - - // Make a new .yaml file - const newSourceFilePath = getFileAPIPathFromNameAndType( - newSourceName, - EntityType.Table, - ); - await runtimeServicePutFile(instanceId, { - path: newSourceFilePath, - blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), - create: true, - createOnly: false, - }); - - const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); - - // Create or update the `.env` file - const newEnvBlob = await updateDotEnvWithSecrets( - queryClient, - rewrittenConnector, - rewrittenFormValues, - "source", - ); - - // Make sure the file has reconciled before testing the connection - await runtimeServicePutFileAndWaitForReconciliation(instanceId, { - path: ".env", - blob: newEnvBlob, - create: true, - createOnly: false, - }); - - // Wait for source resource-level reconciliation - // This must happen after .env reconciliation since sources depend on secrets - try { - await waitForResourceReconciliation( - instanceId, - newSourceName, - ResourceKind.Model, - ); - } catch (error) { - // The source file was already created, so we need to delete it - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - const errorDetails = (error as any).details; - - throw { - message: error.message || "Unable to establish a connection", - details: - errorDetails && errorDetails !== error.message - ? errorDetails - : undefined, - }; - } - - // Check for file errors - // If the model file has errors, rollback the changes - const errorMessage = await fileArtifacts.checkFileErrors( - queryClient, - instanceId, - newSourceFilePath, - ); - if (errorMessage) { - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - throw new Error(errorMessage); - } - - await goto(`/files/${newSourceFilePath}`); -} - -const connectorSubmissions = new Map< - string, - { - promise: Promise; - connectorName: string; - completed: boolean; - } ->(); - async function saveConnectorAnyway( queryClient: QueryClient, connector: V1ConnectorDriver, @@ -495,3 +413,85 @@ export async function submitAddConnectorForm( // Wait for the submission to complete await submissionPromise; } + +export async function submitAddSourceForm( + queryClient: QueryClient, + connector: V1ConnectorDriver, + formValues: AddDataFormValues, +): Promise { + const instanceId = get(runtime).instanceId; + await beforeSubmitForm(instanceId, connector); + + const newSourceName = formValues.name as string; + + const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( + connector, + formValues, + ); + + // Make a new .yaml file + const newSourceFilePath = getFileAPIPathFromNameAndType( + newSourceName, + EntityType.Table, + ); + await runtimeServicePutFile(instanceId, { + path: newSourceFilePath, + blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), + create: true, + createOnly: false, + }); + + const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); + + // Create or update the `.env` file + const newEnvBlob = await updateDotEnvWithSecrets( + queryClient, + rewrittenConnector, + rewrittenFormValues, + "source", + ); + + // Make sure the file has reconciled before testing the connection + await runtimeServicePutFileAndWaitForReconciliation(instanceId, { + path: ".env", + blob: newEnvBlob, + create: true, + createOnly: false, + }); + + // Wait for source resource-level reconciliation + // This must happen after .env reconciliation since sources depend on secrets + try { + await waitForResourceReconciliation( + instanceId, + newSourceName, + ResourceKind.Model, + ); + } catch (error) { + // The source file was already created, so we need to delete it + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + const errorDetails = (error as any).details; + + throw { + message: error.message || "Unable to establish a connection", + details: + errorDetails && errorDetails !== error.message + ? errorDetails + : undefined, + }; + } + + // Check for file errors + // If the model file has errors, rollback the changes + const errorMessage = await fileArtifacts.checkFileErrors( + queryClient, + instanceId, + newSourceFilePath, + ); + if (errorMessage) { + await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); + throw new Error(errorMessage); + } + + await goto(`/files/${newSourceFilePath}`); +} From d4bdae6598e2f9ce7d710f7207adc3a6dbe1dfc4 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 15:43:02 +0800 Subject: [PATCH 018/103] fix azure schema --- web-common/src/features/sources/modal/yupSchemas.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 86d15fb30bc..e1e84c7f750 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -71,7 +71,14 @@ export const getYupSchema = { .required("Google application credentials is required"), }), - azure: yup.object().shape({ + azure_connector: yup.object().shape({ + azure_storage_account: yup.string().optional(), + azure_storage_key: yup.string().optional(), + azure_storage_sas_token: yup.string().optional(), + azure_storage_connection_string: yup.string().optional(), + }), + + azure_source: yup.object().shape({ path: yup .string() .matches( @@ -79,7 +86,6 @@ export const getYupSchema = { "Must be an Azure URI (e.g. azure://container/path)", ) .required("Path is required"), - azure_storage_account: yup.string(), name: yup .string() .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) From 0f8cc468075477d7b8bf905496f8f83468e1354d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 16:01:32 +0800 Subject: [PATCH 019/103] use public instead of skip in gcs --- .../features/sources/modal/AddDataForm.svelte | 30 ++++++++++++++----- .../sources/modal/AddDataFormManager.ts | 14 +++++++++ .../sources/modal/MultiStepAuthForm.svelte | 3 +- .../modal/MultiStepFormRenderer.svelte | 10 +++++++ .../src/features/sources/modal/constants.ts | 7 ++++- .../sources/modal/multi-step-auth-configs.ts | 2 ++ 6 files changed, 57 insertions(+), 9 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 42cf07beaa1..e60dec8036e 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -304,10 +304,28 @@ ? clickhouseSubmitting && saveAnyway : submitting && saveAnyway; + // Track selected auth method for multi-step connectors to adjust UI labels. + // Only initialize when config becomes available; do not reset after user selection. + let selectedAuthMethod: string = ""; + $: if ( + activeMultiStepConfig && + !selectedAuthMethod && + activeMultiStepConfig.authOptions?.length + ) { + selectedAuthMethod = + activeMultiStepConfig.defaultAuthMethod || + activeMultiStepConfig.authOptions?.[0]?.value || + ""; + } + $: if (!activeMultiStepConfig) { + selectedAuthMethod = ""; + } + handleOnUpdate = formManager.makeOnUpdate({ onClose, queryClient, getConnectionTab: () => connectionTab, + getSelectedAuthMethod: () => selectedAuthMethod, setParamsError: (message: string | null, details?: string) => { paramsError = message; paramsErrorDetails = details; @@ -426,6 +444,7 @@ paramsErrors={$paramsErrors} {onStringInputChange} {handleFileUpload} + bind:authMethod={selectedAuthMethod} /> {:else} {/if} - {#if isMultiStepConnector && stepState.step === "connector"} - - {/if} -
diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 005d2e098e3..559bacdd7aa 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -203,6 +203,7 @@ export class AddDataFormManager { submitting: boolean; clickhouseConnectorType?: ClickHouseConnectorType; clickhouseSubmitting?: boolean; + selectedAuthMethod?: string; }): string { const { isConnectorForm, @@ -210,6 +211,7 @@ export class AddDataFormManager { submitting, clickhouseConnectorType, clickhouseSubmitting, + selectedAuthMethod, } = args; const isClickhouse = this.connector.name === "clickhouse"; @@ -224,6 +226,9 @@ export class AddDataFormManager { if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { + if (selectedAuthMethod === "public") { + return submitting ? "Continuing..." : "Continue"; + } return submitting ? "Testing connection..." : "Test and Connect"; } if (this.isMultiStepConnector && step === "source") { @@ -239,6 +244,7 @@ export class AddDataFormManager { onClose: () => void; queryClient: any; getConnectionTab: () => "parameters" | "dsn"; + getSelectedAuthMethod?: () => string | undefined; setParamsError: (message: string | null, details?: string) => void; setDsnError: (message: string | null, details?: string) => void; setShowSaveAnyway?: (value: boolean) => void; @@ -247,6 +253,7 @@ export class AddDataFormManager { onClose, queryClient, getConnectionTab, + getSelectedAuthMethod, setParamsError, setDsnError, setShowSaveAnyway, @@ -278,6 +285,7 @@ export class AddDataFormManager { if (!event.form.valid) return; const values = event.form.data; + const selectedAuthMethod = getSelectedAuthMethod?.(); try { const stepState = get(connectorStepStore) as ConnectorStepState; @@ -285,6 +293,12 @@ export class AddDataFormManager { await submitAddSourceForm(queryClient, connector, values); onClose(); } else if (isMultiStepConnector && stepState.step === "connector") { + // For public auth, skip Test & Connect and go straight to the next step. + if (selectedAuthMethod === "public") { + setConnectorConfig(values); + setStep("source"); + return; + } await submitAddConnectorForm(queryClient, connector, values, false); setConnectorConfig(values); setStep("source"); diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte index 27fd21a34b3..e927cdb68cd 100644 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte @@ -18,7 +18,8 @@ export let excludedKeys: string[] = []; // Keep auth method local to this component; default to provided value or first option. - let authMethod: string = defaultAuthMethod || authOptions?.[0]?.value || ""; + export let authMethod: string = + defaultAuthMethod || authOptions?.[0]?.value || ""; $: hasSingleAuthOption = authOptions?.length === 1; $: if (hasSingleAuthOption && authOptions?.[0]?.value) { authMethod = authOptions[0].value; diff --git a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte b/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte index 0bf95e63347..76cf8e997f7 100644 --- a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte +++ b/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte @@ -11,6 +11,15 @@ export let paramsErrors: Record; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; + + // Bubble the selected auth method to the parent so it can adjust UI. + export let authMethod: string = + config?.defaultAuthMethod || config?.authOptions?.[0]?.value || ""; + + $: if (config && !authMethod) { + authMethod = + config.defaultAuthMethod || config.authOptions?.[0]?.value || ""; + } {#if config} @@ -20,6 +29,7 @@ {paramsErrors} {onStringInputChange} {handleFileUpload} + bind:authMethod authOptions={config.authOptions} defaultAuthMethod={config.defaultAuthMethod || config.authOptions?.[0]?.value} diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index ec7af49839e..1931c9bdd95 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -17,7 +17,7 @@ export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "dsn", label: "Enter connection string" }, ]; -export type GCSAuthMethod = "credentials" | "hmac"; +export type GCSAuthMethod = "public" | "credentials" | "hmac"; export const GCS_AUTH_OPTIONS: { value: GCSAuthMethod; @@ -25,6 +25,11 @@ export const GCS_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, { value: "credentials", label: "GCP credentials", diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index aac8fab3900..cd29f5bf586 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -10,6 +10,7 @@ export const multiStepFormConfigs: Record = { authOptions: GCS_AUTH_OPTIONS, defaultAuthMethod: "credentials", clearFieldsByMethod: { + public: ["google_application_credentials", "key_id", "secret"], credentials: ["key_id", "secret"], hmac: ["google_application_credentials"], }, @@ -20,6 +21,7 @@ export const multiStepFormConfigs: Record = { "name", ], authFieldGroups: { + public: [], credentials: [ { type: "credentials", From ca67542293f90d331478444ca5554457309064fb Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:11:45 +0800 Subject: [PATCH 020/103] public option for object storage connectors --- .../features/sources/modal/AddDataModal.svelte | 2 +- .../features/sources/modal/FormValidation.ts | 8 ++++++++ .../src/features/sources/modal/constants.ts | 18 ++++++++++++++++-- .../sources/modal/multi-step-auth-configs.ts | 14 ++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 42c8289d1b1..289cdf82720 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -121,7 +121,7 @@ // FIXME: excluding salesforce until we implement the table discovery APIs $: isConnectorType = - selectedConnector?.name === "gcs" || + selectedConnector?.implementsObjectStore || selectedConnector?.implementsOlap || selectedConnector?.implementsSqlStore || (selectedConnector?.implementsWarehouse && diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index a3e784ac27c..ee4bfa7e89a 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -19,5 +19,13 @@ export function getValidationSchemaForConnector( } } + // For multi-step connector step, prefer connector-specific schema when present. + if (isMultiStepConnector && formType === "connector") { + const connectorKey = `${name}_connector`; + if (connectorKey in getYupSchema) { + return getYupSchema[connectorKey as keyof typeof getYupSchema]; + } + } + return getYupSchema[name as keyof typeof getYupSchema]; } diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 1931c9bdd95..7e78fdfa751 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -44,7 +44,7 @@ export const GCS_AUTH_OPTIONS: { }, ]; -export type S3AuthMethod = "access_keys"; +export type S3AuthMethod = "access_keys" | "public"; export const S3_AUTH_OPTIONS: { value: S3AuthMethod; @@ -52,6 +52,11 @@ export const S3_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, { value: "access_keys", label: "Access keys", @@ -59,7 +64,11 @@ export const S3_AUTH_OPTIONS: { }, ]; -export type AzureAuthMethod = "account_key" | "sas_token" | "connection_string"; +export type AzureAuthMethod = + | "account_key" + | "sas_token" + | "connection_string" + | "public"; export const AZURE_AUTH_OPTIONS: { value: AzureAuthMethod; @@ -67,6 +76,11 @@ export const AZURE_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ + { + value: "public", + label: "Public", + description: "Access publicly readable blobs without credentials.", + }, { value: "connection_string", label: "Connection String", diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index cd29f5bf586..a1e2d567662 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -58,6 +58,12 @@ export const multiStepFormConfigs: Record = { defaultAuthMethod: "access_keys", clearFieldsByMethod: { access_keys: [], + public: [ + "aws_access_key_id", + "aws_secret_access_key", + "region", + "endpoint", + ], }, excludedKeys: [ "aws_access_key_id", @@ -103,6 +109,7 @@ export const multiStepFormConfigs: Record = { hint: "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", }, ], + public: [], }, }, azure: { @@ -119,6 +126,12 @@ export const multiStepFormConfigs: Record = { "azure_storage_key", "azure_storage_sas_token", ], + public: [ + "azure_storage_account", + "azure_storage_key", + "azure_storage_sas_token", + "azure_storage_connection_string", + ], }, excludedKeys: [ "azure_storage_account", @@ -176,6 +189,7 @@ export const multiStepFormConfigs: Record = { hint: "Shared Access Signature token for the storage account", }, ], + public: [], }, }, }; From c2d1d84b20c295168b38a1f0eb190009004a2d52 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:17:43 +0800 Subject: [PATCH 021/103] unblock public in s3 --- web-common/src/features/sources/modal/yupSchemas.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index e1e84c7f750..8110d84d926 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -6,10 +6,10 @@ import { export const getYupSchema = { s3_connector: yup.object().shape({ - aws_access_key_id: yup.string().required("AWS access key ID is required"), - aws_secret_access_key: yup - .string() - .required("AWS secret access key is required"), + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + region: yup.string().optional(), + endpoint: yup.string().optional(), }), s3_source: yup.object().shape({ From 2db9c08ddecff0ed5c27b82cec7af2e1dfc7cc78 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:24:00 +0800 Subject: [PATCH 022/103] gate save anyway on multi step connector submission --- .../sources/modal/AddDataFormManager.ts | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 559bacdd7aa..1bb8e1b31d1 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -272,23 +272,29 @@ export class AddDataFormManager { >; result?: Extract; }) => { - // For non-ClickHouse connectors, expose Save Anyway when a submission starts + if (!event.form.valid) return; + + const values = event.form.data; + const selectedAuthMethod = getSelectedAuthMethod?.(); + const stepState = get(connectorStepStore) as ConnectorStepState; + + // For non-ClickHouse connectors, expose Save Anyway when a submission starts, + // but skip for multi-step public auth where we bypass submission. if ( isConnectorForm && connector.name !== "clickhouse" && typeof setShowSaveAnyway === "function" && - event?.result + event?.result && + !( + isMultiStepConnector && + stepState?.step === "connector" && + selectedAuthMethod === "public" + ) ) { setShowSaveAnyway(true); } - if (!event.form.valid) return; - - const values = event.form.data; - const selectedAuthMethod = getSelectedAuthMethod?.(); - try { - const stepState = get(connectorStepStore) as ConnectorStepState; if (isMultiStepConnector && stepState.step === "source") { await submitAddSourceForm(queryClient, connector, values); onClose(); From 3a3d3b49a69fca5e873aac50e4bc5bad091f1bc6 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:32:53 +0800 Subject: [PATCH 023/103] import your data right data panel --- .../features/sources/modal/AddDataForm.svelte | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e60dec8036e..5a125fd5775 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -543,29 +543,43 @@
- {#if dsnError || paramsError || clickhouseError} - + {#if dsnError || paramsError || clickhouseError} + + {/if} + + - {/if} - - + + {#if isMultiStepConnector && $connectorStepStore.step === "connector"} +
+ Already connected? +
+ {/if} +
From 833272488b1ca6a492492febc3abb0eda79655db Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:40:10 +0800 Subject: [PATCH 024/103] import data copy for model step 2 --- web-common/src/features/sources/modal/AddDataForm.svelte | 8 +++++--- .../src/features/sources/modal/AddDataFormManager.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 5a125fd5775..788cb706138 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -519,9 +519,11 @@ : submitting} loadingCopy={connector.name === "clickhouse" ? "Connecting..." - : selectedAuthMethod === "public" - ? "Continuing..." - : "Testing connection..."} + : isMultiStepConnector && stepState.step === "source" + ? "Importing data..." + : selectedAuthMethod === "public" + ? "Continuing..." + : "Testing connection..."} form={connector.name === "clickhouse" ? clickhouseFormId : formId} submitForm type="primary" diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 1bb8e1b31d1..fbd6581f17f 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -232,7 +232,7 @@ export class AddDataFormManager { return submitting ? "Testing connection..." : "Test and Connect"; } if (this.isMultiStepConnector && step === "source") { - return submitting ? "Creating model..." : "Test and Add data"; + return submitting ? "Importing data..." : "Import Data"; } return submitting ? "Testing connection..." : "Test and Connect"; } From 54a3e6287db3972faa907ad446559d26ec347d69 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 17:46:53 +0800 Subject: [PATCH 025/103] gate save anyway on step 2 when import data --- web-common/src/features/sources/modal/AddDataFormManager.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index fbd6581f17f..7a6a8797738 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -278,6 +278,7 @@ export class AddDataFormManager { const selectedAuthMethod = getSelectedAuthMethod?.(); const stepState = get(connectorStepStore) as ConnectorStepState; + // FIXME: simplify this logic // For non-ClickHouse connectors, expose Save Anyway when a submission starts, // but skip for multi-step public auth where we bypass submission. if ( @@ -289,7 +290,9 @@ export class AddDataFormManager { isMultiStepConnector && stepState?.step === "connector" && selectedAuthMethod === "public" - ) + ) && + // Do not show Save Anyway on the model (source) step of multi-step flows. + stepState?.step !== "source" ) { setShowSaveAnyway(true); } From a17f504bcf49e1ba5b9b1d79c18f487e52a42641 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 20:05:29 +0800 Subject: [PATCH 026/103] gcs source in yup schema --- web-common/src/features/sources/modal/yupSchemas.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 8110d84d926..710ac3a5bd2 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -23,7 +23,7 @@ export const getYupSchema = { .required(), }), - gcs: yup.object().shape({ + gcs_connector: yup.object().shape({ google_application_credentials: yup.string().optional(), key_id: yup.string().optional(), secret: yup.string().optional(), @@ -33,6 +33,17 @@ export const getYupSchema = { .optional(), }), + gcs_source: yup.object().shape({ + path: yup + .string() + .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") + .optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required(), + }), + https: yup.object().shape({ path: yup .string() From b0ad7ea065f19d6c36885ff778463c14184a6154 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 20:42:02 +0800 Subject: [PATCH 027/103] dynamic validaton for multi step connectors --- .../features/sources/modal/AddDataForm.svelte | 8 ++- .../sources/modal/AddDataFormManager.ts | 17 ++++- .../features/sources/modal/FormValidation.ts | 62 ++++++++++++++++++- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 788cb706138..9f36024ed4e 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -56,6 +56,7 @@ formType, onParamsUpdate: (e: any) => handleOnUpdate(e), onDsnUpdate: (e: any) => handleOnUpdate(e), + getSelectedAuthMethod: () => selectedAuthMethod, }); const isMultiStepConnector = formManager.isMultiStepConnector; @@ -141,7 +142,12 @@ connector.name as keyof typeof multiStepFormConfigs ]; if (!config) return true; - const groups = Object.values(config.authFieldGroups || {}); + // Only validate the currently selected auth method; fall back to default. + const method = + selectedAuthMethod || + config.defaultAuthMethod || + config.authOptions?.[0]?.value; + const groups = method ? [config.authFieldGroups?.[method] || []] : []; if (!groups.length) return false; const hasError = (fieldId: string) => Boolean(($paramsErrors[fieldId] as any)?.length); diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 7a6a8797738..f2ed43ef1f6 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -69,15 +69,25 @@ export class AddDataFormManager { return normalizeConnectorError(this.connector.name ?? "", e); } + private getSelectedAuthMethod?: () => string | undefined; + constructor(args: { connector: V1ConnectorDriver; formType: AddDataFormType; onParamsUpdate: (event: SuperFormUpdateEvent) => void; onDsnUpdate: (event: SuperFormUpdateEvent) => void; + getSelectedAuthMethod?: () => string | undefined; }) { - const { connector, formType, onParamsUpdate, onDsnUpdate } = args; + const { + connector, + formType, + onParamsUpdate, + onDsnUpdate, + getSelectedAuthMethod, + } = args; this.connector = connector; this.formType = formType; + this.getSelectedAuthMethod = getSelectedAuthMethod; // Layout height this.formHeight = TALL_FORM_CONNECTORS.has(connector.name ?? "") @@ -128,7 +138,10 @@ export class AddDataFormManager { const paramsSchemaDef = getValidationSchemaForConnector( connector.name as string, formType, - { isMultiStepConnector: this.isMultiStepConnector }, + { + isMultiStepConnector: this.isMultiStepConnector, + authMethodGetter: this.getSelectedAuthMethod, + }, ); const paramsAdapter = yup(paramsSchemaDef); type ParamsOut = YupInfer; diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index ee4bfa7e89a..a47a004a780 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,14 +1,19 @@ +import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import type { AddDataFormType } from "./types"; +import { multiStepFormConfigs } from "./multi-step-auth-configs"; +import type { AddDataFormType, AuthField } from "./types"; export { dsnSchema }; export function getValidationSchemaForConnector( name: string, formType: AddDataFormType, - opts?: { isMultiStepConnector?: boolean }, + opts?: { + isMultiStepConnector?: boolean; + authMethodGetter?: () => string | undefined; + }, ) { - const { isMultiStepConnector } = opts || {}; + const { isMultiStepConnector, authMethodGetter } = opts || {}; // For multi-step source flows, prefer the connector-specific schema when present // so step 1 (connector) validation doesn't require source-only fields. @@ -21,6 +26,13 @@ export function getValidationSchemaForConnector( // For multi-step connector step, prefer connector-specific schema when present. if (isMultiStepConnector && formType === "connector") { + // Generic dynamic schema based on auth options, driven by config. + const dynamicSchema = makeAuthOptionValidationSchema( + name, + authMethodGetter, + ); + if (dynamicSchema) return dynamicSchema; + const connectorKey = `${name}_connector`; if (connectorKey in getYupSchema) { return getYupSchema[connectorKey as keyof typeof getYupSchema]; @@ -29,3 +41,47 @@ export function getValidationSchemaForConnector( return getYupSchema[name as keyof typeof getYupSchema]; } + +/** + * Build a yup schema that enforces required fields for the selected auth option + * using the multi-step auth config. This keeps validation in sync with the UI + * definitions in constants/multi-step-auth-configs. + */ +function makeAuthOptionValidationSchema( + connectorName: string, + getAuthMethod?: () => string | undefined, +) { + const config = + multiStepFormConfigs[connectorName as keyof typeof multiStepFormConfigs]; + if (!config) return null; + + // Collect all field definitions across auth methods. + const fieldValidations: Record = {}; + + for (const [method, fields] of Object.entries(config.authFieldGroups || {})) { + for (const field of fields as AuthField[]) { + // Only validate concrete input/credential fields. + const required = !(field.optional ?? false); + if (!required) continue; + const label = field.type === "input" ? field.label || field.id : field.id; + // Only apply requirement when the selected auth method matches. + fieldValidations[field.id] = ( + fieldValidations[field.id] || yup.string() + ).test( + `required-${field.id}-${method}`, + `${label} is required`, + (value) => { + if (!getAuthMethod) return true; + const current = getAuthMethod(); + if (current !== method) return true; + return !!value; + }, + ); + } + } + + // If nothing to validate, skip dynamic schema. + if (!Object.keys(fieldValidations).length) return null; + + return yup.object().shape(fieldValidations); +} From ee15857af5d0f8538987c624eadf83b64db5ef5a Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 20:47:57 +0800 Subject: [PATCH 028/103] add comments to the fallback schema --- web-common/src/features/sources/modal/yupSchemas.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 710ac3a5bd2..949cfecbbe6 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,6 +5,8 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. s3_connector: yup.object().shape({ aws_access_key_id: yup.string().optional(), aws_secret_access_key: yup.string().optional(), @@ -23,6 +25,8 @@ export const getYupSchema = { .required(), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. gcs_connector: yup.object().shape({ google_application_credentials: yup.string().optional(), key_id: yup.string().optional(), @@ -82,6 +86,9 @@ export const getYupSchema = { .required("Google application credentials is required"), }), + // Keep these optional here; per-auth required fields are enforced dynamically + // via multi-step auth configs. This schema acts as a safe fallback (e.g. source + // step selection of `${name}_connector`). azure_connector: yup.object().shape({ azure_storage_account: yup.string().optional(), azure_storage_key: yup.string().optional(), From 5e8a21b3c895c01a28dd2a0c404580f03b115671 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:02:52 +0800 Subject: [PATCH 029/103] azure and s3 e2e --- .../tests/connectors/test-connection.spec.ts | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/web-local/tests/connectors/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts index e571c62d3bf..437211c26a2 100644 --- a/web-local/tests/connectors/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -4,6 +4,72 @@ import { test } from "../setup/base"; test.describe("Test Connection", () => { test.use({ project: "Blank" }); + test("Azure connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#azure").click(); + await page.waitForSelector('form[id*="azure"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Default method is Storage Account Key -> requires account + key. + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Storage account" }).fill("acct"); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Access key" }).fill("key"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch to Connection String -> requires connection string, so disabled until filled. + await page.getByRole("radio", { name: "Connection String" }).click(); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Connection string" }) + .fill("DefaultEndpointsProtocol=https;"); + await expect(button).toBeEnabled(); + }); + + test("S3 connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#s3").click(); + await page.waitForSelector('form[id*="s3"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Default method is Access keys -> requires access key id + secret. + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch back to Access keys -> fields cleared, so disabled until refilled. + await page.getByRole("radio", { name: "Access keys" }).click(); + await expect(button).toBeDisabled(); + }); + test("GCS connector - HMAC", async ({ page }) => { // Skip test if environment variables are not set if ( From 8123e2b16f79975648a86321de1bcd6fea7cbc22 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:17:58 +0800 Subject: [PATCH 030/103] update default auth method for azure --- .../src/features/sources/modal/multi-step-auth-configs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index a1e2d567662..9307704f3b3 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -114,7 +114,7 @@ export const multiStepFormConfigs: Record = { }, azure: { authOptions: AZURE_AUTH_OPTIONS, - defaultAuthMethod: "account_key", + defaultAuthMethod: "connection_string", clearFieldsByMethod: { account_key: [ "azure_storage_connection_string", From 1c25ae7ccffff268435d372450f1deb56a2d782c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:30:55 +0800 Subject: [PATCH 031/103] hide save anyway when advacning to the model step for multi step --- web-common/src/features/sources/modal/AddDataForm.svelte | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 9f36024ed4e..f4cf730797d 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -134,6 +134,11 @@ let clickhouseDsnForm; let clickhouseShowSaveAnyway: boolean = false; + // Hide Save Anyway once we advance to the model step in multi-step flows. + $: if (isMultiStepConnector && stepState.step === "source") { + showSaveAnyway = false; + } + $: isSubmitDisabled = (() => { // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) if (isMultiStepConnector && stepState.step === "connector") { From 10546025c072dd0dc47924bd8ebb9066f27721a0 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 21:49:59 +0800 Subject: [PATCH 032/103] fix e2e --- web-local/tests/connectors/test-connection.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web-local/tests/connectors/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts index 437211c26a2..1ed49d8276c 100644 --- a/web-local/tests/connectors/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -16,7 +16,8 @@ test.describe("Test Connection", () => { name: /(Test and Connect|Continue)/, }); - // Default method is Storage Account Key -> requires account + key. + // Select Storage Account Key (default may be different) -> requires account + key. + await page.getByRole("radio", { name: "Storage Account Key" }).click(); await expect(button).toBeDisabled(); await page.getByRole("textbox", { name: "Storage account" }).fill("acct"); From 35b69b7d3bf489ad268e07b938effe698f567b89 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 22:10:39 +0800 Subject: [PATCH 033/103] centralize multi step auth method state to the store --- .../features/sources/modal/AddDataForm.svelte | 50 +++++++++++-------- .../sources/modal/AddDataFormManager.ts | 7 +-- .../sources/modal/connectorStepStore.ts | 16 +++++- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index f4cf730797d..f3a81c0414b 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -19,7 +19,7 @@ } from "./constants"; import { getInitialFormValuesFromProperties } from "../sourceUtils"; - import { connectorStepStore } from "./connectorStepStore"; + import { connectorStepStore, setAuthMethod } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; @@ -28,6 +28,7 @@ import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; import { multiStepFormConfigs } from "./multi-step-auth-configs"; + import { get } from "svelte/store"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -56,13 +57,23 @@ formType, onParamsUpdate: (e: any) => handleOnUpdate(e), onDsnUpdate: (e: any) => handleOnUpdate(e), - getSelectedAuthMethod: () => selectedAuthMethod, + getSelectedAuthMethod: () => + get(connectorStepStore).selectedAuthMethod ?? undefined, }); const isMultiStepConnector = formManager.isMultiStepConnector; const isSourceForm = formManager.isSourceForm; const isConnectorForm = formManager.isConnectorForm; const onlyDsn = hasOnlyDsn(connector, isConnectorForm); + const selectedAuthMethodStore = { + subscribe: (run: (value: string) => void) => + connectorStepStore.subscribe((state) => + run(state.selectedAuthMethod ?? ""), + ), + set: (method: string) => setAuthMethod(method || null), + }; + let selectedAuthMethod: string = ""; + $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; $: stepProperties = isMultiStepConnector && stepState.step === "source" @@ -221,6 +232,22 @@ ] || null : null; + $: if (isMultiStepConnector && activeMultiStepConfig) { + const options = activeMultiStepConfig.authOptions ?? []; + const fallback = + activeMultiStepConfig.defaultAuthMethod || options[0]?.value || null; + const hasValidSelection = options.some( + (option) => option.value === stepState.selectedAuthMethod, + ); + if (!hasValidSelection) { + if (fallback !== stepState.selectedAuthMethod) { + setAuthMethod(fallback ?? null); + } + } + } else if (stepState.selectedAuthMethod) { + setAuthMethod(null); + } + $: isSubmitting = submitting; // Reset errors when form is modified @@ -315,23 +342,6 @@ ? clickhouseSubmitting && saveAnyway : submitting && saveAnyway; - // Track selected auth method for multi-step connectors to adjust UI labels. - // Only initialize when config becomes available; do not reset after user selection. - let selectedAuthMethod: string = ""; - $: if ( - activeMultiStepConfig && - !selectedAuthMethod && - activeMultiStepConfig.authOptions?.length - ) { - selectedAuthMethod = - activeMultiStepConfig.defaultAuthMethod || - activeMultiStepConfig.authOptions?.[0]?.value || - ""; - } - $: if (!activeMultiStepConfig) { - selectedAuthMethod = ""; - } - handleOnUpdate = formManager.makeOnUpdate({ onClose, queryClient, @@ -455,7 +465,7 @@ paramsErrors={$paramsErrors} {onStringInputChange} {handleFileUpload} - bind:authMethod={selectedAuthMethod} + bind:authMethod={$selectedAuthMethodStore} /> {:else} , any, Record>; }; -// Shape of the step store for multi-step connectors -type ConnectorStepState = { - step: "connector" | "source"; - connectorConfig: Record | null; -}; - export class AddDataFormManager { formHeight: string; paramsFormId: string; diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 9ce2451f126..1168a65e532 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -2,12 +2,16 @@ import { writable } from "svelte/store"; export type ConnectorStep = "connector" | "source"; -export const connectorStepStore = writable<{ +export type ConnectorStepState = { step: ConnectorStep; connectorConfig: Record | null; -}>({ + selectedAuthMethod: string | null; +}; + +export const connectorStepStore = writable({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); export function setStep(step: ConnectorStep) { @@ -18,9 +22,17 @@ export function setConnectorConfig(config: Record) { connectorStepStore.update((state) => ({ ...state, connectorConfig: config })); } +export function setAuthMethod(method: string | null) { + connectorStepStore.update((state) => ({ + ...state, + selectedAuthMethod: method, + })); +} + export function resetConnectorStep() { connectorStepStore.set({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); } From ee61f0605a75d77c997a9aebe4ab624a1e142fda Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 23:14:51 +0800 Subject: [PATCH 034/103] clean up save anyway logic --- .../sources/modal/AddDataFormManager.ts | 53 ++++++++++++++----- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 641e2459c8a..17b8abf258a 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -179,6 +179,40 @@ export class AddDataFormManager { return MULTI_STEP_CONNECTORS.includes(this.connector.name ?? ""); } + /** + * Determines whether the "Save Anyway" button should be shown for the current submission. + */ + private shouldShowSaveAnywayButton(args: { + isConnectorForm: boolean; + event?: + | { + result?: Extract; + } + | undefined; + stepState: ConnectorStepState | undefined; + selectedAuthMethod?: string; + }): boolean { + const { isConnectorForm, event, stepState, selectedAuthMethod } = args; + + // Only show for connector forms (not sources) + if (!isConnectorForm) return false; + + // ClickHouse has its own error handling + if (this.connector.name === "clickhouse") return false; + + // Need a submission result to show the button + if (!event?.result) return false; + + // Multi-step connectors: don't show on source step (final step) + if (stepState?.step === "source") return false; + + // Public auth bypasses connection test, so no "Save Anyway" needed + if (stepState?.step === "connector" && selectedAuthMethod === "public") + return false; + + return true; + } + getActiveFormId(args: { connectionTab: "parameters" | "dsn"; onlyDsn: boolean; @@ -286,21 +320,14 @@ export class AddDataFormManager { const selectedAuthMethod = getSelectedAuthMethod?.(); const stepState = get(connectorStepStore) as ConnectorStepState; - // FIXME: simplify this logic - // For non-ClickHouse connectors, expose Save Anyway when a submission starts, - // but skip for multi-step public auth where we bypass submission. if ( - isConnectorForm && - connector.name !== "clickhouse" && typeof setShowSaveAnyway === "function" && - event?.result && - !( - isMultiStepConnector && - stepState?.step === "connector" && - selectedAuthMethod === "public" - ) && - // Do not show Save Anyway on the model (source) step of multi-step flows. - stepState?.step !== "source" + this.shouldShowSaveAnywayButton({ + isConnectorForm, + event, + stepState, + selectedAuthMethod, + }) ) { setShowSaveAnyway(true); } From 680422883759bb62ae283372a0dec49a3e62a795 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 23:22:48 +0800 Subject: [PATCH 035/103] save anyway e2e --- .../tests/connectors/save-anyway.spec.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/web-local/tests/connectors/save-anyway.spec.ts b/web-local/tests/connectors/save-anyway.spec.ts index 4fbb190ef5b..38f89955e37 100644 --- a/web-local/tests/connectors/save-anyway.spec.ts +++ b/web-local/tests/connectors/save-anyway.spec.ts @@ -47,4 +47,39 @@ test.describe("Save Anyway feature", () => { await expect(codeEditor).toContainText("type: connector"); await expect(codeEditor).toContainText("driver: clickhouse"); }); + + test("GCS connector - shows Save Anyway after failed test", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#gcs").click(); + + await page.waitForSelector('form[id*="gcs"]'); + + // Use HMAC auth with invalid credentials to force a failure + await page.getByRole("radio", { name: "HMAC keys" }).click(); + await page.getByRole("textbox", { name: "Access Key ID" }).fill("bad-id"); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("bad-secret"); + + const saveAnywayButton = page.getByRole("button", { + name: "Save Anyway", + }); + + // Should not be visible before submission + await expect(saveAnywayButton).toBeHidden(); + + await page + .getByRole("dialog") + .getByRole("button", { name: "Test and Connect" }) + .click(); + + // Error should surface, and Save Anyway should now be offered + await expect(page.locator(".error-container")).toBeVisible({ + timeout: 15000, + }); + await expect(saveAnywayButton).toBeVisible({ timeout: 15000 }); + }); }); From 0411f34cb0a3d2e5d0ad2f854567af14df1c7c2e Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 9 Dec 2025 23:59:25 +0800 Subject: [PATCH 036/103] isMultiStepConnectorDisabled --- .../features/sources/modal/AddDataForm.svelte | 33 ++++------------- .../src/features/sources/modal/utils.ts | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index f3a81c0414b..b41c05b063d 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -12,7 +12,7 @@ import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; - import { isEmpty } from "./utils"; + import { hasOnlyDsn, isEmpty, isMultiStepConnectorDisabled } from "./utils"; import { CONNECTION_TAB_OPTIONS, type ClickHouseConnectorType, @@ -25,7 +25,6 @@ import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; - import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; import { multiStepFormConfigs } from "./multi-step-auth-configs"; import { get } from "svelte/store"; @@ -72,6 +71,7 @@ ), set: (method: string) => setAuthMethod(method || null), }; + let selectedAuthMethod: string = ""; $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; @@ -153,31 +153,12 @@ $: isSubmitDisabled = (() => { // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) if (isMultiStepConnector && stepState.step === "connector") { - const config = - multiStepFormConfigs[ - connector.name as keyof typeof multiStepFormConfigs - ]; - if (!config) return true; - // Only validate the currently selected auth method; fall back to default. - const method = - selectedAuthMethod || - config.defaultAuthMethod || - config.authOptions?.[0]?.value; - const groups = method ? [config.authFieldGroups?.[method] || []] : []; - if (!groups.length) return false; - const hasError = (fieldId: string) => - Boolean(($paramsErrors[fieldId] as any)?.length); - const groupSatisfied = groups.some((fields) => - fields.every((field: any) => { - const required = !(field.optional ?? false); - if (!required) return true; - const value = $paramsForm[field.id]; - if (isEmpty(value)) return false; - if (hasError(field.id)) return false; - return true; - }), + return isMultiStepConnectorDisabled( + activeMultiStepConfig, + selectedAuthMethod, + $paramsForm, + $paramsErrors, ); - return !groupSatisfied; } if (onlyDsn || connectionTab === "dsn") { diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 260f7f3bef2..1df39029b84 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,6 +1,7 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; +import type { MultiStepFormConfig } from "./types"; /** * Returns true for undefined, null, empty string, or whitespace-only string. @@ -83,6 +84,42 @@ export function hasOnlyDsn( return hasDsn && !hasOthers; } +/** + * Returns true when the active multi-step auth method has missing or invalid + * required fields. Falls back to configured default/first auth method. + */ +export function isMultiStepConnectorDisabled( + config: MultiStepFormConfig | null, + selectedMethod: string, + paramsFormValue: Record, + paramsFormErrors: Record, +) { + if (!config) return true; + + const options = config.authOptions ?? []; + const hasValidSelection = options.some((opt) => opt.value === selectedMethod); + const method = + (hasValidSelection && selectedMethod) || + config.defaultAuthMethod || + options[0]?.value; + + if (!method) return true; + + const fields = config.authFieldGroups?.[method] || []; + // If method isn't known or has no fields, only allow when explicitly public. + if (!fields.length) return method === "public"; + + return !fields.every((field) => { + if (field.optional ?? false) return true; + + const value = paramsFormValue[field.id]; + const errorsForField = paramsFormErrors[field.id] as any; + const hasErrors = Boolean(errorsForField?.length); + + return !isEmpty(value) && !hasErrors; + }); +} + /** * Applies ClickHouse Cloud-specific default requirements for connector values. * - For ClickHouse Cloud: enforces `ssl: true` and `port: "8443"` From 6694210d2ca1f0abfbb3df03732ad6abc3e319f5 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 10 Dec 2025 00:05:55 +0800 Subject: [PATCH 037/103] reorg public option --- .../src/features/sources/modal/constants.ts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 7e78fdfa751..8e02f8a7b44 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -25,11 +25,6 @@ export const GCS_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ - { - value: "public", - label: "Public", - description: "Access publicly readable buckets without credentials.", - }, { value: "credentials", label: "GCP credentials", @@ -42,6 +37,11 @@ export const GCS_AUTH_OPTIONS: { description: "Use HMAC access key and secret for S3-compatible authentication.", }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, ]; export type S3AuthMethod = "access_keys" | "public"; @@ -52,16 +52,16 @@ export const S3_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ - { - value: "public", - label: "Public", - description: "Access publicly readable buckets without credentials.", - }, { value: "access_keys", label: "Access keys", description: "Use AWS access key ID and secret access key.", }, + { + value: "public", + label: "Public", + description: "Access publicly readable buckets without credentials.", + }, ]; export type AzureAuthMethod = @@ -76,11 +76,6 @@ export const AZURE_AUTH_OPTIONS: { description: string; hint?: string; }[] = [ - { - value: "public", - label: "Public", - description: "Access publicly readable blobs without credentials.", - }, { value: "connection_string", label: "Connection String", @@ -96,6 +91,11 @@ export const AZURE_AUTH_OPTIONS: { label: "Shared Access Signature (SAS) Token", description: "Most secure, fine-grained control", }, + { + value: "public", + label: "Public", + description: "Access publicly readable blobs without credentials.", + }, ]; // pre-defined order for sources From a1626e92b5e6189697a7e50def5809c1a548cb21 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 10 Dec 2025 00:10:49 +0800 Subject: [PATCH 038/103] colocate button labels --- .../sources/modal/AddDataFormManager.ts | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 17b8abf258a..b915299e919 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -42,6 +42,12 @@ type SuperFormUpdateEvent = { form: SuperValidated, any, Record>; }; +const BUTTON_LABELS = { + public: { idle: "Continue", submitting: "Continuing..." }, + connector: { idle: "Test and Connect", submitting: "Testing connection..." }, + source: { idle: "Import Data", submitting: "Importing data..." }, +}; + export class AddDataFormManager { formHeight: string; paramsFormId: string; @@ -269,14 +275,22 @@ export class AddDataFormManager { if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { if (selectedAuthMethod === "public") { - return submitting ? "Continuing..." : "Continue"; + return submitting + ? BUTTON_LABELS.public.submitting + : BUTTON_LABELS.public.idle; } - return submitting ? "Testing connection..." : "Test and Connect"; + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } if (this.isMultiStepConnector && step === "source") { - return submitting ? "Importing data..." : "Import Data"; + return submitting + ? BUTTON_LABELS.source.submitting + : BUTTON_LABELS.source.idle; } - return submitting ? "Testing connection..." : "Test and Connect"; + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } return "Test and Add data"; From fdb5d834ef5cb150eebc8a1efb77739d206fca68 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 10 Dec 2025 01:57:39 +0800 Subject: [PATCH 039/103] fixes --- .../src/features/sources/modal/utils.ts | 7 ++-- .../tests/connectors/save-anyway.spec.ts | 35 ------------------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 1df39029b84..c227b4c1b50 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -105,9 +105,12 @@ export function isMultiStepConnectorDisabled( if (!method) return true; + // Selecting "public" should always enable the button for multi-step auth flows. + if (method === "public") return false; + const fields = config.authFieldGroups?.[method] || []; - // If method isn't known or has no fields, only allow when explicitly public. - if (!fields.length) return method === "public"; + // Unknown auth methods or ones without fields stay disabled. + if (!fields.length) return true; return !fields.every((field) => { if (field.optional ?? false) return true; diff --git a/web-local/tests/connectors/save-anyway.spec.ts b/web-local/tests/connectors/save-anyway.spec.ts index 38f89955e37..4fbb190ef5b 100644 --- a/web-local/tests/connectors/save-anyway.spec.ts +++ b/web-local/tests/connectors/save-anyway.spec.ts @@ -47,39 +47,4 @@ test.describe("Save Anyway feature", () => { await expect(codeEditor).toContainText("type: connector"); await expect(codeEditor).toContainText("driver: clickhouse"); }); - - test("GCS connector - shows Save Anyway after failed test", async ({ - page, - }) => { - await page.getByRole("button", { name: "Add Asset" }).click(); - await page.getByRole("menuitem", { name: "Add Data" }).click(); - await page.locator("#gcs").click(); - - await page.waitForSelector('form[id*="gcs"]'); - - // Use HMAC auth with invalid credentials to force a failure - await page.getByRole("radio", { name: "HMAC keys" }).click(); - await page.getByRole("textbox", { name: "Access Key ID" }).fill("bad-id"); - await page - .getByRole("textbox", { name: "Secret Access Key" }) - .fill("bad-secret"); - - const saveAnywayButton = page.getByRole("button", { - name: "Save Anyway", - }); - - // Should not be visible before submission - await expect(saveAnywayButton).toBeHidden(); - - await page - .getByRole("dialog") - .getByRole("button", { name: "Test and Connect" }) - .click(); - - // Error should surface, and Save Anyway should now be offered - await expect(page.locator(".error-container")).toBeVisible({ - timeout: 15000, - }); - await expect(saveAnywayButton).toBeVisible({ timeout: 15000 }); - }); }); From 66bcba2570e4cdca56da7d6a294fc012e38ee5e2 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 12 Dec 2025 00:02:57 +0800 Subject: [PATCH 040/103] use onsubmit to fix validation flicker rerender --- web-common/src/features/sources/modal/AddDataFormManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index b915299e919..101cc1f2d93 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -159,6 +159,7 @@ export class AddDataFormManager { validators: paramsAdapter, onUpdate: onParamsUpdate, resetForm: false, + validationMethod: "onsubmit", }); // Superforms: dsn @@ -170,6 +171,7 @@ export class AddDataFormManager { validators: dsnAdapter, onUpdate: onDsnUpdate, resetForm: false, + validationMethod: "onsubmit", }); } From 119af77d377bbb93f3af68455ca0ba02b0600410 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 12 Dec 2025 00:47:13 +0800 Subject: [PATCH 041/103] fix submission for already connected? --- .../features/sources/modal/AddDataFormManager.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 101cc1f2d93..3a12ac8b682 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -234,7 +234,7 @@ export class AddDataFormManager { handleSkip(): void { const stepState = get(connectorStepStore) as ConnectorStepState; if (!this.isMultiStepConnector || stepState.step !== "connector") return; - setConnectorConfig(get(this.params.form) as Record); + setConnectorConfig({}); setStep("source"); } @@ -330,12 +330,20 @@ export class AddDataFormManager { >; result?: Extract; }) => { - if (!event.form.valid) return; - const values = event.form.data; const selectedAuthMethod = getSelectedAuthMethod?.(); const stepState = get(connectorStepStore) as ConnectorStepState; + // When in the source step of a multi-step flow, the superform still uses + // the connector schema, so it can appear invalid because connector fields + // were intentionally skipped. Allow submission in that case and rely on + // the UI-level required checks for source fields. + if ( + !event.form.valid && + !(isMultiStepConnector && stepState.step === "source") + ) + return; + if ( typeof setShowSaveAnyway === "function" && this.shouldShowSaveAnywayButton({ From b713e8b4f668a911585e5adb1e2ba9bb6c561ab3 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 12 Dec 2025 11:31:20 +0800 Subject: [PATCH 042/103] s3 naming --- .../src/features/entity-management/name-utils.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/web-common/src/features/entity-management/name-utils.ts b/web-common/src/features/entity-management/name-utils.ts index 01f9567a937..52a6d72dc99 100644 --- a/web-common/src/features/entity-management/name-utils.ts +++ b/web-common/src/features/entity-management/name-utils.ts @@ -21,8 +21,22 @@ export function getName(name: string, others: string[]): string { const set = new Set(others.map((other) => other.toLowerCase())); let result = name; + const incrementableSuffix = /(.+)_([0-9]+)$/; while (set.has(result.toLowerCase())) { + // Special-case for "s3": don't roll over to "s4", append suffix instead. + if (name.toLowerCase() === "s3") { + const match = incrementableSuffix.exec(result); + if (match) { + const base = match[1]; + const number = Number.parseInt(match[2], 10) + 1; + result = `${base}_${number}`; + continue; + } + result = `${name}_1`; + continue; + } + result = INCREMENT.exec(result)?.[1] ? result.replace(INCREMENT, (m) => (+m + 1).toString()) : `${result}_1`; From 83bfcdf0e295f27ec54af5d5bd5d933d045eb235 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Mon, 15 Dec 2025 23:58:16 +0800 Subject: [PATCH 043/103] json schema config --- .../features/sources/modal/AddDataForm.svelte | 11 +- .../features/sources/modal/FormValidation.ts | 32 +- .../sources/modal/multi-step-auth-configs.ts | 671 +++++++++++++----- .../src/features/sources/modal/types.ts | 58 ++ 4 files changed, 567 insertions(+), 205 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index b41c05b063d..2356c325327 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -26,7 +26,7 @@ import { AddDataFormManager } from "./AddDataFormManager"; import AddDataFormSection from "./AddDataFormSection.svelte"; - import { multiStepFormConfigs } from "./multi-step-auth-configs"; + import { getMultiStepFormConfig } from "./multi-step-auth-configs"; import { get } from "svelte/store"; export let connector: V1ConnectorDriver; @@ -207,11 +207,10 @@ } })(); - $: activeMultiStepConfig = isMultiStepConnector - ? multiStepFormConfigs[ - connector.name as keyof typeof multiStepFormConfigs - ] || null - : null; + $: activeMultiStepConfig = + isMultiStepConnector && connector.name + ? getMultiStepFormConfig(connector.name) || null + : null; $: if (isMultiStepConnector && activeMultiStepConfig) { const options = activeMultiStepConfig.authOptions ?? []; diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index a47a004a780..88bc40ef303 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,7 @@ import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import { multiStepFormConfigs } from "./multi-step-auth-configs"; -import type { AddDataFormType, AuthField } from "./types"; +import { getMultiStepFormConfig } from "./multi-step-auth-configs"; +import type { AddDataFormType } from "./types"; export { dsnSchema }; @@ -51,24 +51,26 @@ function makeAuthOptionValidationSchema( connectorName: string, getAuthMethod?: () => string | undefined, ) { - const config = - multiStepFormConfigs[connectorName as keyof typeof multiStepFormConfigs]; + const config = getMultiStepFormConfig(connectorName); if (!config) return null; - // Collect all field definitions across auth methods. const fieldValidations: Record = {}; - for (const [method, fields] of Object.entries(config.authFieldGroups || {})) { - for (const field of fields as AuthField[]) { - // Only validate concrete input/credential fields. - const required = !(field.optional ?? false); - if (!required) continue; - const label = field.type === "input" ? field.label || field.id : field.id; - // Only apply requirement when the selected auth method matches. - fieldValidations[field.id] = ( - fieldValidations[field.id] || yup.string() + for (const [method, fields] of Object.entries( + config.requiredFieldsByMethod || {}, + )) { + for (const fieldId of fields) { + const authField = config.authFieldGroups[method]?.find( + (f) => f.id === fieldId, + ); + const label = + config.fieldLabels[fieldId] || + (authField?.type === "input" ? authField.label : authField?.id) || + fieldId; + fieldValidations[fieldId] = ( + fieldValidations[fieldId] || yup.string() ).test( - `required-${field.id}-${method}`, + `required-${fieldId}-${method}`, `${label} is required`, (value) => { if (!getAuthMethod) return true; diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 9307704f3b3..ce2d39e7b7a 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -1,195 +1,498 @@ -import { - AZURE_AUTH_OPTIONS, - GCS_AUTH_OPTIONS, - S3_AUTH_OPTIONS, -} from "./constants"; -import type { MultiStepFormConfig } from "./types"; - -export const multiStepFormConfigs: Record = { - gcs: { - authOptions: GCS_AUTH_OPTIONS, - defaultAuthMethod: "credentials", - clearFieldsByMethod: { - public: ["google_application_credentials", "key_id", "secret"], - credentials: ["key_id", "secret"], - hmac: ["google_application_credentials"], +import type { + AuthField, + AuthOption, + JSONSchemaConditional, + JSONSchemaField, + MultiStepFormConfig, + MultiStepFormSchema, +} from "./types"; + +type VisibleIf = Record< + string, + string | number | boolean | Array +>; + +export const multiStepFormSchemas: Record = { + s3: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + description: "Choose how to authenticate to S3", + enum: ["access_keys", "public"], + default: "access_keys", + "x-display": "radio", + "x-enum-labels": ["Access keys", "Public"], + "x-enum-descriptions": [ + "Use AWS access key ID and secret access key.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + aws_access_key_id: { + type: "string", + title: "Access Key ID", + description: "AWS access key ID for the bucket", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_secret_access_key: { + type: "string", + title: "Secret Access Key", + description: "AWS secret access key for the bucket", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + region: { + type: "string", + title: "Region", + description: + "Rill uses your default AWS region unless you set it explicitly.", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + endpoint: { + type: "string", + title: "Endpoint", + description: + "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", + "x-placeholder": "https://s3.example.com", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + path: { + type: "string", + title: "S3 URI", + description: "Path to your S3 bucket or prefix", + pattern: "^s3://", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-step": "source", + }, }, - excludedKeys: [ - "google_application_credentials", - "key_id", - "secret", - "name", - ], - authFieldGroups: { - public: [], - credentials: [ - { - type: "credentials", - id: "google_application_credentials", - optional: false, - hint: "Upload a JSON key file for a service account with GCS access.", - accept: ".json", - }, - ], - hmac: [ - { - type: "input", - id: "key_id", - label: "Access Key ID", - placeholder: "Enter your HMAC access key ID", - optional: false, - secret: true, - hint: "HMAC access key ID for S3-compatible authentication", - }, - { - type: "input", - id: "secret", - label: "Secret Access Key", - placeholder: "Enter your HMAC secret access key", - optional: false, - secret: true, - hint: "HMAC secret access key for S3-compatible authentication", + allOf: [ + { + if: { properties: { auth_method: { const: "access_keys" } } }, + then: { + required: ["aws_access_key_id", "aws_secret_access_key"], }, - ], - }, + }, + ], }, - s3: { - authOptions: S3_AUTH_OPTIONS, - defaultAuthMethod: "access_keys", - clearFieldsByMethod: { - access_keys: [], - public: [ - "aws_access_key_id", - "aws_secret_access_key", - "region", - "endpoint", - ], + gcs: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["credentials", "hmac", "public"], + default: "credentials", + description: "Choose how to authenticate to GCS", + "x-display": "radio", + "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with GCS access.", + "Use HMAC access key and secret for S3-compatible authentication.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + google_application_credentials: { + type: "string", + title: "Service account key", + description: + "Upload a JSON key file for a service account with GCS access.", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + "x-visible-if": { auth_method: "credentials" }, + }, + key_id: { + type: "string", + title: "Access Key ID", + description: "HMAC access key ID for S3-compatible authentication", + "x-placeholder": "Enter your HMAC access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + secret: { + type: "string", + title: "Secret Access Key", + description: "HMAC secret access key for S3-compatible authentication", + "x-placeholder": "Enter your HMAC secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-step": "source", + }, }, - excludedKeys: [ - "aws_access_key_id", - "aws_secret_access_key", - "region", - "endpoint", - "name", + allOf: [ + { + if: { properties: { auth_method: { const: "credentials" } } }, + then: { required: ["google_application_credentials"] }, + }, + { + if: { properties: { auth_method: { const: "hmac" } } }, + then: { required: ["key_id", "secret"] }, + }, ], - authFieldGroups: { - access_keys: [ - { - type: "input", - id: "aws_access_key_id", - label: "Access Key ID", - placeholder: "Enter AWS access key ID", - optional: false, - secret: true, - hint: "AWS access key ID for the bucket", - }, - { - type: "input", - id: "aws_secret_access_key", - label: "Secret Access Key", - placeholder: "Enter AWS secret access key", - optional: false, - secret: true, - hint: "AWS secret access key for the bucket", - }, - { - type: "input", - id: "region", - label: "Region", - placeholder: "us-east-1", - optional: true, - hint: "Rill uses your default AWS region unless you set it explicitly.", - }, - { - type: "input", - id: "endpoint", - label: "Endpoint", - placeholder: "https://s3.example.com", - optional: true, - hint: "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", - }, - ], - public: [], - }, }, azure: { - authOptions: AZURE_AUTH_OPTIONS, - defaultAuthMethod: "connection_string", - clearFieldsByMethod: { - account_key: [ - "azure_storage_connection_string", - "azure_storage_sas_token", - ], - sas_token: ["azure_storage_connection_string", "azure_storage_key"], - connection_string: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - ], - public: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - "azure_storage_connection_string", - ], + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["connection_string", "account_key", "sas_token", "public"], + default: "connection_string", + description: "Choose how to authenticate to Azure Blob Storage", + "x-display": "radio", + "x-enum-labels": [ + "Connection String", + "Storage Account Key", + "SAS Token", + "Public", + ], + "x-enum-descriptions": [ + "Provide a full Azure Storage connection string.", + "Provide the storage account name and access key.", + "Provide the storage account name and SAS token.", + "Access publicly readable blobs without credentials.", + ], + "x-step": "connector", + }, + azure_storage_connection_string: { + type: "string", + title: "Connection string", + description: "Paste an Azure Storage connection string", + "x-placeholder": "Enter Azure storage connection string", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + azure_storage_account: { + type: "string", + title: "Storage account", + description: "The name of the Azure storage account", + "x-placeholder": "Enter Azure storage account", + "x-step": "connector", + "x-visible-if": { auth_method: ["account_key", "sas_token"] }, + }, + azure_storage_key: { + type: "string", + title: "Access key", + description: "Primary or secondary access key for the storage account", + "x-placeholder": "Enter Azure storage access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "account_key" }, + }, + azure_storage_sas_token: { + type: "string", + title: "SAS token", + description: + "Shared Access Signature token for the storage account (starting with ?sv=)", + "x-placeholder": "Enter Azure SAS token", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "sas_token" }, + }, + path: { + type: "string", + title: "Blob URI", + description: + "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", + pattern: "^https?://", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-step": "source", + }, }, - excludedKeys: [ - "azure_storage_account", - "azure_storage_key", - "azure_storage_sas_token", - "azure_storage_connection_string", - "name", - ], - authFieldGroups: { - connection_string: [ - { - type: "input", - id: "azure_storage_connection_string", - label: "Connection string", - placeholder: "Enter Azure storage connection string", - optional: false, - secret: true, - hint: "Paste an Azure Storage connection string", - }, - ], - account_key: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, - hint: "The name of the Azure storage account", - }, - { - type: "input", - id: "azure_storage_key", - label: "Access key", - placeholder: "Enter Azure storage access key", - optional: false, - secret: true, - hint: "Primary or secondary access key for the storage account", - }, - ], - sas_token: [ - { - type: "input", - id: "azure_storage_account", - label: "Storage account", - placeholder: "Enter Azure storage account", - optional: false, + allOf: [ + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["azure_storage_connection_string"] }, + }, + { + if: { properties: { auth_method: { const: "account_key" } } }, + then: { required: ["azure_storage_account", "azure_storage_key"] }, + }, + { + if: { properties: { auth_method: { const: "sas_token" } } }, + then: { + required: ["azure_storage_account", "azure_storage_sas_token"], }, - { - type: "input", - id: "azure_storage_sas_token", - label: "SAS token", - placeholder: "Enter Azure SAS token", - optional: false, - secret: true, - hint: "Shared Access Signature token for the storage account", - }, - ], - public: [], - }, + }, + ], }, }; + +export function getMultiStepFormConfig( + connectorName: string, +): MultiStepFormConfig | null { + const schema = + multiStepFormSchemas[connectorName as keyof typeof multiStepFormSchemas]; + if (!schema?.properties) return null; + + const authMethodKey = findAuthMethodKey(schema); + if (!authMethodKey) return null; + + const authProperty = schema.properties[authMethodKey]; + const authOptions = buildAuthOptions(authProperty); + if (!authOptions.length) return null; + + const defaultAuthMethod = + authProperty.default !== undefined && authProperty.default !== null + ? String(authProperty.default) + : authOptions[0]?.value; + + const requiredByMethod = buildRequiredByMethod( + schema, + authMethodKey, + authOptions.map((o) => o.value), + ); + const authFieldGroups = buildAuthFieldGroups( + schema, + authMethodKey, + authOptions, + requiredByMethod, + ); + const excludedKeys = buildExcludedKeys( + schema, + authMethodKey, + authFieldGroups, + ); + const clearFieldsByMethod = buildClearFieldsByMethod( + schema, + authMethodKey, + authOptions, + ); + + return { + schema, + authMethodKey, + authOptions, + defaultAuthMethod: defaultAuthMethod || undefined, + clearFieldsByMethod, + excludedKeys, + authFieldGroups, + requiredFieldsByMethod: requiredByMethod, + fieldLabels: buildFieldLabels(schema), + }; +} + +function findAuthMethodKey(schema: MultiStepFormSchema): string | null { + if (!schema.properties) return null; + for (const [key, value] of Object.entries(schema.properties)) { + if (value.enum && value["x-display"] === "radio") { + return key; + } + } + return schema.properties.auth_method ? "auth_method" : null; +} + +function buildAuthOptions(authProperty: JSONSchemaField): AuthOption[] { + if (!authProperty.enum) return []; + const labels = authProperty["x-enum-labels"] ?? []; + const descriptions = authProperty["x-enum-descriptions"] ?? []; + return authProperty.enum.map((value, idx) => ({ + value: String(value), + label: labels[idx] ?? String(value), + description: + descriptions[idx] ?? authProperty.description ?? "Choose an option", + hint: authProperty["x-hint"], + })); +} + +function buildRequiredByMethod( + schema: MultiStepFormSchema, + authMethodKey: string, + methods: string[], +): Record { + const conditionals = schema.allOf ?? []; + const baseRequired = new Set(schema.required ?? []); + const result: Record = {}; + + for (const method of methods) { + const required = new Set(baseRequired); + for (const conditional of conditionals) { + if (!matchesAuthMethod(conditional, authMethodKey, method)) { + conditional.else?.required?.forEach((field) => required.add(field)); + continue; + } + conditional.then?.required?.forEach((field) => required.add(field)); + } + result[method] = Array.from(required); + } + + return result; +} + +function matchesAuthMethod( + conditional: JSONSchemaConditional, + authMethodKey: string, + method: string, +) { + const constValue = + conditional.if?.properties?.[authMethodKey as keyof VisibleIf]?.const; + if (constValue === undefined || constValue === null) return false; + return String(constValue) === method; +} + +function buildAuthFieldGroups( + schema: MultiStepFormSchema, + authMethodKey: string, + authOptions: AuthOption[], + requiredByMethod: Record, +): Record { + const groups: Record = {}; + const properties = schema.properties ?? {}; + + for (const option of authOptions) { + const required = new Set(requiredByMethod[option.value] ?? []); + for (const [key, prop] of Object.entries(properties)) { + if (key === authMethodKey) continue; + if (!isConnectorStep(prop)) continue; + if (!isVisibleForMethod(prop, authMethodKey, option.value)) continue; + + const field: AuthField = toAuthField(key, prop, required.has(key)); + groups[option.value] = [...(groups[option.value] ?? []), field]; + } + } + + return groups; +} + +function buildClearFieldsByMethod( + schema: MultiStepFormSchema, + authMethodKey: string, + authOptions: AuthOption[], +): Record { + const properties = schema.properties ?? {}; + const clear: Record = {}; + + for (const option of authOptions) { + const fields: string[] = []; + for (const [key, prop] of Object.entries(properties)) { + if (key === authMethodKey) continue; + if (!isVisibleForMethod(prop, authMethodKey, option.value)) { + fields.push(key); + } + } + clear[option.value] = fields; + } + + return clear; +} + +function buildExcludedKeys( + schema: MultiStepFormSchema, + authMethodKey: string, + authFieldGroups: Record, +): string[] { + const excluded = new Set([authMethodKey]); + const properties = schema.properties ?? {}; + const groupedFieldKeys = new Set( + Object.values(authFieldGroups) + .flat() + .map((field) => field.id), + ); + + for (const [key, prop] of Object.entries(properties)) { + const step = prop["x-step"]; + if (step === "source") excluded.add(key); + if (groupedFieldKeys.has(key)) excluded.add(key); + } + + return Array.from(excluded); +} + +function buildFieldLabels(schema: MultiStepFormSchema): Record { + const labels: Record = {}; + for (const [key, prop] of Object.entries(schema.properties ?? {})) { + if (prop.title) labels[key] = prop.title; + } + return labels; +} + +function isConnectorStep(prop: JSONSchemaField): boolean { + return (prop["x-step"] ?? "connector") === "connector"; +} + +function isVisibleForMethod( + prop: JSONSchemaField, + authMethodKey: string, + method: string, +): boolean { + const conditions = prop["x-visible-if"]; + if (!conditions) return true; + + const authCondition = conditions[authMethodKey]; + if (authCondition === undefined) return true; + if (Array.isArray(authCondition)) { + return authCondition.map(String).includes(method); + } + return String(authCondition) === method; +} + +function toAuthField( + key: string, + prop: JSONSchemaField, + isRequired: boolean, +): AuthField { + const base = { + id: key, + optional: !isRequired, + hint: prop.description ?? prop["x-hint"], + }; + + if (prop["x-display"] === "file" || prop.format === "file") { + return { + type: "credentials", + accept: prop["x-accept"], + ...base, + }; + } + + return { + type: "input", + label: prop.title ?? key, + placeholder: prop["x-placeholder"], + secret: prop["x-secret"], + ...base, + }; +} diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index b9cd39a8b1a..d2f87c9e3e2 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -27,10 +27,68 @@ export type AuthField = hint?: string; }; +type JSONSchemaVisibleIfValue = + | string + | number + | boolean + | Array; + +export type JSONSchemaField = { + type?: "string" | "number" | "boolean" | "object"; + title?: string; + description?: string; + enum?: Array; + const?: string | number | boolean; + default?: string | number | boolean; + pattern?: string; + format?: string; + properties?: Record; + required?: string[]; + "x-display"?: "radio" | "select" | "textarea" | "file"; + "x-step"?: "connector" | "source"; + "x-secret"?: boolean; + "x-visible-if"?: Record; + "x-enum-labels"?: string[]; + "x-enum-descriptions"?: string[]; + "x-placeholder"?: string; + "x-hint"?: string; + "x-accept"?: string; +}; + +export type JSONSchemaCondition = { + properties?: Record; +}; + +export type JSONSchemaConstraint = { + required?: string[]; +}; + +export type JSONSchemaConditional = { + if?: JSONSchemaCondition; + then?: JSONSchemaConstraint; + else?: JSONSchemaConstraint; +}; + +export type JSONSchemaObject = { + $schema?: string; + type: "object"; + title?: string; + description?: string; + properties?: Record; + required?: string[]; + allOf?: JSONSchemaConditional[]; +}; + +export type MultiStepFormSchema = JSONSchemaObject; + export type MultiStepFormConfig = { + schema: MultiStepFormSchema; + authMethodKey: string; authOptions: AuthOption[]; clearFieldsByMethod: Record; excludedKeys: string[]; authFieldGroups: Record; + requiredFieldsByMethod: Record; + fieldLabels: Record; defaultAuthMethod?: string; }; From b976b96c746600f994384a21b63f689f8dba1b03 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 16 Dec 2025 10:39:49 +0800 Subject: [PATCH 044/103] json schema form renderer --- .../features/sources/modal/AddDataForm.svelte | 62 ++-- .../features/sources/modal/FormValidation.ts | 27 +- .../modal/JSONSchemaFormRenderer.svelte | 246 +++++++++++++++ .../sources/modal/MultiStepAuthForm.svelte | 116 ------- .../modal/MultiStepFormRenderer.svelte | 75 ----- .../sources/modal/multi-step-auth-configs.ts | 283 ++++++------------ .../src/features/sources/modal/utils.ts | 34 ++- 7 files changed, 411 insertions(+), 432 deletions(-) create mode 100644 web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte delete mode 100644 web-common/src/features/sources/modal/MultiStepAuthForm.svelte delete mode 100644 web-common/src/features/sources/modal/MultiStepFormRenderer.svelte diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 2356c325327..66adcf494a5 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -22,11 +22,14 @@ import { connectorStepStore, setAuthMethod } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - import MultiStepFormRenderer from "./MultiStepFormRenderer.svelte"; + import JSONSchemaFormRenderer from "./JSONSchemaFormRenderer.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; import AddDataFormSection from "./AddDataFormSection.svelte"; - import { getMultiStepFormConfig } from "./multi-step-auth-configs"; + import { + getAuthOptionsFromSchema, + getConnectorSchema, + } from "./multi-step-auth-configs"; import { get } from "svelte/store"; export let connector: V1ConnectorDriver; @@ -154,7 +157,7 @@ // Multi-step connectors, connector step: check auth fields (any satisfied group enables button) if (isMultiStepConnector && stepState.step === "connector") { return isMultiStepConnectorDisabled( - activeMultiStepConfig, + activeSchema, selectedAuthMethod, $paramsForm, $paramsErrors, @@ -207,15 +210,18 @@ } })(); - $: activeMultiStepConfig = + $: activeSchema = isMultiStepConnector && connector.name - ? getMultiStepFormConfig(connector.name) || null + ? getConnectorSchema(connector.name) || null : null; - $: if (isMultiStepConnector && activeMultiStepConfig) { - const options = activeMultiStepConfig.authOptions ?? []; - const fallback = - activeMultiStepConfig.defaultAuthMethod || options[0]?.value || null; + $: activeAuthInfo = activeSchema + ? getAuthOptionsFromSchema(activeSchema) + : null; + + $: if (isMultiStepConnector && activeAuthInfo) { + const options = activeAuthInfo.options ?? []; + const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; const hasValidSelection = options.some( (option) => option.value === stepState.selectedAuthMethod, ); @@ -437,12 +443,12 @@ enhance={paramsEnhance} onSubmit={paramsSubmit} > - {#if activeMultiStepConfig} - - + {#if activeSchema} + + {:else} + + {/if} {/if} {:else} diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 88bc40ef303..244710a1d25 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,11 @@ import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; -import { getMultiStepFormConfig } from "./multi-step-auth-configs"; -import type { AddDataFormType } from "./types"; +import { + getConnectorSchema, + getFieldLabel, + getRequiredFieldsByAuthMethod, +} from "./multi-step-auth-configs"; +import type { AddDataFormType, MultiStepFormSchema } from "./types"; export { dsnSchema }; @@ -51,22 +55,17 @@ function makeAuthOptionValidationSchema( connectorName: string, getAuthMethod?: () => string | undefined, ) { - const config = getMultiStepFormConfig(connectorName); - if (!config) return null; + const schema = getConnectorSchema(connectorName); + if (!schema) return null; const fieldValidations: Record = {}; + const requiredByMethod = getRequiredFieldsByAuthMethod(schema, { + step: "connector", + }); - for (const [method, fields] of Object.entries( - config.requiredFieldsByMethod || {}, - )) { + for (const [method, fields] of Object.entries(requiredByMethod || {})) { for (const fieldId of fields) { - const authField = config.authFieldGroups[method]?.find( - (f) => f.id === fieldId, - ); - const label = - config.fieldLabels[fieldId] || - (authField?.type === "input" ? authField.label : authField?.id) || - fieldId; + const label = getFieldLabel(schema as MultiStepFormSchema, fieldId); fieldValidations[fieldId] = ( fieldValidations[fieldId] || yup.string() ).test( diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte new file mode 100644 index 00000000000..5d87829c611 --- /dev/null +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -0,0 +1,246 @@ + + +{#if schema} + {#if step === "connector" && authInfo} + {#if authInfo.options.length > 1} +
+
Authentication method
+ + +
+ {#each visibleFieldsFor(option.value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} +
+ + +
+ {:else if authInfo.options[0]} +
+ {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} +
+ {/if} + {:else} +
+ {#each visibleFieldsFor(authMethod, step) as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} +
+ {/if} +{/if} diff --git a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte b/web-common/src/features/sources/modal/MultiStepAuthForm.svelte deleted file mode 100644 index e927cdb68cd..00000000000 --- a/web-common/src/features/sources/modal/MultiStepAuthForm.svelte +++ /dev/null @@ -1,116 +0,0 @@ - - - -{#if !hasSingleAuthOption} -
-
Authentication method
- - - - - -
-{:else if authOptions?.[0]} -
- -
-{/if} - - -{#each properties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {#if !excluded.has(propertyKey)} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/if} -{/each} diff --git a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte b/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte deleted file mode 100644 index 76cf8e997f7..00000000000 --- a/web-common/src/features/sources/modal/MultiStepFormRenderer.svelte +++ /dev/null @@ -1,75 +0,0 @@ - - -{#if config} - - - {#if config.authFieldGroups?.[option.value]} -
- {#each config.authFieldGroups[option.value] as field (field.id)} - {#if field.type === "credentials"} - - {:else} - - {/if} - {/each} -
- {/if} - - -{/if} diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index ce2d39e7b7a..83e54701444 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -1,9 +1,7 @@ import type { - AuthField, AuthOption, JSONSchemaConditional, JSONSchemaField, - MultiStepFormConfig, MultiStepFormSchema, } from "./types"; @@ -261,61 +259,16 @@ export const multiStepFormSchemas: Record = { }, }; -export function getMultiStepFormConfig( +export function getConnectorSchema( connectorName: string, -): MultiStepFormConfig | null { +): MultiStepFormSchema | null { const schema = multiStepFormSchemas[connectorName as keyof typeof multiStepFormSchemas]; if (!schema?.properties) return null; - - const authMethodKey = findAuthMethodKey(schema); - if (!authMethodKey) return null; - - const authProperty = schema.properties[authMethodKey]; - const authOptions = buildAuthOptions(authProperty); - if (!authOptions.length) return null; - - const defaultAuthMethod = - authProperty.default !== undefined && authProperty.default !== null - ? String(authProperty.default) - : authOptions[0]?.value; - - const requiredByMethod = buildRequiredByMethod( - schema, - authMethodKey, - authOptions.map((o) => o.value), - ); - const authFieldGroups = buildAuthFieldGroups( - schema, - authMethodKey, - authOptions, - requiredByMethod, - ); - const excludedKeys = buildExcludedKeys( - schema, - authMethodKey, - authFieldGroups, - ); - const clearFieldsByMethod = buildClearFieldsByMethod( - schema, - authMethodKey, - authOptions, - ); - - return { - schema, - authMethodKey, - authOptions, - defaultAuthMethod: defaultAuthMethod || undefined, - clearFieldsByMethod, - excludedKeys, - authFieldGroups, - requiredFieldsByMethod: requiredByMethod, - fieldLabels: buildFieldLabels(schema), - }; + return schema; } -function findAuthMethodKey(schema: MultiStepFormSchema): string | null { +export function findAuthMethodKey(schema: MultiStepFormSchema): string | null { if (!schema.properties) return null; for (const [key, value] of Object.entries(schema.properties)) { if (value.enum && value["x-display"] === "radio") { @@ -325,174 +278,122 @@ function findAuthMethodKey(schema: MultiStepFormSchema): string | null { return schema.properties.auth_method ? "auth_method" : null; } -function buildAuthOptions(authProperty: JSONSchemaField): AuthOption[] { - if (!authProperty.enum) return []; +export function getAuthOptionsFromSchema( + schema: MultiStepFormSchema, +): { key: string; options: AuthOption[]; defaultMethod?: string } | null { + const authMethodKey = findAuthMethodKey(schema); + if (!authMethodKey) return null; + const authProperty = schema.properties?.[authMethodKey]; + if (!authProperty?.enum) return null; + const labels = authProperty["x-enum-labels"] ?? []; const descriptions = authProperty["x-enum-descriptions"] ?? []; - return authProperty.enum.map((value, idx) => ({ - value: String(value), - label: labels[idx] ?? String(value), - description: - descriptions[idx] ?? authProperty.description ?? "Choose an option", - hint: authProperty["x-hint"], - })); + const options = + authProperty.enum?.map((value, idx) => ({ + value: String(value), + label: labels[idx] ?? String(value), + description: + descriptions[idx] ?? authProperty.description ?? "Choose an option", + hint: authProperty["x-hint"], + })) ?? []; + + const defaultMethod = + authProperty.default !== undefined && authProperty.default !== null + ? String(authProperty.default) + : options[0]?.value; + + return { + key: authMethodKey, + options, + defaultMethod: defaultMethod || undefined, + }; } -function buildRequiredByMethod( +export function getRequiredFieldsByAuthMethod( schema: MultiStepFormSchema, - authMethodKey: string, - methods: string[], + opts?: { step?: "connector" | "source" }, ): Record { + const authInfo = getAuthOptionsFromSchema(schema); + if (!authInfo) return {}; + const conditionals = schema.allOf ?? []; const baseRequired = new Set(schema.required ?? []); const result: Record = {}; - for (const method of methods) { - const required = new Set(baseRequired); - for (const conditional of conditionals) { - if (!matchesAuthMethod(conditional, authMethodKey, method)) { - conditional.else?.required?.forEach((field) => required.add(field)); - continue; + for (const option of authInfo.options) { + const required = new Set(); + + // Start with base required fields. + baseRequired.forEach((field) => { + if (!opts?.step || isStepMatch(schema, field, opts.step)) { + required.add(field); } - conditional.then?.required?.forEach((field) => required.add(field)); + }); + + // Apply conditionals. + for (const conditional of conditionals) { + const matches = matchesAuthMethod( + conditional, + authInfo.key, + option.value, + ); + const target = matches ? conditional.then : conditional.else; + target?.required?.forEach((field) => { + if (!opts?.step || isStepMatch(schema, field, opts.step)) { + required.add(field); + } + }); } - result[method] = Array.from(required); + + result[option.value] = Array.from(required); } return result; } -function matchesAuthMethod( - conditional: JSONSchemaConditional, - authMethodKey: string, - method: string, -) { - const constValue = - conditional.if?.properties?.[authMethodKey as keyof VisibleIf]?.const; - if (constValue === undefined || constValue === null) return false; - return String(constValue) === method; -} - -function buildAuthFieldGroups( +export function getFieldLabel( schema: MultiStepFormSchema, - authMethodKey: string, - authOptions: AuthOption[], - requiredByMethod: Record, -): Record { - const groups: Record = {}; - const properties = schema.properties ?? {}; - - for (const option of authOptions) { - const required = new Set(requiredByMethod[option.value] ?? []); - for (const [key, prop] of Object.entries(properties)) { - if (key === authMethodKey) continue; - if (!isConnectorStep(prop)) continue; - if (!isVisibleForMethod(prop, authMethodKey, option.value)) continue; - - const field: AuthField = toAuthField(key, prop, required.has(key)); - groups[option.value] = [...(groups[option.value] ?? []), field]; - } - } - - return groups; + key: string, +): string { + return schema.properties?.[key]?.title || key; } -function buildClearFieldsByMethod( +export function isStepMatch( schema: MultiStepFormSchema, - authMethodKey: string, - authOptions: AuthOption[], -): Record { - const properties = schema.properties ?? {}; - const clear: Record = {}; - - for (const option of authOptions) { - const fields: string[] = []; - for (const [key, prop] of Object.entries(properties)) { - if (key === authMethodKey) continue; - if (!isVisibleForMethod(prop, authMethodKey, option.value)) { - fields.push(key); - } - } - clear[option.value] = fields; - } - - return clear; + key: string, + step: "connector" | "source", +): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; + return (prop["x-step"] ?? "connector") === step; } -function buildExcludedKeys( +export function isVisibleForValues( schema: MultiStepFormSchema, - authMethodKey: string, - authFieldGroups: Record, -): string[] { - const excluded = new Set([authMethodKey]); - const properties = schema.properties ?? {}; - const groupedFieldKeys = new Set( - Object.values(authFieldGroups) - .flat() - .map((field) => field.id), - ); - - for (const [key, prop] of Object.entries(properties)) { - const step = prop["x-step"]; - if (step === "source") excluded.add(key); - if (groupedFieldKeys.has(key)) excluded.add(key); - } - - return Array.from(excluded); -} - -function buildFieldLabels(schema: MultiStepFormSchema): Record { - const labels: Record = {}; - for (const [key, prop] of Object.entries(schema.properties ?? {})) { - if (prop.title) labels[key] = prop.title; - } - return labels; -} - -function isConnectorStep(prop: JSONSchemaField): boolean { - return (prop["x-step"] ?? "connector") === "connector"; -} - -function isVisibleForMethod( - prop: JSONSchemaField, - authMethodKey: string, - method: string, + key: string, + values: Record, ): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; const conditions = prop["x-visible-if"]; if (!conditions) return true; - const authCondition = conditions[authMethodKey]; - if (authCondition === undefined) return true; - if (Array.isArray(authCondition)) { - return authCondition.map(String).includes(method); - } - return String(authCondition) === method; + return Object.entries(conditions).every(([depKey, expected]) => { + const actual = values?.[depKey]; + if (Array.isArray(expected)) { + return expected.map(String).includes(String(actual)); + } + return String(actual) === String(expected); + }); } -function toAuthField( - key: string, - prop: JSONSchemaField, - isRequired: boolean, -): AuthField { - const base = { - id: key, - optional: !isRequired, - hint: prop.description ?? prop["x-hint"], - }; - - if (prop["x-display"] === "file" || prop.format === "file") { - return { - type: "credentials", - accept: prop["x-accept"], - ...base, - }; - } - - return { - type: "input", - label: prop.title ?? key, - placeholder: prop["x-placeholder"], - secret: prop["x-secret"], - ...base, - }; +function matchesAuthMethod( + conditional: JSONSchemaConditional, + authMethodKey: string, + method: string, +) { + const constValue = + conditional.if?.properties?.[authMethodKey as keyof VisibleIf]?.const; + if (constValue === undefined || constValue === null) return false; + return String(constValue) === method; } diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index c227b4c1b50..95da0214dc5 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,7 +1,12 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; -import type { MultiStepFormConfig } from "./types"; +import type { MultiStepFormSchema } from "./types"; +import { + getAuthOptionsFromSchema, + getRequiredFieldsByAuthMethod, + isStepMatch, +} from "./multi-step-auth-configs"; /** * Returns true for undefined, null, empty string, or whitespace-only string. @@ -89,18 +94,19 @@ export function hasOnlyDsn( * required fields. Falls back to configured default/first auth method. */ export function isMultiStepConnectorDisabled( - config: MultiStepFormConfig | null, + schema: MultiStepFormSchema | null, selectedMethod: string, paramsFormValue: Record, paramsFormErrors: Record, ) { - if (!config) return true; + if (!schema) return true; - const options = config.authOptions ?? []; + const authInfo = getAuthOptionsFromSchema(schema); + const options = authInfo?.options ?? []; const hasValidSelection = options.some((opt) => opt.value === selectedMethod); const method = (hasValidSelection && selectedMethod) || - config.defaultAuthMethod || + authInfo?.defaultMethod || options[0]?.value; if (!method) return true; @@ -108,17 +114,17 @@ export function isMultiStepConnectorDisabled( // Selecting "public" should always enable the button for multi-step auth flows. if (method === "public") return false; - const fields = config.authFieldGroups?.[method] || []; - // Unknown auth methods or ones without fields stay disabled. - if (!fields.length) return true; - - return !fields.every((field) => { - if (field.optional ?? false) return true; + const requiredByMethod = getRequiredFieldsByAuthMethod(schema, { + step: "connector", + }); + const requiredFields = requiredByMethod[method] ?? []; + if (!requiredFields.length) return true; - const value = paramsFormValue[field.id]; - const errorsForField = paramsFormErrors[field.id] as any; + return !requiredFields.every((fieldId) => { + if (!isStepMatch(schema, fieldId, "connector")) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; const hasErrors = Boolean(errorsForField?.length); - return !isEmpty(value) && !hasErrors; }); } From 9396e928a974af98a463432aa51417438f6bbe5d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 16 Dec 2025 17:19:09 +0800 Subject: [PATCH 045/103] missing placeholder texts --- .../src/features/sources/modal/multi-step-auth-configs.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 83e54701444..11472193ae6 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -70,6 +70,7 @@ export const multiStepFormSchemas: Record = { title: "S3 URI", description: "Path to your S3 bucket or prefix", pattern: "^s3://", + "x-placeholder": "s3://bucket/path", "x-step": "source", }, name: { @@ -77,6 +78,7 @@ export const multiStepFormSchemas: Record = { title: "Model name", description: "Name for the source model", pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", "x-step": "source", }, }, @@ -142,6 +144,7 @@ export const multiStepFormSchemas: Record = { title: "GCS URI", description: "Path to your GCS bucket or prefix", pattern: "^gs://", + "x-placeholder": "gs://bucket/path", "x-step": "source", }, name: { @@ -149,6 +152,7 @@ export const multiStepFormSchemas: Record = { title: "Model name", description: "Name for the source model", pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", "x-step": "source", }, }, @@ -230,6 +234,7 @@ export const multiStepFormSchemas: Record = { description: "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", pattern: "^https?://", + "x-placeholder": "https://account.blob.core.windows.net/container", "x-step": "source", }, name: { @@ -237,6 +242,7 @@ export const multiStepFormSchemas: Record = { title: "Model name", description: "Name for the source model", pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", "x-step": "source", }, }, From 634d2f01e3085f6045d66d73d96a917f469265ee Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Tue, 16 Dec 2025 23:14:00 +0800 Subject: [PATCH 046/103] prettier --- .../src/features/sources/modal/JSONSchemaFormRenderer.svelte | 2 +- .../src/features/sources/modal/multi-step-auth-configs.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 5d87829c611..4cd4d7ad1f4 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -4,7 +4,7 @@ import Radio from "@rilldata/web-common/components/forms/Radio.svelte"; import CredentialsInput from "@rilldata/web-common/components/forms/CredentialsInput.svelte"; import { normalizeErrors } from "./utils"; - import type { JSONSchemaField, MultiStepFormSchema } from "./types"; + import type { MultiStepFormSchema } from "./types"; import { findAuthMethodKey, getAuthOptionsFromSchema, diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 11472193ae6..0bdfad7c71d 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -1,7 +1,6 @@ import type { AuthOption, JSONSchemaConditional, - JSONSchemaField, MultiStepFormSchema, } from "./types"; From e495160ebffdf310239299116e20aa0015be56db Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 00:31:04 +0800 Subject: [PATCH 047/103] fix field spacing --- .../modal/JSONSchemaFormRenderer.svelte | 192 +++++++++--------- 1 file changed, 93 insertions(+), 99 deletions(-) diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 4cd4d7ad1f4..399411b16f6 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -95,108 +95,56 @@ name="multi-auth-method" > -
- {#each visibleFieldsFor(option.value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} -
+ {#each visibleFieldsFor(option.value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} {:else if authInfo.options[0]} -
- {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} -
- {/if} - {:else} -
- {#each visibleFieldsFor(authMethod, step) as [key, prop]} + {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]}
{#if prop["x-display"] === "file" || prop.format === "file"} {/each} -
+ {/if} + {:else} + {#each visibleFieldsFor(authMethod, step) as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} {/if} {/if} From 54ee93fc0438990d1ae8cf4072946179c207e8e9 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 21:45:35 +0800 Subject: [PATCH 048/103] individual schemas --- .../sources/modal/multi-step-auth-configs.ts | 258 +----------------- .../features/sources/modal/schemas/azure.ts | 98 +++++++ .../src/features/sources/modal/schemas/gcs.ts | 78 ++++++ .../src/features/sources/modal/schemas/s3.ts | 82 ++++++ 4 files changed, 264 insertions(+), 252 deletions(-) create mode 100644 web-common/src/features/sources/modal/schemas/azure.ts create mode 100644 web-common/src/features/sources/modal/schemas/gcs.ts create mode 100644 web-common/src/features/sources/modal/schemas/s3.ts diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 0bdfad7c71d..561cad4d708 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -3,6 +3,9 @@ import type { JSONSchemaConditional, MultiStepFormSchema, } from "./types"; +import { azureSchema } from "./schemas/azure"; +import { gcsSchema } from "./schemas/gcs"; +import { s3Schema } from "./schemas/s3"; type VisibleIf = Record< string, @@ -10,258 +13,9 @@ type VisibleIf = Record< >; export const multiStepFormSchemas: Record = { - s3: { - $schema: "http://json-schema.org/draft-07/schema#", - type: "object", - properties: { - auth_method: { - type: "string", - title: "Authentication method", - description: "Choose how to authenticate to S3", - enum: ["access_keys", "public"], - default: "access_keys", - "x-display": "radio", - "x-enum-labels": ["Access keys", "Public"], - "x-enum-descriptions": [ - "Use AWS access key ID and secret access key.", - "Access publicly readable buckets without credentials.", - ], - "x-step": "connector", - }, - aws_access_key_id: { - type: "string", - title: "Access Key ID", - description: "AWS access key ID for the bucket", - "x-placeholder": "Enter AWS access key ID", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - aws_secret_access_key: { - type: "string", - title: "Secret Access Key", - description: "AWS secret access key for the bucket", - "x-placeholder": "Enter AWS secret access key", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - region: { - type: "string", - title: "Region", - description: - "Rill uses your default AWS region unless you set it explicitly.", - "x-placeholder": "us-east-1", - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - endpoint: { - type: "string", - title: "Endpoint", - description: - "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", - "x-placeholder": "https://s3.example.com", - "x-step": "connector", - "x-visible-if": { auth_method: "access_keys" }, - }, - path: { - type: "string", - title: "S3 URI", - description: "Path to your S3 bucket or prefix", - pattern: "^s3://", - "x-placeholder": "s3://bucket/path", - "x-step": "source", - }, - name: { - type: "string", - title: "Model name", - description: "Name for the source model", - pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", - "x-placeholder": "my_model", - "x-step": "source", - }, - }, - allOf: [ - { - if: { properties: { auth_method: { const: "access_keys" } } }, - then: { - required: ["aws_access_key_id", "aws_secret_access_key"], - }, - }, - ], - }, - gcs: { - $schema: "http://json-schema.org/draft-07/schema#", - type: "object", - properties: { - auth_method: { - type: "string", - title: "Authentication method", - enum: ["credentials", "hmac", "public"], - default: "credentials", - description: "Choose how to authenticate to GCS", - "x-display": "radio", - "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], - "x-enum-descriptions": [ - "Upload a JSON key file for a service account with GCS access.", - "Use HMAC access key and secret for S3-compatible authentication.", - "Access publicly readable buckets without credentials.", - ], - "x-step": "connector", - }, - google_application_credentials: { - type: "string", - title: "Service account key", - description: - "Upload a JSON key file for a service account with GCS access.", - format: "file", - "x-display": "file", - "x-accept": ".json", - "x-step": "connector", - "x-visible-if": { auth_method: "credentials" }, - }, - key_id: { - type: "string", - title: "Access Key ID", - description: "HMAC access key ID for S3-compatible authentication", - "x-placeholder": "Enter your HMAC access key ID", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "hmac" }, - }, - secret: { - type: "string", - title: "Secret Access Key", - description: "HMAC secret access key for S3-compatible authentication", - "x-placeholder": "Enter your HMAC secret access key", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "hmac" }, - }, - path: { - type: "string", - title: "GCS URI", - description: "Path to your GCS bucket or prefix", - pattern: "^gs://", - "x-placeholder": "gs://bucket/path", - "x-step": "source", - }, - name: { - type: "string", - title: "Model name", - description: "Name for the source model", - pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", - "x-placeholder": "my_model", - "x-step": "source", - }, - }, - allOf: [ - { - if: { properties: { auth_method: { const: "credentials" } } }, - then: { required: ["google_application_credentials"] }, - }, - { - if: { properties: { auth_method: { const: "hmac" } } }, - then: { required: ["key_id", "secret"] }, - }, - ], - }, - azure: { - $schema: "http://json-schema.org/draft-07/schema#", - type: "object", - properties: { - auth_method: { - type: "string", - title: "Authentication method", - enum: ["connection_string", "account_key", "sas_token", "public"], - default: "connection_string", - description: "Choose how to authenticate to Azure Blob Storage", - "x-display": "radio", - "x-enum-labels": [ - "Connection String", - "Storage Account Key", - "SAS Token", - "Public", - ], - "x-enum-descriptions": [ - "Provide a full Azure Storage connection string.", - "Provide the storage account name and access key.", - "Provide the storage account name and SAS token.", - "Access publicly readable blobs without credentials.", - ], - "x-step": "connector", - }, - azure_storage_connection_string: { - type: "string", - title: "Connection string", - description: "Paste an Azure Storage connection string", - "x-placeholder": "Enter Azure storage connection string", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "connection_string" }, - }, - azure_storage_account: { - type: "string", - title: "Storage account", - description: "The name of the Azure storage account", - "x-placeholder": "Enter Azure storage account", - "x-step": "connector", - "x-visible-if": { auth_method: ["account_key", "sas_token"] }, - }, - azure_storage_key: { - type: "string", - title: "Access key", - description: "Primary or secondary access key for the storage account", - "x-placeholder": "Enter Azure storage access key", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "account_key" }, - }, - azure_storage_sas_token: { - type: "string", - title: "SAS token", - description: - "Shared Access Signature token for the storage account (starting with ?sv=)", - "x-placeholder": "Enter Azure SAS token", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "sas_token" }, - }, - path: { - type: "string", - title: "Blob URI", - description: - "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", - pattern: "^https?://", - "x-placeholder": "https://account.blob.core.windows.net/container", - "x-step": "source", - }, - name: { - type: "string", - title: "Model name", - description: "Name for the source model", - pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", - "x-placeholder": "my_model", - "x-step": "source", - }, - }, - allOf: [ - { - if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["azure_storage_connection_string"] }, - }, - { - if: { properties: { auth_method: { const: "account_key" } } }, - then: { required: ["azure_storage_account", "azure_storage_key"] }, - }, - { - if: { properties: { auth_method: { const: "sas_token" } } }, - then: { - required: ["azure_storage_account", "azure_storage_sas_token"], - }, - }, - ], - }, + s3: s3Schema, + gcs: gcsSchema, + azure: azureSchema, }; export function getConnectorSchema( diff --git a/web-common/src/features/sources/modal/schemas/azure.ts b/web-common/src/features/sources/modal/schemas/azure.ts new file mode 100644 index 00000000000..853ba021e47 --- /dev/null +++ b/web-common/src/features/sources/modal/schemas/azure.ts @@ -0,0 +1,98 @@ +import type { MultiStepFormSchema } from "../types"; + +export const azureSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["connection_string", "account_key", "sas_token", "public"], + default: "connection_string", + description: "Choose how to authenticate to Azure Blob Storage", + "x-display": "radio", + "x-enum-labels": [ + "Connection String", + "Storage Account Key", + "SAS Token", + "Public", + ], + "x-enum-descriptions": [ + "Provide a full Azure Storage connection string.", + "Provide the storage account name and access key.", + "Provide the storage account name and SAS token.", + "Access publicly readable blobs without credentials.", + ], + "x-step": "connector", + }, + azure_storage_connection_string: { + type: "string", + title: "Connection string", + description: "Paste an Azure Storage connection string", + "x-placeholder": "Enter Azure storage connection string", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + azure_storage_account: { + type: "string", + title: "Storage account", + description: "The name of the Azure storage account", + "x-placeholder": "Enter Azure storage account", + "x-step": "connector", + "x-visible-if": { auth_method: ["account_key", "sas_token"] }, + }, + azure_storage_key: { + type: "string", + title: "Access key", + description: "Primary or secondary access key for the storage account", + "x-placeholder": "Enter Azure storage access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "account_key" }, + }, + azure_storage_sas_token: { + type: "string", + title: "SAS token", + description: + "Shared Access Signature token for the storage account (starting with ?sv=)", + "x-placeholder": "Enter Azure SAS token", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "sas_token" }, + }, + path: { + type: "string", + title: "Blob URI", + description: + "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", + pattern: "^https?://", + "x-placeholder": "https://account.blob.core.windows.net/container", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["azure_storage_connection_string"] }, + }, + { + if: { properties: { auth_method: { const: "account_key" } } }, + then: { required: ["azure_storage_account", "azure_storage_key"] }, + }, + { + if: { properties: { auth_method: { const: "sas_token" } } }, + then: { + required: ["azure_storage_account", "azure_storage_sas_token"], + }, + }, + ], +}; diff --git a/web-common/src/features/sources/modal/schemas/gcs.ts b/web-common/src/features/sources/modal/schemas/gcs.ts new file mode 100644 index 00000000000..bb5d7569b3c --- /dev/null +++ b/web-common/src/features/sources/modal/schemas/gcs.ts @@ -0,0 +1,78 @@ +import type { MultiStepFormSchema } from "../types"; + +export const gcsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["credentials", "hmac", "public"], + default: "credentials", + description: "Choose how to authenticate to GCS", + "x-display": "radio", + "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with GCS access.", + "Use HMAC access key and secret for S3-compatible authentication.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + google_application_credentials: { + type: "string", + title: "Service account key", + description: + "Upload a JSON key file for a service account with GCS access.", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + "x-visible-if": { auth_method: "credentials" }, + }, + key_id: { + type: "string", + title: "Access Key ID", + description: "HMAC access key ID for S3-compatible authentication", + "x-placeholder": "Enter your HMAC access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + secret: { + type: "string", + title: "Secret Access Key", + description: "HMAC secret access key for S3-compatible authentication", + "x-placeholder": "Enter your HMAC secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://", + "x-placeholder": "gs://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "credentials" } } }, + then: { required: ["google_application_credentials"] }, + }, + { + if: { properties: { auth_method: { const: "hmac" } } }, + then: { required: ["key_id", "secret"] }, + }, + ], +}; diff --git a/web-common/src/features/sources/modal/schemas/s3.ts b/web-common/src/features/sources/modal/schemas/s3.ts new file mode 100644 index 00000000000..0e29927d306 --- /dev/null +++ b/web-common/src/features/sources/modal/schemas/s3.ts @@ -0,0 +1,82 @@ +import type { MultiStepFormSchema } from "../types"; + +export const s3Schema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + description: "Choose how to authenticate to S3", + enum: ["access_keys", "public"], + default: "access_keys", + "x-display": "radio", + "x-enum-labels": ["Access keys", "Public"], + "x-enum-descriptions": [ + "Use AWS access key ID and secret access key.", + "Access publicly readable buckets without credentials.", + ], + "x-step": "connector", + }, + aws_access_key_id: { + type: "string", + title: "Access Key ID", + description: "AWS access key ID for the bucket", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_secret_access_key: { + type: "string", + title: "Secret Access Key", + description: "AWS secret access key for the bucket", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + region: { + type: "string", + title: "Region", + description: + "Rill uses your default AWS region unless you set it explicitly.", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + endpoint: { + type: "string", + title: "Endpoint", + description: + "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", + "x-placeholder": "https://s3.example.com", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + path: { + type: "string", + title: "S3 URI", + description: "Path to your S3 bucket or prefix", + pattern: "^s3://", + "x-placeholder": "s3://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "access_keys" } } }, + then: { + required: ["aws_access_key_id", "aws_secret_access_key"], + }, + }, + ], +}; From d7b6c8302eddc79f1f962758a4934855e580b3b8 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 22:04:04 +0800 Subject: [PATCH 049/103] templates reorg --- .../sources/modal/multi-step-auth-configs.ts | 6 +- .../src/features/sources/modal/types.ts | 65 ++++--------------- .../modal => templates}/schemas/azure.ts | 2 +- .../modal => templates}/schemas/gcs.ts | 2 +- .../modal => templates}/schemas/s3.ts | 2 +- .../src/features/templates/schemas/types.ts | 53 +++++++++++++++ 6 files changed, 70 insertions(+), 60 deletions(-) rename web-common/src/features/{sources/modal => templates}/schemas/azure.ts (98%) rename web-common/src/features/{sources/modal => templates}/schemas/gcs.ts (97%) rename web-common/src/features/{sources/modal => templates}/schemas/s3.ts (97%) create mode 100644 web-common/src/features/templates/schemas/types.ts diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 561cad4d708..88566546e22 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -3,9 +3,9 @@ import type { JSONSchemaConditional, MultiStepFormSchema, } from "./types"; -import { azureSchema } from "./schemas/azure"; -import { gcsSchema } from "./schemas/gcs"; -import { s3Schema } from "./schemas/s3"; +import { azureSchema } from "../../templates/schemas/azure"; +import { gcsSchema } from "../../templates/schemas/gcs"; +import { s3Schema } from "../../templates/schemas/s3"; type VisibleIf = Record< string, diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index d2f87c9e3e2..8fd405fe6de 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -1,3 +1,14 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; + +export type { + JSONSchemaCondition, + JSONSchemaConditional, + JSONSchemaConstraint, + JSONSchemaField, + JSONSchemaObject, + MultiStepFormSchema, +} from "../../templates/schemas/types"; + export type AddDataFormType = "source" | "connector"; export type ConnectorType = "parameters" | "dsn"; @@ -27,60 +38,6 @@ export type AuthField = hint?: string; }; -type JSONSchemaVisibleIfValue = - | string - | number - | boolean - | Array; - -export type JSONSchemaField = { - type?: "string" | "number" | "boolean" | "object"; - title?: string; - description?: string; - enum?: Array; - const?: string | number | boolean; - default?: string | number | boolean; - pattern?: string; - format?: string; - properties?: Record; - required?: string[]; - "x-display"?: "radio" | "select" | "textarea" | "file"; - "x-step"?: "connector" | "source"; - "x-secret"?: boolean; - "x-visible-if"?: Record; - "x-enum-labels"?: string[]; - "x-enum-descriptions"?: string[]; - "x-placeholder"?: string; - "x-hint"?: string; - "x-accept"?: string; -}; - -export type JSONSchemaCondition = { - properties?: Record; -}; - -export type JSONSchemaConstraint = { - required?: string[]; -}; - -export type JSONSchemaConditional = { - if?: JSONSchemaCondition; - then?: JSONSchemaConstraint; - else?: JSONSchemaConstraint; -}; - -export type JSONSchemaObject = { - $schema?: string; - type: "object"; - title?: string; - description?: string; - properties?: Record; - required?: string[]; - allOf?: JSONSchemaConditional[]; -}; - -export type MultiStepFormSchema = JSONSchemaObject; - export type MultiStepFormConfig = { schema: MultiStepFormSchema; authMethodKey: string; diff --git a/web-common/src/features/sources/modal/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts similarity index 98% rename from web-common/src/features/sources/modal/schemas/azure.ts rename to web-common/src/features/templates/schemas/azure.ts index 853ba021e47..c48aec3fc2b 100644 --- a/web-common/src/features/sources/modal/schemas/azure.ts +++ b/web-common/src/features/templates/schemas/azure.ts @@ -1,4 +1,4 @@ -import type { MultiStepFormSchema } from "../types"; +import type { MultiStepFormSchema } from "./types"; export const azureSchema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", diff --git a/web-common/src/features/sources/modal/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts similarity index 97% rename from web-common/src/features/sources/modal/schemas/gcs.ts rename to web-common/src/features/templates/schemas/gcs.ts index bb5d7569b3c..d41614abf47 100644 --- a/web-common/src/features/sources/modal/schemas/gcs.ts +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -1,4 +1,4 @@ -import type { MultiStepFormSchema } from "../types"; +import type { MultiStepFormSchema } from "./types"; export const gcsSchema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", diff --git a/web-common/src/features/sources/modal/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts similarity index 97% rename from web-common/src/features/sources/modal/schemas/s3.ts rename to web-common/src/features/templates/schemas/s3.ts index 0e29927d306..70dd72c0c61 100644 --- a/web-common/src/features/sources/modal/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -1,4 +1,4 @@ -import type { MultiStepFormSchema } from "../types"; +import type { MultiStepFormSchema } from "./types"; export const s3Schema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts new file mode 100644 index 00000000000..3c21dc4d1fa --- /dev/null +++ b/web-common/src/features/templates/schemas/types.ts @@ -0,0 +1,53 @@ +export type JSONSchemaVisibleIfValue = + | string + | number + | boolean + | Array; + +export type JSONSchemaField = { + type?: "string" | "number" | "boolean" | "object"; + title?: string; + description?: string; + enum?: Array; + const?: string | number | boolean; + default?: string | number | boolean; + pattern?: string; + format?: string; + properties?: Record; + required?: string[]; + "x-display"?: "radio" | "select" | "textarea" | "file"; + "x-step"?: "connector" | "source"; + "x-secret"?: boolean; + "x-visible-if"?: Record; + "x-enum-labels"?: string[]; + "x-enum-descriptions"?: string[]; + "x-placeholder"?: string; + "x-hint"?: string; + "x-accept"?: string; +}; + +export type JSONSchemaCondition = { + properties?: Record; +}; + +export type JSONSchemaConstraint = { + required?: string[]; +}; + +export type JSONSchemaConditional = { + if?: JSONSchemaCondition; + then?: JSONSchemaConstraint; + else?: JSONSchemaConstraint; +}; + +export type JSONSchemaObject = { + $schema?: string; + type: "object"; + title?: string; + description?: string; + properties?: Record; + required?: string[]; + allOf?: JSONSchemaConditional[]; +}; + +export type MultiStepFormSchema = JSONSchemaObject; From 99f076dd12eaac035473da674e29f5444c2b829c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Wed, 17 Dec 2025 22:35:47 +0800 Subject: [PATCH 050/103] schema driven --- .../features/sources/modal/AddDataForm.svelte | 26 +- .../modal/JSONSchemaFormRenderer.svelte | 385 ++++++++---------- .../sources/modal/multi-step-auth-configs.ts | 8 +- .../sources/modal/schema-field-utils.ts | 111 +++++ .../src/features/templates/schemas/types.ts | 3 +- 5 files changed, 318 insertions(+), 215 deletions(-) create mode 100644 web-common/src/features/sources/modal/schema-field-utils.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 66adcf494a5..966b0de61d3 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -220,15 +220,29 @@ : null; $: if (isMultiStepConnector && activeAuthInfo) { + const authKey = activeAuthInfo.key; const options = activeAuthInfo.options ?? []; const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; + const currentValue = ($paramsForm as Record | undefined)?.[ + authKey + ] as string | undefined; const hasValidSelection = options.some( - (option) => option.value === stepState.selectedAuthMethod, + (option) => option.value === currentValue, ); - if (!hasValidSelection) { - if (fallback !== stepState.selectedAuthMethod) { - setAuthMethod(fallback ?? null); - } + const nextValue = (hasValidSelection ? currentValue : fallback) ?? null; + + if (!hasValidSelection && nextValue !== null) { + paramsForm.update( + ($form) => { + if ($form?.[authKey] === nextValue) return $form; + return { ...$form, [authKey]: nextValue }; + }, + { taint: false }, + ); + } + + if (nextValue !== stepState.selectedAuthMethod) { + setAuthMethod(nextValue); } } else if (stepState.selectedAuthMethod) { setAuthMethod(null); @@ -451,7 +465,6 @@ errors={$paramsErrors} {onStringInputChange} {handleFileUpload} - bind:authMethod={$selectedAuthMethodStore} /> {:else} {:else} ; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; - // Bubble the selected auth method to the parent so it can adjust UI. - export let authMethod: string = ""; + $: properties = schema?.properties ?? {}; - $: authInfo = schema ? getAuthOptionsFromSchema(schema) : null; - $: authMethodKey = schema ? authInfo?.key || findAuthMethodKey(schema) : null; - $: requiredByMethodConnector = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "connector" }) - : {}; - $: requiredByMethodSource = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "source" }) - : {}; - - $: if (schema && authInfo && !authMethod) { - authMethod = authInfo.defaultMethod || authInfo.options[0]?.value || ""; + // Apply defaults from the schema into the form when missing. + $: if (schema && form) { + const defaults = schema.properties ?? {}; + form.update( + ($form) => { + let mutated = false; + const next = { ...$form }; + for (const [key, prop] of Object.entries(defaults)) { + if (next[key] === undefined && prop.default !== undefined) { + next[key] = prop.default; + mutated = true; + } + } + return mutated ? next : $form; + }, + { taint: false }, + ); } - // Clear fields that are not visible for the active auth method to avoid - // sending stale values across methods. - $: if (schema && authMethod && step === "connector") { + // Clear fields that are not visible for the current step to avoid + // sending stale values for hidden inputs. + $: if (schema && form) { form.update( ($form) => { - const properties = schema.properties ?? {}; - for (const key of Object.keys(properties)) { - if (key === authMethodKey) continue; - const prop = properties[key]; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== "connector") continue; - const visible = isVisibleForValues(schema, key, { - ...$form, - [authMethodKey ?? "auth_method"]: authMethod, - }); - if (!visible && key in $form) { - $form[key] = ""; + let mutated = false; + const next = { ...$form }; + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, step)) continue; + const visible = isVisibleForValues(schema, key, next); + if (!visible && Object.prototype.hasOwnProperty.call(next, key)) { + next[key] = ""; + mutated = true; } } - return $form; + return mutated ? next : $form; }, { taint: false }, ); } - function visibleFieldsFor( - method: string | undefined, - currentStep: "connector" | "source", - ) { + $: requiredFields = schema + ? computeRequiredFields(schema, { ...$form }, step) + : new Set(); + + function isRequired(key: string) { + return requiredFields.has(key); + } + + function visibleFields(values: Record = { ...$form }) { if (!schema) return []; - const properties = schema.properties ?? {}; - const values = { ...$form, [authMethodKey ?? "auth_method"]: method }; - return Object.entries(properties).filter(([key, prop]) => { - if (authMethodKey && key === authMethodKey) return false; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== currentStep) return false; - return isVisibleForValues(schema, key, values); - }); + return visibleFieldsForValues(schema, values, step); + } + + function isRadioField(prop: JSONSchemaField) { + return Boolean(prop.enum && prop["x-display"] === "radio"); } - function isRequiredFor(method: string | undefined, key: string): boolean { - if (!schema) return false; - const requiredMap = - step === "connector" ? requiredByMethodConnector : requiredByMethodSource; - const requiredSet = requiredMap[method ?? ""] ?? []; - return requiredSet.includes(key); + function radioOptions(prop: JSONSchemaField) { + return ( + prop.enum?.map((value, idx) => ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + })) ?? [] + ); + } + + $: visibleEntries = visibleFields(); + $: radioEntries = visibleEntries.filter(([, prop]) => isRadioField(prop)); + $: radioDependentKeys = schema + ? keysDependingOn( + schema, + radioEntries.map(([key]) => key), + step, + ) + : new Set(); + $: nonRadioEntries = visibleEntries.filter( + ([key, prop]) => !isRadioField(prop) && !radioDependentKeys.has(key), + ); + + function visibleFieldsForRadioOption( + fieldKey: string, + optionValue: string | number | boolean, + ) { + if (!schema) return []; + const values = { ...$form, [fieldKey]: optionValue }; + return visibleFieldsForValues(schema, values, step).filter( + ([key, prop]) => + key !== fieldKey && + (radioDependentKeys.has(key) || isRadioField(prop)) && + matchesStep(prop, step), + ); } {#if schema} - {#if step === "connector" && authInfo} - {#if authInfo.options.length > 1} -
-
Authentication method
+ {#each radioEntries as [key, prop]} +
+
{prop.title ?? key}
+ + + {#each visibleFieldsForRadioOption(key, option.value) as [childKey, childProp]} +
+ {#if childProp["x-display"] === "file" || childProp.format === "file"} + + {:else if childProp.type === "boolean"} + + {:else if isRadioField(childProp)} + + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + + +
+ {/each} + + {#each nonRadioEntries as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if isRadioField(prop)} - - {#each visibleFieldsFor(option.value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - - -
- {:else if authInfo.options[0]} - {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - {/if} - {:else} - {#each visibleFieldsFor(authMethod, step) as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - {/if} + bind:value={$form[key]} + options={radioOptions(prop)} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} {/if} diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 88566546e22..4782b708ef8 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -70,7 +70,7 @@ export function getAuthOptionsFromSchema( export function getRequiredFieldsByAuthMethod( schema: MultiStepFormSchema, - opts?: { step?: "connector" | "source" }, + opts?: { step?: string }, ): Record { const authInfo = getAuthOptionsFromSchema(schema); if (!authInfo) return {}; @@ -120,11 +120,13 @@ export function getFieldLabel( export function isStepMatch( schema: MultiStepFormSchema, key: string, - step: "connector" | "source", + step: string, ): boolean { const prop = schema.properties?.[key]; if (!prop) return false; - return (prop["x-step"] ?? "connector") === step; + const fieldStep = prop["x-step"]; + if (!fieldStep) return true; + return fieldStep === step; } export function isVisibleForValues( diff --git a/web-common/src/features/sources/modal/schema-field-utils.ts b/web-common/src/features/sources/modal/schema-field-utils.ts new file mode 100644 index 00000000000..c335ee3f9e3 --- /dev/null +++ b/web-common/src/features/sources/modal/schema-field-utils.ts @@ -0,0 +1,111 @@ +import type { + JSONSchemaConditional, + JSONSchemaField, + MultiStepFormSchema, +} from "./types"; + +type Step = string | null | undefined; + +export function matchesStep(prop: JSONSchemaField | undefined, step: Step) { + if (!step) return true; + const fieldStep = prop?.["x-step"]; + return fieldStep ? fieldStep === step : true; +} + +export function isVisibleForValues( + schema: MultiStepFormSchema, + key: string, + values: Record, +): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; + const conditions = prop["x-visible-if"]; + if (!conditions) return true; + + return Object.entries(conditions).every(([depKey, expected]) => { + const actual = values?.[depKey]; + if (Array.isArray(expected)) { + return expected.map(String).includes(String(actual)); + } + return String(actual) === String(expected); + }); +} + +export function visibleFieldsForValues( + schema: MultiStepFormSchema, + values: Record, + step?: Step, +): Array<[string, JSONSchemaField]> { + const properties = schema.properties ?? {}; + return Object.entries(properties).filter(([key, prop]) => { + if (!matchesStep(prop, step)) return false; + return isVisibleForValues(schema, key, values); + }); +} + +export function computeRequiredFields( + schema: MultiStepFormSchema, + values: Record, + step?: Step, +): Set { + const required = new Set(); + const properties = schema.properties ?? {}; + + // Base required fields. + for (const field of schema.required ?? []) { + if (!step || matchesStep(properties[field], step)) { + required.add(field); + } + } + + // Conditional required fields driven by `allOf`. + for (const conditional of schema.allOf ?? []) { + const applies = matchesConditional(conditional, values); + const target = applies ? conditional.then : conditional.else; + for (const field of target?.required ?? []) { + if (!step || matchesStep(properties[field], step)) { + required.add(field); + } + } + } + + return required; +} + +export function dependsOnField(prop: JSONSchemaField, dependency: string) { + const conditions = prop["x-visible-if"]; + if (!conditions) return false; + return Object.prototype.hasOwnProperty.call(conditions, dependency); +} + +export function keysDependingOn( + schema: MultiStepFormSchema, + dependencies: string[], + step?: Step, +): Set { + const properties = schema.properties ?? {}; + const result = new Set(); + + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, step)) continue; + if (dependencies.some((dep) => dependsOnField(prop, dep))) { + result.add(key); + } + } + + return result; +} + +function matchesConditional( + conditional: JSONSchemaConditional, + values: Record, +) { + const conditions = conditional.if?.properties; + if (!conditions) return false; + + return Object.entries(conditions).every(([depKey, constraint]) => { + if (!("const" in constraint)) return false; + const actual = values?.[depKey]; + return String(actual) === String(constraint.const); + }); +} diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index 3c21dc4d1fa..fe3442b0adb 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -16,7 +16,8 @@ export type JSONSchemaField = { properties?: Record; required?: string[]; "x-display"?: "radio" | "select" | "textarea" | "file"; - "x-step"?: "connector" | "source"; + // Arbitrary step identifier so renderers can filter without hardcoding. + "x-step"?: string; "x-secret"?: boolean; "x-visible-if"?: Record; "x-enum-labels"?: string[]; From 48a9f92ac0daf25cee3e4401f1a4c33552ae14b8 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 01:18:38 +0800 Subject: [PATCH 051/103] Revert "schema driven" This reverts commit 99f076dd12eaac035473da674e29f5444c2b829c. --- .../features/sources/modal/AddDataForm.svelte | 26 +- .../modal/JSONSchemaFormRenderer.svelte | 385 ++++++++++-------- .../sources/modal/multi-step-auth-configs.ts | 8 +- .../sources/modal/schema-field-utils.ts | 111 ----- .../src/features/templates/schemas/types.ts | 3 +- 5 files changed, 215 insertions(+), 318 deletions(-) delete mode 100644 web-common/src/features/sources/modal/schema-field-utils.ts diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 966b0de61d3..66adcf494a5 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -220,29 +220,15 @@ : null; $: if (isMultiStepConnector && activeAuthInfo) { - const authKey = activeAuthInfo.key; const options = activeAuthInfo.options ?? []; const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; - const currentValue = ($paramsForm as Record | undefined)?.[ - authKey - ] as string | undefined; const hasValidSelection = options.some( - (option) => option.value === currentValue, + (option) => option.value === stepState.selectedAuthMethod, ); - const nextValue = (hasValidSelection ? currentValue : fallback) ?? null; - - if (!hasValidSelection && nextValue !== null) { - paramsForm.update( - ($form) => { - if ($form?.[authKey] === nextValue) return $form; - return { ...$form, [authKey]: nextValue }; - }, - { taint: false }, - ); - } - - if (nextValue !== stepState.selectedAuthMethod) { - setAuthMethod(nextValue); + if (!hasValidSelection) { + if (fallback !== stepState.selectedAuthMethod) { + setAuthMethod(fallback ?? null); + } } } else if (stepState.selectedAuthMethod) { setAuthMethod(null); @@ -465,6 +451,7 @@ errors={$paramsErrors} {onStringInputChange} {handleFileUpload} + bind:authMethod={$selectedAuthMethodStore} /> {:else} {:else} ; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; - $: properties = schema?.properties ?? {}; + // Bubble the selected auth method to the parent so it can adjust UI. + export let authMethod: string = ""; - // Apply defaults from the schema into the form when missing. - $: if (schema && form) { - const defaults = schema.properties ?? {}; - form.update( - ($form) => { - let mutated = false; - const next = { ...$form }; - for (const [key, prop] of Object.entries(defaults)) { - if (next[key] === undefined && prop.default !== undefined) { - next[key] = prop.default; - mutated = true; - } - } - return mutated ? next : $form; - }, - { taint: false }, - ); + $: authInfo = schema ? getAuthOptionsFromSchema(schema) : null; + $: authMethodKey = schema ? authInfo?.key || findAuthMethodKey(schema) : null; + $: requiredByMethodConnector = schema + ? getRequiredFieldsByAuthMethod(schema, { step: "connector" }) + : {}; + $: requiredByMethodSource = schema + ? getRequiredFieldsByAuthMethod(schema, { step: "source" }) + : {}; + + $: if (schema && authInfo && !authMethod) { + authMethod = authInfo.defaultMethod || authInfo.options[0]?.value || ""; } - // Clear fields that are not visible for the current step to avoid - // sending stale values for hidden inputs. - $: if (schema && form) { + // Clear fields that are not visible for the active auth method to avoid + // sending stale values across methods. + $: if (schema && authMethod && step === "connector") { form.update( ($form) => { - let mutated = false; - const next = { ...$form }; - for (const [key, prop] of Object.entries(properties)) { - if (!matchesStep(prop, step)) continue; - const visible = isVisibleForValues(schema, key, next); - if (!visible && Object.prototype.hasOwnProperty.call(next, key)) { - next[key] = ""; - mutated = true; + const properties = schema.properties ?? {}; + for (const key of Object.keys(properties)) { + if (key === authMethodKey) continue; + const prop = properties[key]; + const stepForField = prop["x-step"] ?? "connector"; + if (stepForField !== "connector") continue; + const visible = isVisibleForValues(schema, key, { + ...$form, + [authMethodKey ?? "auth_method"]: authMethod, + }); + if (!visible && key in $form) { + $form[key] = ""; } } - return mutated ? next : $form; + return $form; }, { taint: false }, ); } - $: requiredFields = schema - ? computeRequiredFields(schema, { ...$form }, step) - : new Set(); - - function isRequired(key: string) { - return requiredFields.has(key); - } - - function visibleFields(values: Record = { ...$form }) { + function visibleFieldsFor( + method: string | undefined, + currentStep: "connector" | "source", + ) { if (!schema) return []; - return visibleFieldsForValues(schema, values, step); - } - - function isRadioField(prop: JSONSchemaField) { - return Boolean(prop.enum && prop["x-display"] === "radio"); + const properties = schema.properties ?? {}; + const values = { ...$form, [authMethodKey ?? "auth_method"]: method }; + return Object.entries(properties).filter(([key, prop]) => { + if (authMethodKey && key === authMethodKey) return false; + const stepForField = prop["x-step"] ?? "connector"; + if (stepForField !== currentStep) return false; + return isVisibleForValues(schema, key, values); + }); } - function radioOptions(prop: JSONSchemaField) { - return ( - prop.enum?.map((value, idx) => ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - })) ?? [] - ); - } - - $: visibleEntries = visibleFields(); - $: radioEntries = visibleEntries.filter(([, prop]) => isRadioField(prop)); - $: radioDependentKeys = schema - ? keysDependingOn( - schema, - radioEntries.map(([key]) => key), - step, - ) - : new Set(); - $: nonRadioEntries = visibleEntries.filter( - ([key, prop]) => !isRadioField(prop) && !radioDependentKeys.has(key), - ); - - function visibleFieldsForRadioOption( - fieldKey: string, - optionValue: string | number | boolean, - ) { - if (!schema) return []; - const values = { ...$form, [fieldKey]: optionValue }; - return visibleFieldsForValues(schema, values, step).filter( - ([key, prop]) => - key !== fieldKey && - (radioDependentKeys.has(key) || isRadioField(prop)) && - matchesStep(prop, step), - ); + function isRequiredFor(method: string | undefined, key: string): boolean { + if (!schema) return false; + const requiredMap = + step === "connector" ? requiredByMethodConnector : requiredByMethodSource; + const requiredSet = requiredMap[method ?? ""] ?? []; + return requiredSet.includes(key); } {#if schema} - {#each radioEntries as [key, prop]} -
-
{prop.title ?? key}
- - - {#each visibleFieldsForRadioOption(key, option.value) as [childKey, childProp]} -
- {#if childProp["x-display"] === "file" || childProp.format === "file"} - - {:else if childProp.type === "boolean"} - - {:else if isRadioField(childProp)} - - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - - -
- {/each} - - {#each nonRadioEntries as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if isRadioField(prop)} + {#if step === "connector" && authInfo} + {#if authInfo.options.length > 1} +
+
Authentication method
- {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} + bind:value={authMethod} + options={authInfo.options} + name="multi-auth-method" + > + + {#each visibleFieldsFor(option.value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + + +
+ {:else if authInfo.options[0]} + {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + {/if} + {:else} + {#each visibleFieldsFor(authMethod, step) as [key, prop]} +
+ {#if prop["x-display"] === "file" || prop.format === "file"} + + {:else if prop.type === "boolean"} + + {:else if prop.enum && prop["x-display"] === "radio"} + ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + }))} + name={`${key}-radio`} + /> + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + {/if} {/if} diff --git a/web-common/src/features/sources/modal/multi-step-auth-configs.ts b/web-common/src/features/sources/modal/multi-step-auth-configs.ts index 4782b708ef8..88566546e22 100644 --- a/web-common/src/features/sources/modal/multi-step-auth-configs.ts +++ b/web-common/src/features/sources/modal/multi-step-auth-configs.ts @@ -70,7 +70,7 @@ export function getAuthOptionsFromSchema( export function getRequiredFieldsByAuthMethod( schema: MultiStepFormSchema, - opts?: { step?: string }, + opts?: { step?: "connector" | "source" }, ): Record { const authInfo = getAuthOptionsFromSchema(schema); if (!authInfo) return {}; @@ -120,13 +120,11 @@ export function getFieldLabel( export function isStepMatch( schema: MultiStepFormSchema, key: string, - step: string, + step: "connector" | "source", ): boolean { const prop = schema.properties?.[key]; if (!prop) return false; - const fieldStep = prop["x-step"]; - if (!fieldStep) return true; - return fieldStep === step; + return (prop["x-step"] ?? "connector") === step; } export function isVisibleForValues( diff --git a/web-common/src/features/sources/modal/schema-field-utils.ts b/web-common/src/features/sources/modal/schema-field-utils.ts deleted file mode 100644 index c335ee3f9e3..00000000000 --- a/web-common/src/features/sources/modal/schema-field-utils.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { - JSONSchemaConditional, - JSONSchemaField, - MultiStepFormSchema, -} from "./types"; - -type Step = string | null | undefined; - -export function matchesStep(prop: JSONSchemaField | undefined, step: Step) { - if (!step) return true; - const fieldStep = prop?.["x-step"]; - return fieldStep ? fieldStep === step : true; -} - -export function isVisibleForValues( - schema: MultiStepFormSchema, - key: string, - values: Record, -): boolean { - const prop = schema.properties?.[key]; - if (!prop) return false; - const conditions = prop["x-visible-if"]; - if (!conditions) return true; - - return Object.entries(conditions).every(([depKey, expected]) => { - const actual = values?.[depKey]; - if (Array.isArray(expected)) { - return expected.map(String).includes(String(actual)); - } - return String(actual) === String(expected); - }); -} - -export function visibleFieldsForValues( - schema: MultiStepFormSchema, - values: Record, - step?: Step, -): Array<[string, JSONSchemaField]> { - const properties = schema.properties ?? {}; - return Object.entries(properties).filter(([key, prop]) => { - if (!matchesStep(prop, step)) return false; - return isVisibleForValues(schema, key, values); - }); -} - -export function computeRequiredFields( - schema: MultiStepFormSchema, - values: Record, - step?: Step, -): Set { - const required = new Set(); - const properties = schema.properties ?? {}; - - // Base required fields. - for (const field of schema.required ?? []) { - if (!step || matchesStep(properties[field], step)) { - required.add(field); - } - } - - // Conditional required fields driven by `allOf`. - for (const conditional of schema.allOf ?? []) { - const applies = matchesConditional(conditional, values); - const target = applies ? conditional.then : conditional.else; - for (const field of target?.required ?? []) { - if (!step || matchesStep(properties[field], step)) { - required.add(field); - } - } - } - - return required; -} - -export function dependsOnField(prop: JSONSchemaField, dependency: string) { - const conditions = prop["x-visible-if"]; - if (!conditions) return false; - return Object.prototype.hasOwnProperty.call(conditions, dependency); -} - -export function keysDependingOn( - schema: MultiStepFormSchema, - dependencies: string[], - step?: Step, -): Set { - const properties = schema.properties ?? {}; - const result = new Set(); - - for (const [key, prop] of Object.entries(properties)) { - if (!matchesStep(prop, step)) continue; - if (dependencies.some((dep) => dependsOnField(prop, dep))) { - result.add(key); - } - } - - return result; -} - -function matchesConditional( - conditional: JSONSchemaConditional, - values: Record, -) { - const conditions = conditional.if?.properties; - if (!conditions) return false; - - return Object.entries(conditions).every(([depKey, constraint]) => { - if (!("const" in constraint)) return false; - const actual = values?.[depKey]; - return String(actual) === String(constraint.const); - }); -} diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index fe3442b0adb..3c21dc4d1fa 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -16,8 +16,7 @@ export type JSONSchemaField = { properties?: Record; required?: string[]; "x-display"?: "radio" | "select" | "textarea" | "file"; - // Arbitrary step identifier so renderers can filter without hardcoding. - "x-step"?: string; + "x-step"?: "connector" | "source"; "x-secret"?: boolean; "x-visible-if"?: Record; "x-enum-labels"?: string[]; From efdfd5c7c6cb3c3bb1abca832c04fbe27aa3751a Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 12:21:26 +0800 Subject: [PATCH 052/103] re-add aws_role_arn --- runtime/drivers/s3/s3.go | 18 ++++++++++++++++++ .../src/features/templates/schemas/s3.ts | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 81bd18aaf56..005e7ab4868 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -61,6 +61,24 @@ var spec = drivers.Spec{ Required: false, Hint: "Overrides the S3 endpoint to connect to. This should only be used to connect to S3 compatible services, such as Cloudflare R2 or MinIO.", }, + { + Key: "aws_role_arn", + Type: drivers.StringPropertyType, + Secret: true, + Description: "AWS Role ARN to assume", + }, + { + Key: "aws_role_session_name", + Type: drivers.StringPropertyType, + Secret: true, + Description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + }, + { + Key: "aws_external_id", + Type: drivers.StringPropertyType, + Secret: true, + Description: "Optional external ID to use when assuming an AWS role for cross-account access.", + }, }, SourceProperties: []*drivers.PropertySpec{ { diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index 70dd72c0c61..f5653bf561b 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -54,6 +54,15 @@ export const s3Schema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "access_keys" }, }, + aws_role_arn: { + type: "string", + title: "AWS Role ARN", + description: "AWS Role ARN to assume", + "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, path: { type: "string", title: "S3 URI", From 9cdcfa7d0367601017143c9f203936b3ec66f995 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 13:11:26 +0800 Subject: [PATCH 053/103] fix auth method revalidation --- .../features/sources/modal/AddDataForm.svelte | 15 +- .../modal/JSONSchemaFormRenderer.svelte | 396 +++++++++++------- .../src/features/sources/modal/utils.ts | 10 + 3 files changed, 259 insertions(+), 162 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 66adcf494a5..1efeafe261a 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -28,6 +28,7 @@ import AddDataFormSection from "./AddDataFormSection.svelte"; import { getAuthOptionsFromSchema, + findAuthMethodKey, getConnectorSchema, } from "./multi-step-auth-configs"; import { get } from "svelte/store"; @@ -234,6 +235,18 @@ setAuthMethod(null); } + // Keep auth method store aligned with the form's enum value from the schema. + $: if (isMultiStepConnector && activeSchema) { + const authKey = findAuthMethodKey(activeSchema); + if (authKey) { + const currentValue = $paramsForm?.[authKey] as string | undefined; + const normalized = currentValue ? String(currentValue) : null; + if (normalized !== (stepState.selectedAuthMethod ?? null)) { + setAuthMethod(normalized); + } + } + } + $: isSubmitting = submitting; // Reset errors when form is modified @@ -451,7 +464,6 @@ errors={$paramsErrors} {onStringInputChange} {handleFileUpload} - bind:authMethod={$selectedAuthMethodStore} /> {:else} {:else} ; export let onStringInputChange: (e: Event) => void; export let handleFileUpload: (file: File) => Promise; - // Bubble the selected auth method to the parent so it can adjust UI. - export let authMethod: string = ""; + const radioDisplay = "radio"; - $: authInfo = schema ? getAuthOptionsFromSchema(schema) : null; - $: authMethodKey = schema ? authInfo?.key || findAuthMethodKey(schema) : null; - $: requiredByMethodConnector = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "connector" }) - : {}; - $: requiredByMethodSource = schema - ? getRequiredFieldsByAuthMethod(schema, { step: "source" }) - : {}; + $: stepFilter = step; + $: dependentMap = schema + ? buildDependentMap(schema, stepFilter) + : new Map>(); + $: dependentKeys = new Set( + Array.from(dependentMap.values()).flatMap((entries) => + entries.map(([key]) => key), + ), + ); + $: visibleEntries = schema + ? computeVisibleEntries(schema, stepFilter, $form) + : []; + $: requiredFields = schema + ? computeRequiredFields(schema, $form, stepFilter) + : new Set(); + $: renderOrder = schema + ? computeRenderOrder(visibleEntries, dependentMap, dependentKeys) + : []; - $: if (schema && authInfo && !authMethod) { - authMethod = authInfo.defaultMethod || authInfo.options[0]?.value || ""; + // Seed defaults once when schema-provided defaults exist. + $: if (schema) { + form.update( + ($form) => { + const properties = schema.properties ?? {}; + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, stepFilter)) continue; + const current = $form[key]; + if ( + (current === undefined || current === null) && + prop.default !== undefined + ) { + $form[key] = prop.default; + } + } + return $form; + }, + { taint: false }, + ); } - // Clear fields that are not visible for the active auth method to avoid - // sending stale values across methods. - $: if (schema && authMethod && step === "connector") { + // Clear hidden fields for the active step to avoid stale submissions. + $: if (schema) { form.update( ($form) => { const properties = schema.properties ?? {}; - for (const key of Object.keys(properties)) { - if (key === authMethodKey) continue; - const prop = properties[key]; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== "connector") continue; - const visible = isVisibleForValues(schema, key, { - ...$form, - [authMethodKey ?? "auth_method"]: authMethod, - }); - if (!visible && key in $form) { + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, stepFilter)) continue; + const visible = isVisibleForValues(schema, key, $form); + if (!visible && key in $form && $form[key] !== "") { $form[key] = ""; } } @@ -60,144 +74,210 @@ ); } - function visibleFieldsFor( - method: string | undefined, - currentStep: "connector" | "source", + function matchesStep(prop: JSONSchemaField | undefined, stepValue?: string) { + if (!stepValue) return true; + const propStep = prop?.["x-step"]; + if (!propStep) return true; + return propStep === stepValue; + } + + function isRadioEnum(prop: JSONSchemaField) { + return Boolean(prop.enum && prop["x-display"] === radioDisplay); + } + + function computeVisibleEntries( + currentSchema: MultiStepFormSchema, + currentStep: string | undefined, + values: Record, ) { - if (!schema) return []; - const properties = schema.properties ?? {}; - const values = { ...$form, [authMethodKey ?? "auth_method"]: method }; + const properties = currentSchema.properties ?? {}; return Object.entries(properties).filter(([key, prop]) => { - if (authMethodKey && key === authMethodKey) return false; - const stepForField = prop["x-step"] ?? "connector"; - if (stepForField !== currentStep) return false; - return isVisibleForValues(schema, key, values); + if (!matchesStep(prop, currentStep)) return false; + return isVisibleForValues(currentSchema, key, values); + }); + } + + function matchesCondition( + condition: + | Record + | undefined, + values: Record, + ) { + if (!condition || !Object.keys(condition).length) return false; + return Object.entries(condition).every(([depKey, def]) => { + if (def.const === undefined || def.const === null) return false; + return String(values?.[depKey]) === String(def.const); }); } - function isRequiredFor(method: string | undefined, key: string): boolean { - if (!schema) return false; - const requiredMap = - step === "connector" ? requiredByMethodConnector : requiredByMethodSource; - const requiredSet = requiredMap[method ?? ""] ?? []; - return requiredSet.includes(key); + function computeRequiredFields( + currentSchema: MultiStepFormSchema, + values: Record, + currentStep: string | undefined, + ) { + const properties = currentSchema.properties ?? {}; + const required = new Set(); + (currentSchema.required ?? []).forEach((key) => { + if (matchesStep(properties[key], currentStep)) required.add(key); + }); + + for (const conditional of currentSchema.allOf ?? []) { + const condition = conditional.if?.properties; + const matches = matchesCondition(condition, values); + const branch = matches ? conditional.then : conditional.else; + branch?.required?.forEach((key) => { + if (matchesStep(properties[key], currentStep)) required.add(key); + }); + } + return required; + } + + function computeRenderOrder( + entries: Array<[string, JSONSchemaField]>, + dependents: Map>, + dependentKeySet: Set, + ) { + const result: Array<[string, JSONSchemaField]> = []; + const rendered = new Set(); + + for (const [key, prop] of entries) { + if (rendered.has(key)) continue; + + if (isRadioEnum(prop)) { + rendered.add(key); + dependents.get(key)?.forEach(([childKey]) => rendered.add(childKey)); + result.push([key, prop]); + continue; + } + + if (dependentKeySet.has(key)) continue; + + rendered.add(key); + result.push([key, prop]); + } + + return result; + } + + function buildDependentMap( + currentSchema: MultiStepFormSchema, + currentStep: string | undefined, + ) { + const properties = currentSchema.properties ?? {}; + const map = new Map>(); + + for (const [key, prop] of Object.entries(properties)) { + const visibleIf = prop["x-visible-if"]; + if (!visibleIf) continue; + + for (const controllerKey of Object.keys(visibleIf)) { + const controller = properties[controllerKey]; + if (!controller) continue; + if (!matchesStep(controller, currentStep)) continue; + if (!matchesStep(prop, currentStep)) continue; + + const entries = map.get(controllerKey) ?? []; + entries.push([key, prop]); + map.set(controllerKey, entries); + } + } + + return map; + } + + function getDependentFieldsForOption( + controllerKey: string, + optionValue: string | number | boolean, + ) { + if (!schema) return []; + const dependents = dependentMap.get(controllerKey) ?? []; + const values = { ...$form, [controllerKey]: optionValue }; + return dependents.filter(([key]) => + isVisibleForValues(schema, key, values), + ); + } + + function radioOptions(prop: JSONSchemaField) { + return ( + prop.enum?.map((value, idx) => ({ + value: String(value), + label: prop["x-enum-labels"]?.[idx] ?? String(value), + description: prop["x-enum-descriptions"]?.[idx], + })) ?? [] + ); + } + + function isRequired(key: string) { + return requiredFields.has(key); } {#if schema} - {#if step === "connector" && authInfo} - {#if authInfo.options.length > 1} + {#each renderOrder as [key, prop]} + {#if isRadioEnum(prop)}
-
Authentication method
+ {#if prop.title} +
{prop.title}
+ {/if} - {#each visibleFieldsFor(option.value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} + {#if dependentMap.get(key)?.length} + {#each getDependentFieldsForOption(key, option.value) as [childKey, childProp]} +
+ {#if childProp["x-display"] === "file" || childProp.format === "file"} + + {:else if childProp.type === "boolean"} + + {:else if isRadioEnum(childProp)} + + {:else} + onStringInputChange(e)} + alwaysShowError + /> + {/if} +
+ {/each} + {/if}
- {:else if authInfo.options[0]} - {#each visibleFieldsFor(authMethod || authInfo.options[0].value, "connector") as [key, prop]} -
- {#if prop["x-display"] === "file" || prop.format === "file"} - - {:else if prop.type === "boolean"} - - {:else if prop.enum && prop["x-display"] === "radio"} - ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} - name={`${key}-radio`} - /> - {:else} - onStringInputChange(e)} - alwaysShowError - /> - {/if} -
- {/each} - {/if} - {:else} - {#each visibleFieldsFor(authMethod, step) as [key, prop]} + {:else}
{#if prop["x-display"] === "file" || prop.format === "file"} - {:else if prop.enum && prop["x-display"] === "radio"} + {:else if isRadioEnum(prop)} ({ - value: String(value), - label: prop["x-enum-labels"]?.[idx] ?? String(value), - description: prop["x-enum-descriptions"]?.[idx], - }))} + options={radioOptions(prop)} name={`${key}-radio`} /> {:else} @@ -225,7 +301,7 @@ id={key} label={prop.title ?? key} placeholder={prop["x-placeholder"]} - optional={!isRequiredFor(authMethod, key)} + optional={!isRequired(key)} secret={prop["x-secret"]} hint={prop.description ?? prop["x-hint"]} errors={normalizeErrors(errors?.[key])} @@ -235,6 +311,6 @@ /> {/if}
- {/each} - {/if} + {/if} + {/each} {/if} diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 95da0214dc5..cf29ee61338 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -3,6 +3,7 @@ import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; import type { MultiStepFormSchema } from "./types"; import { + findAuthMethodKey, getAuthOptionsFromSchema, getRequiredFieldsByAuthMethod, isStepMatch, @@ -103,9 +104,18 @@ export function isMultiStepConnectorDisabled( const authInfo = getAuthOptionsFromSchema(schema); const options = authInfo?.options ?? []; + const authKey = authInfo?.key || findAuthMethodKey(schema); + const methodFromForm = + authKey && paramsFormValue?.[authKey] != null + ? String(paramsFormValue[authKey]) + : undefined; const hasValidSelection = options.some((opt) => opt.value === selectedMethod); + const hasValidFormSelection = options.some( + (opt) => opt.value === methodFromForm, + ); const method = (hasValidSelection && selectedMethod) || + (hasValidFormSelection && methodFromForm) || authInfo?.defaultMethod || options[0]?.value; From f8b7aaad0ea98e04498b1e1b4df5be93730cdb1d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 16:47:29 +0800 Subject: [PATCH 054/103] fix e2e --- web-common/src/features/sources/modal/utils.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index cf29ee61338..ddd63333036 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -96,7 +96,7 @@ export function hasOnlyDsn( */ export function isMultiStepConnectorDisabled( schema: MultiStepFormSchema | null, - selectedMethod: string, + _selectedMethod: string, paramsFormValue: Record, paramsFormErrors: Record, ) { @@ -109,12 +109,10 @@ export function isMultiStepConnectorDisabled( authKey && paramsFormValue?.[authKey] != null ? String(paramsFormValue[authKey]) : undefined; - const hasValidSelection = options.some((opt) => opt.value === selectedMethod); const hasValidFormSelection = options.some( (opt) => opt.value === methodFromForm, ); const method = - (hasValidSelection && selectedMethod) || (hasValidFormSelection && methodFromForm) || authInfo?.defaultMethod || options[0]?.value; From 5c19210e6ef37c2f45990c28e606c9157695765d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 17:06:42 +0800 Subject: [PATCH 055/103] fix selected auth method remnant --- .../features/sources/modal/AddDataForm.svelte | 18 ++++++++++++++---- web-common/src/features/sources/modal/utils.ts | 1 - 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 1efeafe261a..cf15a5fa9e8 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -77,6 +77,7 @@ }; let selectedAuthMethod: string = ""; + let activeAuthMethod: string | null = null; $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; $: stepProperties = @@ -159,7 +160,6 @@ if (isMultiStepConnector && stepState.step === "connector") { return isMultiStepConnectorDisabled( activeSchema, - selectedAuthMethod, $paramsForm, $paramsErrors, ); @@ -247,6 +247,16 @@ } } + // Auth method to use for UI (labels, CTA), derived from form first. + $: activeAuthMethod = (() => { + if (!(isMultiStepConnector && activeSchema)) return selectedAuthMethod; + const authKey = findAuthMethodKey(activeSchema); + if (authKey && $paramsForm?.[authKey] != null) { + return String($paramsForm[authKey]); + } + return selectedAuthMethod; + })(); + $: isSubmitting = submitting; // Reset errors when form is modified @@ -345,7 +355,7 @@ onClose, queryClient, getConnectionTab: () => connectionTab, - getSelectedAuthMethod: () => selectedAuthMethod, + getSelectedAuthMethod: () => activeAuthMethod || undefined, setParamsError: (message: string | null, details?: string) => { paramsError = message; paramsErrorDetails = details; @@ -551,7 +561,7 @@ ? "Connecting..." : isMultiStepConnector && stepState.step === "source" ? "Importing data..." - : selectedAuthMethod === "public" + : activeAuthMethod === "public" ? "Continuing..." : "Testing connection..."} form={connector.name === "clickhouse" ? clickhouseFormId : formId} @@ -564,7 +574,7 @@ submitting, clickhouseConnectorType, clickhouseSubmitting, - selectedAuthMethod, + selectedAuthMethod: activeAuthMethod ?? selectedAuthMethod, })}
diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index ddd63333036..ad491e1fb79 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -96,7 +96,6 @@ export function hasOnlyDsn( */ export function isMultiStepConnectorDisabled( schema: MultiStepFormSchema | null, - _selectedMethod: string, paramsFormValue: Record, paramsFormErrors: Record, ) { From 5da442e7ae2e60e97bc7bdf7a51c90805c721a1d Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 18:13:29 +0800 Subject: [PATCH 056/103] fix continue submission --- .../features/sources/modal/AddDataForm.svelte | 16 +++++++++++-- .../sources/modal/AddDataFormManager.ts | 24 ++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index cf15a5fa9e8..4371bf76890 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -102,8 +102,20 @@ // Update form when (re)entering step 1: restore defaults for connector properties $: if (isMultiStepConnector && stepState.step === "connector") { paramsForm.update( - () => - getInitialFormValuesFromProperties(connector.configProperties ?? []), + ($current) => { + const base = getInitialFormValuesFromProperties( + connector.configProperties ?? [], + ); + // Preserve previously selected auth method when returning to connector step. + if (activeSchema) { + const authKey = findAuthMethodKey(activeSchema); + const persisted = stepState.selectedAuthMethod; + if (authKey && persisted) { + base[authKey] = persisted; + } + } + return { ...base, ...$current }; + }, { taint: false }, ); } diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 3a12ac8b682..f72c1668e19 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -36,6 +36,10 @@ import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-clien import type { ClickHouseConnectorType } from "./constants"; import { applyClickHouseCloudRequirements } from "./utils"; import type { ActionResult } from "@sveltejs/kit"; +import { + findAuthMethodKey, + getConnectorSchema, +} from "./multi-step-auth-configs"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { @@ -331,9 +335,27 @@ export class AddDataFormManager { result?: Extract; }) => { const values = event.form.data; - const selectedAuthMethod = getSelectedAuthMethod?.(); + const schema = getConnectorSchema(this.connector.name ?? ""); + const authKey = schema ? findAuthMethodKey(schema) : null; + const selectedAuthMethod = + (authKey && values && values[authKey] != null + ? String(values[authKey]) + : undefined) || + getSelectedAuthMethod?.() || + ""; const stepState = get(connectorStepStore) as ConnectorStepState; + // Fast-path: public auth skips validation/test and goes straight to source step. + if ( + isMultiStepConnector && + stepState.step === "connector" && + selectedAuthMethod === "public" + ) { + setConnectorConfig(values); + setStep("source"); + return; + } + // When in the source step of a multi-step flow, the superform still uses // the connector schema, so it can appear invalid because connector fields // were intentionally skipped. Allow submission in that case and rely on From bdfed2006d49b7e81e129bc6eaa47e0cbe9526b6 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 18:25:02 +0800 Subject: [PATCH 057/103] fix lingering save anyway after submission for public option --- web-common/src/features/sources/modal/AddDataForm.svelte | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 4371bf76890..e2b0920393b 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -78,6 +78,7 @@ let selectedAuthMethod: string = ""; let activeAuthMethod: string | null = null; + let prevAuthMethod: string | null = null; $: selectedAuthMethod = $selectedAuthMethodStore; $: stepState = $connectorStepStore; $: stepProperties = @@ -269,6 +270,13 @@ return selectedAuthMethod; })(); + // Clear Save Anyway state whenever auth method changes (any direction). + $: if (activeAuthMethod !== prevAuthMethod) { + prevAuthMethod = activeAuthMethod; + showSaveAnyway = false; + saveAnyway = false; + } + $: isSubmitting = submitting; // Reset errors when form is modified From 2324101813935e7d5e225e09645c471381fbc297 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Thu, 18 Dec 2025 18:51:30 +0800 Subject: [PATCH 058/103] clear input fields on auth method change --- .../modal/JSONSchemaFormRenderer.svelte | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 6683905fa6e..91a20f76641 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -57,21 +57,32 @@ } // Clear hidden fields for the active step to avoid stale submissions. + // Depend on `$form` so this runs when the auth method (or other values) change. $: if (schema) { - form.update( - ($form) => { - const properties = schema.properties ?? {}; - for (const [key, prop] of Object.entries(properties)) { - if (!matchesStep(prop, stepFilter)) continue; - const visible = isVisibleForValues(schema, key, $form); - if (!visible && key in $form && $form[key] !== "") { - $form[key] = ""; + const currentValues = $form; + const properties = schema.properties ?? {}; + + const shouldClear = Object.entries(properties).some(([key, prop]) => { + if (!matchesStep(prop, stepFilter)) return false; + const visible = isVisibleForValues(schema, key, currentValues); + return !visible && key in currentValues && currentValues[key] !== ""; + }); + + if (shouldClear) { + form.update( + ($form) => { + for (const [key, prop] of Object.entries(properties)) { + if (!matchesStep(prop, stepFilter)) continue; + const visible = isVisibleForValues(schema, key, $form); + if (!visible && key in $form && $form[key] !== "") { + $form[key] = ""; + } } - } - return $form; - }, - { taint: false }, - ); + return $form; + }, + { taint: false }, + ); + } } function matchesStep(prop: JSONSchemaField | undefined, stepValue?: string) { From 1f3d7e2ce48fa64687ccae23e05aba3f7a652c9c Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 19 Dec 2025 15:29:05 +0800 Subject: [PATCH 059/103] explicit group fields --- .../features/sources/modal/AddDataForm.svelte | 8 ++ .../modal/JSONSchemaFormRenderer.svelte | 98 +++++++++---------- .../src/features/templates/schemas/azure.ts | 6 ++ .../src/features/templates/schemas/gcs.ts | 5 + .../src/features/templates/schemas/s3.ts | 10 ++ .../src/features/templates/schemas/types.ts | 5 + 6 files changed, 83 insertions(+), 49 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e2b0920393b..f21e3c54f95 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -236,12 +236,20 @@ $: if (isMultiStepConnector && activeAuthInfo) { const options = activeAuthInfo.options ?? []; const fallback = activeAuthInfo.defaultMethod || options[0]?.value || null; + const authKey = + activeAuthInfo.key || (activeSchema && findAuthMethodKey(activeSchema)); const hasValidSelection = options.some( (option) => option.value === stepState.selectedAuthMethod, ); if (!hasValidSelection) { if (fallback !== stepState.selectedAuthMethod) { setAuthMethod(fallback ?? null); + if (fallback && authKey) { + paramsForm.update(($form) => { + if ($form[authKey] !== fallback) $form[authKey] = fallback; + return $form; + }); + } } } } else if (stepState.selectedAuthMethod) { diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index 91a20f76641..c3ab54c1f9b 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -17,12 +17,12 @@ const radioDisplay = "radio"; $: stepFilter = step; - $: dependentMap = schema - ? buildDependentMap(schema, stepFilter) - : new Map>(); - $: dependentKeys = new Set( - Array.from(dependentMap.values()).flatMap((entries) => - entries.map(([key]) => key), + $: groupedFields = schema + ? buildGroupedFields(schema, stepFilter) + : new Map>(); + $: groupedChildKeys = new Set( + Array.from(groupedFields.values()).flatMap((group) => + Object.values(group).flat(), ), ); $: visibleEntries = schema @@ -32,10 +32,11 @@ ? computeRequiredFields(schema, $form, stepFilter) : new Set(); $: renderOrder = schema - ? computeRenderOrder(visibleEntries, dependentMap, dependentKeys) + ? computeRenderOrder(visibleEntries, groupedChildKeys) : []; - // Seed defaults once when schema-provided defaults exist. + // Seed defaults for initial render: use explicit defaults, and for radio enums + // fall back to first option when no value is set. $: if (schema) { form.update( ($form) => { @@ -43,11 +44,13 @@ for (const [key, prop] of Object.entries(properties)) { if (!matchesStep(prop, stepFilter)) continue; const current = $form[key]; - if ( - (current === undefined || current === null) && - prop.default !== undefined - ) { + const isUnset = + current === undefined || current === null || current === ""; + + if (isUnset && prop.default !== undefined) { $form[key] = prop.default; + } else if (isUnset && isRadioEnum(prop) && prop.enum?.length) { + $form[key] = String(prop.enum[0]); } } return $form; @@ -145,67 +148,64 @@ function computeRenderOrder( entries: Array<[string, JSONSchemaField]>, - dependents: Map>, - dependentKeySet: Set, + groupedChildKeySet: Set, ) { const result: Array<[string, JSONSchemaField]> = []; - const rendered = new Set(); - for (const [key, prop] of entries) { - if (rendered.has(key)) continue; - - if (isRadioEnum(prop)) { - rendered.add(key); - dependents.get(key)?.forEach(([childKey]) => rendered.add(childKey)); - result.push([key, prop]); - continue; - } - - if (dependentKeySet.has(key)) continue; - - rendered.add(key); + if (groupedChildKeySet.has(key)) continue; result.push([key, prop]); } return result; } - function buildDependentMap( + function buildGroupedFields( currentSchema: MultiStepFormSchema, currentStep: string | undefined, - ) { + ): Map> { const properties = currentSchema.properties ?? {}; - const map = new Map>(); + const map = new Map>(); for (const [key, prop] of Object.entries(properties)) { - const visibleIf = prop["x-visible-if"]; - if (!visibleIf) continue; - - for (const controllerKey of Object.keys(visibleIf)) { - const controller = properties[controllerKey]; - if (!controller) continue; - if (!matchesStep(controller, currentStep)) continue; - if (!matchesStep(prop, currentStep)) continue; + const grouped = prop["x-grouped-fields"]; + if (!grouped) continue; + if (!matchesStep(prop, currentStep)) continue; - const entries = map.get(controllerKey) ?? []; - entries.push([key, prop]); - map.set(controllerKey, entries); + const filteredOptions: Record = {}; + const groupedEntries = Object.entries(grouped) as Array< + [string, string[]] + >; + for (const [optionValue, childKeys] of groupedEntries) { + filteredOptions[optionValue] = childKeys.filter((childKey) => { + const childProp = properties[childKey]; + if (!childProp) return false; + return matchesStep(childProp, currentStep); + }); } + map.set(key, filteredOptions); } return map; } - function getDependentFieldsForOption( + function getGroupedFieldsForOption( controllerKey: string, optionValue: string | number | boolean, ) { if (!schema) return []; - const dependents = dependentMap.get(controllerKey) ?? []; + const properties = schema.properties ?? {}; + const childKeys = + groupedFields.get(controllerKey)?.[String(optionValue)] ?? []; const values = { ...$form, [controllerKey]: optionValue }; - return dependents.filter(([key]) => - isVisibleForValues(schema, key, values), - ); + + return childKeys + .map< + [string, JSONSchemaField | undefined] + >((childKey) => [childKey, properties[childKey]]) + .filter( + (entry): entry is [string, JSONSchemaField] => + Boolean(entry[1]) && isVisibleForValues(schema, entry[0], values), + ); } function radioOptions(prop: JSONSchemaField) { @@ -236,8 +236,8 @@ name={`${key}-radio`} > - {#if dependentMap.get(key)?.length} - {#each getDependentFieldsForOption(key, option.value) as [childKey, childProp]} + {#if groupedFields.get(key)} + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]}
{#if childProp["x-display"] === "file" || childProp.format === "file"} ; }; export type JSONSchemaCondition = { From 97dd8fdd5a17d427dd28fa57a3a53990b225fc88 Mon Sep 17 00:00:00 2001 From: Cyrus Goh Date: Fri, 19 Dec 2025 16:44:17 +0800 Subject: [PATCH 060/103] reduce markup dup in the renderer --- .../modal/JSONSchemaFieldControl.svelte | 55 +++++++++ .../modal/JSONSchemaFormRenderer.svelte | 105 +++++------------- 2 files changed, 82 insertions(+), 78 deletions(-) create mode 100644 web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte diff --git a/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte new file mode 100644 index 00000000000..a7c47942c04 --- /dev/null +++ b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte @@ -0,0 +1,55 @@ + + +{#if prop["x-display"] === "file" || prop.format === "file"} + +{:else if prop.type === "boolean"} + +{:else if options?.length} + +{:else} + onStringInputChange(e)} + alwaysShowError + /> +{/if} diff --git a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte index c3ab54c1f9b..6cc9b8c8652 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFormRenderer.svelte @@ -1,9 +1,6 @@ + +{#if stepState.step === "connector"} + + {#if activeSchema} + + {:else} + + {/if} + +{:else} + + {#if activeSchema} + + {:else} + + {/if} + +{/if} From 22bdc58c96db8fdeed71b15798b2e3621551e000 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 15:44:10 -0500 Subject: [PATCH 065/103] feat: Add HTTPS connector with JSON Schema-based authentication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements multi-step HTTPS connector with simplified header-based auth. Frontend changes: - Created https.ts schema with 2 auth options: * Public (no authentication) * Custom Headers (JSON format for Authorization, API keys, etc.) - Registered HTTPS in connector-schemas.ts - Added HTTPS to MULTI_STEP_CONNECTORS - Created Yup validation for https_connector and https_source Backend changes: - Updated runtime/drivers/https/https.go: * Added ConfigProperties to Spec for connector-level headers * Updated FilePaths() to merge connector headers with model headers * DisplayName changed to "HTTPS" with improved description - Maintains backward compatibility with existing header support Auth examples users can provide: {"Authorization": "Bearer my-token"} {"X-API-Key": "my-key", "X-Custom": "value"} Part of Sprint 1: REST APIs migration to universal connector schemas. 🤖 Generated with Claude Code (https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- runtime/drivers/https/https.go | 36 ++++++++++++----- .../sources/modal/connector-schemas.ts | 2 + .../src/features/sources/modal/constants.ts | 2 +- .../src/features/sources/modal/yupSchemas.ts | 17 ++++++++ .../src/features/templates/schemas/https.ts | 40 +++++++++++++++++++ 5 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 web-common/src/features/templates/schemas/https.ts diff --git a/runtime/drivers/https/https.go b/runtime/drivers/https/https.go index fa70e25123f..7ae79b7058e 100644 --- a/runtime/drivers/https/https.go +++ b/runtime/drivers/https/https.go @@ -21,17 +21,25 @@ func init() { } var spec = drivers.Spec{ - DisplayName: "https", - Description: "Connect to a remote file.", + DisplayName: "HTTPS", + Description: "Connect to remote files and REST APIs.", DocsURL: "https://docs.rilldata.com/build/connect/#adding-a-remote-source", - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. + ConfigProperties: []*drivers.PropertySpec{ + { + Key: "headers", + Type: drivers.StringPropertyType, + DisplayName: "HTTP Headers (JSON)", + Description: `HTTP headers as JSON object. Example: {"Authorization": "Bearer my-token"}`, + Placeholder: `{"Authorization": "Bearer my-token"}`, + }, + }, SourceProperties: []*drivers.PropertySpec{ { Key: "path", Type: drivers.StringPropertyType, - DisplayName: "Path", - Description: "Path to the remote file.", - Placeholder: "https://example.com/file.csv", + DisplayName: "URL", + Description: "URL to the remote file or API endpoint", + Placeholder: "https://api.example.com/data", Required: true, }, { @@ -102,8 +110,9 @@ func (d driver) Open(instanceID string, config map[string]any, st *storage.Clien } conn := &Connection{ - config: config, - logger: logger, + config: config, + configProp: cfg, + logger: logger, } return conn, nil } @@ -121,8 +130,9 @@ func (d driver) TertiarySourceConnectors(ctx context.Context, src map[string]any } type Connection struct { - config map[string]any - logger *zap.Logger + config map[string]any + configProp *ConfigProperties + logger *zap.Logger } var _ drivers.Handle = &Connection{} @@ -254,6 +264,12 @@ func (c *Connection) FilePaths(ctx context.Context, src map[string]any) ([]strin return nil, fmt.Errorf("failed to create request for path %s: %w", path, err) } + // Apply connector-level headers first (from config) + for k, v := range c.configProp.Headers { + req.Header.Set(k, v) + } + + // Model-specific headers override connector headers for k, v := range modelProp.Headers { req.Header.Set(k, v) } diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 0b94cffa19f..5ae092b9f21 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -1,12 +1,14 @@ import type { MultiStepFormSchema } from "../../templates/schemas/types"; import { azureSchema } from "../../templates/schemas/azure"; import { gcsSchema } from "../../templates/schemas/gcs"; +import { httpsSchema } from "../../templates/schemas/https"; import { s3Schema } from "../../templates/schemas/s3"; export const multiStepFormSchemas: Record = { s3: s3Schema, gcs: gcsSchema, azure: azureSchema, + https: httpsSchema, }; export function getConnectorSchema( diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 8e02f8a7b44..e183cec0f84 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -126,7 +126,7 @@ export const OLAP_ENGINES = [ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) -export const MULTI_STEP_CONNECTORS = ["gcs", "s3", "azure"]; +export const MULTI_STEP_CONNECTORS = ["gcs", "s3", "azure", "https"]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 949cfecbbe6..80c15c0f6ea 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -48,6 +48,23 @@ export const getYupSchema = { .required(), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + https_connector: yup.object().shape({ + headers: yup.string().optional(), + }), + + https_source: yup.object().shape({ + path: yup + .string() + .matches(/^https?:\/\//, 'Path must start with "http(s)://"') + .required("Path is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Source name is required"), + }), + https: yup.object().shape({ path: yup .string() diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts new file mode 100644 index 00000000000..a10570e2d6b --- /dev/null +++ b/web-common/src/features/templates/schemas/https.ts @@ -0,0 +1,40 @@ +import type { MultiStepFormSchema } from "./types"; + +export const httpsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["public", "headers"], + default: "public", + description: "Choose how to authenticate to the REST API", + "x-display": "radio", + "x-enum-labels": ["Public", "Custom Headers"], + "x-enum-descriptions": [ + "Access publicly available APIs without authentication.", + "Provide custom HTTP headers for authentication (e.g., Authorization, API keys).", + ], + "x-grouped-fields": { + public: [], + headers: ["headers"], + }, + }, + headers: { + type: "string", + title: "HTTP Headers (JSON)", + description: + 'HTTP headers as JSON object. Example: {"Authorization": "Bearer my-token", "X-API-Key": "value"}', + "x-placeholder": '{"Authorization": "Bearer my-token"}', + "x-display": "textarea", + "x-visible-if": { auth_method: "headers" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "headers" } } }, + then: { required: ["headers"] }, + }, + ], +}; From 3ad094317895b7f0c60cf0f907fd034290529f25 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 15:48:16 -0500 Subject: [PATCH 066/103] feat: Add PostgreSQL, MySQL, and SQLite connector schemas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements JSON Schema-based multi-step forms for database connectors. Frontend changes: - Created postgres.ts schema with 2 auth options: * Username & Password (host, port, database, user, password, sslmode) * Connection String (DSN) - Created mysql.ts schema with 2 auth options: * Username & Password (host, port, database, user, password, sslmode) * Connection String (DSN) - Created sqlite.ts schema with 2 location options: * Local File (file path to SQLite database) * Remote URL (URL to SQLite database) - Registered all three schemas in connector-schemas.ts - Added postgres, mysql, sqlite to MULTI_STEP_CONNECTORS - Created Yup validation for _connector variants Backend changes: - PostgreSQL: Already had ConfigProperties defined (no changes needed) - MySQL: Already had ConfigProperties defined (no changes needed) - SQLite: Moved 'db' field from SourceProperties to ConfigProperties * ConfigProperties: db (database path/URL) * SourceProperties: table, name All three connectors now support the two-step flow: Step 1: Configure connector (authentication/database location) Step 2: Select source data (table selection, model naming) Part of Sprint 2: Database Connectors migration to universal schemas. 🤖 Generated with Claude Code (https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- runtime/drivers/sqlite/sqlite.go | 12 +-- .../sources/modal/connector-schemas.ts | 6 ++ .../src/features/sources/modal/constants.ts | 10 ++- .../src/features/sources/modal/yupSchemas.ts | 30 +++++++ .../src/features/templates/schemas/mysql.ts | 89 +++++++++++++++++++ .../features/templates/schemas/postgres.ts | 89 +++++++++++++++++++ .../src/features/templates/schemas/sqlite.ts | 42 +++++++++ 7 files changed, 272 insertions(+), 6 deletions(-) create mode 100644 web-common/src/features/templates/schemas/mysql.ts create mode 100644 web-common/src/features/templates/schemas/postgres.ts create mode 100644 web-common/src/features/templates/schemas/sqlite.ts diff --git a/runtime/drivers/sqlite/sqlite.go b/runtime/drivers/sqlite/sqlite.go index 355299369ab..c7af6223893 100644 --- a/runtime/drivers/sqlite/sqlite.go +++ b/runtime/drivers/sqlite/sqlite.go @@ -85,16 +85,18 @@ func (d driver) Spec() drivers.Spec { DisplayName: "SQLite", Description: "Import data from SQLite into DuckDB.", DocsURL: "https://docs.rilldata.com/build/connectors/data-source/sqlite", - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. - SourceProperties: []*drivers.PropertySpec{ + ConfigProperties: []*drivers.PropertySpec{ { Key: "db", Type: drivers.StringPropertyType, Required: true, - DisplayName: "DB", - Description: "Path to SQLite db file", - Placeholder: "/path/to/sqlite.db", + DisplayName: "Database Path", + Description: "Path or URL to the SQLite database file", + Placeholder: "/path/to/database.db", + Hint: "Local file path or remote URL to the SQLite database", }, + }, + SourceProperties: []*drivers.PropertySpec{ { Key: "table", Type: drivers.StringPropertyType, diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 5ae092b9f21..3815e19cd3f 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -2,13 +2,19 @@ import type { MultiStepFormSchema } from "../../templates/schemas/types"; import { azureSchema } from "../../templates/schemas/azure"; import { gcsSchema } from "../../templates/schemas/gcs"; import { httpsSchema } from "../../templates/schemas/https"; +import { mysqlSchema } from "../../templates/schemas/mysql"; +import { postgresSchema } from "../../templates/schemas/postgres"; import { s3Schema } from "../../templates/schemas/s3"; +import { sqliteSchema } from "../../templates/schemas/sqlite"; export const multiStepFormSchemas: Record = { s3: s3Schema, gcs: gcsSchema, azure: azureSchema, https: httpsSchema, + postgres: postgresSchema, + mysql: mysqlSchema, + sqlite: sqliteSchema, }; export function getConnectorSchema( diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index e183cec0f84..700d18cbf85 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -126,7 +126,15 @@ export const OLAP_ENGINES = [ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) -export const MULTI_STEP_CONNECTORS = ["gcs", "s3", "azure", "https"]; +export const MULTI_STEP_CONNECTORS = [ + "gcs", + "s3", + "azure", + "https", + "postgres", + "mysql", + "sqlite", +]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 80c15c0f6ea..33d9246af87 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -127,6 +127,36 @@ export const getYupSchema = { .required("Source name is required"), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + postgres_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + database: yup.string().optional(), + user: yup.string().optional(), + password: yup.string().optional(), + sslmode: yup.string().optional(), + dsn: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + mysql_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + database: yup.string().optional(), + user: yup.string().optional(), + password: yup.string().optional(), + sslmode: yup.string().optional(), + dsn: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + sqlite_connector: yup.object().shape({ + db: yup.string().optional(), + }), + postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts new file mode 100644 index 00000000000..f191a11c1b6 --- /dev/null +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -0,0 +1,89 @@ +import type { MultiStepFormSchema } from "./types"; + +export const mysqlSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to MySQL", + "x-display": "radio", + "x-enum-labels": ["Username & Password", "Connection String"], + "x-enum-descriptions": [ + "Provide individual connection parameters (host, port, database, username, password).", + "Provide a complete MySQL connection string (DSN).", + ], + "x-grouped-fields": { + parameters: ["host", "port", "database", "user", "password", "sslmode"], + connection_string: ["dsn"], + }, + }, + host: { + type: "string", + title: "Host", + description: "Database server hostname or IP address", + "x-placeholder": "localhost", + "x-visible-if": { auth_method: "parameters" }, + }, + port: { + type: "number", + title: "Port", + description: "Database server port", + default: 3306, + "x-placeholder": "3306", + "x-visible-if": { auth_method: "parameters" }, + }, + database: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "my_database", + "x-visible-if": { auth_method: "parameters" }, + }, + user: { + type: "string", + title: "Username", + description: "Database user", + "x-placeholder": "root", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Database password", + "x-placeholder": "Enter password", + "x-secret": true, + "x-visible-if": { auth_method: "parameters" }, + }, + sslmode: { + type: "string", + title: "SSL Mode", + description: "SSL connection mode", + enum: ["disabled", "preferred", "required", "verify_ca", "verify_identity"], + default: "preferred", + "x-display": "select", + "x-visible-if": { auth_method: "parameters" }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "MySQL connection string (DSN)", + "x-placeholder": "user:password@tcp(host:3306)/dbname?tls=preferred", + "x-secret": true, + "x-visible-if": { auth_method: "connection_string" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["host", "database", "user", "password"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts new file mode 100644 index 00000000000..59b9bf44a76 --- /dev/null +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -0,0 +1,89 @@ +import type { MultiStepFormSchema } from "./types"; + +export const postgresSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to PostgreSQL", + "x-display": "radio", + "x-enum-labels": ["Username & Password", "Connection String"], + "x-enum-descriptions": [ + "Provide individual connection parameters (host, port, database, username, password).", + "Provide a complete PostgreSQL connection string (DSN).", + ], + "x-grouped-fields": { + parameters: ["host", "port", "database", "user", "password", "sslmode"], + connection_string: ["dsn"], + }, + }, + host: { + type: "string", + title: "Host", + description: "Database server hostname or IP address", + "x-placeholder": "localhost", + "x-visible-if": { auth_method: "parameters" }, + }, + port: { + type: "number", + title: "Port", + description: "Database server port", + default: 5432, + "x-placeholder": "5432", + "x-visible-if": { auth_method: "parameters" }, + }, + database: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "my_database", + "x-visible-if": { auth_method: "parameters" }, + }, + user: { + type: "string", + title: "Username", + description: "Database user", + "x-placeholder": "postgres", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Database password", + "x-placeholder": "Enter password", + "x-secret": true, + "x-visible-if": { auth_method: "parameters" }, + }, + sslmode: { + type: "string", + title: "SSL Mode", + description: "SSL connection mode", + enum: ["disable", "require", "verify-ca", "verify-full"], + default: "prefer", + "x-display": "select", + "x-visible-if": { auth_method: "parameters" }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "PostgreSQL connection string (DSN)", + "x-placeholder": "postgres://user:password@host:5432/dbname?sslmode=require", + "x-secret": true, + "x-visible-if": { auth_method: "connection_string" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["host", "database", "user", "password"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts new file mode 100644 index 00000000000..1816cbc44f3 --- /dev/null +++ b/web-common/src/features/templates/schemas/sqlite.ts @@ -0,0 +1,42 @@ +import type { MultiStepFormSchema } from "./types"; + +export const sqliteSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Database location", + enum: ["local_file", "remote_url"], + default: "local_file", + description: "Choose where the SQLite database is located", + "x-display": "radio", + "x-enum-labels": ["Local File", "Remote URL"], + "x-enum-descriptions": [ + "Path to a local SQLite database file on disk.", + "URL to a remote SQLite database file.", + ], + "x-grouped-fields": { + local_file: ["db"], + remote_url: ["db"], + }, + }, + db: { + type: "string", + title: "Database Path", + description: "Path or URL to the SQLite database file", + "x-placeholder": "/path/to/database.db", + "x-visible-if": { auth_method: "local_file" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "local_file" } } }, + then: { required: ["db"] }, + }, + { + if: { properties: { auth_method: { const: "remote_url" } } }, + then: { required: ["db"] }, + }, + ], +}; From 882ae689a0fac184b6b03ed701b2c0ddb0d90782 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 15:56:14 -0500 Subject: [PATCH 067/103] feat: add JSON schemas for cloud warehouse connectors (Sprint 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add multi-step form schemas for Snowflake, BigQuery, Redshift, and Athena connectors: Frontend changes: - Created snowflake.ts schema with password, keypair, and connection_string auth methods - Created bigquery.ts schema with service account auth - Created redshift.ts schema for Redshift Data API (serverless/provisioned) - Created athena.ts schema with AWS credentials and optional role ARN - Registered all four schemas in connector-schemas.ts - Added all four to MULTI_STEP_CONNECTORS constant - Added Yup validation fallback schemas for all four connectors Backend changes: - Snowflake: Added PropertySpecs for private_key and private_key_passphrase, updated mapstructure tags to snake_case - Redshift: Added PropertySpec for cluster_identifier - Athena: Added PropertySpecs for aws_role_arn, region, and workgroup 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- runtime/drivers/athena/athena.go | 21 ++++ runtime/drivers/redshift/redshift.go | 8 ++ runtime/drivers/snowflake/snowflake.go | 33 +++-- .../sources/modal/connector-schemas.ts | 8 ++ .../src/features/sources/modal/constants.ts | 4 + .../src/features/sources/modal/yupSchemas.ts | 44 +++++++ .../src/features/templates/schemas/athena.ts | 47 +++++++ .../features/templates/schemas/bigquery.ts | 45 +++++++ .../features/templates/schemas/redshift.ts | 48 ++++++++ .../features/templates/schemas/snowflake.ts | 116 ++++++++++++++++++ 10 files changed, 366 insertions(+), 8 deletions(-) create mode 100644 web-common/src/features/templates/schemas/athena.ts create mode 100644 web-common/src/features/templates/schemas/bigquery.ts create mode 100644 web-common/src/features/templates/schemas/redshift.ts create mode 100644 web-common/src/features/templates/schemas/snowflake.ts diff --git a/runtime/drivers/athena/athena.go b/runtime/drivers/athena/athena.go index a3b24c2b12f..c8a7fdbb8d4 100644 --- a/runtime/drivers/athena/athena.go +++ b/runtime/drivers/athena/athena.go @@ -60,6 +60,27 @@ var spec = drivers.Spec{ Placeholder: "s3://bucket-name/path/", Required: true, }, + { + Key: "aws_role_arn", + Type: drivers.StringPropertyType, + DisplayName: "IAM Role ARN", + Description: "AWS IAM role ARN to assume (optional)", + Placeholder: "arn:aws:iam::123456789012:role/MyRole", + }, + { + Key: "region", + Type: drivers.StringPropertyType, + DisplayName: "AWS Region", + Description: "AWS region where Athena is configured", + Placeholder: "us-east-1", + }, + { + Key: "workgroup", + Type: drivers.StringPropertyType, + DisplayName: "Workgroup", + Description: "Athena workgroup name (optional)", + Placeholder: "primary", + }, }, ImplementsWarehouse: true, } diff --git a/runtime/drivers/redshift/redshift.go b/runtime/drivers/redshift/redshift.go index 54a03213a44..34bf6a5bdd0 100644 --- a/runtime/drivers/redshift/redshift.go +++ b/runtime/drivers/redshift/redshift.go @@ -73,6 +73,14 @@ var spec = drivers.Spec{ Placeholder: "dev", Required: true, }, + { + Key: "cluster_identifier", + Type: drivers.StringPropertyType, + DisplayName: "Cluster Identifier", + Description: "Redshift provisioned cluster identifier (for provisioned clusters)", + Placeholder: "my-redshift-cluster", + Hint: "Provide either workgroup (for serverless) or cluster identifier (for provisioned clusters)", + }, }, ImplementsWarehouse: true, } diff --git a/runtime/drivers/snowflake/snowflake.go b/runtime/drivers/snowflake/snowflake.go index 8b9dd5ed513..00c22cb7f91 100644 --- a/runtime/drivers/snowflake/snowflake.go +++ b/runtime/drivers/snowflake/snowflake.go @@ -37,7 +37,7 @@ var spec = drivers.Spec{ DisplayName: "Snowflake Connection String", Required: false, DocsURL: "https://docs.rilldata.com/build/connectors/data-source/snowflake", - Placeholder: "@//?warehouse=&role=&authenticator=SNOWFLAKE_JWT&privateKey=", + Placeholder: "@//?warehouse=&role=&authenticator=SNOWFLAKE_JWT&private_key=", Hint: "Can be configured here or by setting the 'connector.snowflake.dsn' environment variable (using '.env' or '--env').", Secret: true, }, @@ -95,6 +95,22 @@ var spec = drivers.Spec{ Placeholder: "your_role", Hint: "The Snowflake role to use (defaults to your default role if not specified)", }, + { + Key: "private_key", + Type: drivers.StringPropertyType, + DisplayName: "Private Key", + Description: "RSA private key in PEM format for key pair authentication", + Placeholder: "-----BEGIN PRIVATE KEY-----\\n...\\n-----END PRIVATE KEY-----", + Secret: true, + }, + { + Key: "private_key_passphrase", + Type: drivers.StringPropertyType, + DisplayName: "Private Key Passphrase", + Description: "Passphrase for encrypted private key (if applicable)", + Placeholder: "Enter passphrase if key is encrypted", + Secret: true, + }, }, ImplementsWarehouse: true, } @@ -108,11 +124,12 @@ type configProperties struct { Password string `mapstructure:"password"` Database string `mapstructure:"database"` Schema string `mapstructure:"schema"` - Warehouse string `mapstructure:"warehouse"` - Role string `mapstructure:"role"` - Authenticator string `mapstructure:"authenticator"` - PrivateKey string `mapstructure:"privateKey"` - ParallelFetchLimit int `mapstructure:"parallel_fetch_limit"` + Warehouse string `mapstructure:"warehouse"` + Role string `mapstructure:"role"` + Authenticator string `mapstructure:"authenticator"` + PrivateKey string `mapstructure:"private_key"` + PrivateKeyPassphrase string `mapstructure:"private_key_passphrase"` + ParallelFetchLimit int `mapstructure:"parallel_fetch_limit"` Extras map[string]any `mapstructure:",remain"` // LogQueries controls whether to log the raw SQL passed to OLAP. @@ -146,7 +163,7 @@ func (c *configProperties) validate() error { set = append(set, "authenticator") } if c.PrivateKey != "" { - set = append(set, "privateKey") + set = append(set, "private_key") } if c.DSN != "" && len(set) > 0 { return fmt.Errorf("snowflake: Only one of 'dsn' or [%s] can be set", strings.Join(set, ", ")) @@ -164,7 +181,7 @@ func (c *configProperties) resolveDSN() (string, error) { } if c.Password == "" && c.PrivateKey == "" { - return "", errors.New("either password or privateKey must be provided") + return "", errors.New("either password or private_key must be provided") } cfg := &gosnowflake.Config{ diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 3815e19cd3f..936dc92deb3 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -1,10 +1,14 @@ import type { MultiStepFormSchema } from "../../templates/schemas/types"; +import { athenaSchema } from "../../templates/schemas/athena"; import { azureSchema } from "../../templates/schemas/azure"; +import { bigquerySchema } from "../../templates/schemas/bigquery"; import { gcsSchema } from "../../templates/schemas/gcs"; import { httpsSchema } from "../../templates/schemas/https"; import { mysqlSchema } from "../../templates/schemas/mysql"; import { postgresSchema } from "../../templates/schemas/postgres"; +import { redshiftSchema } from "../../templates/schemas/redshift"; import { s3Schema } from "../../templates/schemas/s3"; +import { snowflakeSchema } from "../../templates/schemas/snowflake"; import { sqliteSchema } from "../../templates/schemas/sqlite"; export const multiStepFormSchemas: Record = { @@ -15,6 +19,10 @@ export const multiStepFormSchemas: Record = { postgres: postgresSchema, mysql: mysqlSchema, sqlite: sqliteSchema, + snowflake: snowflakeSchema, + bigquery: bigquerySchema, + redshift: redshiftSchema, + athena: athenaSchema, }; export function getConnectorSchema( diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 700d18cbf85..480f26f71d4 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -134,6 +134,10 @@ export const MULTI_STEP_CONNECTORS = [ "postgres", "mysql", "sqlite", + "snowflake", + "bigquery", + "redshift", + "athena", ]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 33d9246af87..5a4c5bfb23d 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -157,6 +157,50 @@ export const getYupSchema = { db: yup.string().optional(), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + snowflake_connector: yup.object().shape({ + account: yup.string().optional(), + user: yup.string().optional(), + password: yup.string().optional(), + private_key: yup.string().optional(), + private_key_passphrase: yup.string().optional(), + warehouse: yup.string().optional(), + database: yup.string().optional(), + schema: yup.string().optional(), + role: yup.string().optional(), + dsn: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + bigquery_connector: yup.object().shape({ + google_application_credentials: yup.string().optional(), + project_id: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + redshift_connector: yup.object().shape({ + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + region: yup.string().optional(), + database: yup.string().optional(), + workgroup: yup.string().optional(), + cluster_identifier: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + athena_connector: yup.object().shape({ + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + aws_role_arn: yup.string().optional(), + region: yup.string().optional(), + workgroup: yup.string().optional(), + output_location: yup.string().optional(), + }), + postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts new file mode 100644 index 00000000000..b50aca2538e --- /dev/null +++ b/web-common/src/features/templates/schemas/athena.ts @@ -0,0 +1,47 @@ +import type { MultiStepFormSchema } from "./types"; + +export const athenaSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS Access Key ID", + description: "AWS access key ID", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + }, + aws_secret_access_key: { + type: "string", + title: "AWS Secret Access Key", + description: "AWS secret access key", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + }, + output_location: { + type: "string", + title: "S3 Output Location", + description: "S3 URI for query results", + "x-placeholder": "s3://my-bucket/athena-results/", + }, + aws_role_arn: { + type: "string", + title: "IAM Role ARN (Optional)", + description: "AWS IAM role ARN to assume (optional)", + "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", + }, + region: { + type: "string", + title: "AWS Region", + description: "AWS region where Athena is configured", + "x-placeholder": "us-east-1", + }, + workgroup: { + type: "string", + title: "Workgroup", + description: "Athena workgroup name (optional)", + "x-placeholder": "primary", + }, + }, + required: ["aws_access_key_id", "aws_secret_access_key", "output_location"], +}; diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts new file mode 100644 index 00000000000..2f24a95b0f5 --- /dev/null +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -0,0 +1,45 @@ +import type { MultiStepFormSchema } from "./types"; + +export const bigquerySchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["service_account"], + default: "service_account", + description: "Choose how to authenticate to BigQuery", + "x-display": "radio", + "x-enum-labels": ["Service Account"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with BigQuery access.", + ], + "x-grouped-fields": { + service_account: ["google_application_credentials", "project_id"], + }, + }, + google_application_credentials: { + type: "string", + title: "Service Account Key", + description: "Upload a JSON key file for a service account with BigQuery access", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-visible-if": { auth_method: "service_account" }, + }, + project_id: { + type: "string", + title: "Project ID", + description: "Google Cloud project ID (optional if specified in credentials)", + "x-placeholder": "my-project-id", + "x-visible-if": { auth_method: "service_account" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "service_account" } } }, + then: { required: ["google_application_credentials"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts new file mode 100644 index 00000000000..fd2b170fe74 --- /dev/null +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -0,0 +1,48 @@ +import type { MultiStepFormSchema } from "./types"; + +export const redshiftSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS Access Key ID", + description: "AWS access key ID for Redshift Data API", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + }, + aws_secret_access_key: { + type: "string", + title: "AWS Secret Access Key", + description: "AWS secret access key for Redshift Data API", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + }, + region: { + type: "string", + title: "AWS Region", + description: "AWS region where the Redshift cluster is located", + "x-placeholder": "us-east-1", + }, + database: { + type: "string", + title: "Database", + description: "Redshift database name", + "x-placeholder": "dev", + }, + workgroup: { + type: "string", + title: "Workgroup", + description: "Redshift Serverless workgroup name (for serverless)", + "x-placeholder": "default-workgroup", + }, + cluster_identifier: { + type: "string", + title: "Cluster Identifier", + description: "Redshift provisioned cluster identifier (for provisioned clusters)", + "x-placeholder": "my-redshift-cluster", + "x-hint": "Provide either workgroup (for serverless) or cluster identifier (for provisioned)", + }, + }, + required: ["aws_access_key_id", "aws_secret_access_key", "database"], +}; diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts new file mode 100644 index 00000000000..e0e03cef786 --- /dev/null +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -0,0 +1,116 @@ +import type { MultiStepFormSchema } from "./types"; + +export const snowflakeSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["password", "keypair", "connection_string"], + default: "password", + description: "Choose how to authenticate to Snowflake", + "x-display": "radio", + "x-enum-labels": ["Username & Password", "Key Pair", "Connection String"], + "x-enum-descriptions": [ + "Authenticate with username and password.", + "Authenticate with RSA key pair (more secure).", + "Provide a complete Snowflake connection string (DSN).", + ], + "x-grouped-fields": { + password: ["account", "user", "password", "warehouse", "database", "schema", "role"], + keypair: ["account", "user", "private_key", "private_key_passphrase", "warehouse", "database", "schema", "role"], + connection_string: ["dsn"], + }, + }, + account: { + type: "string", + title: "Account", + description: "Snowflake account identifier (e.g., abc12345.us-east-1)", + "x-placeholder": "abc12345.us-east-1", + "x-visible-if": { auth_method: ["password", "keypair"] }, + }, + user: { + type: "string", + title: "Username", + description: "Snowflake username", + "x-placeholder": "Enter username", + "x-visible-if": { auth_method: ["password", "keypair"] }, + }, + password: { + type: "string", + title: "Password", + description: "Snowflake password", + "x-placeholder": "Enter password", + "x-secret": true, + "x-visible-if": { auth_method: "password" }, + }, + private_key: { + type: "string", + title: "Private Key", + description: "RSA private key in PEM format", + "x-placeholder": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----", + "x-display": "textarea", + "x-secret": true, + "x-visible-if": { auth_method: "keypair" }, + }, + private_key_passphrase: { + type: "string", + title: "Private Key Passphrase", + description: "Optional passphrase for encrypted private key", + "x-placeholder": "Enter passphrase if key is encrypted", + "x-secret": true, + "x-visible-if": { auth_method: "keypair" }, + }, + warehouse: { + type: "string", + title: "Warehouse", + description: "Snowflake warehouse name", + "x-placeholder": "COMPUTE_WH", + "x-visible-if": { auth_method: ["password", "keypair"] }, + }, + database: { + type: "string", + title: "Database", + description: "Snowflake database name", + "x-placeholder": "MY_DATABASE", + "x-visible-if": { auth_method: ["password", "keypair"] }, + }, + schema: { + type: "string", + title: "Schema", + description: "Snowflake schema name", + "x-placeholder": "PUBLIC", + "x-visible-if": { auth_method: ["password", "keypair"] }, + }, + role: { + type: "string", + title: "Role", + description: "Optional Snowflake role to assume", + "x-placeholder": "ANALYST", + "x-visible-if": { auth_method: ["password", "keypair"] }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "Snowflake connection string (DSN)", + "x-placeholder": "user:password@account/database/schema?warehouse=wh", + "x-secret": true, + "x-visible-if": { auth_method: "connection_string" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "password" } } }, + then: { required: ["account", "user", "password"] }, + }, + { + if: { properties: { auth_method: { const: "keypair" } } }, + then: { required: ["account", "user", "private_key"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; From 54975a90e4190a5daeabe95f9ddcc919d37fba10 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 16:16:57 -0500 Subject: [PATCH 068/103] feat: add JSON schemas for OLAP engine connectors (Sprint 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add multi-step form schemas for DuckDB, MotherDuck, Druid, and Pinot connectors: Frontend changes: - Created duckdb.ts schema with path, attach, and mode options - Created motherduck.ts schema with path, token, schema_name, and mode - Created druid.ts schema with parameters vs connection_string auth methods - Created pinot.ts schema with parameters vs connection_string auth methods - Registered all four schemas in connector-schemas.ts - Added all four to MULTI_STEP_CONNECTORS constant - Added Yup validation fallback schemas for all four connectors - Skipped ClickHouse (already has custom form: AddClickHouseForm.svelte) Backend changes: - No PropertySpec changes needed (all drivers already have complete ConfigProperties) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .../sources/modal/connector-schemas.ts | 8 ++ .../src/features/sources/modal/constants.ts | 4 + .../src/features/sources/modal/yupSchemas.ts | 41 ++++++++ .../src/features/templates/schemas/druid.ts | 79 ++++++++++++++++ .../src/features/templates/schemas/duckdb.ts | 34 +++++++ .../features/templates/schemas/motherduck.ts | 41 ++++++++ .../src/features/templates/schemas/pinot.ts | 93 +++++++++++++++++++ 7 files changed, 300 insertions(+) create mode 100644 web-common/src/features/templates/schemas/druid.ts create mode 100644 web-common/src/features/templates/schemas/duckdb.ts create mode 100644 web-common/src/features/templates/schemas/motherduck.ts create mode 100644 web-common/src/features/templates/schemas/pinot.ts diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 936dc92deb3..d4289b07501 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -2,9 +2,13 @@ import type { MultiStepFormSchema } from "../../templates/schemas/types"; import { athenaSchema } from "../../templates/schemas/athena"; import { azureSchema } from "../../templates/schemas/azure"; import { bigquerySchema } from "../../templates/schemas/bigquery"; +import { druidSchema } from "../../templates/schemas/druid"; +import { duckdbSchema } from "../../templates/schemas/duckdb"; import { gcsSchema } from "../../templates/schemas/gcs"; import { httpsSchema } from "../../templates/schemas/https"; +import { motherduckSchema } from "../../templates/schemas/motherduck"; import { mysqlSchema } from "../../templates/schemas/mysql"; +import { pinotSchema } from "../../templates/schemas/pinot"; import { postgresSchema } from "../../templates/schemas/postgres"; import { redshiftSchema } from "../../templates/schemas/redshift"; import { s3Schema } from "../../templates/schemas/s3"; @@ -23,6 +27,10 @@ export const multiStepFormSchemas: Record = { bigquery: bigquerySchema, redshift: redshiftSchema, athena: athenaSchema, + duckdb: duckdbSchema, + motherduck: motherduckSchema, + druid: druidSchema, + pinot: pinotSchema, }; export function getConnectorSchema( diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 480f26f71d4..d31a985111b 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -138,6 +138,10 @@ export const MULTI_STEP_CONNECTORS = [ "bigquery", "redshift", "athena", + "duckdb", + "motherduck", + "druid", + "pinot", ]; export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 5a4c5bfb23d..94bcce2d630 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -201,6 +201,47 @@ export const getYupSchema = { output_location: yup.string().optional(), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + duckdb_connector: yup.object().shape({ + path: yup.string().optional(), + attach: yup.string().optional(), + mode: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + motherduck_connector: yup.object().shape({ + path: yup.string().optional(), + token: yup.string().optional(), + schema_name: yup.string().optional(), + mode: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + druid_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + username: yup.string().optional(), + password: yup.string().optional(), + ssl: yup.boolean().optional(), + dsn: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + pinot_connector: yup.object().shape({ + broker_host: yup.string().optional(), + broker_port: yup.number().optional(), + controller_host: yup.string().optional(), + controller_port: yup.number().optional(), + username: yup.string().optional(), + password: yup.string().optional(), + ssl: yup.boolean().optional(), + dsn: yup.string().optional(), + }), + postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts new file mode 100644 index 00000000000..c6ef00f2435 --- /dev/null +++ b/web-common/src/features/templates/schemas/druid.ts @@ -0,0 +1,79 @@ +import type { MultiStepFormSchema } from "./types"; + +export const druidSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to Druid", + "x-display": "radio", + "x-enum-labels": ["Connection Parameters", "Connection String"], + "x-enum-descriptions": [ + "Provide individual connection parameters (host, port, etc.)", + "Provide a complete Druid connection string (DSN)", + ], + "x-grouped-fields": { + parameters: ["host", "port", "username", "password", "ssl"], + connection_string: ["dsn"], + }, + }, + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the Druid server", + "x-placeholder": "localhost", + "x-visible-if": { auth_method: "parameters" }, + }, + port: { + type: "number", + title: "Port", + description: "Port number of the Druid server", + "x-placeholder": "8888", + "x-visible-if": { auth_method: "parameters" }, + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the Druid server (optional)", + "x-placeholder": "default", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the Druid server (optional)", + "x-placeholder": "Enter password", + "x-secret": true, + "x-visible-if": { auth_method: "parameters" }, + }, + ssl: { + type: "boolean", + title: "Use SSL", + description: "Use SSL to connect to the Druid server", + default: true, + "x-visible-if": { auth_method: "parameters" }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "Druid connection string (DSN)", + "x-placeholder": "https://example.com/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=username&avaticaPassword=password", + "x-secret": true, + "x-visible-if": { auth_method: "connection_string" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["host", "ssl"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts new file mode 100644 index 00000000000..cbb6f3f0b1b --- /dev/null +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -0,0 +1,34 @@ +import type { MultiStepFormSchema } from "./types"; + +export const duckdbSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Database Path", + description: "Path to external DuckDB database file", + "x-placeholder": "/path/to/main.db", + }, + attach: { + type: "string", + title: "Attach (Advanced)", + description: "Attach to an existing DuckDB database with options (alternative to path)", + "x-placeholder": "'ducklake:metadata.ducklake' AS my_ducklake(DATA_PATH 'datafiles')", + }, + mode: { + type: "string", + title: "Connection Mode", + description: "Database access mode", + enum: ["read", "readwrite"], + default: "read", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], + }, + }, + required: ["path"], +}; diff --git a/web-common/src/features/templates/schemas/motherduck.ts b/web-common/src/features/templates/schemas/motherduck.ts new file mode 100644 index 00000000000..23a0ffd3625 --- /dev/null +++ b/web-common/src/features/templates/schemas/motherduck.ts @@ -0,0 +1,41 @@ +import type { MultiStepFormSchema } from "./types"; + +export const motherduckSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Database Path", + description: "Path to MotherDuck database (must be prefixed with 'md:')", + "x-placeholder": "md:my_db", + }, + token: { + type: "string", + title: "MotherDuck Token", + description: "Your MotherDuck authentication token", + "x-placeholder": "Enter your MotherDuck token", + "x-secret": true, + }, + schema_name: { + type: "string", + title: "Schema Name", + description: "Default schema used by the MotherDuck database", + "x-placeholder": "main", + }, + mode: { + type: "string", + title: "Connection Mode", + description: "Database access mode", + enum: ["read", "readwrite"], + default: "read", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], + }, + }, + required: ["path", "token", "schema_name"], +}; diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts new file mode 100644 index 00000000000..8a1f8dbb685 --- /dev/null +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -0,0 +1,93 @@ +import type { MultiStepFormSchema } from "./types"; + +export const pinotSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to Pinot", + "x-display": "radio", + "x-enum-labels": ["Connection Parameters", "Connection String"], + "x-enum-descriptions": [ + "Provide individual connection parameters (broker, controller, etc.)", + "Provide a complete Pinot connection string (DSN)", + ], + "x-grouped-fields": { + parameters: ["broker_host", "broker_port", "controller_host", "controller_port", "username", "password", "ssl"], + connection_string: ["dsn"], + }, + }, + broker_host: { + type: "string", + title: "Broker Host", + description: "Hostname or IP address of the Pinot broker server", + "x-placeholder": "localhost", + "x-visible-if": { auth_method: "parameters" }, + }, + broker_port: { + type: "number", + title: "Broker Port", + description: "Port number of the Pinot broker server", + "x-placeholder": "8000", + "x-visible-if": { auth_method: "parameters" }, + }, + controller_host: { + type: "string", + title: "Controller Host", + description: "Hostname or IP address of the Pinot controller server", + "x-placeholder": "localhost", + "x-visible-if": { auth_method: "parameters" }, + }, + controller_port: { + type: "number", + title: "Controller Port", + description: "Port number of the Pinot controller server", + "x-placeholder": "9000", + "x-visible-if": { auth_method: "parameters" }, + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the Pinot server (optional)", + "x-placeholder": "default", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the Pinot server (optional)", + "x-placeholder": "Enter password", + "x-secret": true, + "x-visible-if": { auth_method: "parameters" }, + }, + ssl: { + type: "boolean", + title: "Use SSL", + description: "Use SSL to connect to the Pinot server", + default: true, + "x-visible-if": { auth_method: "parameters" }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "Pinot connection string (DSN)", + "x-placeholder": "http(s)://username:password@localhost:8000?controller=localhost:9000", + "x-secret": true, + "x-visible-if": { auth_method: "connection_string" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["broker_host", "controller_host", "ssl"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; From 7a7477a4757011a8e7f77112aceb1e1bf7811f27 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 17:06:18 -0500 Subject: [PATCH 069/103] fix: simplify BigQuery and SQLite schemas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BigQuery: Remove unnecessary auth_method radio (only one auth method exists) - SQLite: Simplify to just local database path (not remote URL) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .../features/templates/schemas/bigquery.ts | 24 +------------- .../src/features/templates/schemas/sqlite.ts | 31 ++----------------- 2 files changed, 3 insertions(+), 52 deletions(-) diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts index 2f24a95b0f5..d64323110a9 100644 --- a/web-common/src/features/templates/schemas/bigquery.ts +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -4,21 +4,6 @@ export const bigquerySchema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", type: "object", properties: { - auth_method: { - type: "string", - title: "Authentication method", - enum: ["service_account"], - default: "service_account", - description: "Choose how to authenticate to BigQuery", - "x-display": "radio", - "x-enum-labels": ["Service Account"], - "x-enum-descriptions": [ - "Upload a JSON key file for a service account with BigQuery access.", - ], - "x-grouped-fields": { - service_account: ["google_application_credentials", "project_id"], - }, - }, google_application_credentials: { type: "string", title: "Service Account Key", @@ -26,20 +11,13 @@ export const bigquerySchema: MultiStepFormSchema = { format: "file", "x-display": "file", "x-accept": ".json", - "x-visible-if": { auth_method: "service_account" }, }, project_id: { type: "string", title: "Project ID", description: "Google Cloud project ID (optional if specified in credentials)", "x-placeholder": "my-project-id", - "x-visible-if": { auth_method: "service_account" }, }, }, - allOf: [ - { - if: { properties: { auth_method: { const: "service_account" } } }, - then: { required: ["google_application_credentials"] }, - }, - ], + required: ["google_application_credentials"], }; diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts index 1816cbc44f3..f66cd73232d 100644 --- a/web-common/src/features/templates/schemas/sqlite.ts +++ b/web-common/src/features/templates/schemas/sqlite.ts @@ -4,39 +4,12 @@ export const sqliteSchema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", type: "object", properties: { - auth_method: { - type: "string", - title: "Database location", - enum: ["local_file", "remote_url"], - default: "local_file", - description: "Choose where the SQLite database is located", - "x-display": "radio", - "x-enum-labels": ["Local File", "Remote URL"], - "x-enum-descriptions": [ - "Path to a local SQLite database file on disk.", - "URL to a remote SQLite database file.", - ], - "x-grouped-fields": { - local_file: ["db"], - remote_url: ["db"], - }, - }, db: { type: "string", title: "Database Path", - description: "Path or URL to the SQLite database file", + description: "Path to the SQLite database file", "x-placeholder": "/path/to/database.db", - "x-visible-if": { auth_method: "local_file" }, }, }, - allOf: [ - { - if: { properties: { auth_method: { const: "local_file" } } }, - then: { required: ["db"] }, - }, - { - if: { properties: { auth_method: { const: "remote_url" } } }, - then: { required: ["db"] }, - }, - ], + required: ["db"], }; From c3f3a7623884bc898a56c69ee613e2e0e4421337 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 17:07:17 -0500 Subject: [PATCH 070/103] fix: update BigQuery credentials field label to 'GCP Credentials' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- web-common/src/features/templates/schemas/bigquery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts index d64323110a9..1cdeeb366a3 100644 --- a/web-common/src/features/templates/schemas/bigquery.ts +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -6,7 +6,7 @@ export const bigquerySchema: MultiStepFormSchema = { properties: { google_application_credentials: { type: "string", - title: "Service Account Key", + title: "GCP Credentials", description: "Upload a JSON key file for a service account with BigQuery access", format: "file", "x-display": "file", From aaac4b25f1238e9ee95bbe582226447cea3fe3a2 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 17:08:12 -0500 Subject: [PATCH 071/103] fix: add table and source name fields to SQLite schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add table and name fields to SQLite schema with proper step separation: - db: connector step (database path) - table: source step (table name) - name: source step (source name) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .../src/features/templates/schemas/sqlite.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts index f66cd73232d..f0f9fbd5d25 100644 --- a/web-common/src/features/templates/schemas/sqlite.ts +++ b/web-common/src/features/templates/schemas/sqlite.ts @@ -9,7 +9,22 @@ export const sqliteSchema: MultiStepFormSchema = { title: "Database Path", description: "Path to the SQLite database file", "x-placeholder": "/path/to/database.db", + "x-step": "connector", + }, + table: { + type: "string", + title: "Table", + description: "SQLite table name", + "x-placeholder": "my_table", + "x-step": "source", + }, + name: { + type: "string", + title: "Source Name", + description: "Name for this source", + "x-placeholder": "my_sqlite_source", + "x-step": "source", }, }, - required: ["db"], + required: ["db", "table", "name"], }; From 806d23b948537abba768192425aae3476727cf68 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 17:15:02 -0500 Subject: [PATCH 072/103] fix: resolve multi-step connector validation and UI issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix several critical issues with multi-step connector forms: 1. **Enable validation for schemas without auth_method selector**: - BigQuery, Athena, Redshift now properly enable "Test and Connect" button - Added fallback path in isMultiStepConnectorDisabled() for schemas without radio enum - Check top-level required fields when no auth method selector exists 2. **Add x-step annotations to all connector fields**: - MySQL and PostgreSQL schemas now have "x-step": "connector" on all fields - HTTPS schema now has "x-step": "connector" on auth_method and headers - Matches pattern used by cloud storage connectors (S3, GCS, Azure) - Ensures proper validation and connector preview updates These changes fix: - Athena/BigQuery/Redshift button not enabling when fields filled - MySQL/PostgreSQL connection string fields not updating connector preview - HTTPS public auth method working like cloud storage connectors 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- web-common/src/features/sources/modal/utils.ts | 16 ++++++++++++++++ .../src/features/templates/schemas/https.ts | 2 ++ .../src/features/templates/schemas/mysql.ts | 8 ++++++++ .../src/features/templates/schemas/postgres.ts | 8 ++++++++ 4 files changed, 34 insertions(+) diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 4ba3427df79..1f1224a8b75 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -102,6 +102,22 @@ export function isMultiStepConnectorDisabled( if (!schema) return true; const authInfo = getRadioEnumOptions(schema); + + // Handle schemas without auth method radio selector (e.g., BigQuery, Athena) + if (!authInfo) { + const requiredFields = (schema.required ?? []).filter((fieldId) => + isStepMatch(schema, fieldId, "connector"), + ); + if (!requiredFields.length) return false; + + return !requiredFields.every((fieldId) => { + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); + } + const options = authInfo?.options ?? []; const authKey = authInfo?.key || findRadioEnumKey(schema); const methodFromForm = diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts index a10570e2d6b..bddd563e963 100644 --- a/web-common/src/features/templates/schemas/https.ts +++ b/web-common/src/features/templates/schemas/https.ts @@ -20,6 +20,7 @@ export const httpsSchema: MultiStepFormSchema = { public: [], headers: ["headers"], }, + "x-step": "connector", }, headers: { type: "string", @@ -28,6 +29,7 @@ export const httpsSchema: MultiStepFormSchema = { 'HTTP headers as JSON object. Example: {"Authorization": "Bearer my-token", "X-API-Key": "value"}', "x-placeholder": '{"Authorization": "Bearer my-token"}', "x-display": "textarea", + "x-step": "connector", "x-visible-if": { auth_method: "headers" }, }, }, diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts index f191a11c1b6..f6375c3d07d 100644 --- a/web-common/src/features/templates/schemas/mysql.ts +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -20,12 +20,14 @@ export const mysqlSchema: MultiStepFormSchema = { parameters: ["host", "port", "database", "user", "password", "sslmode"], connection_string: ["dsn"], }, + "x-step": "connector", }, host: { type: "string", title: "Host", description: "Database server hostname or IP address", "x-placeholder": "localhost", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, port: { @@ -34,6 +36,7 @@ export const mysqlSchema: MultiStepFormSchema = { description: "Database server port", default: 3306, "x-placeholder": "3306", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, database: { @@ -41,6 +44,7 @@ export const mysqlSchema: MultiStepFormSchema = { title: "Database", description: "Database name", "x-placeholder": "my_database", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, user: { @@ -48,6 +52,7 @@ export const mysqlSchema: MultiStepFormSchema = { title: "Username", description: "Database user", "x-placeholder": "root", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, password: { @@ -56,6 +61,7 @@ export const mysqlSchema: MultiStepFormSchema = { description: "Database password", "x-placeholder": "Enter password", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, sslmode: { @@ -65,6 +71,7 @@ export const mysqlSchema: MultiStepFormSchema = { enum: ["disabled", "preferred", "required", "verify_ca", "verify_identity"], default: "preferred", "x-display": "select", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, dsn: { @@ -73,6 +80,7 @@ export const mysqlSchema: MultiStepFormSchema = { description: "MySQL connection string (DSN)", "x-placeholder": "user:password@tcp(host:3306)/dbname?tls=preferred", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, }, diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts index 59b9bf44a76..bd5e688ba51 100644 --- a/web-common/src/features/templates/schemas/postgres.ts +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -20,12 +20,14 @@ export const postgresSchema: MultiStepFormSchema = { parameters: ["host", "port", "database", "user", "password", "sslmode"], connection_string: ["dsn"], }, + "x-step": "connector", }, host: { type: "string", title: "Host", description: "Database server hostname or IP address", "x-placeholder": "localhost", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, port: { @@ -34,6 +36,7 @@ export const postgresSchema: MultiStepFormSchema = { description: "Database server port", default: 5432, "x-placeholder": "5432", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, database: { @@ -41,6 +44,7 @@ export const postgresSchema: MultiStepFormSchema = { title: "Database", description: "Database name", "x-placeholder": "my_database", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, user: { @@ -48,6 +52,7 @@ export const postgresSchema: MultiStepFormSchema = { title: "Username", description: "Database user", "x-placeholder": "postgres", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, password: { @@ -56,6 +61,7 @@ export const postgresSchema: MultiStepFormSchema = { description: "Database password", "x-placeholder": "Enter password", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, sslmode: { @@ -65,6 +71,7 @@ export const postgresSchema: MultiStepFormSchema = { enum: ["disable", "require", "verify-ca", "verify-full"], default: "prefer", "x-display": "select", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, dsn: { @@ -73,6 +80,7 @@ export const postgresSchema: MultiStepFormSchema = { description: "PostgreSQL connection string (DSN)", "x-placeholder": "postgres://user:password@host:5432/dbname?sslmode=require", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, }, From 480c2812ded7021a1d0c5f8e1193bd131f6cac31 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 17:20:27 -0500 Subject: [PATCH 073/103] fix: revert SQLite to single-form source (not multi-step connector) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite should be a simple source creation form with all fields (db, table, name) in one UI, not split into connector and source steps. Changes: - Removed SQLite from MULTI_STEP_CONNECTORS array - Removed SQLite schema and registration - Reverted sqlite.go to have all fields in SourceProperties (no ConfigProperties) - Removed sqlite_connector Yup validation SQLite now works like local_file - a single form for source creation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- runtime/drivers/sqlite/sqlite.go | 7 ++--- .../sources/modal/connector-schemas.ts | 2 -- .../src/features/sources/modal/constants.ts | 1 - .../src/features/sources/modal/yupSchemas.ts | 6 ---- .../src/features/templates/schemas/sqlite.ts | 30 ------------------- 5 files changed, 2 insertions(+), 44 deletions(-) delete mode 100644 web-common/src/features/templates/schemas/sqlite.ts diff --git a/runtime/drivers/sqlite/sqlite.go b/runtime/drivers/sqlite/sqlite.go index c7af6223893..e48c6aa3556 100644 --- a/runtime/drivers/sqlite/sqlite.go +++ b/runtime/drivers/sqlite/sqlite.go @@ -85,18 +85,15 @@ func (d driver) Spec() drivers.Spec { DisplayName: "SQLite", Description: "Import data from SQLite into DuckDB.", DocsURL: "https://docs.rilldata.com/build/connectors/data-source/sqlite", - ConfigProperties: []*drivers.PropertySpec{ + SourceProperties: []*drivers.PropertySpec{ { Key: "db", Type: drivers.StringPropertyType, Required: true, DisplayName: "Database Path", - Description: "Path or URL to the SQLite database file", + Description: "Path to SQLite database file", Placeholder: "/path/to/database.db", - Hint: "Local file path or remote URL to the SQLite database", }, - }, - SourceProperties: []*drivers.PropertySpec{ { Key: "table", Type: drivers.StringPropertyType, diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index d4289b07501..48bf715c02b 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -13,7 +13,6 @@ import { postgresSchema } from "../../templates/schemas/postgres"; import { redshiftSchema } from "../../templates/schemas/redshift"; import { s3Schema } from "../../templates/schemas/s3"; import { snowflakeSchema } from "../../templates/schemas/snowflake"; -import { sqliteSchema } from "../../templates/schemas/sqlite"; export const multiStepFormSchemas: Record = { s3: s3Schema, @@ -22,7 +21,6 @@ export const multiStepFormSchemas: Record = { https: httpsSchema, postgres: postgresSchema, mysql: mysqlSchema, - sqlite: sqliteSchema, snowflake: snowflakeSchema, bigquery: bigquerySchema, redshift: redshiftSchema, diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index d31a985111b..62bf3995949 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -133,7 +133,6 @@ export const MULTI_STEP_CONNECTORS = [ "https", "postgres", "mysql", - "sqlite", "snowflake", "bigquery", "redshift", diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 94bcce2d630..eb76fbec783 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -151,12 +151,6 @@ export const getYupSchema = { dsn: yup.string().optional(), }), - // Keep base auth fields optional; per-method required fields come from - // multi-step auth configs. This schema is a safe fallback. - sqlite_connector: yup.object().shape({ - db: yup.string().optional(), - }), - // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. snowflake_connector: yup.object().shape({ diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts deleted file mode 100644 index f0f9fbd5d25..00000000000 --- a/web-common/src/features/templates/schemas/sqlite.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { MultiStepFormSchema } from "./types"; - -export const sqliteSchema: MultiStepFormSchema = { - $schema: "http://json-schema.org/draft-07/schema#", - type: "object", - properties: { - db: { - type: "string", - title: "Database Path", - description: "Path to the SQLite database file", - "x-placeholder": "/path/to/database.db", - "x-step": "connector", - }, - table: { - type: "string", - title: "Table", - description: "SQLite table name", - "x-placeholder": "my_table", - "x-step": "source", - }, - name: { - type: "string", - title: "Source Name", - description: "Name for this source", - "x-placeholder": "my_sqlite_source", - "x-step": "source", - }, - }, - required: ["db", "table", "name"], -}; From af1743763696f003a1a0a3356ac94c341d2e71c0 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 17:22:26 -0500 Subject: [PATCH 074/103] fix: add defensive check in validation to handle null form values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add paramsFormValue null check and improve comment clarity in isMultiStepConnectorDisabled. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- web-common/src/features/sources/modal/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 1f1224a8b75..11c9db9d3a1 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -99,7 +99,7 @@ export function isMultiStepConnectorDisabled( paramsFormValue: Record, paramsFormErrors: Record, ) { - if (!schema) return true; + if (!schema || !paramsFormValue) return true; const authInfo = getRadioEnumOptions(schema); @@ -141,8 +141,11 @@ export function isMultiStepConnectorDisabled( step: "connector", }); const requiredFields = requiredByMethod[method] ?? []; + + // If no required fields found, button should be disabled (something is wrong) if (!requiredFields.length) return true; + // Check if all required fields are filled and have no errors return !requiredFields.every((fieldId) => { if (!isStepMatch(schema, fieldId, "connector")) return true; const value = paramsFormValue[fieldId]; From 574f18b3b937e9718f97c0bb43139bc101fe80b8 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 18:35:43 -0500 Subject: [PATCH 075/103] commit --- runtime/drivers/clickhouse/clickhouse.go | 1 - runtime/drivers/duckdb/duckdb.go | 2 - .../sources/modal/AddDataFormManager.ts | 43 ++++++++- .../sources/modal/AddDataModal.svelte | 3 +- .../modal/JSONSchemaFieldControl.svelte | 15 ++++ .../modal/MultiStepConnectorFlow.svelte | 21 ++--- .../src/features/sources/modal/utils.ts | 35 ++++++-- .../src/features/sources/modal/yupSchemas.ts | 90 +++++++++++++++++++ .../src/features/sources/sourceUtils.ts | 35 ++++---- .../templates/JSONSchemaFormRenderer.svelte | 82 ++++++++++++++++- .../src/features/templates/schemas/athena.ts | 42 ++++++++- .../features/templates/schemas/bigquery.ts | 20 ++++- .../src/features/templates/schemas/druid.ts | 27 +++++- .../src/features/templates/schemas/duckdb.ts | 24 ++++- .../src/features/templates/schemas/https.ts | 23 ++++- .../features/templates/schemas/motherduck.ts | 22 ++++- .../src/features/templates/schemas/mysql.ts | 17 +++- .../src/features/templates/schemas/pinot.ts | 29 +++++- .../features/templates/schemas/postgres.ts | 22 ++++- .../features/templates/schemas/redshift.ts | 24 ++++- .../src/features/templates/schemas/s3.ts | 22 +++++ .../features/templates/schemas/snowflake.ts | 33 ++++++- .../src/features/templates/schemas/types.ts | 1 + 23 files changed, 571 insertions(+), 62 deletions(-) diff --git a/runtime/drivers/clickhouse/clickhouse.go b/runtime/drivers/clickhouse/clickhouse.go index 432fce80ef9..0ad2879ea3a 100644 --- a/runtime/drivers/clickhouse/clickhouse.go +++ b/runtime/drivers/clickhouse/clickhouse.go @@ -58,7 +58,6 @@ var spec = drivers.Spec{ Description: "Set the mode for the ClickHouse connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, - NoPrompt: true, }, { Key: "dsn", diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 12ee6fbaaf0..9317638f0bd 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -64,7 +64,6 @@ var spec = drivers.Spec{ Description: "Set the mode for the DuckDB connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, - NoPrompt: true, }, }, SourceProperties: []*drivers.PropertySpec{ @@ -110,7 +109,6 @@ var motherduckSpec = drivers.Spec{ Description: "Set the mode for the DuckDB connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, - NoPrompt: true, }, { Key: "schema_name", diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 0c8a9933fe4..a5af0d14cb3 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -32,12 +32,15 @@ import { import { get } from "svelte/store"; import { compileConnectorYAML } from "../../connectors/code-utils"; import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; -import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-client"; +import { + ConnectorDriverPropertyType, + type ConnectorDriverProperty, +} from "@rilldata/web-common/runtime-client"; import type { ClickHouseConnectorType } from "./constants"; import { applyClickHouseCloudRequirements } from "./utils"; import type { ActionResult } from "@sveltejs/kit"; import { getConnectorSchema } from "./connector-schemas"; -import { findRadioEnumKey } from "../../templates/schema-utils"; +import { findRadioEnumKey, isVisibleForValues } from "../../templates/schema-utils"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { @@ -502,10 +505,38 @@ export class AddDataFormManager { : filteredParamsProperties; const getConnectorYamlPreview = (values: Record) => { - const orderedProperties = + let orderedProperties = onlyDsn || connectionTab === "dsn" ? filteredDsnProperties : connectorPropertiesForPreview; + + // For multi-step connectors with schemas, build properties from schema + if (isMultiStepConnector && stepState?.step === "connector") { + const schema = getConnectorSchema(connector.name); + if (schema) { + // Build ordered properties from schema fields that are visible and on connector step + const schemaProperties: ConnectorDriverProperty[] = []; + const properties = schema.properties ?? {}; + + for (const [key, prop] of Object.entries(properties)) { + // Only include connector step fields that are currently visible + if (prop["x-step"] === "connector" && isVisibleForValues(schema, key, values)) { + schemaProperties.push({ + key, + type: prop.type === "number" + ? ConnectorDriverPropertyType.TYPE_NUMBER + : prop.type === "boolean" + ? ConnectorDriverPropertyType.TYPE_BOOLEAN + : ConnectorDriverPropertyType.TYPE_STRING, + secret: prop["x-secret"] || false, + }); + } + } + + orderedProperties = schemaProperties; + } + } + return compileConnectorYAML(connector, values, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; @@ -557,6 +588,12 @@ export class AddDataFormManager { connector, filteredValues, ); + + // For multi-step connectors on source step, always show model YAML + if (isMultiStepConnector && stepState?.step === "source") { + return compileSourceYAML(rewrittenConnector, rewrittenFormValues); + } + const isRewrittenToDuckDb = rewrittenConnector.name === "duckdb"; if (isRewrittenToDuckDb) { return compileSourceYAML(rewrittenConnector, rewrittenFormValues); diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 289cdf82720..3b30a2dc874 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -21,7 +21,7 @@ import DuplicateSource from "./DuplicateSource.svelte"; import LocalSourceUpload from "./LocalSourceUpload.svelte"; import RequestConnectorForm from "./RequestConnectorForm.svelte"; - import { OLAP_ENGINES, ALL_CONNECTORS, SOURCES } from "./constants"; + import { OLAP_ENGINES, ALL_CONNECTORS, SOURCES, MULTI_STEP_CONNECTORS } from "./constants"; import { ICONS } from "./icons"; import { resetConnectorStep } from "./connectorStepStore"; @@ -121,6 +121,7 @@ // FIXME: excluding salesforce until we implement the table discovery APIs $: isConnectorType = + MULTI_STEP_CONNECTORS.includes(selectedConnector?.name ?? "") || selectedConnector?.implementsObjectStore || selectedConnector?.implementsOlap || selectedConnector?.implementsSqlStore || diff --git a/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte index a7c47942c04..b2d712374eb 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte @@ -2,6 +2,7 @@ import Input from "@rilldata/web-common/components/forms/Input.svelte"; import Checkbox from "@rilldata/web-common/components/forms/Checkbox.svelte"; import Radio from "@rilldata/web-common/components/forms/Radio.svelte"; + import Select from "@rilldata/web-common/components/forms/Select.svelte"; import CredentialsInput from "@rilldata/web-common/components/forms/CredentialsInput.svelte"; import { normalizeErrors } from "./utils"; import type { JSONSchemaField } from "./types"; @@ -18,6 +19,9 @@ | Array<{ value: string; label: string; description?: string }> | undefined; export let name: string | undefined; + + $: isSelectEnum = prop.enum && prop["x-display"] === "select"; + $: isTextarea = prop["x-display"] === "textarea"; {#if prop["x-display"] === "file" || prop.format === "file"} @@ -39,6 +43,15 @@ /> {:else if options?.length} +{:else if isSelectEnum && options} + onStringInputChange(e)} alwaysShowError diff --git a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte index 49f1c06a473..0c0babc807e 100644 --- a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte +++ b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte @@ -76,19 +76,13 @@ } // Restore defaults (and persisted auth) when returning to connector step. - $: if (stepState.step === "connector") { + // For schema-based connectors, JSONSchemaFormRenderer handles defaults + $: if (stepState.step === "connector" && !activeSchema) { paramsForm.update( ($current) => { const base = getInitialFormValuesFromProperties( connector.configProperties ?? [], ); - if (activeSchema) { - const authKey = findRadioEnumKey(activeSchema); - const persisted = stepState.selectedAuthMethod; - if (authKey && persisted) { - base[authKey] = persisted; - } - } return { ...base, ...$current }; }, { taint: false }, @@ -102,7 +96,7 @@ $: activeAuthInfo = activeSchema ? getRadioEnumOptions(activeSchema) : null; // Ensure we always have a valid auth method selection for the active schema. - $: if (activeSchema && activeAuthInfo) { + $: if (activeSchema && activeAuthInfo && stepState.step === "connector") { const options = activeAuthInfo.options ?? []; const fallback = activeAuthInfo.defaultValue || options[0]?.value || null; const authKey = activeAuthInfo.key || findRadioEnumKey(activeSchema); @@ -114,13 +108,15 @@ setAuthMethod(fallback ?? null); if (fallback && authKey) { paramsForm.update(($form) => { - if ($form[authKey] !== fallback) $form[authKey] = fallback; + if ($form[authKey] !== fallback) { + $form[authKey] = fallback; + } return $form; - }); + }, { taint: false }); } } } - } else if (stepState.selectedAuthMethod) { + } else if (stepState.selectedAuthMethod && !activeAuthInfo) { setAuthMethod(null); } @@ -151,6 +147,7 @@ activeSchema, $paramsForm, $paramsErrors, + stepState.step, ); $: primaryButtonLabel = formManager.getPrimaryButtonLabel({ isConnectorForm: formManager.isConnectorForm, diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 11c9db9d3a1..ac9eb28448f 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -98,6 +98,7 @@ export function isMultiStepConnectorDisabled( schema: MultiStepFormSchema | null, paramsFormValue: Record, paramsFormErrors: Record, + currentStep: "connector" | "source" = "connector", ) { if (!schema || !paramsFormValue) return true; @@ -106,7 +107,7 @@ export function isMultiStepConnectorDisabled( // Handle schemas without auth method radio selector (e.g., BigQuery, Athena) if (!authInfo) { const requiredFields = (schema.required ?? []).filter((fieldId) => - isStepMatch(schema, fieldId, "connector"), + isStepMatch(schema, fieldId, currentStep), ); if (!requiredFields.length) return false; @@ -137,17 +138,41 @@ export function isMultiStepConnectorDisabled( // Selecting "public" should always enable the button for multi-step auth flows. if (method === "public") return false; + // When on source step and auth method isn't set (user skipped connector step), + // validate all source step required fields regardless of auth method + if (currentStep === "source" && !methodFromForm) { + const allSourceRequired = new Set(); + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { + step: currentStep, + }); + + // Collect all source step required fields across all auth methods + for (const fields of Object.values(requiredByMethod)) { + fields.forEach((field) => allSourceRequired.add(field)); + } + + const sourceRequiredFields = Array.from(allSourceRequired); + if (!sourceRequiredFields.length) return false; + + return !sourceRequiredFields.every((fieldId) => { + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); + } + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { - step: "connector", + step: currentStep, }); const requiredFields = requiredByMethod[method] ?? []; - // If no required fields found, button should be disabled (something is wrong) - if (!requiredFields.length) return true; + // If no required fields found for this step, button should be enabled + if (!requiredFields.length) return false; // Check if all required fields are filled and have no errors return !requiredFields.every((fieldId) => { - if (!isStepMatch(schema, fieldId, "connector")) return true; + if (!isStepMatch(schema, fieldId, currentStep)) return true; const value = paramsFormValue[fieldId]; const errorsForField = paramsFormErrors[fieldId] as any; const hasErrors = Boolean(errorsForField?.length); diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index eb76fbec783..afa445d0811 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -79,12 +79,38 @@ export const getYupSchema = { duckdb: yup.object().shape({ path: yup.string().required("path is required"), attach: yup.string().optional(), + sql: yup.string().optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .optional(), + }), + + duckdb_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), }), motherduck: yup.object().shape({ token: yup.string().required("Token is required"), path: yup.string().required("Path is required"), schema_name: yup.string().required("Schema name is required"), + sql: yup.string().optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .optional(), + }), + + motherduck_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), }), sqlite: yup.object().shape({ @@ -139,6 +165,14 @@ export const getYupSchema = { dsn: yup.string().optional(), }), + postgres_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. mysql_connector: yup.object().shape({ @@ -151,6 +185,14 @@ export const getYupSchema = { dsn: yup.string().optional(), }), + mysql_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. snowflake_connector: yup.object().shape({ @@ -166,6 +208,14 @@ export const getYupSchema = { dsn: yup.string().optional(), }), + snowflake_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. bigquery_connector: yup.object().shape({ @@ -173,6 +223,14 @@ export const getYupSchema = { project_id: yup.string().optional(), }), + bigquery_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. redshift_connector: yup.object().shape({ @@ -184,6 +242,14 @@ export const getYupSchema = { cluster_identifier: yup.string().optional(), }), + redshift_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. athena_connector: yup.object().shape({ @@ -195,6 +261,14 @@ export const getYupSchema = { output_location: yup.string().optional(), }), + athena_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. duckdb_connector: yup.object().shape({ @@ -223,6 +297,14 @@ export const getYupSchema = { dsn: yup.string().optional(), }), + druid_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + // Keep base auth fields optional; per-method required fields come from // multi-step auth configs. This schema is a safe fallback. pinot_connector: yup.object().shape({ @@ -236,6 +318,14 @@ export const getYupSchema = { dsn: yup.string().optional(), }), + pinot_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 480709142f6..a005f94ac2a 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -57,11 +57,12 @@ export function compileSourceYAML( } if (key === "sql") { - // For SQL, we want to use a multi-line string - return `${key}: |\n ${value + // For SQL, we want to use a multi-line string and add a dev section + const sqlLines = value .split("\n") - .map((line) => `${line}`) - .join("\n")}`; + .map((line) => ` ${line}`) + .join("\n"); + return `${key}: |\n${sqlLines}\n\ndev:\n ${key}: |\n${sqlLines}\n limit 10000`; } const isStringProperty = stringPropertyKeys.includes(key); @@ -212,36 +213,38 @@ export function prepareSourceFormData( // Create a copy of form values to avoid mutating the original const processedValues = { ...formValues }; + // Apply DuckDB rewrite logic FIRST (before stripping connector properties) + // This is important for connectors like SQLite that need connector properties + // to build the SQL query before they're removed. + const [rewrittenConnector, rewrittenFormValues] = maybeRewriteToDuckDb( + connector, + processedValues, + ); + // Strip connector configuration keys from the source form values to prevent // leaking connector-level fields (e.g., credentials) into the model file. if (connector.configProperties) { const connectorPropertyKeys = new Set( connector.configProperties.map((p) => p.key).filter(Boolean), ); - for (const key of Object.keys(processedValues)) { + for (const key of Object.keys(rewrittenFormValues)) { if (connectorPropertyKeys.has(key)) { - delete processedValues[key]; + delete rewrittenFormValues[key]; } } } // Handle placeholder values for required source properties - if (connector.sourceProperties) { - for (const prop of connector.sourceProperties) { - if (prop.key && prop.required && !(prop.key in processedValues)) { + if (rewrittenConnector.sourceProperties) { + for (const prop of rewrittenConnector.sourceProperties) { + if (prop.key && prop.required && !(prop.key in rewrittenFormValues)) { if (prop.placeholder) { - processedValues[prop.key] = prop.placeholder; + rewrittenFormValues[prop.key] = prop.placeholder; } } } } - // Apply DuckDB rewrite logic - const [rewrittenConnector, rewrittenFormValues] = maybeRewriteToDuckDb( - connector, - processedValues, - ); - return [rewrittenConnector, rewrittenFormValues]; } diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte index 525169d0430..85f787134e1 100644 --- a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -35,6 +35,12 @@ $: renderOrder = schema ? computeRenderOrder(visibleEntries, groupedChildKeys) : []; + $: regularFields = renderOrder.filter( + ([_, prop]) => !prop["x-advanced"], + ); + $: advancedFields = renderOrder.filter( + ([_, prop]) => prop["x-advanced"], + ); // Seed defaults for initial render: use explicit defaults, and for radio enums // fall back to first option when no value is set. @@ -93,6 +99,14 @@ return Boolean(prop.enum && prop["x-display"] === radioDisplay); } + function isSelectEnum(prop: JSONSchemaField) { + return Boolean(prop.enum && prop["x-display"] === "select"); + } + + function hasEnumOptions(prop: JSONSchemaField) { + return isRadioEnum(prop) || isSelectEnum(prop); + } + function computeVisibleEntries( currentSchema: MultiStepFormSchema, currentStep: string | undefined, @@ -217,7 +231,7 @@ {#if schema} - {#each renderOrder as [key, prop]} + {#each regularFields as [key, prop]} {#if isRadioEnum(prop)}
{#if prop.title} @@ -241,7 +255,7 @@ bind:checked={$form[childKey]} {onStringInputChange} {handleFileUpload} - options={isRadioEnum(childProp) + options={hasEnumOptions(childProp) ? radioOptions(childProp) : undefined} name={`${childKey}-radio`} @@ -263,10 +277,72 @@ bind:checked={$form[key]} {onStringInputChange} {handleFileUpload} - options={isRadioEnum(prop) ? radioOptions(prop) : undefined} + options={hasEnumOptions(prop) ? radioOptions(prop) : undefined} name={`${key}-radio`} />
{/if} {/each} + + {#if advancedFields.length > 0} +
+ + Advanced Configuration + +
+ {#each advancedFields as [key, prop]} + {#if isRadioEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + + {#if groupedFields.get(key)} + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} +
+ +
+ {/each} + {/if} + + +
+ {:else} +
+ +
+ {/if} + {/each} +
+
+ {/if} {/if} diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts index b50aca2538e..6a6aefe0596 100644 --- a/web-common/src/features/templates/schemas/athena.ts +++ b/web-common/src/features/templates/schemas/athena.ts @@ -10,6 +10,7 @@ export const athenaSchema: MultiStepFormSchema = { description: "AWS access key ID", "x-placeholder": "Enter AWS access key ID", "x-secret": true, + "x-step": "connector", }, aws_secret_access_key: { type: "string", @@ -17,31 +18,70 @@ export const athenaSchema: MultiStepFormSchema = { description: "AWS secret access key", "x-placeholder": "Enter AWS secret access key", "x-secret": true, + "x-step": "connector", }, output_location: { type: "string", title: "S3 Output Location", description: "S3 URI for query results", "x-placeholder": "s3://my-bucket/athena-results/", + "x-step": "connector", }, aws_role_arn: { type: "string", title: "IAM Role ARN (Optional)", description: "AWS IAM role ARN to assume (optional)", "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", + "x-step": "connector", + }, + role_session_name: { + type: "string", + title: "Role Session Name", + description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + "x-placeholder": "my-session-name", + "x-secret": true, + "x-step": "connector", + "x-advanced": true, + }, + external_id: { + type: "string", + title: "External ID", + description: "Optional external ID to use when assuming an AWS role for cross-account access.", + "x-placeholder": "external-id-123", + "x-secret": true, + "x-step": "connector", + "x-advanced": true, }, region: { type: "string", title: "AWS Region", description: "AWS region where Athena is configured", "x-placeholder": "us-east-1", + "x-step": "connector", }, workgroup: { type: "string", title: "Workgroup", description: "Athena workgroup name (optional)", "x-placeholder": "primary", + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Athena", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", }, }, - required: ["aws_access_key_id", "aws_secret_access_key", "output_location"], + required: ["aws_access_key_id", "aws_secret_access_key", "output_location", "sql", "name"], }; diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts index 1cdeeb366a3..a5b0b1beac9 100644 --- a/web-common/src/features/templates/schemas/bigquery.ts +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -11,13 +11,31 @@ export const bigquerySchema: MultiStepFormSchema = { format: "file", "x-display": "file", "x-accept": ".json", + "x-step": "connector", }, project_id: { type: "string", title: "Project ID", description: "Google Cloud project ID (optional if specified in credentials)", "x-placeholder": "my-project-id", + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from BigQuery", + "x-placeholder": "SELECT * FROM `project.dataset.table`;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", }, }, - required: ["google_application_credentials"], + required: ["google_application_credentials", "sql", "name"], }; diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts index c6ef00f2435..68a98d132d9 100644 --- a/web-common/src/features/templates/schemas/druid.ts +++ b/web-common/src/features/templates/schemas/druid.ts @@ -20,12 +20,14 @@ export const druidSchema: MultiStepFormSchema = { parameters: ["host", "port", "username", "password", "ssl"], connection_string: ["dsn"], }, + "x-step": "connector", }, host: { type: "string", title: "Host", description: "Hostname or IP address of the Druid server", "x-placeholder": "localhost", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, port: { @@ -33,6 +35,7 @@ export const druidSchema: MultiStepFormSchema = { title: "Port", description: "Port number of the Druid server", "x-placeholder": "8888", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, username: { @@ -40,6 +43,7 @@ export const druidSchema: MultiStepFormSchema = { title: "Username", description: "Username to connect to the Druid server (optional)", "x-placeholder": "default", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, password: { @@ -48,6 +52,7 @@ export const druidSchema: MultiStepFormSchema = { description: "Password to connect to the Druid server (optional)", "x-placeholder": "Enter password", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, ssl: { @@ -55,6 +60,7 @@ export const druidSchema: MultiStepFormSchema = { title: "Use SSL", description: "Use SSL to connect to the Druid server", default: true, + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, dsn: { @@ -63,17 +69,34 @@ export const druidSchema: MultiStepFormSchema = { description: "Druid connection string (DSN)", "x-placeholder": "https://example.com/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=username&avaticaPassword=password", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Druid", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, }, allOf: [ { if: { properties: { auth_method: { const: "parameters" } } }, - then: { required: ["host", "ssl"] }, + then: { required: ["host", "ssl", "sql", "name"] }, }, { if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["dsn"] }, + then: { required: ["dsn", "sql", "name"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts index cbb6f3f0b1b..30f929be0d9 100644 --- a/web-common/src/features/templates/schemas/duckdb.ts +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -9,12 +9,15 @@ export const duckdbSchema: MultiStepFormSchema = { title: "Database Path", description: "Path to external DuckDB database file", "x-placeholder": "/path/to/main.db", + "x-step": "connector", }, attach: { type: "string", - title: "Attach (Advanced)", + title: "Attach", description: "Attach to an existing DuckDB database with options (alternative to path)", "x-placeholder": "'ducklake:metadata.ducklake' AS my_ducklake(DATA_PATH 'datafiles')", + "x-step": "connector", + "x-advanced": true, }, mode: { type: "string", @@ -28,7 +31,24 @@ export const duckdbSchema: MultiStepFormSchema = { "Only read operations are allowed (recommended for security)", "Enable model creation and table mutations", ], + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from DuckDB", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", }, }, - required: ["path"], + required: ["path", "sql", "name"], }; diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts index bddd563e963..ace8518400f 100644 --- a/web-common/src/features/templates/schemas/https.ts +++ b/web-common/src/features/templates/schemas/https.ts @@ -17,7 +17,6 @@ export const httpsSchema: MultiStepFormSchema = { "Provide custom HTTP headers for authentication (e.g., Authorization, API keys).", ], "x-grouped-fields": { - public: [], headers: ["headers"], }, "x-step": "connector", @@ -32,11 +31,31 @@ export const httpsSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "headers" }, }, + path: { + type: "string", + title: "URL", + description: "HTTP(S) URL to fetch data from", + pattern: "^https?://", + "x-placeholder": "https://api.example.com/data", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, }, allOf: [ { if: { properties: { auth_method: { const: "headers" } } }, - then: { required: ["headers"] }, + then: { required: ["headers", "path", "name"] }, + }, + { + if: { properties: { auth_method: { const: "public" } } }, + then: { required: ["path", "name"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/motherduck.ts b/web-common/src/features/templates/schemas/motherduck.ts index 23a0ffd3625..4f425e3401c 100644 --- a/web-common/src/features/templates/schemas/motherduck.ts +++ b/web-common/src/features/templates/schemas/motherduck.ts @@ -9,6 +9,7 @@ export const motherduckSchema: MultiStepFormSchema = { title: "Database Path", description: "Path to MotherDuck database (must be prefixed with 'md:')", "x-placeholder": "md:my_db", + "x-step": "connector", }, token: { type: "string", @@ -16,12 +17,14 @@ export const motherduckSchema: MultiStepFormSchema = { description: "Your MotherDuck authentication token", "x-placeholder": "Enter your MotherDuck token", "x-secret": true, + "x-step": "connector", }, schema_name: { type: "string", title: "Schema Name", description: "Default schema used by the MotherDuck database", "x-placeholder": "main", + "x-step": "connector", }, mode: { type: "string", @@ -35,7 +38,24 @@ export const motherduckSchema: MultiStepFormSchema = { "Only read operations are allowed (recommended for security)", "Enable model creation and table mutations", ], + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from MotherDuck", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", }, }, - required: ["path", "token", "schema_name"], + required: ["path", "token", "schema_name", "sql", "name"], }; diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts index f6375c3d07d..c9cb531feb8 100644 --- a/web-common/src/features/templates/schemas/mysql.ts +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -68,7 +68,7 @@ export const mysqlSchema: MultiStepFormSchema = { type: "string", title: "SSL Mode", description: "SSL connection mode", - enum: ["disabled", "preferred", "required", "verify_ca", "verify_identity"], + enum: ["disabled", "preferred", "required"], default: "preferred", "x-display": "select", "x-step": "connector", @@ -83,6 +83,21 @@ export const mysqlSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from MySQL", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, }, allOf: [ { diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts index 8a1f8dbb685..75dfb899533 100644 --- a/web-common/src/features/templates/schemas/pinot.ts +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -20,12 +20,14 @@ export const pinotSchema: MultiStepFormSchema = { parameters: ["broker_host", "broker_port", "controller_host", "controller_port", "username", "password", "ssl"], connection_string: ["dsn"], }, + "x-step": "connector", }, broker_host: { type: "string", title: "Broker Host", description: "Hostname or IP address of the Pinot broker server", "x-placeholder": "localhost", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, broker_port: { @@ -33,6 +35,7 @@ export const pinotSchema: MultiStepFormSchema = { title: "Broker Port", description: "Port number of the Pinot broker server", "x-placeholder": "8000", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, controller_host: { @@ -40,6 +43,7 @@ export const pinotSchema: MultiStepFormSchema = { title: "Controller Host", description: "Hostname or IP address of the Pinot controller server", "x-placeholder": "localhost", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, controller_port: { @@ -47,6 +51,7 @@ export const pinotSchema: MultiStepFormSchema = { title: "Controller Port", description: "Port number of the Pinot controller server", "x-placeholder": "9000", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, username: { @@ -54,6 +59,7 @@ export const pinotSchema: MultiStepFormSchema = { title: "Username", description: "Username to connect to the Pinot server (optional)", "x-placeholder": "default", + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, password: { @@ -62,6 +68,7 @@ export const pinotSchema: MultiStepFormSchema = { description: "Password to connect to the Pinot server (optional)", "x-placeholder": "Enter password", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, ssl: { @@ -69,6 +76,7 @@ export const pinotSchema: MultiStepFormSchema = { title: "Use SSL", description: "Use SSL to connect to the Pinot server", default: true, + "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, dsn: { @@ -77,17 +85,34 @@ export const pinotSchema: MultiStepFormSchema = { description: "Pinot connection string (DSN)", "x-placeholder": "http(s)://username:password@localhost:8000?controller=localhost:9000", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Pinot", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, }, allOf: [ { if: { properties: { auth_method: { const: "parameters" } } }, - then: { required: ["broker_host", "controller_host", "ssl"] }, + then: { required: ["broker_host", "controller_host", "ssl", "sql", "name"] }, }, { if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["dsn"] }, + then: { required: ["dsn", "sql", "name"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts index bd5e688ba51..6fd18e66620 100644 --- a/web-common/src/features/templates/schemas/postgres.ts +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -68,7 +68,7 @@ export const postgresSchema: MultiStepFormSchema = { type: "string", title: "SSL Mode", description: "SSL connection mode", - enum: ["disable", "require", "verify-ca", "verify-full"], + enum: ["disable", "allow", "prefer", "require"], default: "prefer", "x-display": "select", "x-step": "connector", @@ -83,15 +83,31 @@ export const postgresSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from PostgreSQL", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, }, allOf: [ { if: { properties: { auth_method: { const: "parameters" } } }, - then: { required: ["host", "database", "user", "password"] }, + then: { required: ["host", "database", "user", "password", "sql", "name"] }, }, { if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["dsn"] }, + then: { required: ["dsn", "sql", "name"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts index fd2b170fe74..f746efac156 100644 --- a/web-common/src/features/templates/schemas/redshift.ts +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -10,6 +10,7 @@ export const redshiftSchema: MultiStepFormSchema = { description: "AWS access key ID for Redshift Data API", "x-placeholder": "Enter AWS access key ID", "x-secret": true, + "x-step": "connector", }, aws_secret_access_key: { type: "string", @@ -17,24 +18,28 @@ export const redshiftSchema: MultiStepFormSchema = { description: "AWS secret access key for Redshift Data API", "x-placeholder": "Enter AWS secret access key", "x-secret": true, + "x-step": "connector", }, region: { type: "string", title: "AWS Region", description: "AWS region where the Redshift cluster is located", "x-placeholder": "us-east-1", + "x-step": "connector", }, database: { type: "string", title: "Database", description: "Redshift database name", "x-placeholder": "dev", + "x-step": "connector", }, workgroup: { type: "string", title: "Workgroup", description: "Redshift Serverless workgroup name (for serverless)", "x-placeholder": "default-workgroup", + "x-step": "connector", }, cluster_identifier: { type: "string", @@ -42,7 +47,24 @@ export const redshiftSchema: MultiStepFormSchema = { description: "Redshift provisioned cluster identifier (for provisioned clusters)", "x-placeholder": "my-redshift-cluster", "x-hint": "Provide either workgroup (for serverless) or cluster identifier (for provisioned)", + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Redshift", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", }, }, - required: ["aws_access_key_id", "aws_secret_access_key", "database"], + required: ["aws_access_key_id", "aws_secret_access_key", "database", "sql", "name"], }; diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index 4f20794d307..e84a9147221 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -23,6 +23,8 @@ export const s3Schema: MultiStepFormSchema = { "region", "endpoint", "aws_role_arn", + "aws_role_session_name", + "aws_external_id", ], public: [], }, @@ -73,6 +75,26 @@ export const s3Schema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "access_keys" }, }, + aws_role_session_name: { + type: "string", + title: "Role Session Name", + description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + "x-placeholder": "my-session-name", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, + }, + aws_external_id: { + type: "string", + title: "External ID", + description: "Optional external ID to use when assuming an AWS role for cross-account access.", + "x-placeholder": "external-id-123", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, + }, path: { type: "string", title: "S3 URI", diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index e0e03cef786..6a2ea70671f 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -22,12 +22,14 @@ export const snowflakeSchema: MultiStepFormSchema = { keypair: ["account", "user", "private_key", "private_key_passphrase", "warehouse", "database", "schema", "role"], connection_string: ["dsn"], }, + "x-step": "connector", }, account: { type: "string", title: "Account", description: "Snowflake account identifier (e.g., abc12345.us-east-1)", "x-placeholder": "abc12345.us-east-1", + "x-step": "connector", "x-visible-if": { auth_method: ["password", "keypair"] }, }, user: { @@ -35,6 +37,7 @@ export const snowflakeSchema: MultiStepFormSchema = { title: "Username", description: "Snowflake username", "x-placeholder": "Enter username", + "x-step": "connector", "x-visible-if": { auth_method: ["password", "keypair"] }, }, password: { @@ -43,6 +46,7 @@ export const snowflakeSchema: MultiStepFormSchema = { description: "Snowflake password", "x-placeholder": "Enter password", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "password" }, }, private_key: { @@ -52,6 +56,7 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-placeholder": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----", "x-display": "textarea", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "keypair" }, }, private_key_passphrase: { @@ -60,6 +65,7 @@ export const snowflakeSchema: MultiStepFormSchema = { description: "Optional passphrase for encrypted private key", "x-placeholder": "Enter passphrase if key is encrypted", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "keypair" }, }, warehouse: { @@ -67,6 +73,7 @@ export const snowflakeSchema: MultiStepFormSchema = { title: "Warehouse", description: "Snowflake warehouse name", "x-placeholder": "COMPUTE_WH", + "x-step": "connector", "x-visible-if": { auth_method: ["password", "keypair"] }, }, database: { @@ -74,6 +81,7 @@ export const snowflakeSchema: MultiStepFormSchema = { title: "Database", description: "Snowflake database name", "x-placeholder": "MY_DATABASE", + "x-step": "connector", "x-visible-if": { auth_method: ["password", "keypair"] }, }, schema: { @@ -81,6 +89,7 @@ export const snowflakeSchema: MultiStepFormSchema = { title: "Schema", description: "Snowflake schema name", "x-placeholder": "PUBLIC", + "x-step": "connector", "x-visible-if": { auth_method: ["password", "keypair"] }, }, role: { @@ -88,6 +97,7 @@ export const snowflakeSchema: MultiStepFormSchema = { title: "Role", description: "Optional Snowflake role to assume", "x-placeholder": "ANALYST", + "x-step": "connector", "x-visible-if": { auth_method: ["password", "keypair"] }, }, dsn: { @@ -96,21 +106,38 @@ export const snowflakeSchema: MultiStepFormSchema = { description: "Snowflake connection string (DSN)", "x-placeholder": "user:password@account/database/schema?warehouse=wh", "x-secret": true, + "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Snowflake", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, }, allOf: [ { if: { properties: { auth_method: { const: "password" } } }, - then: { required: ["account", "user", "password"] }, + then: { required: ["account", "user", "password", "sql", "name"] }, }, { if: { properties: { auth_method: { const: "keypair" } } }, - then: { required: ["account", "user", "private_key"] }, + then: { required: ["account", "user", "private_key", "sql", "name"] }, }, { if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["dsn"] }, + then: { required: ["dsn", "sql", "name"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index 9c1fafbc464..2bc367eeb69 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -24,6 +24,7 @@ export type JSONSchemaField = { "x-placeholder"?: string; "x-hint"?: string; "x-accept"?: string; + "x-advanced"?: boolean; /** * Explicit grouping for radio/select options: maps an option value to the * child field keys that should render beneath that option. From 126e9037e15a823818e2ccbd62e6c9c125270f15 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 19:00:32 -0500 Subject: [PATCH 076/103] clean up --- asd | Bin 0 -> 12288 bytes .../src/features/connectors/code-utils.ts | 13 +-- .../src/features/sources/modal/constants.ts | 82 +----------------- .../src/features/sources/sourceUtils.ts | 6 +- .../src/features/templates/schemas/https.ts | 2 +- .../src/features/templates/schemas/mysql.ts | 5 +- .../features/templates/schemas/postgres.ts | 8 +- .../features/templates/schemas/snowflake.ts | 6 +- 8 files changed, 24 insertions(+), 98 deletions(-) create mode 100644 asd diff --git a/asd b/asd new file mode 100644 index 0000000000000000000000000000000000000000..2230283a26663375035808f7b530f35f957d8973 GIT binary patch literal 12288 zcmeI#F$%&^3;@tCICv4k>C>V@ofPUOxOoDL-oRN`!O4pVUcnQ1AJL%{!BTK)mzO0W z2?XAj{p&h9UE+PY$ybZ;UmVgniR0$oA}zLAwwq0p<{c9P1PBlyK!5-N0t5&UAV8o! zfyZ%pJFn-@>f-w+s(uLlvK*AD_Z`$K|9_1{g8%^n1PBlyK!5-N0t5(jD&YU`l;3dz U1PBlyK!5-N0t5&UAkd1y2hRI9P5=M^ literal 0 HcmV?d00001 diff --git a/web-common/src/features/connectors/code-utils.ts b/web-common/src/features/connectors/code-utils.ts index edb0bcb4e06..ed64c824008 100644 --- a/web-common/src/features/connectors/code-utils.ts +++ b/web-common/src/features/connectors/code-utils.ts @@ -62,16 +62,17 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; properties = properties.filter(options.fieldFilter); } - // Get the secret property keys + // Get the secret property keys from the properties being used (orderedProperties or configProperties) + const propertiesForTypeChecking = options?.orderedProperties ?? connector.configProperties ?? []; const secretPropertyKeys = - connector.configProperties - ?.filter((property) => property.secret) + propertiesForTypeChecking + .filter((property) => property.secret) .map((property) => property.key) || []; - // Get the string property keys + // Get the string property keys from the properties being used const stringPropertyKeys = - connector.configProperties - ?.filter( + propertiesForTypeChecking + .filter( (property) => property.type === ConnectorDriverPropertyType.TYPE_STRING, ) .map((property) => property.key) || []; diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 62bf3995949..b51a5a482c9 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -17,87 +17,6 @@ export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "dsn", label: "Enter connection string" }, ]; -export type GCSAuthMethod = "public" | "credentials" | "hmac"; - -export const GCS_AUTH_OPTIONS: { - value: GCSAuthMethod; - label: string; - description: string; - hint?: string; -}[] = [ - { - value: "credentials", - label: "GCP credentials", - description: - "Upload a JSON key file for a service account with GCS access.", - }, - { - value: "hmac", - label: "HMAC keys", - description: - "Use HMAC access key and secret for S3-compatible authentication.", - }, - { - value: "public", - label: "Public", - description: "Access publicly readable buckets without credentials.", - }, -]; - -export type S3AuthMethod = "access_keys" | "public"; - -export const S3_AUTH_OPTIONS: { - value: S3AuthMethod; - label: string; - description: string; - hint?: string; -}[] = [ - { - value: "access_keys", - label: "Access keys", - description: "Use AWS access key ID and secret access key.", - }, - { - value: "public", - label: "Public", - description: "Access publicly readable buckets without credentials.", - }, -]; - -export type AzureAuthMethod = - | "account_key" - | "sas_token" - | "connection_string" - | "public"; - -export const AZURE_AUTH_OPTIONS: { - value: AzureAuthMethod; - label: string; - description: string; - hint?: string; -}[] = [ - { - value: "connection_string", - label: "Connection String", - description: "Alternative for cloud deployment", - }, - { - value: "account_key", - label: "Storage Account Key", - description: "Recommended for cloud deployment", - }, - { - value: "sas_token", - label: "Shared Access Signature (SAS) Token", - description: "Most secure, fine-grained control", - }, - { - value: "public", - label: "Public", - description: "Access publicly readable blobs without credentials.", - }, -]; - // pre-defined order for sources export const SOURCES = [ "athena", @@ -149,4 +68,5 @@ export const TALL_FORM_CONNECTORS = new Set([ "clickhouse", "snowflake", "salesforce", + "postgres" ]); diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index a005f94ac2a..6b71aa34224 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -62,7 +62,11 @@ export function compileSourceYAML( .split("\n") .map((line) => ` ${line}`) .join("\n"); - return `${key}: |\n${sqlLines}\n\ndev:\n ${key}: |\n${sqlLines}\n limit 10000`; + const devSqlLines = value + .split("\n") + .map((line) => ` ${line}`) + .join("\n"); + return `${key}: |\n${sqlLines}\n\ndev:\n ${key}: |\n${devSqlLines}\n limit 10000`; } const isStringProperty = stringPropertyKeys.includes(key); diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts index ace8518400f..297380e3a65 100644 --- a/web-common/src/features/templates/schemas/https.ts +++ b/web-common/src/features/templates/schemas/https.ts @@ -11,7 +11,7 @@ export const httpsSchema: MultiStepFormSchema = { default: "public", description: "Choose how to authenticate to the REST API", "x-display": "radio", - "x-enum-labels": ["Public", "Custom Headers"], + "x-enum-labels": ["Custom Headers", "Public"], "x-enum-descriptions": [ "Access publicly available APIs without authentication.", "Provide custom HTTP headers for authentication (e.g., Authorization, API keys).", diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts index c9cb531feb8..1963137ec8a 100644 --- a/web-common/src/features/templates/schemas/mysql.ts +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -17,7 +17,7 @@ export const mysqlSchema: MultiStepFormSchema = { "Provide a complete MySQL connection string (DSN).", ], "x-grouped-fields": { - parameters: ["host", "port", "database", "user", "password", "sslmode"], + parameters: ["host", "port", "database", "user", "password", "ssl-mode"], connection_string: ["dsn"], }, "x-step": "connector", @@ -64,7 +64,7 @@ export const mysqlSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, - sslmode: { + "ssl-mode": { type: "string", title: "SSL Mode", description: "SSL connection mode", @@ -88,6 +88,7 @@ export const mysqlSchema: MultiStepFormSchema = { title: "SQL Query", description: "SQL query to extract data from MySQL", "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", "x-step": "source", }, name: { diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts index 6fd18e66620..fa97526e0d2 100644 --- a/web-common/src/features/templates/schemas/postgres.ts +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -17,7 +17,7 @@ export const postgresSchema: MultiStepFormSchema = { "Provide a complete PostgreSQL connection string (DSN).", ], "x-grouped-fields": { - parameters: ["host", "port", "database", "user", "password", "sslmode"], + parameters: ["host", "port", "dbname", "user", "password", "sslmode"], connection_string: ["dsn"], }, "x-step": "connector", @@ -39,7 +39,7 @@ export const postgresSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, - database: { + dbname: { type: "string", title: "Database", description: "Database name", @@ -103,11 +103,11 @@ export const postgresSchema: MultiStepFormSchema = { allOf: [ { if: { properties: { auth_method: { const: "parameters" } } }, - then: { required: ["host", "database", "user", "password", "sql", "name"] }, + then: { required: ["host", "dbname", "user", "password"] }, }, { if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["dsn", "sql", "name"] }, + then: { required: ["dsn"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index 6a2ea70671f..849c9ae2fe9 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -129,15 +129,15 @@ export const snowflakeSchema: MultiStepFormSchema = { allOf: [ { if: { properties: { auth_method: { const: "password" } } }, - then: { required: ["account", "user", "password", "sql", "name"] }, + then: { required: ["account", "user", "password"] }, }, { if: { properties: { auth_method: { const: "keypair" } } }, - then: { required: ["account", "user", "private_key", "sql", "name"] }, + then: { required: ["account", "user", "private_key"] }, }, { if: { properties: { auth_method: { const: "connection_string" } } }, - then: { required: ["dsn", "sql", "name"] }, + then: { required: ["dsn"] }, }, ], }; From 77c705c2ae245a393e4dd59cc0ed4f49dca707ad Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 19:23:59 -0500 Subject: [PATCH 077/103] advanced --- .../src/features/connectors/code-utils.ts | 5 ++ .../sources/modal/AddDataFormManager.ts | 71 ++++++++++++------- .../features/templates/schemas/bigquery.ts | 8 +++ .../src/features/templates/schemas/https.ts | 6 +- .../src/features/templates/schemas/mysql.ts | 12 +++- .../features/templates/schemas/postgres.ts | 12 +++- .../features/templates/schemas/redshift.ts | 8 +++ .../features/templates/schemas/snowflake.ts | 22 +++++- 8 files changed, 109 insertions(+), 35 deletions(-) diff --git a/web-common/src/features/connectors/code-utils.ts b/web-common/src/features/connectors/code-utils.ts index ed64c824008..2f31fc0ecab 100644 --- a/web-common/src/features/connectors/code-utils.ts +++ b/web-common/src/features/connectors/code-utils.ts @@ -82,6 +82,11 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; .filter((property) => { if (!property.key) return false; const value = formValues[property.key]; + + // Secret fields should always be shown (with env variable placeholder) even if empty + const isSecretProperty = secretPropertyKeys.includes(property.key); + if (isSecretProperty) return true; + if (value === undefined) return false; // Filter out empty strings for optional fields if (typeof value === "string" && value.trim() === "") return false; diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index a5af0d14cb3..55bdf61018d 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -505,36 +505,57 @@ export class AddDataFormManager { : filteredParamsProperties; const getConnectorYamlPreview = (values: Record) => { - let orderedProperties = - onlyDsn || connectionTab === "dsn" - ? filteredDsnProperties - : connectorPropertiesForPreview; + let orderedProperties: ConnectorDriverProperty[] = []; // For multi-step connectors with schemas, build properties from schema - if (isMultiStepConnector && stepState?.step === "connector") { - const schema = getConnectorSchema(connector.name); - if (schema) { - // Build ordered properties from schema fields that are visible and on connector step - const schemaProperties: ConnectorDriverProperty[] = []; - const properties = schema.properties ?? {}; - - for (const [key, prop] of Object.entries(properties)) { - // Only include connector step fields that are currently visible - if (prop["x-step"] === "connector" && isVisibleForValues(schema, key, values)) { - schemaProperties.push({ - key, - type: prop.type === "number" - ? ConnectorDriverPropertyType.TYPE_NUMBER - : prop.type === "boolean" - ? ConnectorDriverPropertyType.TYPE_BOOLEAN - : ConnectorDriverPropertyType.TYPE_STRING, - secret: prop["x-secret"] || false, - }); - } + const schema = isMultiStepConnector && stepState?.step === "connector" + ? getConnectorSchema(connector.name) + : null; + + if (schema) { + // Build ordered properties from schema fields that are visible and on connector step + const schemaProperties: ConnectorDriverProperty[] = []; + const properties = schema.properties ?? {}; + + // Find the auth method radio field key (e.g., "auth_method") - we don't want to include this in YAML + const authMethodKey = findRadioEnumKey(schema); + + // Ensure auth_method has a value for visibility checks (use actual value or fallback to default) + let valuesForVisibility = { ...values }; + if (authMethodKey && !valuesForVisibility[authMethodKey]) { + const authProp = properties[authMethodKey]; + if (authProp?.default) { + valuesForVisibility[authMethodKey] = authProp.default; + } else if (authProp?.enum?.length) { + valuesForVisibility[authMethodKey] = authProp.enum[0]; } + } - orderedProperties = schemaProperties; + for (const [key, prop] of Object.entries(properties)) { + // Skip the auth_method field itself - it's just a UI control field + if (key === authMethodKey) continue; + + // Only include connector step fields that are currently visible + if (prop["x-step"] === "connector" && isVisibleForValues(schema, key, valuesForVisibility)) { + schemaProperties.push({ + key, + type: prop.type === "number" + ? ConnectorDriverPropertyType.TYPE_NUMBER + : prop.type === "boolean" + ? ConnectorDriverPropertyType.TYPE_BOOLEAN + : ConnectorDriverPropertyType.TYPE_STRING, + secret: prop["x-secret"] || false, + }); + } } + + orderedProperties = schemaProperties; + } else { + // Non-schema connectors use the old DSN/parameters tab system + orderedProperties = + onlyDsn || connectionTab === "dsn" + ? filteredDsnProperties + : connectorPropertiesForPreview; } return compileConnectorYAML(connector, values, { diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts index a5b0b1beac9..3fd0d8ad569 100644 --- a/web-common/src/features/templates/schemas/bigquery.ts +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -20,6 +20,14 @@ export const bigquerySchema: MultiStepFormSchema = { "x-placeholder": "my-project-id", "x-step": "connector", }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, sql: { type: "string", title: "SQL Query", diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts index 297380e3a65..c7dadab1847 100644 --- a/web-common/src/features/templates/schemas/https.ts +++ b/web-common/src/features/templates/schemas/https.ts @@ -7,7 +7,7 @@ export const httpsSchema: MultiStepFormSchema = { auth_method: { type: "string", title: "Authentication method", - enum: ["public", "headers"], + enum: ["headers", "public"], default: "public", description: "Choose how to authenticate to the REST API", "x-display": "radio", @@ -51,11 +51,11 @@ export const httpsSchema: MultiStepFormSchema = { allOf: [ { if: { properties: { auth_method: { const: "headers" } } }, - then: { required: ["headers", "path", "name"] }, + then: { required: ["headers" }, }, { if: { properties: { auth_method: { const: "public" } } }, - then: { required: ["path", "name"] }, + then: { required: [] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts index 1963137ec8a..89c9e614123 100644 --- a/web-common/src/features/templates/schemas/mysql.ts +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -17,8 +17,8 @@ export const mysqlSchema: MultiStepFormSchema = { "Provide a complete MySQL connection string (DSN).", ], "x-grouped-fields": { - parameters: ["host", "port", "database", "user", "password", "ssl-mode"], - connection_string: ["dsn"], + parameters: ["host", "port", "database", "user", "password", "ssl-mode", "log_queries"], + connection_string: ["dsn", "log_queries"], }, "x-step": "connector", }, @@ -83,6 +83,14 @@ export const mysqlSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, sql: { type: "string", title: "SQL Query", diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts index fa97526e0d2..cf209bd0a48 100644 --- a/web-common/src/features/templates/schemas/postgres.ts +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -17,8 +17,8 @@ export const postgresSchema: MultiStepFormSchema = { "Provide a complete PostgreSQL connection string (DSN).", ], "x-grouped-fields": { - parameters: ["host", "port", "dbname", "user", "password", "sslmode"], - connection_string: ["dsn"], + parameters: ["host", "port", "dbname", "user", "password", "sslmode", "log_queries"], + connection_string: ["dsn", "log_queries"], }, "x-step": "connector", }, @@ -83,6 +83,14 @@ export const postgresSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, sql: { type: "string", title: "SQL Query", diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts index f746efac156..25dec44e8ec 100644 --- a/web-common/src/features/templates/schemas/redshift.ts +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -49,6 +49,14 @@ export const redshiftSchema: MultiStepFormSchema = { "x-hint": "Provide either workgroup (for serverless) or cluster identifier (for provisioned)", "x-step": "connector", }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, sql: { type: "string", title: "SQL Query", diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index 849c9ae2fe9..395d071a399 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -18,9 +18,9 @@ export const snowflakeSchema: MultiStepFormSchema = { "Provide a complete Snowflake connection string (DSN).", ], "x-grouped-fields": { - password: ["account", "user", "password", "warehouse", "database", "schema", "role"], - keypair: ["account", "user", "private_key", "private_key_passphrase", "warehouse", "database", "schema", "role"], - connection_string: ["dsn"], + password: ["account", "user", "password", "warehouse", "database", "schema", "role", "log_queries", "parallel_fetch_limit"], + keypair: ["account", "user", "private_key", "private_key_passphrase", "warehouse", "database", "schema", "role", "log_queries", "parallel_fetch_limit"], + connection_string: ["dsn", "log_queries", "parallel_fetch_limit"], }, "x-step": "connector", }, @@ -109,6 +109,22 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + parallel_fetch_limit: { + type: "number", + title: "Parallel Fetch Limit", + description: "Maximum number of parallel fetches for query results", + "x-placeholder": "10", + "x-step": "connector", + "x-advanced": true, + }, sql: { type: "string", title: "SQL Query", From 73e5bd70dfd32817983360e47c866e401e909e79 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 19:36:25 -0500 Subject: [PATCH 078/103] rmeove clickhouse --- .../sources/modal/AddClickHouseForm.svelte | 506 ------------------ .../features/sources/modal/AddDataForm.svelte | 92 +--- .../sources/modal/AddDataFormManager.ts | 77 +-- .../sources/modal/connector-schemas.ts | 2 + .../src/features/sources/modal/constants.ts | 1 + .../src/features/sources/sourceUtils.ts | 27 + .../features/templates/schemas/clickhouse.ts | 139 +++++ .../src/features/templates/schemas/https.ts | 3 +- 8 files changed, 191 insertions(+), 656 deletions(-) delete mode 100644 web-common/src/features/sources/modal/AddClickHouseForm.svelte create mode 100644 web-common/src/features/templates/schemas/clickhouse.ts diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte deleted file mode 100644 index 3248cf7487d..00000000000 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ /dev/null @@ -1,506 +0,0 @@ - - -
-
- - {#if connectorType === "rill-managed"} -
- -
- {/if} -
- - {#if connectorType === "self-hosted" || connectorType === "clickhouse-cloud"} - - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {@const isPortField = propertyKey === "port"} - {@const isSSLField = propertyKey === "ssl"} - -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - options={connectorType === "clickhouse-cloud" && isPortField - ? [ - { value: "8443", label: "8443 (HTTPS)" }, - { value: "9440", label: "9440 (Native Secure)" }, - ] - : undefined} - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- - -
- {#each dsnProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- -
- {/each} -
- - - {:else} - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- {/if} -
diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 0bb6d33fb77..eb8065ff448 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -8,16 +8,12 @@ import type { SuperValidated } from "sveltekit-superforms"; import type { AddDataFormType, ConnectorType } from "./types"; - import AddClickHouseForm from "./AddClickHouseForm.svelte"; import MultiStepConnectorFlow from "./MultiStepConnectorFlow.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; import { hasOnlyDsn, isEmpty } from "./utils"; - import { - CONNECTION_TAB_OPTIONS, - type ClickHouseConnectorType, - } from "./constants"; + import { CONNECTION_TAB_OPTIONS } from "./constants"; import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; @@ -107,16 +103,6 @@ let dsnError: string | null = null; let dsnErrorDetails: string | undefined = undefined; - let clickhouseError: string | null = null; - let clickhouseErrorDetails: string | undefined = undefined; - let clickhouseFormId: string = ""; - let clickhouseSubmitting: boolean; - let clickhouseIsSubmitDisabled: boolean; - let clickhouseConnectorType: ClickHouseConnectorType = "self-hosted"; - let clickhouseParamsForm; - let clickhouseDsnForm; - let clickhouseShowSaveAnyway: boolean = false; - // Hide Save Anyway once we advance to the model step in multi-step flows. $: if (isMultiStepConnector && stepState.step === "source") { showSaveAnyway = false; @@ -176,13 +162,10 @@ isConnectorForm, step: stepState.step, submitting, - clickhouseConnectorType, - clickhouseSubmitting, selectedAuthMethod: activeAuthMethod ?? undefined, }); $: primaryLoadingCopy = (() => { - if (connector.name === "clickhouse") return "Connecting..."; if (isMultiStepConnector) return multiStepLoadingCopy; return activeAuthMethod === "public" ? "Continuing..." @@ -229,33 +212,15 @@ // For other connectors, use manager helper saveAnyway = true; const values = - connector.name === "clickhouse" - ? connectionTab === "dsn" - ? $clickhouseDsnForm - : $clickhouseParamsForm - : onlyDsn || connectionTab === "dsn" - ? $dsnForm - : $paramsForm; - if (connector.name === "clickhouse") { - clickhouseSubmitting = true; - } + onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; const result = await formManager.saveConnectorAnyway({ queryClient, values, - clickhouseConnectorType, }); if (result.ok) { onClose(); } else { - if (connector.name === "clickhouse") { - if (connectionTab === "dsn") { - dsnError = result.message; - dsnErrorDetails = result.details; - } else { - paramsError = result.message; - paramsErrorDetails = result.details; - } - } else if (onlyDsn || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { dsnError = result.message; dsnErrorDetails = result.details; } else { @@ -264,9 +229,6 @@ } } saveAnyway = false; - if (connector.name === "clickhouse") { - clickhouseSubmitting = false; - } } $: yamlPreview = formManager.computeYamlPreview({ @@ -279,16 +241,9 @@ isConnectorForm, paramsFormValues: $paramsForm, dsnFormValues: $dsnForm, - clickhouseConnectorType, - clickhouseParamsValues: $clickhouseParamsForm, - clickhouseDsnValues: $clickhouseDsnForm, }); - $: isClickhouse = connector.name === "clickhouse"; - $: shouldShowSaveAnywayButton = - isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway); - $: saveAnywayLoading = isClickhouse - ? clickhouseSubmitting && saveAnyway - : submitting && saveAnyway; + $: shouldShowSaveAnywayButton = isConnectorForm && showSaveAnyway; + $: saveAnywayLoading = submitting && saveAnyway; handleOnUpdate = formManager.makeOnUpdate({ onClose, @@ -328,24 +283,7 @@
- {#if connector.name === "clickhouse"} - { - clickhouseError = error; - clickhouseErrorDetails = details; - }} - bind:formId={clickhouseFormId} - bind:isSubmitting={clickhouseSubmitting} - bind:isSubmitDisabled={clickhouseIsSubmitDisabled} - bind:connectorType={clickhouseConnectorType} - bind:connectionTab - bind:paramsForm={clickhouseParamsForm} - bind:dsnForm={clickhouseDsnForm} - bind:showSaveAnyway={clickhouseShowSaveAnyway} - /> - {:else if hasDsnFormOption} + {#if hasDsnFormOption} @@ -479,13 +413,11 @@ class="add-data-side-panel flex flex-col gap-6 p-6 bg-surface w-full max-w-full border-l-0 border-t mt-6 pl-0 pt-6 md:w-96 md:min-w-[320px] md:max-w-[400px] md:border-l md:border-t-0 md:mt-0 md:pl-6 justify-between" >
- {#if dsnError || paramsError || clickhouseError} + {#if dsnError || paramsError} ; dsnFormValues: Record; - clickhouseConnectorType?: ClickHouseConnectorType; - clickhouseParamsValues?: Record; - clickhouseDsnValues?: Record; }): string { const connector = this.connector; const { @@ -494,9 +477,6 @@ export class AddDataFormManager { isConnectorForm, paramsFormValues, dsnFormValues, - clickhouseConnectorType, - clickhouseParamsValues, - clickhouseDsnValues, } = ctx; const connectorPropertiesForPreview = @@ -567,30 +547,6 @@ export class AddDataFormManager { }); }; - const getClickHouseYamlPreview = ( - values: Record, - chType: ClickHouseConnectorType | undefined, - ) => { - // Convert to managed boolean and apply CH Cloud requirements for preview - const managed = chType === "rill-managed"; - const previewValues = { ...values, managed } as Record; - const finalValues = applyClickHouseCloudRequirements( - connector.name, - chType as ClickHouseConnectorType, - previewValues, - ); - return compileConnectorYAML(connector, finalValues, { - fieldFilter: (property) => { - if (onlyDsn || connectionTab === "dsn") return true; - return !property.noPrompt; - }, - orderedProperties: - connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, - }); - }; - const getSourceYamlPreview = (values: Record) => { // For multi-step connectors in step 2, filter out connector properties let filteredValues = values; @@ -622,15 +578,6 @@ export class AddDataFormManager { return getConnectorYamlPreview(rewrittenFormValues); }; - // ClickHouse special-case - if (connector.name === "clickhouse") { - const values = - connectionTab === "dsn" - ? clickhouseDsnValues || {} - : clickhouseParamsValues || {}; - return getClickHouseYamlPreview(values, clickhouseConnectorType); - } - // Multi-step connectors if (isMultiStepConnector) { if (stepState?.step === "connector") { @@ -651,20 +598,14 @@ export class AddDataFormManager { } /** - * Save connector anyway (non-ClickHouse), returning a result object for the caller to handle. + * Save connector anyway, bypassing validation. Returns a result object for the caller to handle. */ async saveConnectorAnyway(args: { queryClient: any; values: Record; - clickhouseConnectorType?: ClickHouseConnectorType; }): Promise<{ ok: true } | { ok: false; message: string; details?: string }> { - const { queryClient, values, clickhouseConnectorType } = args; - const processedValues = applyClickHouseCloudRequirements( - this.connector.name, - (clickhouseConnectorType as ClickHouseConnectorType) || - ("self-hosted" as ClickHouseConnectorType), - values, - ); + const { queryClient, values } = args; + const processedValues = prepareConnectorFormData(this.connector, values); try { await submitAddConnectorForm( queryClient, diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 48bf715c02b..d7e6ef6b795 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -2,6 +2,7 @@ import type { MultiStepFormSchema } from "../../templates/schemas/types"; import { athenaSchema } from "../../templates/schemas/athena"; import { azureSchema } from "../../templates/schemas/azure"; import { bigquerySchema } from "../../templates/schemas/bigquery"; +import { clickhouseSchema } from "../../templates/schemas/clickhouse"; import { druidSchema } from "../../templates/schemas/druid"; import { duckdbSchema } from "../../templates/schemas/duckdb"; import { gcsSchema } from "../../templates/schemas/gcs"; @@ -25,6 +26,7 @@ export const multiStepFormSchemas: Record = { bigquery: bigquerySchema, redshift: redshiftSchema, athena: athenaSchema, + clickhouse: clickhouseSchema, duckdb: duckdbSchema, motherduck: motherduckSchema, druid: druidSchema, diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index b51a5a482c9..6c0cece0607 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -56,6 +56,7 @@ export const MULTI_STEP_CONNECTORS = [ "bigquery", "redshift", "athena", + "clickhouse", "duckdb", "motherduck", "druid", diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 6b71aa34224..f717a25c130 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -206,6 +206,33 @@ export function maybeRewriteToDuckDb( return [connectorCopy, formValues]; } +/** + * Prepare connector form values before submission. + * Handles special transformations like ClickHouse auth_method → managed. + */ +export function prepareConnectorFormData( + connector: V1ConnectorDriver, + formValues: Record, +): Record { + const processedValues = { ...formValues }; + + // ClickHouse: translate auth_method to managed boolean + if (connector.name === "clickhouse" && processedValues.auth_method) { + const authMethod = processedValues.auth_method as string; + processedValues.managed = authMethod === "rill-managed"; + + // Set mode to readwrite for managed ClickHouse + if (processedValues.managed) { + processedValues.mode = "readwrite"; + } + + // Remove the UI-only auth_method field + delete processedValues.auth_method; + } + + return processedValues; +} + /** * Process form data for sources, including DuckDB rewrite logic and placeholder handling. * This serves as a single source of truth for both preview and submission. diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts new file mode 100644 index 00000000000..e877b767599 --- /dev/null +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -0,0 +1,139 @@ +import type { MultiStepFormSchema } from "./types"; + +export const clickhouseSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection type", + enum: ["rill-managed", "self-hosted", "connection-string"], + default: "self-hosted", + description: "Choose how to connect to ClickHouse", + "x-display": "radio", + "x-enum-labels": ["Rill-managed ClickHouse", "Self-hosted", "Connection String"], + "x-enum-descriptions": [ + "Use a managed ClickHouse instance (starts embedded ClickHouse in development).", + "Connect to your own ClickHouse server or ClickHouse Cloud.", + "Provide a complete ClickHouse connection string (DSN).", + ], + "x-grouped-fields": { + "rill-managed": ["mode"], + "self-hosted": ["host", "port", "username", "password", "database", "ssl", "cluster", "mode"], + "connection-string": ["dsn", "mode"], + }, + "x-step": "connector", + }, + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the ClickHouse server", + "x-placeholder": "your-instance.clickhouse.cloud", + "x-step": "connector", + "x-visible-if": { auth_method: "self-hosted" }, + }, + port: { + type: "number", + title: "Port", + description: "Port number of the ClickHouse server", + default: 9000, + "x-placeholder": "9000", + "x-hint": "Default: 9000 (native TCP), 8123 (HTTP). Secure: 9440 (TCP+TLS), 8443 (HTTPS)", + "x-step": "connector", + "x-visible-if": { auth_method: "self-hosted" }, + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the ClickHouse server", + default: "default", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { auth_method: "self-hosted" }, + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the ClickHouse server", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "self-hosted" }, + }, + database: { + type: "string", + title: "Database", + description: "Name of the ClickHouse database to connect to", + default: "default", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { auth_method: "self-hosted" }, + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL to connect to the ClickHouse server", + default: true, + "x-hint": "Enable SSL for secure connections. ClickHouse Cloud always uses SSL.", + "x-step": "connector", + "x-visible-if": { auth_method: "self-hosted" }, + }, + cluster: { + type: "string", + title: "Cluster", + description: "Cluster name for distributed tables", + "x-placeholder": "Cluster name", + "x-hint": "If set, Rill will create models as distributed tables in the cluster", + "x-step": "connector", + "x-visible-if": { auth_method: "self-hosted" }, + "x-advanced": true, + }, + mode: { + type: "string", + title: "Mode", + description: "Connection mode for the ClickHouse database", + enum: ["read", "readwrite"], + default: "read", + "x-display": "select", + "x-hint": "Set to 'readwrite' to enable model creation and table mutations", + "x-step": "connector", + "x-visible-if": { auth_method: ["self-hosted", "connection-string"] }, + "x-advanced": true, + }, + dsn: { + type: "string", + title: "Connection String", + description: "ClickHouse connection string (DSN)", + "x-placeholder": "clickhouse://localhost:9000?username=default&password=password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection-string" }, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from ClickHouse", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "self-hosted" } } }, + then: { required: ["host", "username"] }, + }, + { + if: { properties: { auth_method: { const: "connection-string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts index c7dadab1847..c8ec0746a28 100644 --- a/web-common/src/features/templates/schemas/https.ts +++ b/web-common/src/features/templates/schemas/https.ts @@ -27,7 +27,6 @@ export const httpsSchema: MultiStepFormSchema = { description: 'HTTP headers as JSON object. Example: {"Authorization": "Bearer my-token", "X-API-Key": "value"}', "x-placeholder": '{"Authorization": "Bearer my-token"}', - "x-display": "textarea", "x-step": "connector", "x-visible-if": { auth_method: "headers" }, }, @@ -51,7 +50,7 @@ export const httpsSchema: MultiStepFormSchema = { allOf: [ { if: { properties: { auth_method: { const: "headers" } } }, - then: { required: ["headers" }, + then: { required: ["headers"] }, }, { if: { properties: { auth_method: { const: "public" } } }, From 91d6eba04cb2a879467dbc103abcd9fd8ce09396 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 19:44:10 -0500 Subject: [PATCH 079/103] to be cont'd --- .../modal/MultiStepConnectorFlow.svelte | 49 +++++++++++++++ .../src/features/sources/sourceUtils.ts | 18 +++++- .../features/templates/schemas/clickhouse.ts | 61 ++++++++++--------- 3 files changed, 96 insertions(+), 32 deletions(-) diff --git a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte index 0c0babc807e..4c8a278b849 100644 --- a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte +++ b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte @@ -55,9 +55,15 @@ let activeAuthInfo: ReturnType | null = null; let stepProperties: ConnectorDriverProperty[] | undefined = undefined; let selectedAuthMethod = ""; + let previousAuthMethod: string | null = null; $: selectedAuthMethod = $selectedAuthMethodStore; + // Initialize previousAuthMethod on first load + $: if (previousAuthMethod === null && selectedAuthMethod) { + previousAuthMethod = selectedAuthMethod; + } + // Compute which properties to show for the current step. $: stepProperties = stepState.step === "source" @@ -132,6 +138,49 @@ } } + // Clear form when auth method changes (e.g., switching from parameters to DSN). + $: if (activeSchema && selectedAuthMethod !== previousAuthMethod && previousAuthMethod !== null && previousAuthMethod !== "") { + const authKey = findRadioEnumKey(activeSchema); + if (authKey) { + // Get default values for the new auth method + const defaults: Record = {}; + + // Set auth_method to the new value + defaults[authKey] = selectedAuthMethod; + + // Add default values from schema for fields visible in this auth method + if (activeSchema.properties) { + for (const [key, prop] of Object.entries(activeSchema.properties)) { + if (key === authKey) continue; // Already set + + // Check if this field is visible for the current auth method + const visibleIf = prop["x-visible-if"]; + if (visibleIf && authKey in visibleIf) { + const expectedValue = visibleIf[authKey]; + const matches = Array.isArray(expectedValue) + ? expectedValue.includes(selectedAuthMethod) + : expectedValue === selectedAuthMethod; + + if (matches && prop.default !== undefined) { + defaults[key] = prop.default; + } + } + } + } + + // Update form with cleared values + paramsForm.update(() => defaults, { taint: false }); + + // Clear any form errors + paramsErrors.update(() => ({})); + } + } + + // Track previous auth method for comparison + $: if (selectedAuthMethod) { + previousAuthMethod = selectedAuthMethod; + } + // Active auth method for UI (button labels/loading). $: activeAuthMethod = (() => { if (!(activeSchema && paramsForm)) return selectedAuthMethod; diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index f717a25c130..72fa4d2a41b 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -219,11 +219,23 @@ export function prepareConnectorFormData( // ClickHouse: translate auth_method to managed boolean if (connector.name === "clickhouse" && processedValues.auth_method) { const authMethod = processedValues.auth_method as string; - processedValues.managed = authMethod === "rill-managed"; - // Set mode to readwrite for managed ClickHouse - if (processedValues.managed) { + if (authMethod === "rill-managed") { + // Rill-managed: set managed=true, mode=readwrite + processedValues.managed = true; processedValues.mode = "readwrite"; + } else if (authMethod === "clickhouse-cloud") { + // ClickHouse Cloud: set managed=false, ssl=true, normalize port field + processedValues.managed = false; + processedValues.ssl = true; + // Port field for ClickHouse Cloud is just "port" (no rename needed) + } else if (authMethod === "self-managed") { + // Self-managed: set managed=false, normalize port_self_managed -> port + processedValues.managed = false; + if (processedValues.port_self_managed !== undefined) { + processedValues.port = processedValues.port_self_managed; + delete processedValues.port_self_managed; + } } // Remove the UI-only auth_method field diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index e877b767599..5468576dc26 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -7,20 +7,20 @@ export const clickhouseSchema: MultiStepFormSchema = { auth_method: { type: "string", title: "Connection type", - enum: ["rill-managed", "self-hosted", "connection-string"], - default: "self-hosted", + enum: ["clickhouse-cloud", "self-managed", "rill-managed"], + default: "self-managed", description: "Choose how to connect to ClickHouse", "x-display": "radio", - "x-enum-labels": ["Rill-managed ClickHouse", "Self-hosted", "Connection String"], + "x-enum-labels": ["ClickHouse Cloud", "Self-managed", "Rill-managed"], "x-enum-descriptions": [ + "Connect to ClickHouse Cloud (SSL required, uses secure ports).", + "Connect to your own self-hosted ClickHouse server.", "Use a managed ClickHouse instance (starts embedded ClickHouse in development).", - "Connect to your own ClickHouse server or ClickHouse Cloud.", - "Provide a complete ClickHouse connection string (DSN).", ], "x-grouped-fields": { + "clickhouse-cloud": ["host", "port", "username", "password", "database", "cluster", "mode"], + "self-managed": ["host", "port_self_managed", "username", "password", "database", "ssl", "cluster", "mode"], "rill-managed": ["mode"], - "self-hosted": ["host", "port", "username", "password", "database", "ssl", "cluster", "mode"], - "connection-string": ["dsn", "mode"], }, "x-step": "connector", }, @@ -30,9 +30,21 @@ export const clickhouseSchema: MultiStepFormSchema = { description: "Hostname or IP address of the ClickHouse server", "x-placeholder": "your-instance.clickhouse.cloud", "x-step": "connector", - "x-visible-if": { auth_method: "self-hosted" }, + "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, }, port: { + type: "number", + title: "Port", + description: "Port number of the ClickHouse server", + enum: [8443, 9440], + default: 8443, + "x-display": "select", + "x-enum-labels": ["8443 (HTTPS)", "9440 (Native Secure)"], + "x-hint": "ClickHouse Cloud uses secure ports only", + "x-step": "connector", + "x-visible-if": { auth_method: "clickhouse-cloud" }, + }, + port_self_managed: { type: "number", title: "Port", description: "Port number of the ClickHouse server", @@ -40,7 +52,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "9000", "x-hint": "Default: 9000 (native TCP), 8123 (HTTP). Secure: 9440 (TCP+TLS), 8443 (HTTPS)", "x-step": "connector", - "x-visible-if": { auth_method: "self-hosted" }, + "x-visible-if": { auth_method: "self-managed" }, }, username: { type: "string", @@ -49,7 +61,7 @@ export const clickhouseSchema: MultiStepFormSchema = { default: "default", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: "self-hosted" }, + "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, }, password: { type: "string", @@ -58,7 +70,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "Enter password", "x-secret": true, "x-step": "connector", - "x-visible-if": { auth_method: "self-hosted" }, + "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, }, database: { type: "string", @@ -67,16 +79,16 @@ export const clickhouseSchema: MultiStepFormSchema = { default: "default", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: "self-hosted" }, + "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, }, ssl: { type: "boolean", title: "SSL", description: "Use SSL to connect to the ClickHouse server", default: true, - "x-hint": "Enable SSL for secure connections. ClickHouse Cloud always uses SSL.", + "x-hint": "Enable SSL for secure connections", "x-step": "connector", - "x-visible-if": { auth_method: "self-hosted" }, + "x-visible-if": { auth_method: "self-managed" }, }, cluster: { type: "string", @@ -85,7 +97,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "Cluster name", "x-hint": "If set, Rill will create models as distributed tables in the cluster", "x-step": "connector", - "x-visible-if": { auth_method: "self-hosted" }, + "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, "x-advanced": true, }, mode: { @@ -97,18 +109,9 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-display": "select", "x-hint": "Set to 'readwrite' to enable model creation and table mutations", "x-step": "connector", - "x-visible-if": { auth_method: ["self-hosted", "connection-string"] }, + "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed", "rill-managed"] }, "x-advanced": true, }, - dsn: { - type: "string", - title: "Connection String", - description: "ClickHouse connection string (DSN)", - "x-placeholder": "clickhouse://localhost:9000?username=default&password=password", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { auth_method: "connection-string" }, - }, sql: { type: "string", title: "SQL Query", @@ -128,12 +131,12 @@ export const clickhouseSchema: MultiStepFormSchema = { }, allOf: [ { - if: { properties: { auth_method: { const: "self-hosted" } } }, - then: { required: ["host", "username"] }, + if: { properties: { auth_method: { const: "clickhouse-cloud" } } }, + then: { required: ["host", "port", "username", "password", "database"] }, }, { - if: { properties: { auth_method: { const: "connection-string" } } }, - then: { required: ["dsn"] }, + if: { properties: { auth_method: { const: "self-managed" } } }, + then: { required: ["host", "username"] }, }, ], }; From ecc12c54a1e5dc9be138b8e86f9f3115807f1356 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 20:02:30 -0500 Subject: [PATCH 080/103] cleanup CH forms --- .../sources/modal/connector-schemas.ts | 2 + .../src/features/sources/modal/constants.ts | 16 +--- .../src/features/sources/modal/utils.ts | 24 +---- .../src/features/sources/modal/yupSchemas.ts | 33 +++++++ .../templates/schemas/clickhouse-cloud.ts | 90 +++++++++++++++++++ .../features/templates/schemas/clickhouse.ts | 51 +++++------ 6 files changed, 152 insertions(+), 64 deletions(-) create mode 100644 web-common/src/features/templates/schemas/clickhouse-cloud.ts diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index d7e6ef6b795..42fbc1a123e 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -3,6 +3,7 @@ import { athenaSchema } from "../../templates/schemas/athena"; import { azureSchema } from "../../templates/schemas/azure"; import { bigquerySchema } from "../../templates/schemas/bigquery"; import { clickhouseSchema } from "../../templates/schemas/clickhouse"; +import { clickhouseCloudSchema } from "../../templates/schemas/clickhouse-cloud"; import { druidSchema } from "../../templates/schemas/druid"; import { duckdbSchema } from "../../templates/schemas/duckdb"; import { gcsSchema } from "../../templates/schemas/gcs"; @@ -27,6 +28,7 @@ export const multiStepFormSchemas: Record = { redshift: redshiftSchema, athena: athenaSchema, clickhouse: clickhouseSchema, + clickhousecloud: clickhouseCloudSchema, duckdb: duckdbSchema, motherduck: motherduckSchema, druid: druidSchema, diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 6c0cece0607..b1c92e32786 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -1,17 +1,3 @@ -export type ClickHouseConnectorType = - | "rill-managed" - | "self-hosted" - | "clickhouse-cloud"; - -export const CONNECTOR_TYPE_OPTIONS: { - value: ClickHouseConnectorType; - label: string; -}[] = [ - { value: "rill-managed", label: "Rill-managed ClickHouse" }, - { value: "self-hosted", label: "Self-hosted ClickHouse" }, - { value: "clickhouse-cloud", label: "ClickHouse Cloud" }, -]; - export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "parameters", label: "Enter parameters" }, { value: "dsn", label: "Enter connection string" }, @@ -36,6 +22,7 @@ export const SOURCES = [ export const OLAP_ENGINES = [ "clickhouse", + "clickhousecloud", "motherduck", "duckdb", "druid", @@ -57,6 +44,7 @@ export const MULTI_STEP_CONNECTORS = [ "redshift", "athena", "clickhouse", + "clickhousecloud", "duckdb", "motherduck", "druid", diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index ac9eb28448f..b470afd6144 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,6 +1,5 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; -import type { ClickHouseConnectorType } from "./constants"; import type { MultiStepFormSchema } from "./types"; import { findRadioEnumKey, @@ -180,25 +179,4 @@ export function isMultiStepConnectorDisabled( }); } -/** - * Applies ClickHouse Cloud-specific default requirements for connector values. - * - For ClickHouse Cloud: enforces `ssl: true` - * - Otherwise returns values unchanged - */ -export function applyClickHouseCloudRequirements( - connectorName: string | undefined, - connectorType: ClickHouseConnectorType, - values: Record, -): Record { - // Only force SSL for ClickHouse Cloud when the user is using individual params. - // DSN strings encapsulate their own protocol, so we should not inject `ssl` there. - const isDsnBased = "dsn" in values; - const shouldEnforceSSL = - connectorName === "clickhouse" && - connectorType === "clickhouse-cloud" && - !isDsnBased; - if (shouldEnforceSSL) { - return { ...values, ssl: true } as Record; - } - return values; -} + diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index afa445d0811..9735b56c679 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -408,6 +408,39 @@ export const getYupSchema = { // .required("Connector name is required"), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + clickhousecloud_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + username: yup.string().optional(), + password: yup.string().optional(), + database: yup.string().optional(), + cluster: yup.string().optional(), + mode: yup.string().optional(), + }), + + clickhousecloud_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + clickhousecloud: yup.object().shape({ + host: yup.string(), + port: yup + .string() // Purposefully using a string input, not a numeric input + .matches(/^\d+$/, "Port must be a number"), + username: yup.string(), + password: yup.string(), + database: yup.string(), + cluster: yup.string(), + mode: yup.string(), + name: yup.string(), // Required for typing + }), + druid: yup.object().shape({ host: yup .string() diff --git a/web-common/src/features/templates/schemas/clickhouse-cloud.ts b/web-common/src/features/templates/schemas/clickhouse-cloud.ts new file mode 100644 index 00000000000..a730784fd7e --- /dev/null +++ b/web-common/src/features/templates/schemas/clickhouse-cloud.ts @@ -0,0 +1,90 @@ +import type { MultiStepFormSchema } from "./types"; + +export const clickhouseCloudSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the ClickHouse Cloud server", + "x-placeholder": "your-instance.clickhouse.cloud", + "x-step": "connector", + }, + port: { + type: "number", + title: "Port", + description: "Port number of the ClickHouse Cloud server", + enum: [8443, 9440], + default: 8443, + "x-display": "select", + "x-enum-labels": ["8443 (HTTPS)", "9440 (Native Secure)"], + "x-hint": "ClickHouse Cloud uses secure ports only", + "x-step": "connector", + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the ClickHouse Cloud server", + default: "default", + "x-placeholder": "default", + "x-step": "connector", + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the ClickHouse Cloud server", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + }, + database: { + type: "string", + title: "Database", + description: "Name of the ClickHouse database to connect to", + default: "default", + "x-placeholder": "default", + "x-step": "connector", + }, + cluster: { + type: "string", + title: "Cluster", + description: "Cluster name for distributed tables", + "x-placeholder": "Cluster name", + "x-hint": "If set, Rill will create models as distributed tables in the cluster", + "x-step": "connector", + "x-advanced": true, + }, + mode: { + type: "string", + title: "Connection Mode", + description: "Database access mode", + enum: ["read", "readwrite"], + default: "read", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from ClickHouse", + "x-placeholder": "SELECT * FROM my_table;", + "x-display": "textarea", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["host", "port", "username", "password", "database"], +}; diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index 5468576dc26..50079e1cfca 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -7,18 +7,16 @@ export const clickhouseSchema: MultiStepFormSchema = { auth_method: { type: "string", title: "Connection type", - enum: ["clickhouse-cloud", "self-managed", "rill-managed"], + enum: ["self-managed", "rill-managed"], default: "self-managed", description: "Choose how to connect to ClickHouse", "x-display": "radio", - "x-enum-labels": ["ClickHouse Cloud", "Self-managed", "Rill-managed"], + "x-enum-labels": ["Self-managed", "Rill-managed"], "x-enum-descriptions": [ - "Connect to ClickHouse Cloud (SSL required, uses secure ports).", "Connect to your own self-hosted ClickHouse server.", "Use a managed ClickHouse instance (starts embedded ClickHouse in development).", ], "x-grouped-fields": { - "clickhouse-cloud": ["host", "port", "username", "password", "database", "cluster", "mode"], "self-managed": ["host", "port_self_managed", "username", "password", "database", "ssl", "cluster", "mode"], "rill-managed": ["mode"], }, @@ -30,21 +28,9 @@ export const clickhouseSchema: MultiStepFormSchema = { description: "Hostname or IP address of the ClickHouse server", "x-placeholder": "your-instance.clickhouse.cloud", "x-step": "connector", - "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, + "x-visible-if": { auth_method: [ "self-managed"] }, }, port: { - type: "number", - title: "Port", - description: "Port number of the ClickHouse server", - enum: [8443, 9440], - default: 8443, - "x-display": "select", - "x-enum-labels": ["8443 (HTTPS)", "9440 (Native Secure)"], - "x-hint": "ClickHouse Cloud uses secure ports only", - "x-step": "connector", - "x-visible-if": { auth_method: "clickhouse-cloud" }, - }, - port_self_managed: { type: "number", title: "Port", description: "Port number of the ClickHouse server", @@ -61,7 +47,7 @@ export const clickhouseSchema: MultiStepFormSchema = { default: "default", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, + "x-visible-if": { auth_method: ["self-managed"] }, }, password: { type: "string", @@ -70,7 +56,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "Enter password", "x-secret": true, "x-step": "connector", - "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, + "x-visible-if": { auth_method: ["self-managed"] }, }, database: { type: "string", @@ -79,7 +65,7 @@ export const clickhouseSchema: MultiStepFormSchema = { default: "default", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, + "x-visible-if": { auth_method: ["self-managed"] }, }, ssl: { type: "boolean", @@ -97,20 +83,31 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "Cluster name", "x-hint": "If set, Rill will create models as distributed tables in the cluster", "x-step": "connector", - "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed"] }, + "x-visible-if": { auth_method: ["self-managed"] }, "x-advanced": true, }, mode: { type: "string", - title: "Mode", - description: "Connection mode for the ClickHouse database", + title: "Connection Mode", + description: "Database access mode", enum: ["read", "readwrite"], default: "read", - "x-display": "select", - "x-hint": "Set to 'readwrite' to enable model creation and table mutations", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], "x-step": "connector", - "x-visible-if": { auth_method: ["clickhouse-cloud", "self-managed", "rill-managed"] }, - "x-advanced": true, + }, + managed: { + type: "boolean", + title: "Managed", + description: "Use a managed ClickHouse instance", + default: true, + "x-hint": "Starts an embedded ClickHouse server in development", + "x-step": "connector", + "x-visible-if": { auth_method: ["rill-managed"] }, }, sql: { type: "string", From 8bc88f33c53ce2dd70dec56256d7848634e8936f Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 20:13:09 -0500 Subject: [PATCH 081/103] to be cont;d still som eissues with preview, consolidate clichosue form, need to add managed duckdb option (like managed CH}, e23 testing for all connectors and adding data --- .../features/connectors/connectors-utils.ts | 8 +++- .../sources/modal/AddDataFormManager.ts | 4 +- .../sources/modal/AddDataModal.svelte | 43 ++++++++++++------- .../src/features/sources/modal/constants.ts | 1 + .../src/features/sources/sourceUtils.ts | 22 +++++----- .../features/templates/schemas/clickhouse.ts | 18 +++----- 6 files changed, 53 insertions(+), 43 deletions(-) diff --git a/web-common/src/features/connectors/connectors-utils.ts b/web-common/src/features/connectors/connectors-utils.ts index b82c0c2532d..af7e562b848 100644 --- a/web-common/src/features/connectors/connectors-utils.ts +++ b/web-common/src/features/connectors/connectors-utils.ts @@ -139,8 +139,12 @@ export function getConnectorIconKey(connector: V1AnalyzedConnector): string { /** * Determines the driver name for a connector. - * Special case: MotherDuck connectors use "duckdb" as the driver name. + * Special cases: + * - MotherDuck connectors use "duckdb" as the driver name. + * - ClickHouse Cloud connectors use "clickhouse" as the driver name. */ export function getDriverNameForConnector(connectorName: string): string { - return connectorName === "motherduck" ? "duckdb" : connectorName; + if (connectorName === "motherduck") return "duckdb"; + if (connectorName === "clickhousecloud") return "clickhouse"; + return connectorName; } diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index dff63ff7c95..fcab34ceb69 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -208,8 +208,8 @@ export class AddDataFormManager { // Only show for connector forms (not sources) if (!isConnectorForm) return false; - // ClickHouse has its own error handling - if (this.connector.name === "clickhouse") return false; + // ClickHouse and ClickHouse Cloud have their own error handling + if (this.connector.name === "clickhouse" || this.connector.name === "clickhousecloud") return false; // Need a submission result to show the button if (!event?.result) return false; diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 3b30a2dc874..cd1b9370b2a 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -34,22 +34,33 @@ query: { // arrange connectors in the way we would like to display them select: (data) => { - data.connectors = - data.connectors && - data.connectors - .filter( - // Only show connectors in SOURCES or OLAP_ENGINES - (a) => - a.name && - (SOURCES.includes(a.name) || OLAP_ENGINES.includes(a.name)), - ) - .sort( - // CAST SAFETY: we have filtered out any connectors that - // don't have a `name` in the previous filter - (a, b) => - ALL_CONNECTORS.indexOf(a.name as string) - - ALL_CONNECTORS.indexOf(b.name as string), - ); + let connectors = data.connectors || []; + + // Clone clickhouse connector to create clickhousecloud (frontend-only) + const clickhouseConnector = connectors.find(c => c.name === "clickhouse"); + if (clickhouseConnector) { + const clickhouseCloudConnector: V1ConnectorDriver = { + ...clickhouseConnector, + name: "clickhousecloud", + displayName: "ClickHouse Cloud", + }; + connectors = [...connectors, clickhouseCloudConnector]; + } + + data.connectors = connectors + .filter( + // Only show connectors in SOURCES or OLAP_ENGINES + (a) => + a.name && + (SOURCES.includes(a.name) || OLAP_ENGINES.includes(a.name)), + ) + .sort( + // CAST SAFETY: we have filtered out any connectors that + // don't have a `name` in the previous filter + (a, b) => + ALL_CONNECTORS.indexOf(a.name as string) - + ALL_CONNECTORS.indexOf(b.name as string), + ); return data; }, }, diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index b1c92e32786..14f2dc23a26 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -55,6 +55,7 @@ export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; export const TALL_FORM_CONNECTORS = new Set([ "clickhouse", + "clickhousecloud", "snowflake", "salesforce", "postgres" diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 72fa4d2a41b..81e975d7501 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -6,6 +6,7 @@ import { type V1Source, } from "@rilldata/web-common/runtime-client"; import { makeDotEnvConnectorKey } from "../connectors/code-utils"; +import { getDriverNameForConnector } from "../connectors/connectors-utils"; import { sanitizeEntityName } from "../entity-management/name-utils"; // Helper text that we put at the top of every Model YAML file @@ -51,7 +52,7 @@ export function compileSourceYAML( if (isSecretProperty) { // For source files, we include secret properties return `${key}: "{{ .env.${makeDotEnvConnectorKey( - connector.name as string, + getDriverNameForConnector(connector.name as string), key, )} }}"`; } @@ -79,7 +80,7 @@ export function compileSourceYAML( .join("\n"); return ( - `${SOURCE_MODEL_FILE_TOP}\n\nconnector: ${connector.name}\n\n` + + `${SOURCE_MODEL_FILE_TOP}\n\nconnector: ${getDriverNameForConnector(connector.name as string)}\n\n` + compiledKeyValues ); } @@ -224,24 +225,21 @@ export function prepareConnectorFormData( // Rill-managed: set managed=true, mode=readwrite processedValues.managed = true; processedValues.mode = "readwrite"; - } else if (authMethod === "clickhouse-cloud") { - // ClickHouse Cloud: set managed=false, ssl=true, normalize port field - processedValues.managed = false; - processedValues.ssl = true; - // Port field for ClickHouse Cloud is just "port" (no rename needed) } else if (authMethod === "self-managed") { - // Self-managed: set managed=false, normalize port_self_managed -> port + // Self-managed: set managed=false processedValues.managed = false; - if (processedValues.port_self_managed !== undefined) { - processedValues.port = processedValues.port_self_managed; - delete processedValues.port_self_managed; - } } // Remove the UI-only auth_method field delete processedValues.auth_method; } + // ClickHouse Cloud: always use managed=false and ssl=true + if (connector.name === "clickhousecloud") { + processedValues.managed = false; + processedValues.ssl = true; + } + return processedValues; } diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index 50079e1cfca..de4901db0b2 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -17,7 +17,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "Use a managed ClickHouse instance (starts embedded ClickHouse in development).", ], "x-grouped-fields": { - "self-managed": ["host", "port_self_managed", "username", "password", "database", "ssl", "cluster", "mode"], + "self-managed": ["host", "port", "username", "password", "database", "ssl", "cluster", "mode"], "rill-managed": ["mode"], }, "x-step": "connector", @@ -26,9 +26,9 @@ export const clickhouseSchema: MultiStepFormSchema = { type: "string", title: "Host", description: "Hostname or IP address of the ClickHouse server", - "x-placeholder": "your-instance.clickhouse.cloud", + "x-placeholder": "your-server.clickhouse.com", "x-step": "connector", - "x-visible-if": { auth_method: [ "self-managed"] }, + "x-visible-if": { auth_method: "self-managed" }, }, port: { type: "number", @@ -47,7 +47,7 @@ export const clickhouseSchema: MultiStepFormSchema = { default: "default", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: ["self-managed"] }, + "x-visible-if": { auth_method: "self-managed" }, }, password: { type: "string", @@ -56,7 +56,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "Enter password", "x-secret": true, "x-step": "connector", - "x-visible-if": { auth_method: ["self-managed"] }, + "x-visible-if": { auth_method: "self-managed" }, }, database: { type: "string", @@ -65,7 +65,7 @@ export const clickhouseSchema: MultiStepFormSchema = { default: "default", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: ["self-managed"] }, + "x-visible-if": { auth_method: "self-managed" }, }, ssl: { type: "boolean", @@ -83,7 +83,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "Cluster name", "x-hint": "If set, Rill will create models as distributed tables in the cluster", "x-step": "connector", - "x-visible-if": { auth_method: ["self-managed"] }, + "x-visible-if": { auth_method: "self-managed" }, "x-advanced": true, }, mode: { @@ -127,10 +127,6 @@ export const clickhouseSchema: MultiStepFormSchema = { }, }, allOf: [ - { - if: { properties: { auth_method: { const: "clickhouse-cloud" } } }, - then: { required: ["host", "port", "username", "password", "database"] }, - }, { if: { properties: { auth_method: { const: "self-managed" } } }, then: { required: ["host", "username"] }, From d030140131427102d52972be4d40c923c0760f41 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Fri, 19 Dec 2025 20:18:00 -0500 Subject: [PATCH 082/103] Create ClickHouseCloud.svelte --- .../icons/connectors/ClickHouseCloud.svelte | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 web-common/src/components/icons/connectors/ClickHouseCloud.svelte diff --git a/web-common/src/components/icons/connectors/ClickHouseCloud.svelte b/web-common/src/components/icons/connectors/ClickHouseCloud.svelte new file mode 100644 index 00000000000..1186726b3c1 --- /dev/null +++ b/web-common/src/components/icons/connectors/ClickHouseCloud.svelte @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + From 2faa3381f3cb3998f0400435182cbcfe444b9c69 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 09:48:38 -0500 Subject: [PATCH 083/103] https://jam.dev/c/0fa9e14e-4e81-4f2e-86fe-8db39aff7a18 - To DOs fix connector preview fix placeholde rtext make sql and name required in model page populate snowflakws UI --- .../features/sources/modal/AddDataForm.svelte | 54 ++-- .../sources/modal/AddDataFormManager.ts | 16 +- .../modal/JSONSchemaFieldControl.svelte | 4 +- .../modal/MultiStepConnectorFlow.svelte | 24 -- .../templates/JSONSchemaFormRenderer.svelte | 285 ++++++++++++++++-- .../src/features/templates/schemas/athena.ts | 2 +- .../features/templates/schemas/bigquery.ts | 1 - .../templates/schemas/clickhouse-cloud.ts | 49 ++- .../features/templates/schemas/clickhouse.ts | 71 +++-- .../src/features/templates/schemas/druid.ts | 28 +- .../src/features/templates/schemas/duckdb.ts | 14 +- .../features/templates/schemas/motherduck.ts | 14 +- .../src/features/templates/schemas/mysql.ts | 17 +- .../src/features/templates/schemas/pinot.ts | 28 +- .../features/templates/schemas/postgres.ts | 15 +- .../features/templates/schemas/redshift.ts | 1 - .../features/templates/schemas/snowflake.ts | 59 ++-- .../src/features/templates/schemas/types.ts | 2 +- 18 files changed, 475 insertions(+), 209 deletions(-) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index eb8065ff448..8bac71e17a1 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -61,12 +61,12 @@ let activeAuthMethod: string | null = null; let prevAuthMethod: string | null = null; let stepState = $connectorStepStore; - let multiStepSubmitDisabled = false; - let multiStepButtonLabel = ""; - let multiStepLoadingCopy = ""; + let multiStepSubmitDisabled = true; + let multiStepButtonLabel = "Test and Connect"; + let multiStepLoadingCopy = "Testing connection..."; let shouldShowSkipLink = false; - let primaryButtonLabel = ""; - let primaryLoadingCopy = ""; + let primaryButtonLabel = "Test and Connect"; + let primaryLoadingCopy = "Testing connection..."; $: stepState = $connectorStepStore; @@ -283,7 +283,28 @@
- {#if hasDsnFormOption} + {#if isMultiStepConnector} + + {:else if hasDsnFormOption} - {:else if isMultiStepConnector} - {:else} -{:else if options?.length} - {:else if isSelectEnum && options} | null = null; - let stepProperties: ConnectorDriverProperty[] | undefined = undefined; let selectedAuthMethod = ""; let previousAuthMethod: string | null = null; @@ -64,12 +62,6 @@ previousAuthMethod = selectedAuthMethod; } - // Compute which properties to show for the current step. - $: stepProperties = - stepState.step === "source" - ? (connector.sourceProperties ?? []) - : properties; - // Initialize source step values from stored connector config. $: if (stepState.step === "source" && stepState.connectorConfig) { const sourceProperties = connector.sourceProperties ?? []; @@ -229,14 +221,6 @@ {onStringInputChange} {handleFileUpload} /> - {:else} - {/if} {:else} @@ -254,14 +238,6 @@ {onStringInputChange} {handleFileUpload} /> - {:else} - {/if} {/if} diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte index 85f787134e1..a1b31dbdae4 100644 --- a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -1,5 +1,7 @@ - - - + diff --git a/web-common/src/features/connectors/connector-icon-mapping.ts b/web-common/src/features/connectors/connector-icon-mapping.ts index a5e581f0395..e5bd344eef5 100644 --- a/web-common/src/features/connectors/connector-icon-mapping.ts +++ b/web-common/src/features/connectors/connector-icon-mapping.ts @@ -15,8 +15,8 @@ import ClickHouseCloudIcon from "../../components/icons/connectors/ClickHouseClo export const connectorIconMapping = { athena: AthenaIcon, bigquery: GoogleBigQueryIcon, - clickhouse: ClickHouseIcon, clickhousecloud: ClickHouseCloudIcon, + clickhouse: ClickHouseIcon, motherduck: MotherDuckIcon, druid: ApacheDruidIcon, duckdb: DuckDbIcon, diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index 42fbc1a123e..cac04d47eb0 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -27,8 +27,8 @@ export const multiStepFormSchemas: Record = { bigquery: bigquerySchema, redshift: redshiftSchema, athena: athenaSchema, - clickhouse: clickhouseSchema, clickhousecloud: clickhouseCloudSchema, + clickhouse: clickhouseSchema, duckdb: duckdbSchema, motherduck: motherduckSchema, druid: druidSchema, diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 14f2dc23a26..908cf8d772a 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -21,8 +21,8 @@ export const SOURCES = [ ]; export const OLAP_ENGINES = [ - "clickhouse", "clickhousecloud", + "clickhouse", "motherduck", "duckdb", "druid", @@ -43,8 +43,8 @@ export const MULTI_STEP_CONNECTORS = [ "bigquery", "redshift", "athena", - "clickhouse", "clickhousecloud", + "clickhouse", "duckdb", "motherduck", "druid", diff --git a/web-common/src/features/sources/modal/icons.ts b/web-common/src/features/sources/modal/icons.ts index 9624685b2a5..2fd0e092ebf 100644 --- a/web-common/src/features/sources/modal/icons.ts +++ b/web-common/src/features/sources/modal/icons.ts @@ -16,7 +16,7 @@ import Postgres from "../../../components/icons/connectors/Postgres.svelte"; import Salesforce from "../../../components/icons/connectors/Salesforce.svelte"; import Snowflake from "../../../components/icons/connectors/Snowflake.svelte"; import SQLite from "../../../components/icons/connectors/SQLite.svelte"; -import ClickHouseCloud from "../../../components/icons/connectors/ClickHouseCloudIcon.svelte"; +import ClickHouseCloud from "../../../components/icons/connectors/ClickHouseCloud.svelte"; export const ICONS = { gcs: GoogleCloudStorage, @@ -34,8 +34,8 @@ export const ICONS = { salesforce: Salesforce, local_file: LocalFile, https: Https, - clickhouse: ClickHouse, clickhousecloud: ClickHouseCloud, + clickhouse: ClickHouse, druid: ApacheDruid, pinot: ApachePinot, }; From 954443118286d87377d42c4b52aa049643659dd3 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 11:35:19 -0500 Subject: [PATCH 085/103] fixing bugs, but not sur eif best way --- .../src/features/connectors/code-utils.ts | 13 +- .../sources/modal/AddDataFormManager.ts | 82 ++++++++---- .../modal/JSONSchemaFieldControl.svelte | 1 + .../modal/MultiStepConnectorFlow.svelte | 10 +- .../src/features/sources/modal/constants.ts | 2 +- .../src/features/sources/modal/utils.ts | 3 + .../src/features/sources/sourceUtils.ts | 122 +++++++++++++++--- .../src/features/templates/schema-utils.ts | 20 ++- .../src/features/templates/schemas/athena.ts | 17 ++- .../templates/schemas/clickhouse-cloud.ts | 38 +++--- .../features/templates/schemas/clickhouse.ts | 1 - .../features/templates/schemas/redshift.ts | 3 + .../src/features/templates/schemas/types.ts | 1 + 13 files changed, 237 insertions(+), 76 deletions(-) diff --git a/web-common/src/features/connectors/code-utils.ts b/web-common/src/features/connectors/code-utils.ts index 2f31fc0ecab..d3ef02bceea 100644 --- a/web-common/src/features/connectors/code-utils.ts +++ b/web-common/src/features/connectors/code-utils.ts @@ -83,9 +83,10 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; if (!property.key) return false; const value = formValues[property.key]; - // Secret fields should always be shown (with env variable placeholder) even if empty + // Secret fields should be shown with env variable placeholder if they exist in formValues + // Don't include secrets that weren't provided (e.g., password when using DSN) const isSecretProperty = secretPropertyKeys.includes(property.key); - if (isSecretProperty) return true; + if (isSecretProperty && value !== undefined) return true; if (value === undefined) return false; // Filter out empty strings for optional fields @@ -102,7 +103,7 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; }) .map((property) => { const key = property.key as string; - const value = formValues[key] as string; + const value = formValues[key]; const isSecretProperty = secretPropertyKeys.includes(key); if (isSecretProperty) { @@ -113,12 +114,14 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; )} }}"`; } + // At this point, value is guaranteed to be defined due to the filter above + const stringValue = value as string; const isStringProperty = stringPropertyKeys.includes(key); if (isStringProperty) { - return `${key}: "${value}"`; + return `${key}: "${stringValue}"`; } - return `${key}: ${value}`; + return `${key}: ${stringValue}`; }) .join("\n"); diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 0a30611ad6a..75dfa3c183e 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -38,7 +38,7 @@ import { } from "@rilldata/web-common/runtime-client"; import type { ActionResult } from "@sveltejs/kit"; import { getConnectorSchema } from "./connector-schemas"; -import { findRadioEnumKey, isVisibleForValues } from "../../templates/schema-utils"; +import { findRadioEnumKey, findGroupedEnumKeys, isVisibleForValues } from "../../templates/schema-utils"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { @@ -48,6 +48,7 @@ type SuperFormUpdateEvent = { const BUTTON_LABELS = { public: { idle: "Continue", submitting: "Continuing..." }, connector: { idle: "Test and Connect", submitting: "Testing connection..." }, + connectorReadOnly: { idle: "Test and Add Connector", submitting: "Testing connection..." }, source: { idle: "Import Data", submitting: "Importing data..." }, }; @@ -150,9 +151,27 @@ export class AddDataFormManager { const paramsAdapter = yup(paramsSchemaDef); type ParamsOut = YupInfer; type ParamsIn = YupInferIn; - const initialFormValues = getInitialFormValuesFromProperties( - this.properties, - ); + + // For schema-based connectors, use schema defaults instead of backend properties + // to avoid pulling defaults from the backend driver + const schema = connector.name ? getConnectorSchema(connector.name) : null; + let initialFormValues: Record; + + if (schema?.properties) { + // Extract defaults from schema + initialFormValues = {}; + for (const [key, prop] of Object.entries(schema.properties)) { + if (prop.default !== undefined) { + initialFormValues[key] = prop.default; + } + } + } else { + // Fall back to backend properties for non-schema connectors + initialFormValues = getInitialFormValuesFromProperties( + this.properties, + ); + } + const paramsDefaults = defaults( initialFormValues as Partial, paramsAdapter, @@ -208,9 +227,6 @@ export class AddDataFormManager { // Only show for connector forms (not sources) if (!isConnectorForm) return false; - // ClickHouse and ClickHouse Cloud have their own error handling - if (this.connector.name === "clickhouse" || this.connector.name === "clickhouse-cloud") return false; - // Need a submission result to show the button if (!event?.result) return false; @@ -255,12 +271,14 @@ export class AddDataFormManager { step: "connector" | "source" | string; submitting: boolean; selectedAuthMethod?: string; + mode?: string; }): string { const { isConnectorForm, step, submitting, selectedAuthMethod, + mode, } = args; if (isConnectorForm) { @@ -270,6 +288,12 @@ export class AddDataFormManager { ? BUTTON_LABELS.public.submitting : BUTTON_LABELS.public.idle; } + // For read-only mode, show "Test and Add Connector" instead of "Test and Connect" + if (mode === "read") { + return submitting + ? BUTTON_LABELS.connectorReadOnly.submitting + : BUTTON_LABELS.connectorReadOnly.idle; + } return submitting ? BUTTON_LABELS.connector.submitting : BUTTON_LABELS.connector.idle; @@ -376,6 +400,15 @@ export class AddDataFormManager { } const preparedValues = prepareConnectorFormData(connector, values); await submitAddConnectorForm(queryClient, connector, preparedValues, false); + + // If mode is "read" (read-only), close without going to source step + // Only advance to source step when mode is "readwrite" + const mode = values?.mode as string | undefined; + if (mode === "read") { + onClose(); + return; + } + setConnectorConfig(preparedValues); setStep("source"); return; @@ -488,7 +521,7 @@ export class AddDataFormManager { let orderedProperties: ConnectorDriverProperty[] = []; // For multi-step connectors with schemas, build properties from schema - const schema = isMultiStepConnector && stepState?.step === "connector" + const schema = isMultiStepConnector && stepState?.step === "connector" && connector.name ? getConnectorSchema(connector.name) : null; @@ -497,37 +530,34 @@ export class AddDataFormManager { const schemaProperties: ConnectorDriverProperty[] = []; const properties = schema.properties ?? {}; - // Find the auth method radio field key (e.g., "auth_method") - we don't want to include this in YAML + // Find all grouped enum keys (radio or tabs with grouped fields) - these are UI control fields we don't want in YAML + const groupedEnumKeys = findGroupedEnumKeys(schema); const authMethodKey = findRadioEnumKey(schema); - // Ensure auth_method has a value for visibility checks (use actual value or fallback to default) + // Ensure all grouped enum keys have values for visibility checks (use actual value or fallback to default) let valuesForVisibility = { ...values }; - if (authMethodKey && !valuesForVisibility[authMethodKey]) { - const authProp = properties[authMethodKey]; - if (authProp?.default) { - valuesForVisibility[authMethodKey] = authProp.default; - } else if (authProp?.enum?.length) { - valuesForVisibility[authMethodKey] = authProp.enum[0]; + for (const enumKey of groupedEnumKeys) { + if (!valuesForVisibility[enumKey]) { + const enumProp = properties[enumKey]; + if (enumProp?.default) { + valuesForVisibility[enumKey] = enumProp.default; + } else if (enumProp?.enum?.length) { + valuesForVisibility[enumKey] = enumProp.enum[0]; + } } } for (const [key, prop] of Object.entries(properties)) { - // Skip the auth_method field itself - it's just a UI control field - if (key === authMethodKey) continue; + // Skip grouped enum keys (auth_method, connection_method, etc.) - they're just UI control fields + if (groupedEnumKeys.includes(key)) continue; // Only include connector step fields that are currently visible if (prop["x-step"] === "connector" && isVisibleForValues(schema, key, valuesForVisibility)) { const value = values[key]; const isSecret = prop["x-secret"] || false; - const hasDefault = prop.default !== undefined; - const matchesDefault = hasDefault && value === prop.default; - - // Skip fields that match their default value unless they're secret - // Secret fields should always be shown - if (!isSecret && matchesDefault) continue; - // Also skip if value is undefined/null/empty string (unless secret) - if (!isSecret && (value === undefined || value === null || value === "")) continue; + // Skip if value is undefined/null/empty string + if (value === undefined || value === null || value === "") continue; schemaProperties.push({ key, diff --git a/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte index abd17814bc7..6466f5132f7 100644 --- a/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte +++ b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte @@ -40,6 +40,7 @@ label={prop.title ?? id} hint={prop.description ?? prop["x-hint"]} {optional} + disabled={prop["x-readonly"]} /> {:else if isSelectEnum && options}
{#if childProp.title} @@ -311,6 +314,33 @@
{/if} {/each} + {#if advancedGrouped.length > 0} +
+ + Advanced Configuration + +
+ {#each advancedGrouped as [childKey, childProp]} +
+ +
+ {/each} +
+
+ {/if} {/if} @@ -324,7 +354,10 @@ {#if groupedFields.get(key)} {#each tabOptions(prop) as option} - {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} + {@const allGroupedForTabOption = getGroupedFieldsForOption(key, option.value)} + {@const regularGroupedTab = allGroupedForTabOption.filter(([_, prop]) => !prop["x-advanced"])} + {@const advancedGroupedTab = allGroupedForTabOption.filter(([_, prop]) => prop["x-advanced"])} + {#each regularGroupedTab as [childKey, childProp]} {#if isRadioEnum(childProp)}
{#if childProp.title} @@ -378,6 +411,33 @@
{/if} {/each} + {#if advancedGroupedTab.length > 0} +
+ + Advanced Configuration + +
+ {#each advancedGroupedTab as [childKey, childProp]} +
+ +
+ {/each} +
+
+ {/if} {/each} {/if} diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts index 721e11206ae..11654d1fa5c 100644 --- a/web-common/src/features/templates/schemas/athena.ts +++ b/web-common/src/features/templates/schemas/athena.ts @@ -70,6 +70,22 @@ export const athenaSchema: MultiStepFormSchema = { "x-step": "connector", "x-advanced": true, }, + aws_access_token: { + type: "string", + title: "AWS Access Token", + description: "AWS access token for authentication (optional)", + "x-placeholder": "Enter AWS access token", + "x-secret": true, + "x-step": "connector", + "x-advanced": true, + }, + allow_host_access: { + type: "boolean", + title: "Allow Host Access", + description: "Allow the connector to access the host's network (optional)", + "x-step": "connector", + "x-advanced": true, + }, sql: { type: "string", title: "SQL Query", diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts index 7e397c6aa1d..c1b8b88bfb4 100644 --- a/web-common/src/features/templates/schemas/azure.ts +++ b/web-common/src/features/templates/schemas/azure.ts @@ -67,6 +67,23 @@ export const azureSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "sas_token" }, }, + path_prefixes: { + type: "string", + title: "Prefixes", + description: + "List of prefixes to filter the blobs (e.g., ['logs/', 'data/'])", + "x-step": "connector", + "x-advanced": true, + }, + allow_host_access: { + type: "boolean", + title: "Allow host access", + description: + "Allow access to the source from the host, useful for local development", + default: false, + "x-step": "connector", + "x-advanced": true, + }, path: { type: "string", title: "Blob URI", diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts index 246a5906454..72ae99c5d5e 100644 --- a/web-common/src/features/templates/schemas/bigquery.ts +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -20,6 +20,14 @@ export const bigquerySchema: MultiStepFormSchema = { "x-placeholder": "my-project-id", "x-step": "connector", }, + allow_host_access: { + type: "boolean", + title: "Allow Host Access", + description: "Allow the connector to access the host machine (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, log_queries: { type: "boolean", title: "Log Queries", diff --git a/web-common/src/features/templates/schemas/clickhouse-cloud.ts b/web-common/src/features/templates/schemas/clickhouse-cloud.ts index 6cefc471ea5..50531529ebc 100644 --- a/web-common/src/features/templates/schemas/clickhouse-cloud.ts +++ b/web-common/src/features/templates/schemas/clickhouse-cloud.ts @@ -71,7 +71,6 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { "x-hint": "If set, Rill will create models as distributed tables in the cluster", "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, - "x-advanced": true, }, ssl: { type: "boolean", @@ -83,6 +82,99 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, + database_whitelist: { + type: "string", + title: "Database Whitelist", + description: "List of allowed databases", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, + }, + optimize_temporary_tables_before_partition_replace: { + type: "boolean", + title: "Optimize Temporary Tables", + description: "Optimize temporary tables before partition replace", + default: false, + "x-advanced": true, + "x-step": "connector", + }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Log all queries executed by Rill", + default: false, + "x-advanced": true, + "x-step": "connector", + }, + query_settings_override: { + type: "object", + title: "Query Settings Override", + description: "Override default query settings", + "x-advanced": true, + "x-step": "connector", + }, + query_settings: { + type: "object", + title: "Query Settings", + description: "Custom query settings", + "x-advanced": true, + "x-step": "connector", + }, + embed_port: { + type: "string", + title: "Embed Port", + description: "Port number for embedding the ClickHouse Cloud server", + "x-step": "connector", + "x-advanced": true, + }, + can_scale_to_zero: { + type: "boolean", + title: "Can Scale to Zero", + description: "Enable scaling to zero", + default: false, + "x-advanced": true, + "x-step": "connector", + }, + max_open_conns: { + type: "number", + title: "Max Open Connections", + description: "Maximum number of open connections", + default: 100, + "x-advanced": true, + "x-step": "connector", + }, + max_idle_conns: { + type: "number", + title: "Max Idle Connections", + description: "Maximum number of idle connections", + default: 10, + "x-advanced": true, + "x-step": "connector", + }, + dial_timeout: { + type: "string", + title: "Dial Timeout", + description: "Timeout for establishing a connection", + default: "30s", + "x-advanced": true, + "x-step": "connector", + }, + conn_max_lifetime: { + type: "string", + title: "Connection Max Lifetime", + description: "Maximum lifetime of a connection", + default: "30m", + "x-advanced": true, + "x-step": "connector", + }, + read_timeout: { + type: "string", + title: "Read Timeout", + description: "Timeout for reading from the connection", + default: "30s", + "x-advanced": true, + "x-step": "connector", + }, dsn: { type: "string", title: "Connection String", diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index d3e7e4dae70..0ae310279a4 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -1,3 +1,4 @@ +import { manual } from "prismjs"; import type { MultiStepFormSchema } from "./types"; export const clickhouseSchema: MultiStepFormSchema = { @@ -124,6 +125,109 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "self-managed" }, }, + managed: { + type: "boolean", + title: "Managed", + description: "Enable managed mode for the ClickHouse server", + default: true, + "x-readonly": true, + "x-hint": "Enable managed mode to manage the server automatically", + "x-step": "connector", + "x-visible-if": { auth_method: "rill-managed" }, + }, + database_whitelist: { + type: "string", + title: "Database Whitelist", + description: "List of allowed databases", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, + }, + optimize_temporary_tables_before_partition_replace: { + type: "boolean", + title: "Optimize Temporary Tables", + description: "Optimize temporary tables before partition replace", + default: false, + "x-advanced": true, + "x-step": "connector", + }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Log all queries executed by Rill", + default: false, + "x-advanced": true, + "x-step": "connector", + }, + query_settings_override: { + type: "object", + title: "Query Settings Override", + description: "Override default query settings", + "x-advanced": true, + "x-step": "connector", + }, + query_settings: { + type: "object", + title: "Query Settings", + description: "Custom query settings", + "x-advanced": true, + "x-step": "connector", + }, + embed_port: { + type: "string", + title: "Embed Port", + description: "Port number for embedding the ClickHouse Cloud server", + "x-step": "connector", + "x-advanced": true, + }, + can_scale_to_zero: { + type: "boolean", + title: "Can Scale to Zero", + description: "Enable scaling to zero", + default: false, + "x-advanced": true, + "x-step": "connector", + }, + max_open_conns: { + type: "number", + title: "Max Open Connections", + description: "Maximum number of open connections", + default: 100, + "x-advanced": true, + "x-step": "connector", + }, + max_idle_conns: { + type: "number", + title: "Max Idle Connections", + description: "Maximum number of idle connections", + default: 10, + "x-advanced": true, + "x-step": "connector", + }, + dial_timeout: { + type: "string", + title: "Dial Timeout", + description: "Timeout for establishing a connection", + default: "30s", + "x-advanced": true, + "x-step": "connector", + }, + conn_max_lifetime: { + type: "string", + title: "Connection Max Lifetime", + description: "Maximum lifetime of a connection", + default: "30m", + "x-advanced": true, + "x-step": "connector", + }, + read_timeout: { + type: "string", + title: "Read Timeout", + description: "Timeout for reading from the connection", + default: "30s", + "x-advanced": true, + "x-step": "connector", + }, sql: { type: "string", title: "SQL Query", @@ -163,7 +267,7 @@ export const clickhouseSchema: MultiStepFormSchema = { }, { if: { properties: { mode: { const: "readwrite" } } }, - then: { required: ["sql", "name"] }, + then: { required: ["managed", "sql", "name"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts index a0737bdf82d..3240cfd14d6 100644 --- a/web-common/src/features/templates/schemas/druid.ts +++ b/web-common/src/features/templates/schemas/druid.ts @@ -51,6 +51,30 @@ export const druidSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, + max_open_conns: { + type: "number", + title: "Maximum open connections", + description: "Maximum number of open connections to the Druid server", + default: 10, + "x-step": "connector", + "x-advanced": true, + }, + skip_version_check: { + type: "boolean", + title: "Skip version check", + description: "Skip the version check when connecting to the Druid server", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + skip_query_priority: { + type: "boolean", + title: "Skip query priority", + description: "Skip the query priority when connecting to the Druid server", + default: false, + "x-step": "connector", + "x-advanced": true, + }, ssl: { type: "boolean", title: "Use SSL", diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts index 3a4079e6fc7..0b0f23f6a09 100644 --- a/web-common/src/features/templates/schemas/gcs.ts +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -61,6 +61,23 @@ export const gcsSchema: MultiStepFormSchema = { "x-placeholder": "gs://bucket/path", "x-step": "source", }, + path_prefixes: { + type: "string", + title: "Path prefixes", + description: + "List of prefixes to filter the files in the GCS bucket. Leave empty to include all files.", + "x-step": "source", + "x-advanced": true, + }, + allow_host_access: { + type: "boolean", + title: "Allow host access", + description: + "Allow access to the GCS bucket from the host machine. This is useful for debugging and testing.", + default: false, + "x-step": "source", + "x-advanced": true, + }, name: { type: "string", title: "Model name", diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts index e6bda2b9d11..74076a24e7b 100644 --- a/web-common/src/features/templates/schemas/https.ts +++ b/web-common/src/features/templates/schemas/https.ts @@ -38,6 +38,17 @@ export const httpsSchema: MultiStepFormSchema = { "x-placeholder": "https://api.example.com/data", "x-step": "source", }, + format: { + type: "string", + title: "Data Format", + description: "Format of the data returned by the API", + enum: ["json", "csv"], + default: "json", + "x-display": "radio", + "x-enum-labels": ["JSON", "CSV"], + "x-enum-descriptions": ["JSON format", "CSV format"], + "x-step": "source", + }, name: { type: "string", title: "Model Name", diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts index d0a436aaa65..69cdf63a519 100644 --- a/web-common/src/features/templates/schemas/mysql.ts +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -69,6 +69,7 @@ export const mysqlSchema: MultiStepFormSchema = { "x-display": "select", "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, }, dsn: { type: "string", diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts index b5897bf4ce7..86294583ba7 100644 --- a/web-common/src/features/templates/schemas/pinot.ts +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -41,6 +41,7 @@ export const pinotSchema: MultiStepFormSchema = { "x-placeholder": "localhost", "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, }, controller_port: { type: "number", @@ -49,6 +50,7 @@ export const pinotSchema: MultiStepFormSchema = { "x-placeholder": "9000", "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, }, username: { type: "string", diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts index 7908243b38c..7ebbba49857 100644 --- a/web-common/src/features/templates/schemas/postgres.ts +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -69,6 +69,7 @@ export const postgresSchema: MultiStepFormSchema = { "x-display": "select", "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, }, dsn: { type: "string", diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts index 5de1fbdf161..b73b01e612a 100644 --- a/web-common/src/features/templates/schemas/redshift.ts +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -52,6 +52,14 @@ export const redshiftSchema: MultiStepFormSchema = { "x-step": "connector", "x-advanced": true, }, + allow_host_access: { + type: "boolean", + title: "Allow Host Access", + description: "Allow the connector to access the host's network (useful for private clusters)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, log_queries: { type: "boolean", title: "Log Queries", diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index 0844d1bd245..aefcb911dd7 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -65,6 +65,7 @@ export const s3Schema: MultiStepFormSchema = { "x-placeholder": "https://s3.example.com", "x-step": "connector", "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, }, aws_role_arn: { type: "string", @@ -74,6 +75,7 @@ export const s3Schema: MultiStepFormSchema = { "x-secret": true, "x-step": "connector", "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, }, aws_role_session_name: { type: "string", diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index f7aac97293e..7c56d904078 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -112,6 +112,15 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { connection_method: "parameters" }, }, + authenticator: { + type: "string", + title: "Authenticator", + description: "Optional authenticator method (e.g., 'externalbrowser')", + "x-placeholder": "externalbrowser", + "x-step": "connector", + "x-advanced": true, + "x-visible-if": { connection_method: "parameters" }, + }, dsn: { type: "string", title: "Connection String", From 104497e0d0dfba1f2c599b10690fba44274a425d Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 13:57:20 -0500 Subject: [PATCH 088/103] missing keys --- .../src/features/templates/schemas/gcs.ts | 20 ++++++++--------- .../src/features/templates/schemas/s3.ts | 20 +++++++++++++++++ .../features/templates/schemas/snowflake.ts | 22 +++++++++---------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts index 0b0f23f6a09..803d99b869d 100644 --- a/web-common/src/features/templates/schemas/gcs.ts +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -53,20 +53,12 @@ export const gcsSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "hmac" }, }, - path: { - type: "string", - title: "GCS URI", - description: "Path to your GCS bucket or prefix", - pattern: "^gs://", - "x-placeholder": "gs://bucket/path", - "x-step": "source", - }, path_prefixes: { type: "string", title: "Path prefixes", description: "List of prefixes to filter the files in the GCS bucket. Leave empty to include all files.", - "x-step": "source", + "x-step": "connector", "x-advanced": true, }, allow_host_access: { @@ -75,9 +67,17 @@ export const gcsSchema: MultiStepFormSchema = { description: "Allow access to the GCS bucket from the host machine. This is useful for debugging and testing.", default: false, - "x-step": "source", + "x-step": "connector", "x-advanced": true, }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://", + "x-placeholder": "gs://bucket/path", + "x-step": "source", + }, name: { type: "string", title: "Model name", diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index aefcb911dd7..42f5f3873b6 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -25,6 +25,8 @@ export const s3Schema: MultiStepFormSchema = { "aws_role_arn", "aws_role_session_name", "aws_external_id", + "path_prefixes", + "allow_host_access", ], public: [], }, @@ -97,6 +99,24 @@ export const s3Schema: MultiStepFormSchema = { "x-visible-if": { auth_method: "access_keys" }, "x-advanced": true, }, + path_prefixes: { + type: "string", + title: "Path prefixes", + description: "Prefixes to include in the model", + "x-step": "connector", + "x-advanced": true, + "x-visible-if": { auth_method: "access_keys" }, + }, + allow_host_access: { + type: "boolean", + title: "Allow host access", + description: + "Allow the Rill instance to access the S3 bucket from the host machine.", + default: false, + "x-step": "connector", + "x-advanced": true, + "x-visible-if": { auth_method: "access_keys" }, + }, path: { type: "string", title: "S3 URI", diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index 7c56d904078..cbae41c76e5 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -14,7 +14,7 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-enum-labels": ["Enter parameters", "Enter connection string"], "x-grouped-fields": { parameters: ["auth_method"], - connection_string: ["dsn", "log_queries", "parallel_fetch_limit"], + connection_string: ["dsn"], }, "x-step": "connector", }, @@ -31,8 +31,8 @@ export const snowflakeSchema: MultiStepFormSchema = { "Authenticate with RSA key pair (more secure).", ], "x-grouped-fields": { - password: ["account", "user", "password", "warehouse", "database", "schema", "role", "log_queries", "parallel_fetch_limit"], - keypair: ["account", "user", "private_key", "private_key_passphrase", "warehouse", "database", "schema", "role", "log_queries", "parallel_fetch_limit"], + password: ["account", "user", "password", "warehouse", "database", "schema", "role"], + keypair: ["account", "user", "private_key", "private_key_passphrase", "warehouse", "database", "schema", "role"], }, "x-step": "connector", "x-visible-if": { connection_method: "parameters" }, @@ -130,14 +130,6 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { connection_method: "connection_string" }, }, - log_queries: { - type: "boolean", - title: "Log Queries", - description: "Enable logging of all SQL queries (useful for debugging)", - default: false, - "x-step": "connector", - "x-advanced": true, - }, parallel_fetch_limit: { type: "number", title: "Parallel Fetch Limit", @@ -146,6 +138,14 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-step": "connector", "x-advanced": true, }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, sql: { type: "string", title: "SQL Query", From 3f389637cc81c22e5e711307a7c245bf867ce054 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 13:59:30 -0500 Subject: [PATCH 089/103] prefix --- web-common/src/features/templates/schemas/azure.ts | 1 + web-common/src/features/templates/schemas/gcs.ts | 1 + web-common/src/features/templates/schemas/s3.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts index c1b8b88bfb4..9b09cc01538 100644 --- a/web-common/src/features/templates/schemas/azure.ts +++ b/web-common/src/features/templates/schemas/azure.ts @@ -72,6 +72,7 @@ export const azureSchema: MultiStepFormSchema = { title: "Prefixes", description: "List of prefixes to filter the blobs (e.g., ['logs/', 'data/'])", + "x-placeholder": "['logs/', 'data/']", "x-step": "connector", "x-advanced": true, }, diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts index 803d99b869d..0dc629a0ed7 100644 --- a/web-common/src/features/templates/schemas/gcs.ts +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -58,6 +58,7 @@ export const gcsSchema: MultiStepFormSchema = { title: "Path prefixes", description: "List of prefixes to filter the files in the GCS bucket. Leave empty to include all files.", + "x-placeholder": "['logs/', 'data/']", "x-step": "connector", "x-advanced": true, }, diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index 42f5f3873b6..02a1a1d42e6 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -103,6 +103,7 @@ export const s3Schema: MultiStepFormSchema = { type: "string", title: "Path prefixes", description: "Prefixes to include in the model", + "x-placeholder": "['logs/', 'data/']", "x-step": "connector", "x-advanced": true, "x-visible-if": { auth_method: "access_keys" }, From 3f665f627a9bbe15c55872edd568c8d560be8027 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 14:00:02 -0500 Subject: [PATCH 090/103] prettier --- asd | Bin 12288 -> 0 bytes .../icons/connectors/ClickHouseCloud.svelte | 55 ++++++++-- .../src/features/connectors/code-utils.ts | 3 +- .../features/sources/modal/AddDataForm.svelte | 13 ++- .../sources/modal/AddDataFormManager.ts | 77 +++++++++----- .../sources/modal/AddDataModal.svelte | 11 +- .../modal/MultiStepConnectorFlow.svelte | 22 ++-- .../src/features/sources/modal/constants.ts | 7 +- .../src/features/sources/modal/utils.ts | 2 - .../src/features/sources/sourceUtils.ts | 6 +- .../templates/JSONSchemaFormRenderer.svelte | 96 +++++++++++++----- .../src/features/templates/schema-utils.ts | 6 +- .../src/features/templates/schemas/athena.ts | 17 +++- .../src/features/templates/schemas/azure.ts | 16 ++- .../features/templates/schemas/bigquery.ts | 11 +- .../templates/schemas/clickhouse-cloud.ts | 21 +++- .../features/templates/schemas/clickhouse.ts | 59 ++++++++--- .../src/features/templates/schemas/druid.ts | 6 +- .../src/features/templates/schemas/duckdb.ts | 6 +- .../src/features/templates/schemas/https.ts | 2 +- .../src/features/templates/schemas/mysql.ts | 15 ++- .../src/features/templates/schemas/pinot.ts | 13 ++- .../features/templates/schemas/postgres.ts | 13 ++- .../features/templates/schemas/redshift.ts | 17 +++- .../src/features/templates/schemas/s3.ts | 13 ++- .../features/templates/schemas/snowflake.ts | 39 +++++-- 26 files changed, 406 insertions(+), 140 deletions(-) delete mode 100644 asd diff --git a/asd b/asd deleted file mode 100644 index 2230283a26663375035808f7b530f35f957d8973..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI#F$%&^3;@tCICv4k>C>V@ofPUOxOoDL-oRN`!O4pVUcnQ1AJL%{!BTK)mzO0W z2?XAj{p&h9UE+PY$ybZ;UmVgniR0$oA}zLAwwq0p<{c9P1PBlyK!5-N0t5&UAV8o! zfyZ%pJFn-@>f-w+s(uLlvK*AD_Z`$K|9_1{g8%^n1PBlyK!5-N0t5(jD&YU`l;3dz U1PBlyK!5-N0t5&UAkd1y2hRI9P5=M^ diff --git a/web-common/src/components/icons/connectors/ClickHouseCloud.svelte b/web-common/src/components/icons/connectors/ClickHouseCloud.svelte index c227ea752f1..9c74e3a6f6d 100644 --- a/web-common/src/components/icons/connectors/ClickHouseCloud.svelte +++ b/web-common/src/components/icons/connectors/ClickHouseCloud.svelte @@ -1,11 +1,46 @@ - - - - - - - - - - + + + + + + + + + + diff --git a/web-common/src/features/connectors/code-utils.ts b/web-common/src/features/connectors/code-utils.ts index d3ef02bceea..f82f1e5f02a 100644 --- a/web-common/src/features/connectors/code-utils.ts +++ b/web-common/src/features/connectors/code-utils.ts @@ -63,7 +63,8 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; } // Get the secret property keys from the properties being used (orderedProperties or configProperties) - const propertiesForTypeChecking = options?.orderedProperties ?? connector.configProperties ?? []; + const propertiesForTypeChecking = + options?.orderedProperties ?? connector.configProperties ?? []; const secretPropertyKeys = propertiesForTypeChecking .filter((property) => property.secret) diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 8bac71e17a1..e589a1c8e31 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -211,8 +211,7 @@ // For other connectors, use manager helper saveAnyway = true; - const values = - onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; + const values = onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; const result = await formManager.saveConnectorAnyway({ queryClient, values, @@ -415,12 +414,12 @@
{#if dsnError || paramsError} {/if} diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 5310902456e..3150f06a107 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -33,14 +33,22 @@ import { } from "./connectorStepStore"; import { get } from "svelte/store"; import { compileConnectorYAML } from "../../connectors/code-utils"; -import { compileSourceYAML, prepareSourceFormData, prepareConnectorFormData } from "../sourceUtils"; +import { + compileSourceYAML, + prepareSourceFormData, + prepareConnectorFormData, +} from "../sourceUtils"; import { ConnectorDriverPropertyType, type ConnectorDriverProperty, } from "@rilldata/web-common/runtime-client"; import type { ActionResult } from "@sveltejs/kit"; import { getConnectorSchema } from "./connector-schemas"; -import { findRadioEnumKey, findGroupedEnumKeys, isVisibleForValues } from "../../templates/schema-utils"; +import { + findRadioEnumKey, + findGroupedEnumKeys, + isVisibleForValues, +} from "../../templates/schema-utils"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { @@ -50,7 +58,10 @@ type SuperFormUpdateEvent = { const BUTTON_LABELS = { public: { idle: "Continue", submitting: "Continuing..." }, connector: { idle: "Test and Connect", submitting: "Testing connection..." }, - connectorReadOnly: { idle: "Test and Add Connector", submitting: "Testing connection..." }, + connectorReadOnly: { + idle: "Test and Add Connector", + submitting: "Testing connection...", + }, source: { idle: "Import Data", submitting: "Importing data..." }, }; @@ -105,7 +116,6 @@ export class AddDataFormManager { this.formHeight = FORM_HEIGHT_DEFAULT; } - // IDs this.paramsFormId = `add-data-${connector.name}-form`; this.dsnFormId = `add-data-${connector.name}-dsn-form`; @@ -174,9 +184,7 @@ export class AddDataFormManager { } } else { // Fall back to backend properties for non-schema connectors - initialFormValues = getInitialFormValuesFromProperties( - this.properties, - ); + initialFormValues = getInitialFormValuesFromProperties(this.properties); } const paramsDefaults = defaults( @@ -285,13 +293,8 @@ export class AddDataFormManager { selectedAuthMethod?: string; mode?: string; }): string { - const { - isConnectorForm, - step, - submitting, - selectedAuthMethod, - mode, - } = args; + const { isConnectorForm, step, submitting, selectedAuthMethod, mode } = + args; if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { @@ -411,7 +414,12 @@ export class AddDataFormManager { return; } const preparedValues = prepareConnectorFormData(connector, values); - await submitAddConnectorForm(queryClient, connector, preparedValues, false); + await submitAddConnectorForm( + queryClient, + connector, + preparedValues, + false, + ); // If mode is "read" (read-only), close without going to source step // Only advance to source step when mode is "readwrite" @@ -429,7 +437,12 @@ export class AddDataFormManager { onClose(); } else { const preparedValues = prepareConnectorFormData(connector, values); - await submitAddConnectorForm(queryClient, connector, preparedValues, false); + await submitAddConnectorForm( + queryClient, + connector, + preparedValues, + false, + ); onClose(); } } catch (e) { @@ -533,9 +546,12 @@ export class AddDataFormManager { let orderedProperties: ConnectorDriverProperty[] = []; // For multi-step connectors with schemas, build properties from schema - const schema = isMultiStepConnector && stepState?.step === "connector" && connector.name - ? getConnectorSchema(connector.name) - : null; + const schema = + isMultiStepConnector && + stepState?.step === "connector" && + connector.name + ? getConnectorSchema(connector.name) + : null; if (schema) { // Build ordered properties from schema fields that are visible and on connector step @@ -564,7 +580,10 @@ export class AddDataFormManager { if (groupedEnumKeys.includes(key)) continue; // Only include connector step fields that are currently visible - if (prop["x-step"] === "connector" && isVisibleForValues(schema, key, valuesForVisibility)) { + if ( + prop["x-step"] === "connector" && + isVisibleForValues(schema, key, valuesForVisibility) + ) { const value = values[key]; const isSecret = prop["x-secret"] || false; @@ -573,11 +592,12 @@ export class AddDataFormManager { schemaProperties.push({ key, - type: prop.type === "number" - ? ConnectorDriverPropertyType.TYPE_NUMBER - : prop.type === "boolean" - ? ConnectorDriverPropertyType.TYPE_BOOLEAN - : ConnectorDriverPropertyType.TYPE_STRING, + type: + prop.type === "number" + ? ConnectorDriverPropertyType.TYPE_NUMBER + : prop.type === "boolean" + ? ConnectorDriverPropertyType.TYPE_BOOLEAN + : ConnectorDriverPropertyType.TYPE_STRING, secret: isSecret, }); } @@ -610,12 +630,15 @@ export class AddDataFormManager { ); // Also filter out grouped enum keys (auth_method, connection_method, mode, etc.) - const schema = connector.name ? getConnectorSchema(connector.name) : null; + const schema = connector.name + ? getConnectorSchema(connector.name) + : null; const groupedEnumKeys = schema ? findGroupedEnumKeys(schema) : []; filteredValues = Object.fromEntries( Object.entries(values).filter( - ([key]) => !connectorPropertyKeys.has(key) && !groupedEnumKeys.includes(key), + ([key]) => + !connectorPropertyKeys.has(key) && !groupedEnumKeys.includes(key), ), ); } diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index cd1b9370b2a..cfff6ef215b 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -21,7 +21,12 @@ import DuplicateSource from "./DuplicateSource.svelte"; import LocalSourceUpload from "./LocalSourceUpload.svelte"; import RequestConnectorForm from "./RequestConnectorForm.svelte"; - import { OLAP_ENGINES, ALL_CONNECTORS, SOURCES, MULTI_STEP_CONNECTORS } from "./constants"; + import { + OLAP_ENGINES, + ALL_CONNECTORS, + SOURCES, + MULTI_STEP_CONNECTORS, + } from "./constants"; import { ICONS } from "./icons"; import { resetConnectorStep } from "./connectorStepStore"; @@ -37,7 +42,9 @@ let connectors = data.connectors || []; // Clone clickhouse connector to create clickhousecloud (frontend-only) - const clickhouseConnector = connectors.find(c => c.name === "clickhouse"); + const clickhouseConnector = connectors.find( + (c) => c.name === "clickhouse", + ); if (clickhouseConnector) { const clickhouseCloudConnector: V1ConnectorDriver = { ...clickhouseConnector, diff --git a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte index a8d87e3b0a2..989956ba657 100644 --- a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte +++ b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte @@ -105,12 +105,15 @@ if (fallback !== stepState.selectedAuthMethod) { setAuthMethod(fallback ?? null); if (fallback && authKey) { - paramsForm.update(($form) => { - if ($form[authKey] !== fallback) { - $form[authKey] = fallback; - } - return $form; - }, { taint: false }); + paramsForm.update( + ($form) => { + if ($form[authKey] !== fallback) { + $form[authKey] = fallback; + } + return $form; + }, + { taint: false }, + ); } } } @@ -131,7 +134,12 @@ } // Clear form when auth method changes (e.g., switching from parameters to DSN). - $: if (activeSchema && selectedAuthMethod !== previousAuthMethod && previousAuthMethod !== null && previousAuthMethod !== "") { + $: if ( + activeSchema && + selectedAuthMethod !== previousAuthMethod && + previousAuthMethod !== null && + previousAuthMethod !== "" + ) { const authKey = findRadioEnumKey(activeSchema); if (authKey) { // Get default values for the new auth method diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 6dca54028d8..5453813c369 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -52,7 +52,7 @@ export const MULTI_STEP_CONNECTORS = [ ]; export const FORM_HEIGHT_TALL = "max-h-[55.5rem] min-h-[38.5rem]"; -export const FORM_HEIGHT_MEDIUM= "max-h-[47.5rem] min-h-[34.5rem]"; +export const FORM_HEIGHT_MEDIUM = "max-h-[47.5rem] min-h-[34.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; export const MEDIUM_FORM_CONNECTORS = new Set([ "clickhousecloud", @@ -62,7 +62,4 @@ export const MEDIUM_FORM_CONNECTORS = new Set([ "mysql", "pinot", ]); -export const TALL_FORM_CONNECTORS = new Set([ - "clickhouse", - "snowflake", -]); +export const TALL_FORM_CONNECTORS = new Set(["clickhouse", "snowflake"]); diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 208c6c79ecb..ccaeefaecf3 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -181,5 +181,3 @@ export function isMultiStepConnectorDisabled( return !isEmpty(value) && !hasErrors; }); } - - diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 7bd7484e6ff..b4c78fd870d 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -289,13 +289,15 @@ export function prepareConnectorFormData( if (connector.name === "clickhousecloud") { filteredValues.managed = false; // Only set ssl=true if it's in the filtered values (i.e., using parameters tab) - if ('ssl' in filteredValues) { + if ("ssl" in filteredValues) { filteredValues.ssl = true; } } // Replace with filtered values - Object.keys(processedValues).forEach(key => delete processedValues[key]); + Object.keys(processedValues).forEach( + (key) => delete processedValues[key], + ); Object.assign(processedValues, filteredValues); // Remove the grouped enum keys themselves - they're UI-only fields diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte index f27219384a4..96e218baf7c 100644 --- a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -37,12 +37,8 @@ $: renderOrder = schema ? computeRenderOrder(visibleEntries, groupedChildKeys) : []; - $: regularFields = renderOrder.filter( - ([_, prop]) => !prop["x-advanced"], - ); - $: advancedFields = renderOrder.filter( - ([_, prop]) => prop["x-advanced"], - ); + $: regularFields = renderOrder.filter(([_, prop]) => !prop["x-advanced"]); + $: advancedFields = renderOrder.filter(([_, prop]) => prop["x-advanced"]); // Seed defaults for initial render: use explicit defaults, and for radio enums // fall back to first option when no value is set. @@ -259,16 +255,29 @@ > {#if groupedFields.get(key)} - {@const allGroupedForOption = getGroupedFieldsForOption(key, option.value)} - {@const regularGrouped = allGroupedForOption.filter(([_, prop]) => !prop["x-advanced"])} - {@const advancedGrouped = allGroupedForOption.filter(([_, prop]) => prop["x-advanced"])} + {@const allGroupedForOption = getGroupedFieldsForOption( + key, + option.value, + )} + {@const regularGrouped = allGroupedForOption.filter( + ([_, prop]) => !prop["x-advanced"], + )} + {@const advancedGrouped = allGroupedForOption.filter( + ([_, prop]) => prop["x-advanced"], + )} {#each regularGrouped as [childKey, childProp]} {#if isTabsEnum(childProp)}
{#if childProp.title} -
{childProp.title}
+
+ {childProp.title} +
{/if} - + {#if groupedFields.get(childKey)} {#each tabOptions(childProp) as tabOption} @@ -316,7 +325,9 @@ {/each} {#if advancedGrouped.length > 0}
- + Advanced Configuration
@@ -350,25 +361,41 @@ {#if prop.title}
{prop.title}
{/if} - + {#if groupedFields.get(key)} {#each tabOptions(prop) as option} - {@const allGroupedForTabOption = getGroupedFieldsForOption(key, option.value)} - {@const regularGroupedTab = allGroupedForTabOption.filter(([_, prop]) => !prop["x-advanced"])} - {@const advancedGroupedTab = allGroupedForTabOption.filter(([_, prop]) => prop["x-advanced"])} + {@const allGroupedForTabOption = getGroupedFieldsForOption( + key, + option.value, + )} + {@const regularGroupedTab = allGroupedForTabOption.filter( + ([_, prop]) => !prop["x-advanced"], + )} + {@const advancedGroupedTab = allGroupedForTabOption.filter( + ([_, prop]) => prop["x-advanced"], + )} {#each regularGroupedTab as [childKey, childProp]} {#if isRadioEnum(childProp)}
{#if childProp.title} -
{childProp.title}
+
+ {childProp.title} +
{/if} - + {#if groupedFields.get(childKey)} {#each getGroupedFieldsForOption(childKey, radioOption.value) as [grandchildKey, grandchildProp]}
@@ -413,7 +440,9 @@ {/each} {#if advancedGroupedTab.length > 0}
- + Advanced Configuration
@@ -463,7 +492,9 @@ {#if advancedFields.length > 0}
- + Advanced Configuration
@@ -484,9 +515,15 @@ {#if isTabsEnum(childProp)}
{#if childProp.title} -
{childProp.title}
+
+ {childProp.title} +
{/if} - + {#if groupedFields.get(childKey)} {#each tabOptions(childProp) as tabOption} @@ -541,7 +578,11 @@ {#if prop.title}
{prop.title}
{/if} - + {#if groupedFields.get(key)} {#each tabOptions(prop) as option} @@ -549,14 +590,19 @@ {#if isRadioEnum(childProp)}
{#if childProp.title} -
{childProp.title}
+
+ {childProp.title} +
{/if} - + {#if groupedFields.get(childKey)} {#each getGroupedFieldsForOption(childKey, radioOption.value) as [grandchildKey, grandchildProp]}
diff --git a/web-common/src/features/templates/schema-utils.ts b/web-common/src/features/templates/schema-utils.ts index 04a098d1c07..30ea9436346 100644 --- a/web-common/src/features/templates/schema-utils.ts +++ b/web-common/src/features/templates/schema-utils.ts @@ -57,7 +57,11 @@ export function findRadioEnumKey(schema: MultiStepFormSchema): string | null { // Return radio or tabs fields that have grouped fields - those are auth/connection method selectors // Standalone radio fields (like "mode") should not be considered auth method keys const display = value["x-display"]; - if (value.enum && (display === "radio" || display === "tabs") && value["x-grouped-fields"]) { + if ( + value.enum && + (display === "radio" || display === "tabs") && + value["x-grouped-fields"] + ) { return key; } } diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts index 11654d1fa5c..2255a6dd39d 100644 --- a/web-common/src/features/templates/schemas/athena.ts +++ b/web-common/src/features/templates/schemas/athena.ts @@ -46,7 +46,8 @@ export const athenaSchema: MultiStepFormSchema = { role_session_name: { type: "string", title: "Role Session Name", - description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + description: + "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", "x-placeholder": "my-session-name", "x-secret": true, "x-step": "connector", @@ -55,7 +56,8 @@ export const athenaSchema: MultiStepFormSchema = { external_id: { type: "string", title: "External ID", - description: "Optional external ID to use when assuming an AWS role for cross-account access.", + description: + "Optional external ID to use when assuming an AWS role for cross-account access.", "x-placeholder": "external-id-123", "x-secret": true, "x-step": "connector", @@ -82,7 +84,8 @@ export const athenaSchema: MultiStepFormSchema = { allow_host_access: { type: "boolean", title: "Allow Host Access", - description: "Allow the connector to access the host's network (optional)", + description: + "Allow the connector to access the host's network (optional)", "x-step": "connector", "x-advanced": true, }, @@ -102,5 +105,11 @@ export const athenaSchema: MultiStepFormSchema = { "x-step": "source", }, }, - required: ["aws_access_key_id", "aws_secret_access_key", "output_location", "sql", "name"], + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "output_location", + "sql", + "name", + ], }; diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts index 9b09cc01538..b30e5b50bc6 100644 --- a/web-common/src/features/templates/schemas/azure.ts +++ b/web-common/src/features/templates/schemas/azure.ts @@ -111,12 +111,24 @@ export const azureSchema: MultiStepFormSchema = { }, { if: { properties: { auth_method: { const: "account_key" } } }, - then: { required: ["azure_storage_account", "azure_storage_key", "path", "name"] }, + then: { + required: [ + "azure_storage_account", + "azure_storage_key", + "path", + "name", + ], + }, }, { if: { properties: { auth_method: { const: "sas_token" } } }, then: { - required: ["azure_storage_account", "azure_storage_sas_token", "path", "name"], + required: [ + "azure_storage_account", + "azure_storage_sas_token", + "path", + "name", + ], }, }, { diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts index 72ae99c5d5e..cc0389c1b4a 100644 --- a/web-common/src/features/templates/schemas/bigquery.ts +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -7,7 +7,8 @@ export const bigquerySchema: MultiStepFormSchema = { google_application_credentials: { type: "string", title: "GCP Credentials", - description: "Upload a JSON key file for a service account with BigQuery access", + description: + "Upload a JSON key file for a service account with BigQuery access", format: "file", "x-display": "file", "x-accept": ".json", @@ -16,14 +17,16 @@ export const bigquerySchema: MultiStepFormSchema = { project_id: { type: "string", title: "Project ID", - description: "Google Cloud project ID (optional if specified in credentials)", + description: + "Google Cloud project ID (optional if specified in credentials)", "x-placeholder": "my-project-id", "x-step": "connector", }, - allow_host_access: { + allow_host_access: { type: "boolean", title: "Allow Host Access", - description: "Allow the connector to access the host machine (useful for debugging)", + description: + "Allow the connector to access the host machine (useful for debugging)", default: false, "x-step": "connector", "x-advanced": true, diff --git a/web-common/src/features/templates/schemas/clickhouse-cloud.ts b/web-common/src/features/templates/schemas/clickhouse-cloud.ts index 50531529ebc..92f2bbda0fc 100644 --- a/web-common/src/features/templates/schemas/clickhouse-cloud.ts +++ b/web-common/src/features/templates/schemas/clickhouse-cloud.ts @@ -13,7 +13,16 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { "x-display": "tabs", "x-enum-labels": ["Username & Password", "Connection String"], "x-grouped-fields": { - parameters: ["host", "port", "database", "username", "password", "ssl", "cluster", "mode"], + parameters: [ + "host", + "port", + "database", + "username", + "password", + "ssl", + "cluster", + "mode", + ], connection_string: ["dsn", "mode"], }, "x-step": "connector", @@ -68,7 +77,8 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { title: "Cluster", description: "Cluster name for distributed tables", "x-placeholder": "Cluster name", - "x-hint": "If set, Rill will create models as distributed tables in the cluster", + "x-hint": + "If set, Rill will create models as distributed tables in the cluster", "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, }, @@ -179,7 +189,8 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "string", title: "Connection String", description: "ClickHouse connection string (DSN)", - "x-placeholder": "https://default@your-instance.clickhouse.cloud:8443/default", + "x-placeholder": + "https://default@your-instance.clickhouse.cloud:8443/default", "x-secret": true, "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, @@ -214,7 +225,7 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { "x-placeholder": "my_model", "x-step": "source", "x-visible-if": { mode: "readwrite" }, - } + }, }, allOf: [ { @@ -230,4 +241,4 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { then: { required: ["sql", "name"] }, }, ], -}; \ No newline at end of file +}; diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index 0ae310279a4..c997b1eadac 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -32,7 +32,16 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-display": "tabs", "x-enum-labels": ["Enter parameters", "Enter connection string"], "x-grouped-fields": { - parameters: ["host", "port", "username", "password", "database", "ssl", "cluster", "mode"], + parameters: [ + "host", + "port", + "username", + "password", + "database", + "ssl", + "cluster", + "mode", + ], connection_string: ["dsn", "mode"], }, "x-step": "connector", @@ -44,7 +53,10 @@ export const clickhouseSchema: MultiStepFormSchema = { description: "Hostname or IP address of the ClickHouse server", "x-placeholder": "your-server.clickhouse.com", "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "parameters" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, }, port: { type: "number", @@ -52,9 +64,13 @@ export const clickhouseSchema: MultiStepFormSchema = { description: "Port number of the ClickHouse server", default: 9000, "x-placeholder": "9000", - "x-hint": "Default: 9000 (native TCP), 8123 (HTTP). Secure: 9440 (TCP+TLS), 8443 (HTTPS)", + "x-hint": + "Default: 9000 (native TCP), 8123 (HTTP). Secure: 9440 (TCP+TLS), 8443 (HTTPS)", "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "parameters" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, }, username: { type: "string", @@ -62,7 +78,10 @@ export const clickhouseSchema: MultiStepFormSchema = { description: "Username to connect to the ClickHouse server", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "parameters" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, }, password: { type: "string", @@ -71,7 +90,10 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "Enter password", "x-secret": true, "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "parameters" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, }, database: { type: "string", @@ -80,7 +102,10 @@ export const clickhouseSchema: MultiStepFormSchema = { default: "default", "x-placeholder": "default", "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "parameters" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, }, ssl: { type: "boolean", @@ -89,16 +114,23 @@ export const clickhouseSchema: MultiStepFormSchema = { default: true, "x-hint": "Enable SSL for secure connections", "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "parameters" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, }, cluster: { type: "string", title: "Cluster", description: "Cluster name for distributed tables", "x-placeholder": "Cluster name", - "x-hint": "If set, Rill will create models as distributed tables in the cluster", + "x-hint": + "If set, Rill will create models as distributed tables in the cluster", "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "parameters" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, "x-advanced": true, }, dsn: { @@ -108,7 +140,10 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "clickhouse://username:password@host:port/database", "x-secret": true, "x-step": "connector", - "x-visible-if": { auth_method: "self-managed", connection_method: "connection_string" }, + "x-visible-if": { + auth_method: "self-managed", + connection_method: "connection_string", + }, }, mode: { type: "string", @@ -135,7 +170,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "rill-managed" }, }, - database_whitelist: { + database_whitelist: { type: "string", title: "Database Whitelist", description: "List of allowed databases", diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts index 3240cfd14d6..e0a3d3fb25f 100644 --- a/web-common/src/features/templates/schemas/druid.ts +++ b/web-common/src/features/templates/schemas/druid.ts @@ -70,7 +70,8 @@ export const druidSchema: MultiStepFormSchema = { skip_query_priority: { type: "boolean", title: "Skip query priority", - description: "Skip the query priority when connecting to the Druid server", + description: + "Skip the query priority when connecting to the Druid server", default: false, "x-step": "connector", "x-advanced": true, @@ -87,7 +88,8 @@ export const druidSchema: MultiStepFormSchema = { type: "string", title: "Connection String", description: "Druid connection string (DSN)", - "x-placeholder": "https://example.com/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=username&avaticaPassword=password", + "x-placeholder": + "https://example.com/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=username&avaticaPassword=password", "x-secret": true, "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts index f6f14a26da9..e93adb085c9 100644 --- a/web-common/src/features/templates/schemas/duckdb.ts +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -14,8 +14,10 @@ export const duckdbSchema: MultiStepFormSchema = { attach: { type: "string", title: "Attach", - description: "Attach to an existing DuckDB database with options (alternative to path)", - "x-placeholder": "'ducklake:metadata.ducklake' AS my_ducklake(DATA_PATH 'datafiles')", + description: + "Attach to an existing DuckDB database with options (alternative to path)", + "x-placeholder": + "'ducklake:metadata.ducklake' AS my_ducklake(DATA_PATH 'datafiles')", "x-step": "connector", "x-advanced": true, }, diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts index 74076a24e7b..ca3299876d0 100644 --- a/web-common/src/features/templates/schemas/https.ts +++ b/web-common/src/features/templates/schemas/https.ts @@ -44,7 +44,7 @@ export const httpsSchema: MultiStepFormSchema = { description: "Format of the data returned by the API", enum: ["json", "csv"], default: "json", - "x-display": "radio", + "x-display": "radio", "x-enum-labels": ["JSON", "CSV"], "x-enum-descriptions": ["JSON format", "CSV format"], "x-step": "source", diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts index 69cdf63a519..33cafa3326f 100644 --- a/web-common/src/features/templates/schemas/mysql.ts +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -13,7 +13,16 @@ export const mysqlSchema: MultiStepFormSchema = { "x-display": "tabs", "x-enum-labels": ["Enter parameters", "Enter connection string"], "x-grouped-fields": { - parameters: ["", "host", "port", "database", "user", "password", "ssl-mode", "log_queries"], + parameters: [ + "", + "host", + "port", + "database", + "user", + "password", + "ssl-mode", + "log_queries", + ], connection_string: ["dsn", "log_queries"], }, "x-step": "connector", @@ -108,7 +117,9 @@ export const mysqlSchema: MultiStepFormSchema = { allOf: [ { if: { properties: { auth_method: { const: "parameters" } } }, - then: { required: ["host", "database", "user", "password", "sql", "name"] }, + then: { + required: ["host", "database", "user", "password", "sql", "name"], + }, }, { if: { properties: { auth_method: { const: "connection_string" } } }, diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts index 86294583ba7..273080b9127 100644 --- a/web-common/src/features/templates/schemas/pinot.ts +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -13,7 +13,15 @@ export const pinotSchema: MultiStepFormSchema = { "x-display": "tabs", "x-enum-labels": ["Enter parameters", "Enter connection string"], "x-grouped-fields": { - parameters: ["broker_host", "broker_port", "controller_host", "controller_port", "username", "password", "ssl"], + parameters: [ + "broker_host", + "broker_port", + "controller_host", + "controller_port", + "username", + "password", + "ssl", + ], connection_string: ["dsn"], }, "x-step": "connector", @@ -81,7 +89,8 @@ export const pinotSchema: MultiStepFormSchema = { type: "string", title: "Connection String", description: "Pinot connection string (DSN)", - "x-placeholder": "http(s)://username:password@localhost:8000?controller=localhost:9000", + "x-placeholder": + "http(s)://username:password@localhost:8000?controller=localhost:9000", "x-secret": true, "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts index 7ebbba49857..6629a505ed2 100644 --- a/web-common/src/features/templates/schemas/postgres.ts +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -13,7 +13,15 @@ export const postgresSchema: MultiStepFormSchema = { "x-display": "tabs", "x-enum-labels": ["Enter parameters", "Enter connection string"], "x-grouped-fields": { - parameters: ["host", "port", "dbname", "user", "password", "sslmode", "log_queries"], + parameters: [ + "host", + "port", + "dbname", + "user", + "password", + "sslmode", + "log_queries", + ], connection_string: ["dsn", "log_queries"], }, "x-step": "connector", @@ -75,7 +83,8 @@ export const postgresSchema: MultiStepFormSchema = { type: "string", title: "Connection String", description: "PostgreSQL connection string (DSN)", - "x-placeholder": "postgres://user:password@host:5432/dbname?sslmode=require", + "x-placeholder": + "postgres://user:password@host:5432/dbname?sslmode=require", "x-secret": true, "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts index b73b01e612a..94a20829f7c 100644 --- a/web-common/src/features/templates/schemas/redshift.ts +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -46,16 +46,19 @@ export const redshiftSchema: MultiStepFormSchema = { cluster_identifier: { type: "string", title: "Cluster Identifier", - description: "Redshift provisioned cluster identifier (for provisioned clusters)", + description: + "Redshift provisioned cluster identifier (for provisioned clusters)", "x-placeholder": "my-redshift-cluster", - "x-hint": "Provide either workgroup (for serverless) or cluster identifier (for provisioned)", + "x-hint": + "Provide either workgroup (for serverless) or cluster identifier (for provisioned)", "x-step": "connector", "x-advanced": true, }, allow_host_access: { type: "boolean", title: "Allow Host Access", - description: "Allow the connector to access the host's network (useful for private clusters)", + description: + "Allow the connector to access the host's network (useful for private clusters)", default: false, "x-step": "connector", "x-advanced": true, @@ -84,5 +87,11 @@ export const redshiftSchema: MultiStepFormSchema = { "x-step": "source", }, }, - required: ["aws_access_key_id", "aws_secret_access_key", "database", "sql", "name"], + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "database", + "sql", + "name", + ], }; diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index 02a1a1d42e6..1b5dc293fe5 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -82,7 +82,8 @@ export const s3Schema: MultiStepFormSchema = { aws_role_session_name: { type: "string", title: "Role Session Name", - description: "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + description: + "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", "x-placeholder": "my-session-name", "x-secret": true, "x-step": "connector", @@ -92,7 +93,8 @@ export const s3Schema: MultiStepFormSchema = { aws_external_id: { type: "string", title: "External ID", - description: "Optional external ID to use when assuming an AWS role for cross-account access.", + description: + "Optional external ID to use when assuming an AWS role for cross-account access.", "x-placeholder": "external-id-123", "x-secret": true, "x-step": "connector", @@ -140,7 +142,12 @@ export const s3Schema: MultiStepFormSchema = { { if: { properties: { auth_method: { const: "access_keys" } } }, then: { - required: ["aws_access_key_id", "aws_secret_access_key", "path", "name"], + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "path", + "name", + ], }, }, { diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index cbae41c76e5..0e21f1977cb 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -31,8 +31,25 @@ export const snowflakeSchema: MultiStepFormSchema = { "Authenticate with RSA key pair (more secure).", ], "x-grouped-fields": { - password: ["account", "user", "password", "warehouse", "database", "schema", "role"], - keypair: ["account", "user", "private_key", "private_key_passphrase", "warehouse", "database", "schema", "role"], + password: [ + "account", + "user", + "password", + "warehouse", + "database", + "schema", + "role", + ], + keypair: [ + "account", + "user", + "private_key", + "private_key_passphrase", + "warehouse", + "database", + "schema", + "role", + ], }, "x-step": "connector", "x-visible-if": { connection_method: "parameters" }, @@ -60,16 +77,23 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-placeholder": "Enter password", "x-secret": true, "x-step": "connector", - "x-visible-if": { connection_method: "parameters", auth_method: "password" }, + "x-visible-if": { + connection_method: "parameters", + auth_method: "password", + }, }, private_key: { type: "string", title: "Private Key", description: "RSA private key in PEM format", - "x-placeholder": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----", + "x-placeholder": + "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----", "x-secret": true, "x-step": "connector", - "x-visible-if": { connection_method: "parameters", auth_method: "keypair" }, + "x-visible-if": { + connection_method: "parameters", + auth_method: "keypair", + }, }, private_key_passphrase: { type: "string", @@ -78,7 +102,10 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-placeholder": "Enter passphrase if key is encrypted", "x-secret": true, "x-step": "connector", - "x-visible-if": { connection_method: "parameters", auth_method: "keypair" }, + "x-visible-if": { + connection_method: "parameters", + auth_method: "keypair", + }, }, warehouse: { type: "string", From 229d205809cef8f0badb0f76279efcfee86b0f6d Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 14:14:53 -0500 Subject: [PATCH 091/103] ch ui modal --- .../templates/schemas/clickhouse-cloud.ts | 30 +++++++------ .../features/templates/schemas/clickhouse.ts | 43 +++++++++++++------ 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/web-common/src/features/templates/schemas/clickhouse-cloud.ts b/web-common/src/features/templates/schemas/clickhouse-cloud.ts index 92f2bbda0fc..6cde48ccc4e 100644 --- a/web-common/src/features/templates/schemas/clickhouse-cloud.ts +++ b/web-common/src/features/templates/schemas/clickhouse-cloud.ts @@ -96,23 +96,24 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "string", title: "Database Whitelist", description: "List of allowed databases", + "x-placeholder": "db1,db2", "x-step": "connector", "x-visible-if": { auth_method: "parameters" }, "x-advanced": true, }, optimize_temporary_tables_before_partition_replace: { - type: "boolean", + type: "string", title: "Optimize Temporary Tables", description: "Optimize temporary tables before partition replace", - default: false, + "x-placeholder": "true", "x-advanced": true, "x-step": "connector", }, log_queries: { - type: "boolean", + type: "string", title: "Log Queries", description: "Log all queries executed by Rill", - default: false, + "x-placeholder": "false", "x-advanced": true, "x-step": "connector", }, @@ -120,6 +121,7 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "object", title: "Query Settings Override", description: "Override default query settings", + "x-placeholder": "key1=value1,key2=value2", "x-advanced": true, "x-step": "connector", }, @@ -127,6 +129,7 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "object", title: "Query Settings", description: "Custom query settings", + "x-placeholder": "key1=value1,key2=value2", "x-advanced": true, "x-step": "connector", }, @@ -134,30 +137,31 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "string", title: "Embed Port", description: "Port number for embedding the ClickHouse Cloud server", + "x-placeholder": "8443", "x-step": "connector", "x-advanced": true, }, can_scale_to_zero: { - type: "boolean", + type: "string", title: "Can Scale to Zero", description: "Enable scaling to zero", - default: false, + "x-placeholder": "false", "x-advanced": true, "x-step": "connector", }, max_open_conns: { - type: "number", + type: "string", title: "Max Open Connections", description: "Maximum number of open connections", - default: 100, + "x-placeholder": "100", "x-advanced": true, "x-step": "connector", }, max_idle_conns: { - type: "number", + type: "string", title: "Max Idle Connections", description: "Maximum number of idle connections", - default: 10, + "x-placeholder": "10", "x-advanced": true, "x-step": "connector", }, @@ -165,7 +169,7 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "string", title: "Dial Timeout", description: "Timeout for establishing a connection", - default: "30s", + "x-placeholder": "30s", "x-advanced": true, "x-step": "connector", }, @@ -173,7 +177,7 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "string", title: "Connection Max Lifetime", description: "Maximum lifetime of a connection", - default: "30m", + "x-placeholder": "30m", "x-advanced": true, "x-step": "connector", }, @@ -181,7 +185,7 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { type: "string", title: "Read Timeout", description: "Timeout for reading from the connection", - default: "30s", + "x-placeholder": "30s", "x-advanced": true, "x-step": "connector", }, diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index c997b1eadac..0d75538069b 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -174,94 +174,109 @@ export const clickhouseSchema: MultiStepFormSchema = { type: "string", title: "Database Whitelist", description: "List of allowed databases", + "x-placeholder": "db1,db2", "x-step": "connector", - "x-visible-if": { auth_method: "parameters" }, + "x-visible-if": { connection_method: "self-managed" }, "x-advanced": true, }, optimize_temporary_tables_before_partition_replace: { - type: "boolean", + type: "string", title: "Optimize Temporary Tables", description: "Optimize temporary tables before partition replace", - default: false, + "x-placeholder": "true", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, log_queries: { - type: "boolean", + type: "string", title: "Log Queries", description: "Log all queries executed by Rill", - default: false, + "x-placeholder": "false", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, query_settings_override: { type: "object", title: "Query Settings Override", description: "Override default query settings", + "x-placeholder": "key1=value1,key2=value2", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, query_settings: { type: "object", title: "Query Settings", description: "Custom query settings", + "x-placeholder": "key1=value1,key2=value2", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, embed_port: { type: "string", title: "Embed Port", description: "Port number for embedding the ClickHouse Cloud server", + "x-placeholder": "8443", "x-step": "connector", "x-advanced": true, + "x-visible-if": { connection_method: "self-managed" } }, can_scale_to_zero: { - type: "boolean", + type: "string", title: "Can Scale to Zero", description: "Enable scaling to zero", - default: false, + "x-placeholder": "false", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, max_open_conns: { - type: "number", + type: "string", title: "Max Open Connections", description: "Maximum number of open connections", - default: 100, + "x-placeholder": "100", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, max_idle_conns: { - type: "number", + type: "string", title: "Max Idle Connections", description: "Maximum number of idle connections", - default: 10, + "x-placeholder": "10", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, dial_timeout: { type: "string", title: "Dial Timeout", description: "Timeout for establishing a connection", - default: "30s", + "x-placeholder": "30s", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, conn_max_lifetime: { type: "string", title: "Connection Max Lifetime", description: "Maximum lifetime of a connection", - default: "30m", + "x-placeholder": "30m", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, read_timeout: { type: "string", title: "Read Timeout", description: "Timeout for reading from the connection", - default: "30s", + "x-placeholder": "30s", "x-advanced": true, "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" } }, sql: { type: "string", From 3f56853eecd1d1ab8f62e80d3096afa0e39f575a Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 14:22:09 -0500 Subject: [PATCH 092/103] prettier --- .../features/templates/schemas/clickhouse.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index 0d75538069b..dd5d67d332d 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -186,7 +186,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "true", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, log_queries: { type: "string", @@ -195,7 +195,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "false", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, query_settings_override: { type: "object", @@ -204,7 +204,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "key1=value1,key2=value2", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, query_settings: { type: "object", @@ -213,7 +213,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "key1=value1,key2=value2", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, embed_port: { type: "string", @@ -222,7 +222,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "8443", "x-step": "connector", "x-advanced": true, - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, can_scale_to_zero: { type: "string", @@ -231,7 +231,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "false", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, max_open_conns: { type: "string", @@ -240,7 +240,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "100", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, max_idle_conns: { type: "string", @@ -249,7 +249,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "10", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, dial_timeout: { type: "string", @@ -258,7 +258,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "30s", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, conn_max_lifetime: { type: "string", @@ -267,7 +267,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "30m", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, read_timeout: { type: "string", @@ -276,7 +276,7 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-placeholder": "30s", "x-advanced": true, "x-step": "connector", - "x-visible-if": { connection_method: "self-managed" } + "x-visible-if": { connection_method: "self-managed" }, }, sql: { type: "string", From d69a6a9f2b6143360484e08cb545fceb6cf81ff1 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:21:53 -0500 Subject: [PATCH 093/103] revert snowflake and redshift changes --- runtime/drivers/redshift/redshift.go | 8 -------- runtime/drivers/snowflake/snowflake.go | 21 ++++++--------------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/runtime/drivers/redshift/redshift.go b/runtime/drivers/redshift/redshift.go index 34bf6a5bdd0..54a03213a44 100644 --- a/runtime/drivers/redshift/redshift.go +++ b/runtime/drivers/redshift/redshift.go @@ -73,14 +73,6 @@ var spec = drivers.Spec{ Placeholder: "dev", Required: true, }, - { - Key: "cluster_identifier", - Type: drivers.StringPropertyType, - DisplayName: "Cluster Identifier", - Description: "Redshift provisioned cluster identifier (for provisioned clusters)", - Placeholder: "my-redshift-cluster", - Hint: "Provide either workgroup (for serverless) or cluster identifier (for provisioned clusters)", - }, }, ImplementsWarehouse: true, } diff --git a/runtime/drivers/snowflake/snowflake.go b/runtime/drivers/snowflake/snowflake.go index 00c22cb7f91..028d10644de 100644 --- a/runtime/drivers/snowflake/snowflake.go +++ b/runtime/drivers/snowflake/snowflake.go @@ -37,7 +37,7 @@ var spec = drivers.Spec{ DisplayName: "Snowflake Connection String", Required: false, DocsURL: "https://docs.rilldata.com/build/connectors/data-source/snowflake", - Placeholder: "@//?warehouse=&role=&authenticator=SNOWFLAKE_JWT&private_key=", + Placeholder: "@//?warehouse=&role=&authenticator=SNOWFLAKE_JWT&privateKey=", Hint: "Can be configured here or by setting the 'connector.snowflake.dsn' environment variable (using '.env' or '--env').", Secret: true, }, @@ -96,21 +96,13 @@ var spec = drivers.Spec{ Hint: "The Snowflake role to use (defaults to your default role if not specified)", }, { - Key: "private_key", + Key: "privateKey", Type: drivers.StringPropertyType, DisplayName: "Private Key", Description: "RSA private key in PEM format for key pair authentication", Placeholder: "-----BEGIN PRIVATE KEY-----\\n...\\n-----END PRIVATE KEY-----", Secret: true, - }, - { - Key: "private_key_passphrase", - Type: drivers.StringPropertyType, - DisplayName: "Private Key Passphrase", - Description: "Passphrase for encrypted private key (if applicable)", - Placeholder: "Enter passphrase if key is encrypted", - Secret: true, - }, + } }, ImplementsWarehouse: true, } @@ -127,8 +119,7 @@ type configProperties struct { Warehouse string `mapstructure:"warehouse"` Role string `mapstructure:"role"` Authenticator string `mapstructure:"authenticator"` - PrivateKey string `mapstructure:"private_key"` - PrivateKeyPassphrase string `mapstructure:"private_key_passphrase"` + PrivateKey string `mapstructure:"privateKey"` ParallelFetchLimit int `mapstructure:"parallel_fetch_limit"` Extras map[string]any `mapstructure:",remain"` @@ -163,7 +154,7 @@ func (c *configProperties) validate() error { set = append(set, "authenticator") } if c.PrivateKey != "" { - set = append(set, "private_key") + set = append(set, "privateKey") } if c.DSN != "" && len(set) > 0 { return fmt.Errorf("snowflake: Only one of 'dsn' or [%s] can be set", strings.Join(set, ", ")) @@ -181,7 +172,7 @@ func (c *configProperties) resolveDSN() (string, error) { } if c.Password == "" && c.PrivateKey == "" { - return "", errors.New("either password or private_key must be provided") + return "", errors.New("either password or privateKey must be provided") } cfg := &gosnowflake.Config{ From 6da2a63dfa0c463f7fefe7d11c2ae8739e3e7124 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:24:08 -0500 Subject: [PATCH 094/103] Update snowflake.ts --- .../src/features/templates/schemas/snowflake.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index 0e21f1977cb..00d2affacb6 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -95,18 +95,6 @@ export const snowflakeSchema: MultiStepFormSchema = { auth_method: "keypair", }, }, - private_key_passphrase: { - type: "string", - title: "Private Key Passphrase", - description: "Optional passphrase for encrypted private key", - "x-placeholder": "Enter passphrase if key is encrypted", - "x-secret": true, - "x-step": "connector", - "x-visible-if": { - connection_method: "parameters", - auth_method: "keypair", - }, - }, warehouse: { type: "string", title: "Warehouse", From 5b2ab4c9b9ba4ba95dfe3b902ae13ba0e55fc884 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:49:23 -0500 Subject: [PATCH 095/103] nit --- runtime/drivers/clickhouse/clickhouse.go | 1 + runtime/drivers/duckdb/duckdb.go | 2 ++ runtime/drivers/snowflake/snowflake.go | 2 +- web-common/src/features/sources/modal/AddDataFormManager.ts | 1 + web-common/src/features/templates/schemas/snowflake.ts | 4 ++-- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/runtime/drivers/clickhouse/clickhouse.go b/runtime/drivers/clickhouse/clickhouse.go index 0ad2879ea3a..7062fb05183 100644 --- a/runtime/drivers/clickhouse/clickhouse.go +++ b/runtime/drivers/clickhouse/clickhouse.go @@ -58,6 +58,7 @@ var spec = drivers.Spec{ Description: "Set the mode for the ClickHouse connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, + NoPrompt: true, }, { Key: "dsn", diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 9317638f0bd..65542438642 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -64,6 +64,7 @@ var spec = drivers.Spec{ Description: "Set the mode for the DuckDB connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, + NoPrompt: true, }, }, SourceProperties: []*drivers.PropertySpec{ @@ -109,6 +110,7 @@ var motherduckSpec = drivers.Spec{ Description: "Set the mode for the DuckDB connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, + NoPrompt: true, }, { Key: "schema_name", diff --git a/runtime/drivers/snowflake/snowflake.go b/runtime/drivers/snowflake/snowflake.go index 028d10644de..7af121e98c1 100644 --- a/runtime/drivers/snowflake/snowflake.go +++ b/runtime/drivers/snowflake/snowflake.go @@ -102,7 +102,7 @@ var spec = drivers.Spec{ Description: "RSA private key in PEM format for key pair authentication", Placeholder: "-----BEGIN PRIVATE KEY-----\\n...\\n-----END PRIVATE KEY-----", Secret: true, - } + }, }, ImplementsWarehouse: true, } diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index 3150f06a107..d43e658bc19 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -590,6 +590,7 @@ export class AddDataFormManager { // Skip if value is undefined/null/empty string if (value === undefined || value === null || value === "") continue; + // Should we add other properties, like Array for list of strings, Ie path_prefix, etc. schemaProperties.push({ key, type: diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts index 00d2affacb6..f064c97a58e 100644 --- a/web-common/src/features/templates/schemas/snowflake.ts +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -154,10 +154,10 @@ export const snowflakeSchema: MultiStepFormSchema = { "x-advanced": true, }, log_queries: { - type: "boolean", + type: "string", title: "Log Queries", description: "Enable logging of all SQL queries (useful for debugging)", - default: false, + "x-placeholder": "false", "x-step": "connector", "x-advanced": true, }, From 95d64d8757dc10a98f4ee189891b8922e8c849fd Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:52:28 -0500 Subject: [PATCH 096/103] revert --- runtime/drivers/azure/azure.go | 1 + runtime/drivers/clickhouse/clickhouse.go | 2 +- runtime/drivers/duckdb/duckdb.go | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/runtime/drivers/azure/azure.go b/runtime/drivers/azure/azure.go index 9bcd743bef8..3208ad8a981 100644 --- a/runtime/drivers/azure/azure.go +++ b/runtime/drivers/azure/azure.go @@ -47,6 +47,7 @@ var spec = drivers.Spec{ Secret: true, }, }, + // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ { Key: "path", diff --git a/runtime/drivers/clickhouse/clickhouse.go b/runtime/drivers/clickhouse/clickhouse.go index 7062fb05183..432fce80ef9 100644 --- a/runtime/drivers/clickhouse/clickhouse.go +++ b/runtime/drivers/clickhouse/clickhouse.go @@ -58,7 +58,7 @@ var spec = drivers.Spec{ Description: "Set the mode for the ClickHouse connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, - NoPrompt: true, + NoPrompt: true, }, { Key: "dsn", diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 65542438642..12ee6fbaaf0 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -64,7 +64,7 @@ var spec = drivers.Spec{ Description: "Set the mode for the DuckDB connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, - NoPrompt: true, + NoPrompt: true, }, }, SourceProperties: []*drivers.PropertySpec{ @@ -110,7 +110,7 @@ var motherduckSpec = drivers.Spec{ Description: "Set the mode for the DuckDB connection. By default, it is set to 'read' which allows only read operations. Set to 'readwrite' to enable model creation and table mutations.", Placeholder: modeReadOnly, Default: modeReadOnly, - NoPrompt: true, + NoPrompt: true, }, { Key: "schema_name", From 9c6c8a9d135d925e30cc9f946230e98e72fabd27 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:53:27 -0500 Subject: [PATCH 097/103] nit --- runtime/drivers/snowflake/snowflake.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/drivers/snowflake/snowflake.go b/runtime/drivers/snowflake/snowflake.go index 7af121e98c1..67c7ed257c9 100644 --- a/runtime/drivers/snowflake/snowflake.go +++ b/runtime/drivers/snowflake/snowflake.go @@ -116,11 +116,11 @@ type configProperties struct { Password string `mapstructure:"password"` Database string `mapstructure:"database"` Schema string `mapstructure:"schema"` - Warehouse string `mapstructure:"warehouse"` - Role string `mapstructure:"role"` - Authenticator string `mapstructure:"authenticator"` - PrivateKey string `mapstructure:"privateKey"` - ParallelFetchLimit int `mapstructure:"parallel_fetch_limit"` + Warehouse string `mapstructure:"warehouse"` + Role string `mapstructure:"role"` + Authenticator string `mapstructure:"authenticator"` + PrivateKey string `mapstructure:"privateKey"` + ParallelFetchLimit int `mapstructure:"parallel_fetch_limit"` Extras map[string]any `mapstructure:",remain"` // LogQueries controls whether to log the raw SQL passed to OLAP. From 3f5d3f8c3cb57e731260f60bf6fc91b584f50973 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 15:54:55 -0500 Subject: [PATCH 098/103] Update sqlite.go --- runtime/drivers/sqlite/sqlite.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/drivers/sqlite/sqlite.go b/runtime/drivers/sqlite/sqlite.go index e48c6aa3556..9e9ba986f80 100644 --- a/runtime/drivers/sqlite/sqlite.go +++ b/runtime/drivers/sqlite/sqlite.go @@ -85,6 +85,7 @@ func (d driver) Spec() drivers.Spec { DisplayName: "SQLite", Description: "Import data from SQLite into DuckDB.", DocsURL: "https://docs.rilldata.com/build/connectors/data-source/sqlite", + // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. SourceProperties: []*drivers.PropertySpec{ { Key: "db", From 654a62182fa1472ba8973593d46b0f41fd3a6b56 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Mon, 22 Dec 2025 16:41:50 -0500 Subject: [PATCH 099/103] nit --- .../features/templates/schemas/clickhouse.ts | 1 - .../src/features/templates/schemas/duckdb.ts | 30 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index dd5d67d332d..4306eb5689d 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -1,4 +1,3 @@ -import { manual } from "prismjs"; import type { MultiStepFormSchema } from "./types"; export const clickhouseSchema: MultiStepFormSchema = { diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts index e93adb085c9..b1e869904f2 100644 --- a/web-common/src/features/templates/schemas/duckdb.ts +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -4,6 +4,24 @@ export const duckdbSchema: MultiStepFormSchema = { $schema: "http://json-schema.org/draft-07/schema#", type: "object", properties: { + auth_method: { + type: "string", + title: "Connection type", + enum: ["self-managed", "rill-managed"], + default: "self-managed", + description: "Choose how to connect to DuckDB", + "x-display": "radio", + "x-enum-labels": ["Self-managed", "Rill-managed"], + "x-enum-descriptions": [ + "Connect to your own self-hosted DuckDB server.", + "Use a managed DuckDB instance hosted by Rill.", + ], + "x-grouped-fields": { + "self-managed": ["path", "attach", "mode"], + "rill-managed": ["managed"], + }, + "x-step": "connector", + }, path: { type: "string", title: "Database Path", @@ -26,7 +44,7 @@ export const duckdbSchema: MultiStepFormSchema = { title: "Connection Mode", description: "Database access mode", enum: ["read", "readwrite"], - default: "read", + default: "readwrite", "x-display": "radio", "x-enum-labels": ["Read-only", "Read-write"], "x-enum-descriptions": [ @@ -35,6 +53,16 @@ export const duckdbSchema: MultiStepFormSchema = { ], "x-step": "connector", }, + managed: { + type: "boolean", + title: "Managed", + description: "Enable managed mode for the ClickHouse server", + default: true, + "x-readonly": true, + "x-hint": "Enable managed mode to manage the server automatically", + "x-step": "connector", + "x-visible-if": { auth_method: "rill-managed" }, + }, sql: { type: "string", title: "SQL Query", From 3da5c17397e59b35b88d1f9a13b1838255849acb Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 23 Dec 2025 15:15:12 -0500 Subject: [PATCH 100/103] x-step explorer, create metrics with AI for read only --- .../sources/modal/AddDataExplorer.svelte | 142 ++++++++++++++++++ .../features/sources/modal/AddDataForm.svelte | 21 +-- .../sources/modal/AddDataFormManager.ts | 15 +- .../sources/modal/connectorStepStore.ts | 2 +- .../templates/schemas/clickhouse-cloud.ts | 7 + .../features/templates/schemas/clickhouse.ts | 27 ++-- .../src/features/templates/schemas/druid.ts | 6 + .../src/features/templates/schemas/duckdb.ts | 7 + .../features/templates/schemas/motherduck.ts | 7 + .../src/features/templates/schemas/pinot.ts | 6 + .../src/features/templates/schemas/types.ts | 2 +- 11 files changed, 212 insertions(+), 30 deletions(-) create mode 100644 web-common/src/features/sources/modal/AddDataExplorer.svelte diff --git a/web-common/src/features/sources/modal/AddDataExplorer.svelte b/web-common/src/features/sources/modal/AddDataExplorer.svelte new file mode 100644 index 00000000000..2beca06bc91 --- /dev/null +++ b/web-common/src/features/sources/modal/AddDataExplorer.svelte @@ -0,0 +1,142 @@ + + +
+
+
+

+ Select a table to explore +

+

+ Choose a table from your {connector.displayName ?? connector.name} connector + to generate metrics and create a dashboard. +

+
+ +
+ {#if analyzedConnector} +
    + +
+ {:else if $analyzedConnectors.isLoading} +
Loading connector...
+ {:else} +
No tables found.
+ {/if} +
+ + {#if selectedTable} +
+

+ Selected: {selectedTable.table} +

+
+ {/if} +
+ +
+ + + +
+
diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index e589a1c8e31..5a9f4314595 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -18,6 +18,7 @@ import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; + import AddDataExplorer from "./AddDataExplorer.svelte"; import { AddDataFormManager } from "./AddDataFormManager"; import AddDataFormSection from "./AddDataFormSection.svelte"; @@ -274,20 +275,21 @@ } -
- -
+{#if stepState.step === "explorer"} + +{:else} +
+
- {#if isMultiStepConnector} +
+ {#if isMultiStepConnector}
+{/if} diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index d43e658bc19..3ffac6acb59 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -24,6 +24,7 @@ import { MULTI_STEP_CONNECTORS, MEDIUM_FORM_CONNECTORS, TALL_FORM_CONNECTORS, + OLAP_ENGINES, } from "./constants"; import { connectorStepStore, @@ -421,11 +422,17 @@ export class AddDataFormManager { false, ); - // If mode is "read" (read-only), close without going to source step - // Only advance to source step when mode is "readwrite" + // If mode is "read" (read-only), navigate to explorer step. + // For OLAP connectors without a mode field (Pinot, Druid), also go to explorer. + // Only advance to source step when mode is "readwrite" or undefined (rill-managed). const mode = values?.mode as string | undefined; - if (mode === "read") { - onClose(); + const schema = getConnectorSchema(connector.name ?? ""); + const hasModeField = schema?.properties?.mode !== undefined; + const isReadOnlyOlap = !hasModeField && OLAP_ENGINES.includes(connector.name ?? ""); + + if (mode === "read" || isReadOnlyOlap) { + setConnectorConfig(preparedValues); + setStep("explorer"); return; } diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 1168a65e532..9f58cb53645 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -1,6 +1,6 @@ import { writable } from "svelte/store"; -export type ConnectorStep = "connector" | "source"; +export type ConnectorStep = "connector" | "source" | "explorer"; export type ConnectorStepState = { step: ConnectorStep; diff --git a/web-common/src/features/templates/schemas/clickhouse-cloud.ts b/web-common/src/features/templates/schemas/clickhouse-cloud.ts index 6cde48ccc4e..cc25b09f72b 100644 --- a/web-common/src/features/templates/schemas/clickhouse-cloud.ts +++ b/web-common/src/features/templates/schemas/clickhouse-cloud.ts @@ -230,6 +230,13 @@ export const clickhouseCloudSchema: MultiStepFormSchema = { "x-step": "source", "x-visible-if": { mode: "readwrite" }, }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, }, allOf: [ { diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts index 4306eb5689d..7793493a28e 100644 --- a/web-common/src/features/templates/schemas/clickhouse.ts +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -294,29 +294,26 @@ export const clickhouseSchema: MultiStepFormSchema = { "x-step": "source", "x-visible-if": { mode: "readwrite" }, }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, }, allOf: [ { - if: { - properties: { - auth_method: { const: "self-managed" }, - connection_method: { const: "parameters" }, - }, - }, - then: { required: ["host", "username"] }, + if: { properties: { auth_method: { const: "self-managed" } } }, + then: { required: [] }, }, { - if: { - properties: { - auth_method: { const: "self-managed" }, - connection_method: { const: "connection_string" }, - }, - }, - then: { required: ["dsn"] }, + if: { properties: { auth_method: { const: "rill-managed" } } }, + then: { required: [] }, }, { if: { properties: { mode: { const: "readwrite" } } }, - then: { required: ["managed", "sql", "name"] }, + then: { required: ["sql", "name"] }, }, ], }; diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts index e0a3d3fb25f..6214ab0da8f 100644 --- a/web-common/src/features/templates/schemas/druid.ts +++ b/web-common/src/features/templates/schemas/druid.ts @@ -94,6 +94,12 @@ export const druidSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + }, }, allOf: [ { diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts index b1e869904f2..679f6b3256a 100644 --- a/web-common/src/features/templates/schemas/duckdb.ts +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -80,6 +80,13 @@ export const duckdbSchema: MultiStepFormSchema = { "x-step": "source", "x-visible-if": { mode: "readwrite" }, }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, }, allOf: [ { diff --git a/web-common/src/features/templates/schemas/motherduck.ts b/web-common/src/features/templates/schemas/motherduck.ts index 7431c32012b..6f91a841be9 100644 --- a/web-common/src/features/templates/schemas/motherduck.ts +++ b/web-common/src/features/templates/schemas/motherduck.ts @@ -57,6 +57,13 @@ export const motherduckSchema: MultiStepFormSchema = { "x-step": "source", "x-visible-if": { mode: "readwrite" }, }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, }, allOf: [ { diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts index 273080b9127..800321b7761 100644 --- a/web-common/src/features/templates/schemas/pinot.ts +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -95,6 +95,12 @@ export const pinotSchema: MultiStepFormSchema = { "x-step": "connector", "x-visible-if": { auth_method: "connection_string" }, }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + }, }, allOf: [ { diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index 27706b7670e..0b5839bc046 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -16,7 +16,7 @@ export type JSONSchemaField = { properties?: Record; required?: string[]; "x-display"?: "radio" | "select" | "textarea" | "file" | "tabs"; - "x-step"?: "connector" | "source"; + "x-step"?: "connector" | "source" | "explorer"; "x-secret"?: boolean; "x-readonly"?: boolean; "x-visible-if"?: Record; From 1068d12f5a9dd25ba8c2e21eb6fb63eacde96320 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 23 Dec 2025 18:50:02 -0500 Subject: [PATCH 101/103] pulled new changed from cyrus/shared-multi-step-form --- web-common/src/features/templates/schemas/azure.ts | 6 +++++- web-common/src/features/templates/schemas/gcs.ts | 5 ++++- web-common/src/features/templates/schemas/s3.ts | 5 ++++- web-common/src/features/templates/schemas/types.ts | 8 ++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts index b30e5b50bc6..5c0d144b28b 100644 --- a/web-common/src/features/templates/schemas/azure.ts +++ b/web-common/src/features/templates/schemas/azure.ts @@ -90,7 +90,11 @@ export const azureSchema: MultiStepFormSchema = { title: "Blob URI", description: "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", - pattern: "^https?://", + pattern: "^https?://[A-Za-z0-9.-]+\\.blob\\.core\\.windows\\.net/.+", + errorMessage: { + pattern: + "Enter a blob URL like https://account.blob.core.windows.net/container/path", + }, "x-placeholder": "https://account.blob.core.windows.net/container", "x-step": "source", }, diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts index 0dc629a0ed7..c5586dbe34f 100644 --- a/web-common/src/features/templates/schemas/gcs.ts +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -75,7 +75,10 @@ export const gcsSchema: MultiStepFormSchema = { type: "string", title: "GCS URI", description: "Path to your GCS bucket or prefix", - pattern: "^gs://", + pattern: "^gs://[^/]+(/.*)?$", + errorMessage: { + pattern: "Enter a GCS URI like gs://bucket or gs://bucket/path", + }, "x-placeholder": "gs://bucket/path", "x-step": "source", }, diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts index 1b5dc293fe5..61d87f14103 100644 --- a/web-common/src/features/templates/schemas/s3.ts +++ b/web-common/src/features/templates/schemas/s3.ts @@ -124,7 +124,10 @@ export const s3Schema: MultiStepFormSchema = { type: "string", title: "S3 URI", description: "Path to your S3 bucket or prefix", - pattern: "^s3://", + pattern: "^s3://[^/]+(/.*)?$", + errorMessage: { + pattern: "Enter an S3 URI like s3://bucket or s3://bucket/path", + }, "x-placeholder": "s3://bucket/path", "x-step": "source", }, diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts index 0b5839bc046..6366cdad2fa 100644 --- a/web-common/src/features/templates/schemas/types.ts +++ b/web-common/src/features/templates/schemas/types.ts @@ -13,6 +13,14 @@ export type JSONSchemaField = { default?: string | number | boolean; pattern?: string; format?: string; + /** + * Custom error messages for validation failures. + * Use `pattern` key to provide a human-readable message when pattern validation fails. + */ + errorMessage?: { + pattern?: string; + format?: string; + }; properties?: Record; required?: string[]; "x-display"?: "radio" | "select" | "textarea" | "file" | "tabs"; From e0f8ab8cb19c78335a20b2906d5af7753e7b4681 Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 23 Dec 2025 18:58:22 -0500 Subject: [PATCH 102/103] Update submitAddDataForm.ts --- .../sources/modal/submitAddDataForm.ts | 82 ------------------- 1 file changed, 82 deletions(-) diff --git a/web-common/src/features/sources/modal/submitAddDataForm.ts b/web-common/src/features/sources/modal/submitAddDataForm.ts index df31a03c755..c6ec540ad6e 100644 --- a/web-common/src/features/sources/modal/submitAddDataForm.ts +++ b/web-common/src/features/sources/modal/submitAddDataForm.ts @@ -489,85 +489,3 @@ export async function submitAddConnectorForm( // Wait for the submission to complete await submissionPromise; } - -export async function submitAddSourceForm( - queryClient: QueryClient, - connector: V1ConnectorDriver, - formValues: AddDataFormValues, -): Promise { - const instanceId = get(runtime).instanceId; - await beforeSubmitForm(instanceId, connector); - - const newSourceName = formValues.name as string; - - const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( - connector, - formValues, - ); - - // Make a new .yaml file - const newSourceFilePath = getFileAPIPathFromNameAndType( - newSourceName, - EntityType.Table, - ); - await runtimeServicePutFile(instanceId, { - path: newSourceFilePath, - blob: compileSourceYAML(rewrittenConnector, rewrittenFormValues), - create: true, - createOnly: false, - }); - - const originalEnvBlob = await getOriginalEnvBlob(queryClient, instanceId); - - // Create or update the `.env` file - const newEnvBlob = await updateDotEnvWithSecrets( - queryClient, - rewrittenConnector, - rewrittenFormValues, - "source", - ); - - // Make sure the file has reconciled before testing the connection - await runtimeServicePutFileAndWaitForReconciliation(instanceId, { - path: ".env", - blob: newEnvBlob, - create: true, - createOnly: false, - }); - - // Wait for source resource-level reconciliation - // This must happen after .env reconciliation since sources depend on secrets - try { - await waitForResourceReconciliation( - instanceId, - newSourceName, - ResourceKind.Model, - ); - } catch (error) { - // The source file was already created, so we need to delete it - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - const errorDetails = (error as any).details; - - throw { - message: error.message || "Unable to establish a connection", - details: - errorDetails && errorDetails !== error.message - ? errorDetails - : undefined, - }; - } - - // Check for file errors - // If the model file has errors, rollback the changes - const errorMessage = await fileArtifacts.checkFileErrors( - queryClient, - instanceId, - newSourceFilePath, - ); - if (errorMessage) { - await rollbackChanges(instanceId, newSourceFilePath, originalEnvBlob); - throw new Error(errorMessage); - } - - await goto(`/files/${newSourceFilePath}`); -} From 030a430f44550b6ca69c9de32571a356fe1f97ca Mon Sep 17 00:00:00 2001 From: royendo <67675319+royendo@users.noreply.github.com> Date: Tue, 23 Dec 2025 19:37:45 -0500 Subject: [PATCH 103/103] all ts, missed these three --- runtime/drivers/athena/athena.go | 7 -- .../sources/modal/connector-schemas.ts | 7 ++ .../src/features/sources/modal/yupSchemas.ts | 1 - .../src/features/templates/schemas/athena.ts | 8 -- .../features/templates/schemas/local_file.ts | 34 +++++++++ .../features/templates/schemas/salesforce.ts | 74 +++++++++++++++++++ .../src/features/templates/schemas/sqlite.ts | 31 ++++++++ 7 files changed, 146 insertions(+), 16 deletions(-) create mode 100644 web-common/src/features/templates/schemas/local_file.ts create mode 100644 web-common/src/features/templates/schemas/salesforce.ts create mode 100644 web-common/src/features/templates/schemas/sqlite.ts diff --git a/runtime/drivers/athena/athena.go b/runtime/drivers/athena/athena.go index c8a7fdbb8d4..53fc444f021 100644 --- a/runtime/drivers/athena/athena.go +++ b/runtime/drivers/athena/athena.go @@ -60,13 +60,6 @@ var spec = drivers.Spec{ Placeholder: "s3://bucket-name/path/", Required: true, }, - { - Key: "aws_role_arn", - Type: drivers.StringPropertyType, - DisplayName: "IAM Role ARN", - Description: "AWS IAM role ARN to assume (optional)", - Placeholder: "arn:aws:iam::123456789012:role/MyRole", - }, { Key: "region", Type: drivers.StringPropertyType, diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index cac04d47eb0..90b228320c2 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -8,13 +8,16 @@ import { druidSchema } from "../../templates/schemas/druid"; import { duckdbSchema } from "../../templates/schemas/duckdb"; import { gcsSchema } from "../../templates/schemas/gcs"; import { httpsSchema } from "../../templates/schemas/https"; +import { localFileSchema } from "../../templates/schemas/local_file"; import { motherduckSchema } from "../../templates/schemas/motherduck"; import { mysqlSchema } from "../../templates/schemas/mysql"; import { pinotSchema } from "../../templates/schemas/pinot"; import { postgresSchema } from "../../templates/schemas/postgres"; import { redshiftSchema } from "../../templates/schemas/redshift"; import { s3Schema } from "../../templates/schemas/s3"; +import { salesforceSchema } from "../../templates/schemas/salesforce"; import { snowflakeSchema } from "../../templates/schemas/snowflake"; +import { sqliteSchema } from "../../templates/schemas/sqlite"; export const multiStepFormSchemas: Record = { s3: s3Schema, @@ -33,6 +36,10 @@ export const multiStepFormSchemas: Record = { motherduck: motherduckSchema, druid: druidSchema, pinot: pinotSchema, + // Source-only connectors (no multi-step flow) + salesforce: salesforceSchema, + sqlite: sqliteSchema, + local_file: localFileSchema, }; export function getConnectorSchema( diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 9735b56c679..becb927a448 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -255,7 +255,6 @@ export const getYupSchema = { athena_connector: yup.object().shape({ aws_access_key_id: yup.string().optional(), aws_secret_access_key: yup.string().optional(), - aws_role_arn: yup.string().optional(), region: yup.string().optional(), workgroup: yup.string().optional(), output_location: yup.string().optional(), diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts index 2255a6dd39d..245ee2a842c 100644 --- a/web-common/src/features/templates/schemas/athena.ts +++ b/web-common/src/features/templates/schemas/athena.ts @@ -27,14 +27,6 @@ export const athenaSchema: MultiStepFormSchema = { "x-placeholder": "s3://my-bucket/athena-results/", "x-step": "connector", }, - aws_role_arn: { - type: "string", - title: "IAM Role ARN (Optional)", - description: "AWS IAM role ARN to assume (optional)", - "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", - "x-step": "connector", - "x-advanced": true, - }, region: { type: "string", title: "AWS Region", diff --git a/web-common/src/features/templates/schemas/local_file.ts b/web-common/src/features/templates/schemas/local_file.ts new file mode 100644 index 00000000000..aba507a9503 --- /dev/null +++ b/web-common/src/features/templates/schemas/local_file.ts @@ -0,0 +1,34 @@ +import type { MultiStepFormSchema } from "./types"; + +export const localFileSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "Path or URL to file", + "x-placeholder": "/path/to/file.csv", + "x-step": "source", + }, + format: { + type: "string", + title: "Format", + description: "File format. Inferred from extension if not set.", + enum: ["csv", "parquet", "json", "ndjson"], + "x-display": "select", + "x-enum-labels": ["CSV", "Parquet", "JSON", "NDJSON"], + "x-placeholder": "csv", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_local_source", + "x-step": "source", + }, + }, + required: ["path", "name"], +}; diff --git a/web-common/src/features/templates/schemas/salesforce.ts b/web-common/src/features/templates/schemas/salesforce.ts new file mode 100644 index 00000000000..429279a584c --- /dev/null +++ b/web-common/src/features/templates/schemas/salesforce.ts @@ -0,0 +1,74 @@ +import type { MultiStepFormSchema } from "./types"; + +export const salesforceSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + username: { + type: "string", + title: "Username", + description: "Salesforce username", + "x-placeholder": "Enter your Salesforce username", + "x-step": "source", + }, + password: { + type: "string", + title: "Password", + description: "Salesforce password", + "x-placeholder": "Enter your Salesforce password", + "x-secret": true, + "x-step": "source", + }, + key: { + type: "string", + title: "Security Token", + description: "Salesforce security token", + "x-placeholder": "Enter your security token", + "x-secret": true, + "x-step": "source", + }, + endpoint: { + type: "string", + title: "Endpoint", + description: "Salesforce endpoint URL (optional, defaults to production)", + "x-placeholder": "https://login.salesforce.com", + "x-step": "source", + "x-advanced": true, + }, + client_id: { + type: "string", + title: "Client ID", + description: "Connected App client ID (optional)", + "x-placeholder": "Enter client ID", + "x-step": "source", + "x-advanced": true, + }, + soql: { + type: "string", + title: "SOQL Query", + description: "SOQL Query to extract data from Salesforce", + "x-placeholder": "SELECT Id, CreatedDate, Name FROM Opportunity", + "x-hint": + "Write a SOQL query to retrieve data from your Salesforce object.", + "x-step": "source", + }, + sobject: { + type: "string", + title: "SObject", + description: "SObject to query in Salesforce", + "x-placeholder": "Opportunity", + "x-hint": + "Enter the name of the Salesforce object you want to query (e.g., Opportunity, Lead, Account).", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_salesforce_source", + "x-step": "source", + }, + }, + required: ["soql", "sobject", "name"], +}; diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts new file mode 100644 index 00000000000..6d4b63b349a --- /dev/null +++ b/web-common/src/features/templates/schemas/sqlite.ts @@ -0,0 +1,31 @@ +import type { MultiStepFormSchema } from "./types"; + +export const sqliteSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + db: { + type: "string", + title: "Database Path", + description: "Path to SQLite database file", + "x-placeholder": "/path/to/database.db", + "x-step": "source", + }, + table: { + type: "string", + title: "Table", + description: "SQLite table name", + "x-placeholder": "my_table", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_sqlite_source", + "x-step": "source", + }, + }, + required: ["db", "table", "name"], +};