diff --git a/runtime/drivers/athena/athena.go b/runtime/drivers/athena/athena.go index a3b24c2b12f..53fc444f021 100644 --- a/runtime/drivers/athena/athena.go +++ b/runtime/drivers/athena/athena.go @@ -60,6 +60,20 @@ var spec = drivers.Spec{ Placeholder: "s3://bucket-name/path/", Required: true, }, + { + Key: "region", + Type: drivers.StringPropertyType, + DisplayName: "AWS Region", + Description: "AWS region where Athena is configured", + Placeholder: "us-east-1", + }, + { + Key: "workgroup", + Type: drivers.StringPropertyType, + DisplayName: "Workgroup", + Description: "Athena workgroup name (optional)", + Placeholder: "primary", + }, }, ImplementsWarehouse: true, } diff --git a/runtime/drivers/azure/azure.go b/runtime/drivers/azure/azure.go index dbfd51ffd90..3208ad8a981 100644 --- a/runtime/drivers/azure/azure.go +++ b/runtime/drivers/azure/azure.go @@ -39,9 +39,12 @@ var spec = drivers.Spec{ Secret: true, }, { - Key: "azure_storage_connection_string", - Type: drivers.StringPropertyType, - Secret: true, + Key: "azure_storage_connection_string", + Type: drivers.StringPropertyType, + DisplayName: "Azure Connection String", + Description: "Azure connection string for storage account", + Placeholder: "Paste your Azure connection string here", + Secret: true, }, }, // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. diff --git a/runtime/drivers/https/https.go b/runtime/drivers/https/https.go index fa70e25123f..7ae79b7058e 100644 --- a/runtime/drivers/https/https.go +++ b/runtime/drivers/https/https.go @@ -21,17 +21,25 @@ func init() { } var spec = drivers.Spec{ - DisplayName: "https", - Description: "Connect to a remote file.", + DisplayName: "HTTPS", + Description: "Connect to remote files and REST APIs.", DocsURL: "https://docs.rilldata.com/build/connect/#adding-a-remote-source", - // Important: Any edits to the below properties must be accompanied by changes to the client-side form validation schemas. + ConfigProperties: []*drivers.PropertySpec{ + { + Key: "headers", + Type: drivers.StringPropertyType, + DisplayName: "HTTP Headers (JSON)", + Description: `HTTP headers as JSON object. Example: {"Authorization": "Bearer my-token"}`, + Placeholder: `{"Authorization": "Bearer my-token"}`, + }, + }, SourceProperties: []*drivers.PropertySpec{ { Key: "path", Type: drivers.StringPropertyType, - DisplayName: "Path", - Description: "Path to the remote file.", - Placeholder: "https://example.com/file.csv", + DisplayName: "URL", + Description: "URL to the remote file or API endpoint", + Placeholder: "https://api.example.com/data", Required: true, }, { @@ -102,8 +110,9 @@ func (d driver) Open(instanceID string, config map[string]any, st *storage.Clien } conn := &Connection{ - config: config, - logger: logger, + config: config, + configProp: cfg, + logger: logger, } return conn, nil } @@ -121,8 +130,9 @@ func (d driver) TertiarySourceConnectors(ctx context.Context, src map[string]any } type Connection struct { - config map[string]any - logger *zap.Logger + config map[string]any + configProp *ConfigProperties + logger *zap.Logger } var _ drivers.Handle = &Connection{} @@ -254,6 +264,12 @@ func (c *Connection) FilePaths(ctx context.Context, src map[string]any) ([]strin return nil, fmt.Errorf("failed to create request for path %s: %w", path, err) } + // Apply connector-level headers first (from config) + for k, v := range c.configProp.Headers { + req.Header.Set(k, v) + } + + // Model-specific headers override connector headers for k, v := range modelProp.Headers { req.Header.Set(k, v) } diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 90e84dfd0ca..005e7ab4868 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -26,14 +26,22 @@ var spec = drivers.Spec{ DocsURL: "https://docs.rilldata.com/build/connectors/data-source/s3", ConfigProperties: []*drivers.PropertySpec{ { - Key: "aws_access_key_id", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_access_key_id", + Type: drivers.StringPropertyType, + DisplayName: "AWS access key ID", + Description: "AWS access key ID for explicit credentials", + Placeholder: "Enter your AWS access key ID", + Secret: true, + Required: true, }, { - Key: "aws_secret_access_key", - Type: drivers.StringPropertyType, - Secret: true, + Key: "aws_secret_access_key", + Type: drivers.StringPropertyType, + DisplayName: "AWS secret access key", + Description: "AWS secret access key for explicit credentials", + Placeholder: "Enter your AWS secret access key", + Secret: true, + Required: true, }, { Key: "region", diff --git a/runtime/drivers/snowflake/snowflake.go b/runtime/drivers/snowflake/snowflake.go index 8b9dd5ed513..67c7ed257c9 100644 --- a/runtime/drivers/snowflake/snowflake.go +++ b/runtime/drivers/snowflake/snowflake.go @@ -95,6 +95,14 @@ var spec = drivers.Spec{ Placeholder: "your_role", Hint: "The Snowflake role to use (defaults to your default role if not specified)", }, + { + Key: "privateKey", + Type: drivers.StringPropertyType, + DisplayName: "Private Key", + Description: "RSA private key in PEM format for key pair authentication", + Placeholder: "-----BEGIN PRIVATE KEY-----\\n...\\n-----END PRIVATE KEY-----", + Secret: true, + }, }, ImplementsWarehouse: true, } diff --git a/runtime/drivers/sqlite/sqlite.go b/runtime/drivers/sqlite/sqlite.go index 355299369ab..9e9ba986f80 100644 --- a/runtime/drivers/sqlite/sqlite.go +++ b/runtime/drivers/sqlite/sqlite.go @@ -91,9 +91,9 @@ func (d driver) Spec() drivers.Spec { Key: "db", Type: drivers.StringPropertyType, Required: true, - DisplayName: "DB", - Description: "Path to SQLite db file", - Placeholder: "/path/to/sqlite.db", + DisplayName: "Database Path", + Description: "Path to SQLite database file", + Placeholder: "/path/to/database.db", }, { Key: "table", diff --git a/web-common/src/components/icons/connectors/ClickHouseCloud.svelte b/web-common/src/components/icons/connectors/ClickHouseCloud.svelte new file mode 100644 index 00000000000..9c74e3a6f6d --- /dev/null +++ b/web-common/src/components/icons/connectors/ClickHouseCloud.svelte @@ -0,0 +1,46 @@ + + + + + + + + + + + diff --git a/web-common/src/features/connectors/code-utils.ts b/web-common/src/features/connectors/code-utils.ts index edb0bcb4e06..f82f1e5f02a 100644 --- a/web-common/src/features/connectors/code-utils.ts +++ b/web-common/src/features/connectors/code-utils.ts @@ -62,16 +62,18 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; properties = properties.filter(options.fieldFilter); } - // Get the secret property keys + // Get the secret property keys from the properties being used (orderedProperties or configProperties) + const propertiesForTypeChecking = + options?.orderedProperties ?? connector.configProperties ?? []; const secretPropertyKeys = - connector.configProperties - ?.filter((property) => property.secret) + propertiesForTypeChecking + .filter((property) => property.secret) .map((property) => property.key) || []; - // Get the string property keys + // Get the string property keys from the properties being used const stringPropertyKeys = - connector.configProperties - ?.filter( + propertiesForTypeChecking + .filter( (property) => property.type === ConnectorDriverPropertyType.TYPE_STRING, ) .map((property) => property.key) || []; @@ -81,6 +83,12 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; .filter((property) => { if (!property.key) return false; const value = formValues[property.key]; + + // Secret fields should be shown with env variable placeholder if they exist in formValues + // Don't include secrets that weren't provided (e.g., password when using DSN) + const isSecretProperty = secretPropertyKeys.includes(property.key); + if (isSecretProperty && value !== undefined) return true; + if (value === undefined) return false; // Filter out empty strings for optional fields if (typeof value === "string" && value.trim() === "") return false; @@ -96,7 +104,7 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; }) .map((property) => { const key = property.key as string; - const value = formValues[key] as string; + const value = formValues[key]; const isSecretProperty = secretPropertyKeys.includes(key); if (isSecretProperty) { @@ -107,12 +115,14 @@ driver: ${getDriverNameForConnector(connector.name as string)}`; )} }}"`; } + // At this point, value is guaranteed to be defined due to the filter above + const stringValue = value as string; const isStringProperty = stringPropertyKeys.includes(key); if (isStringProperty) { - return `${key}: "${value}"`; + return `${key}: "${stringValue}"`; } - return `${key}: ${value}`; + return `${key}: ${stringValue}`; }) .join("\n"); diff --git a/web-common/src/features/connectors/connector-icon-mapping.ts b/web-common/src/features/connectors/connector-icon-mapping.ts index a5e581f0395..e5bd344eef5 100644 --- a/web-common/src/features/connectors/connector-icon-mapping.ts +++ b/web-common/src/features/connectors/connector-icon-mapping.ts @@ -15,8 +15,8 @@ import ClickHouseCloudIcon from "../../components/icons/connectors/ClickHouseClo export const connectorIconMapping = { athena: AthenaIcon, bigquery: GoogleBigQueryIcon, - clickhouse: ClickHouseIcon, clickhousecloud: ClickHouseCloudIcon, + clickhouse: ClickHouseIcon, motherduck: MotherDuckIcon, druid: ApacheDruidIcon, duckdb: DuckDbIcon, diff --git a/web-common/src/features/connectors/connectors-utils.ts b/web-common/src/features/connectors/connectors-utils.ts index b82c0c2532d..af7e562b848 100644 --- a/web-common/src/features/connectors/connectors-utils.ts +++ b/web-common/src/features/connectors/connectors-utils.ts @@ -139,8 +139,12 @@ export function getConnectorIconKey(connector: V1AnalyzedConnector): string { /** * Determines the driver name for a connector. - * Special case: MotherDuck connectors use "duckdb" as the driver name. + * Special cases: + * - MotherDuck connectors use "duckdb" as the driver name. + * - ClickHouse Cloud connectors use "clickhouse" as the driver name. */ export function getDriverNameForConnector(connectorName: string): string { - return connectorName === "motherduck" ? "duckdb" : connectorName; + if (connectorName === "motherduck") return "duckdb"; + if (connectorName === "clickhousecloud") return "clickhouse"; + return connectorName; } diff --git a/web-common/src/features/entity-management/name-utils.ts b/web-common/src/features/entity-management/name-utils.ts index 01f9567a937..52a6d72dc99 100644 --- a/web-common/src/features/entity-management/name-utils.ts +++ b/web-common/src/features/entity-management/name-utils.ts @@ -21,8 +21,22 @@ export function getName(name: string, others: string[]): string { const set = new Set(others.map((other) => other.toLowerCase())); let result = name; + const incrementableSuffix = /(.+)_([0-9]+)$/; while (set.has(result.toLowerCase())) { + // Special-case for "s3": don't roll over to "s4", append suffix instead. + if (name.toLowerCase() === "s3") { + const match = incrementableSuffix.exec(result); + if (match) { + const base = match[1]; + const number = Number.parseInt(match[2], 10) + 1; + result = `${base}_${number}`; + continue; + } + result = `${name}_1`; + continue; + } + result = INCREMENT.exec(result)?.[1] ? result.replace(INCREMENT, (m) => (+m + 1).toString()) : `${result}_1`; diff --git a/web-common/src/features/sources/modal/AddClickHouseForm.svelte b/web-common/src/features/sources/modal/AddClickHouseForm.svelte deleted file mode 100644 index 3248cf7487d..00000000000 --- a/web-common/src/features/sources/modal/AddClickHouseForm.svelte +++ /dev/null @@ -1,506 +0,0 @@ - - -
-
- - {#if connectorType === "rill-managed"} -
- -
- {/if} -
- - {#if connectorType === "self-hosted" || connectorType === "clickhouse-cloud"} - - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {@const isPortField = propertyKey === "port"} - {@const isSSLField = propertyKey === "ssl"} - -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - options={connectorType === "clickhouse-cloud" && isPortField - ? [ - { value: "8443", label: "8443 (HTTPS)" }, - { value: "9440", label: "9440 (Native Secure)" }, - ] - : undefined} - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- - -
- {#each dsnProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- -
- {/each} -
- - - {:else} - -
- {#each filteredProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/each} -
- {/if} -
diff --git a/web-common/src/features/sources/modal/AddDataExplorer.svelte b/web-common/src/features/sources/modal/AddDataExplorer.svelte new file mode 100644 index 00000000000..2beca06bc91 --- /dev/null +++ b/web-common/src/features/sources/modal/AddDataExplorer.svelte @@ -0,0 +1,142 @@ + + +
+
+
+

+ Select a table to explore +

+

+ Choose a table from your {connector.displayName ?? connector.name} connector + to generate metrics and create a dashboard. +

+
+ +
+ {#if analyzedConnector} +
    + +
+ {:else if $analyzedConnectors.isLoading} +
Loading connector...
+ {:else} +
No tables found.
+ {/if} +
+ + {#if selectedTable} +
+

+ Selected: {selectedTable.table} +

+
+ {/if} +
+ +
+ + + +
+
diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index ee081dc8f34..5a9f4314595 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -8,24 +8,21 @@ import type { SuperValidated } from "sveltekit-superforms"; import type { AddDataFormType, ConnectorType } from "./types"; - import AddClickHouseForm from "./AddClickHouseForm.svelte"; + import MultiStepConnectorFlow from "./MultiStepConnectorFlow.svelte"; import NeedHelpText from "./NeedHelpText.svelte"; import Tabs from "@rilldata/web-common/components/forms/Tabs.svelte"; import { TabsContent } from "@rilldata/web-common/components/tabs"; - import { isEmpty } from "./utils"; - import { - CONNECTION_TAB_OPTIONS, - type ClickHouseConnectorType, - } from "./constants"; - import { getInitialFormValuesFromProperties } from "../sourceUtils"; + import { hasOnlyDsn, isEmpty } from "./utils"; + import { CONNECTION_TAB_OPTIONS } from "./constants"; import { connectorStepStore } from "./connectorStepStore"; import FormRenderer from "./FormRenderer.svelte"; import YamlPreview from "./YamlPreview.svelte"; - import GCSMultiStepForm from "./GCSMultiStepForm.svelte"; + import AddDataExplorer from "./AddDataExplorer.svelte"; + import { AddDataFormManager } from "./AddDataFormManager"; - import { hasOnlyDsn } from "./utils"; import AddDataFormSection from "./AddDataFormSection.svelte"; + import { get } from "svelte/store"; export let connector: V1ConnectorDriver; export let formType: AddDataFormType; @@ -54,45 +51,31 @@ formType, onParamsUpdate: (e: any) => handleOnUpdate(e), onDsnUpdate: (e: any) => handleOnUpdate(e), + getSelectedAuthMethod: () => + get(connectorStepStore).selectedAuthMethod ?? undefined, }); const isMultiStepConnector = formManager.isMultiStepConnector; const isSourceForm = formManager.isSourceForm; const isConnectorForm = formManager.isConnectorForm; const onlyDsn = hasOnlyDsn(connector, isConnectorForm); - $: stepState = $connectorStepStore; - $: stepProperties = - isMultiStepConnector && stepState.step === "source" - ? (connector.sourceProperties ?? []) - : properties; - $: if ( - isMultiStepConnector && - stepState.step === "source" && - stepState.connectorConfig - ) { - // Initialize form with source properties and default values - const sourceProperties = connector.sourceProperties ?? []; - const initialValues = getInitialFormValuesFromProperties(sourceProperties); - - // Merge with stored connector config - const combinedValues = { ...stepState.connectorConfig, ...initialValues }; - - paramsForm.update(() => combinedValues, { taint: false }); - } + let activeAuthMethod: string | null = null; + let prevAuthMethod: string | null = null; + let stepState = $connectorStepStore; + let multiStepSubmitDisabled = true; + let multiStepButtonLabel = "Test and Connect"; + let multiStepLoadingCopy = "Testing connection..."; + let shouldShowSkipLink = false; + let primaryButtonLabel = "Test and Connect"; + let primaryLoadingCopy = "Testing connection..."; - // Update form when (re)entering step 1: restore defaults for connector properties - $: if (isMultiStepConnector && stepState.step === "connector") { - paramsForm.update( - () => - getInitialFormValuesFromProperties(connector.configProperties ?? []), - { taint: false }, - ); - } + $: stepState = $connectorStepStore; // Form 1: Individual parameters const paramsFormId = formManager.paramsFormId; const properties = formManager.properties; const filteredParamsProperties = formManager.filteredParamsProperties; + let multiStepFormId = paramsFormId; const { form: paramsForm, errors: paramsErrors, @@ -121,17 +104,16 @@ let dsnError: string | null = null; let dsnErrorDetails: string | undefined = undefined; - let clickhouseError: string | null = null; - let clickhouseErrorDetails: string | undefined = undefined; - let clickhouseFormId: string = ""; - let clickhouseSubmitting: boolean; - let clickhouseIsSubmitDisabled: boolean; - let clickhouseConnectorType: ClickHouseConnectorType = "self-hosted"; - let clickhouseParamsForm; - let clickhouseDsnForm; - let clickhouseShowSaveAnyway: boolean = false; + // Hide Save Anyway once we advance to the model step in multi-step flows. + $: if (isMultiStepConnector && stepState.step === "source") { + showSaveAnyway = false; + } $: isSubmitDisabled = (() => { + if (isMultiStepConnector) { + return multiStepSubmitDisabled; + } + if (onlyDsn || connectionTab === "dsn") { // DSN form: check required DSN properties for (const property of dsnProperties) { @@ -149,12 +131,7 @@ return false; } else { // Parameters form: check required properties - // Use stepProperties for multi-step connectors, otherwise use properties - const propertiesToCheck = isMultiStepConnector - ? stepProperties - : properties; - - for (const property of propertiesToCheck) { + for (const property of properties) { if (property.required) { const key = String(property.key); const value = $paramsForm[key]; @@ -168,7 +145,9 @@ } })(); - $: formId = formManager.getActiveFormId({ connectionTab, onlyDsn }); + $: formId = isMultiStepConnector + ? multiStepFormId || formManager.getActiveFormId({ connectionTab, onlyDsn }) + : formManager.getActiveFormId({ connectionTab, onlyDsn }); $: submitting = (() => { if (onlyDsn || connectionTab === "dsn") { @@ -178,6 +157,29 @@ } })(); + $: primaryButtonLabel = isMultiStepConnector + ? multiStepButtonLabel + : formManager.getPrimaryButtonLabel({ + isConnectorForm, + step: stepState.step, + submitting, + selectedAuthMethod: activeAuthMethod ?? undefined, + }); + + $: primaryLoadingCopy = (() => { + if (isMultiStepConnector) return multiStepLoadingCopy; + return activeAuthMethod === "public" + ? "Continuing..." + : "Testing connection..."; + })(); + + // Clear Save Anyway state whenever auth method changes (any direction). + $: if (activeAuthMethod !== prevAuthMethod) { + prevAuthMethod = activeAuthMethod; + showSaveAnyway = false; + saveAnyway = false; + } + $: isSubmitting = submitting; // Reset errors when form is modified @@ -210,34 +212,15 @@ // For other connectors, use manager helper saveAnyway = true; - const values = - connector.name === "clickhouse" - ? connectionTab === "dsn" - ? $clickhouseDsnForm - : $clickhouseParamsForm - : onlyDsn || connectionTab === "dsn" - ? $dsnForm - : $paramsForm; - if (connector.name === "clickhouse") { - clickhouseSubmitting = true; - } + const values = onlyDsn || connectionTab === "dsn" ? $dsnForm : $paramsForm; const result = await formManager.saveConnectorAnyway({ queryClient, values, - clickhouseConnectorType, }); if (result.ok) { onClose(); } else { - if (connector.name === "clickhouse") { - if (connectionTab === "dsn") { - dsnError = result.message; - dsnErrorDetails = result.details; - } else { - paramsError = result.message; - paramsErrorDetails = result.details; - } - } else if (onlyDsn || connectionTab === "dsn") { + if (onlyDsn || connectionTab === "dsn") { dsnError = result.message; dsnErrorDetails = result.details; } else { @@ -246,9 +229,6 @@ } } saveAnyway = false; - if (connector.name === "clickhouse") { - clickhouseSubmitting = false; - } } $: yamlPreview = formManager.computeYamlPreview({ @@ -261,21 +241,15 @@ isConnectorForm, paramsFormValues: $paramsForm, dsnFormValues: $dsnForm, - clickhouseConnectorType, - clickhouseParamsValues: $clickhouseParamsForm, - clickhouseDsnValues: $clickhouseDsnForm, }); - $: isClickhouse = connector.name === "clickhouse"; - $: shouldShowSaveAnywayButton = - isConnectorForm && (showSaveAnyway || clickhouseShowSaveAnyway); - $: saveAnywayLoading = isClickhouse - ? clickhouseSubmitting && saveAnyway - : submitting && saveAnyway; + $: shouldShowSaveAnywayButton = isConnectorForm && showSaveAnyway; + $: saveAnywayLoading = submitting && saveAnyway; handleOnUpdate = formManager.makeOnUpdate({ onClose, queryClient, getConnectionTab: () => connectionTab, + getSelectedAuthMethod: () => activeAuthMethod || undefined, setParamsError: (message: string | null, details?: string) => { paramsError = message; paramsErrorDetails = details; @@ -301,30 +275,35 @@ } -
- -
+{#if stepState.step === "explorer"} + +{:else} +
+
- {#if connector.name === "clickhouse"} - + {#if isMultiStepConnector} + { - clickhouseError = error; - clickhouseErrorDetails = details; - }} - bind:formId={clickhouseFormId} - bind:isSubmitting={clickhouseSubmitting} - bind:isSubmitDisabled={clickhouseIsSubmitDisabled} - bind:connectorType={clickhouseConnectorType} - bind:connectionTab - bind:paramsForm={clickhouseParamsForm} - bind:dsnForm={clickhouseDsnForm} - bind:showSaveAnyway={clickhouseShowSaveAnyway} + {formManager} + {paramsForm} + {paramsErrors} + {paramsEnhance} + {paramsSubmit} + {paramsFormId} + {onStringInputChange} + {handleFileUpload} + submitting={$paramsSubmitting} + bind:activeAuthMethod + bind:isSubmitDisabled={multiStepSubmitDisabled} + bind:primaryButtonLabel={multiStepButtonLabel} + bind:primaryLoadingCopy={multiStepLoadingCopy} + bind:formId={multiStepFormId} + bind:shouldShowSkipLink /> {:else if hasDsnFormOption} - {:else if isMultiStepConnector} - {#if stepState.step === "connector"} - - - - - {:else} - - - - - {/if} {:else} {/if} - {#if isMultiStepConnector && stepState.step === "connector"} - - {/if} -
@@ -482,32 +411,45 @@
- {#if dsnError || paramsError || clickhouseError} - + {#if dsnError || paramsError} + + {/if} + + - {/if} - - + + {#if shouldShowSkipLink} +
+ Already connected? +
+ {/if} +
+{/if} diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index e2d0f9c4c10..3ffac6acb59 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -20,31 +20,50 @@ import { normalizeConnectorError } from "./utils"; import { FORM_HEIGHT_DEFAULT, FORM_HEIGHT_TALL, + FORM_HEIGHT_MEDIUM, MULTI_STEP_CONNECTORS, + MEDIUM_FORM_CONNECTORS, TALL_FORM_CONNECTORS, + OLAP_ENGINES, } from "./constants"; import { connectorStepStore, setConnectorConfig, setStep, + type ConnectorStepState, } from "./connectorStepStore"; import { get } from "svelte/store"; import { compileConnectorYAML } from "../../connectors/code-utils"; -import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; -import type { ConnectorDriverProperty } from "@rilldata/web-common/runtime-client"; -import type { ClickHouseConnectorType } from "./constants"; -import { applyClickHouseCloudRequirements } from "./utils"; +import { + compileSourceYAML, + prepareSourceFormData, + prepareConnectorFormData, +} from "../sourceUtils"; +import { + ConnectorDriverPropertyType, + type ConnectorDriverProperty, +} from "@rilldata/web-common/runtime-client"; import type { ActionResult } from "@sveltejs/kit"; +import { getConnectorSchema } from "./connector-schemas"; +import { + findRadioEnumKey, + findGroupedEnumKeys, + isVisibleForValues, +} from "../../templates/schema-utils"; // Minimal onUpdate event type carrying Superforms's validated form type SuperFormUpdateEvent = { form: SuperValidated, any, Record>; }; -// Shape of the step store for multi-step connectors -type ConnectorStepState = { - step: "connector" | "source"; - connectorConfig: Record | null; +const BUTTON_LABELS = { + public: { idle: "Continue", submitting: "Continuing..." }, + connector: { idle: "Test and Connect", submitting: "Testing connection..." }, + connectorReadOnly: { + idle: "Test and Add Connector", + submitting: "Testing connection...", + }, + source: { idle: "Import Data", submitting: "Importing data..." }, }; export class AddDataFormManager { @@ -69,20 +88,34 @@ export class AddDataFormManager { return normalizeConnectorError(this.connector.name ?? "", e); } + private getSelectedAuthMethod?: () => string | undefined; + constructor(args: { connector: V1ConnectorDriver; formType: AddDataFormType; onParamsUpdate: (event: SuperFormUpdateEvent) => void; onDsnUpdate: (event: SuperFormUpdateEvent) => void; + getSelectedAuthMethod?: () => string | undefined; }) { - const { connector, formType, onParamsUpdate, onDsnUpdate } = args; + const { + connector, + formType, + onParamsUpdate, + onDsnUpdate, + getSelectedAuthMethod, + } = args; this.connector = connector; this.formType = formType; + this.getSelectedAuthMethod = getSelectedAuthMethod; // Layout height - this.formHeight = TALL_FORM_CONNECTORS.has(connector.name ?? "") - ? FORM_HEIGHT_TALL - : FORM_HEIGHT_DEFAULT; + if (TALL_FORM_CONNECTORS.has(connector.name ?? "")) { + this.formHeight = FORM_HEIGHT_TALL; + } else if (MEDIUM_FORM_CONNECTORS.has(connector.name ?? "")) { + this.formHeight = FORM_HEIGHT_MEDIUM; + } else { + this.formHeight = FORM_HEIGHT_DEFAULT; + } // IDs this.paramsFormId = `add-data-${connector.name}-form`; @@ -127,13 +160,34 @@ export class AddDataFormManager { // Superforms: params const paramsSchemaDef = getValidationSchemaForConnector( connector.name as string, + formType, + { + isMultiStepConnector: this.isMultiStepConnector, + authMethodGetter: this.getSelectedAuthMethod, + }, ); const paramsAdapter = yup(paramsSchemaDef); type ParamsOut = YupInfer; type ParamsIn = YupInferIn; - const initialFormValues = getInitialFormValuesFromProperties( - this.properties, - ); + + // For schema-based connectors, use schema defaults instead of backend properties + // to avoid pulling defaults from the backend driver + const schema = connector.name ? getConnectorSchema(connector.name) : null; + let initialFormValues: Record; + + if (schema?.properties) { + // Extract defaults from schema + initialFormValues = {}; + for (const [key, prop] of Object.entries(schema.properties)) { + if (prop.default !== undefined) { + initialFormValues[key] = prop.default; + } + } + } else { + // Fall back to backend properties for non-schema connectors + initialFormValues = getInitialFormValuesFromProperties(this.properties); + } + const paramsDefaults = defaults( initialFormValues as Partial, paramsAdapter, @@ -143,6 +197,7 @@ export class AddDataFormManager { validators: paramsAdapter, onUpdate: onParamsUpdate, resetForm: false, + validationMethod: "onsubmit", }); // Superforms: dsn @@ -154,6 +209,7 @@ export class AddDataFormManager { validators: dsnAdapter, onUpdate: onDsnUpdate, resetForm: false, + validationMethod: "onsubmit", }); } @@ -169,6 +225,37 @@ export class AddDataFormManager { return MULTI_STEP_CONNECTORS.includes(this.connector.name ?? ""); } + /** + * Determines whether the "Save Anyway" button should be shown for the current submission. + */ + private shouldShowSaveAnywayButton(args: { + isConnectorForm: boolean; + event?: + | { + result?: Extract; + } + | undefined; + stepState: ConnectorStepState | undefined; + selectedAuthMethod?: string; + }): boolean { + const { isConnectorForm, event, stepState, selectedAuthMethod } = args; + + // Only show for connector forms (not sources) + if (!isConnectorForm) return false; + + // Need a submission result to show the button + if (!event?.result) return false; + + // Multi-step connectors: don't show on source step (final step) + if (stepState?.step === "source") return false; + + // Public auth bypasses connection test, so no "Save Anyway" needed + if (stepState?.step === "connector" && selectedAuthMethod === "public") + return false; + + return true; + } + getActiveFormId(args: { connectionTab: "parameters" | "dsn"; onlyDsn: boolean; @@ -182,7 +269,12 @@ export class AddDataFormManager { handleSkip(): void { const stepState = get(connectorStepStore) as ConnectorStepState; if (!this.isMultiStepConnector || stepState.step !== "connector") return; - setConnectorConfig(get(this.params.form) as Record); + + const formValues = get(this.params.form); + const mode = formValues?.mode as string | undefined; + + // Preserve the form values (especially mode) when navigating to source step + setConnectorConfig(formValues); setStep("source"); } @@ -199,35 +291,37 @@ export class AddDataFormManager { isConnectorForm: boolean; step: "connector" | "source" | string; submitting: boolean; - clickhouseConnectorType?: ClickHouseConnectorType; - clickhouseSubmitting?: boolean; + selectedAuthMethod?: string; + mode?: string; }): string { - const { - isConnectorForm, - step, - submitting, - clickhouseConnectorType, - clickhouseSubmitting, - } = args; - const isClickhouse = this.connector.name === "clickhouse"; - - if (isClickhouse) { - if (clickhouseConnectorType === "rill-managed") { - return clickhouseSubmitting ? "Connecting..." : "Connect"; - } - return clickhouseSubmitting - ? "Testing connection..." - : "Test and Connect"; - } + const { isConnectorForm, step, submitting, selectedAuthMethod, mode } = + args; if (isConnectorForm) { if (this.isMultiStepConnector && step === "connector") { - return submitting ? "Testing connection..." : "Test and Connect"; + if (selectedAuthMethod === "public") { + return submitting + ? BUTTON_LABELS.public.submitting + : BUTTON_LABELS.public.idle; + } + // For read-only mode, show "Test and Add Connector" instead of "Test and Connect" + if (mode === "read") { + return submitting + ? BUTTON_LABELS.connectorReadOnly.submitting + : BUTTON_LABELS.connectorReadOnly.idle; + } + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } if (this.isMultiStepConnector && step === "source") { - return submitting ? "Creating model..." : "Test and Add data"; + return submitting + ? BUTTON_LABELS.source.submitting + : BUTTON_LABELS.source.idle; } - return submitting ? "Testing connection..." : "Test and Connect"; + return submitting + ? BUTTON_LABELS.connector.submitting + : BUTTON_LABELS.connector.idle; } return "Test and Add data"; @@ -237,6 +331,7 @@ export class AddDataFormManager { onClose: () => void; queryClient: any; getConnectionTab: () => "parameters" | "dsn"; + getSelectedAuthMethod?: () => string | undefined; setParamsError: (message: string | null, details?: string) => void; setDsnError: (message: string | null, details?: string) => void; setShowSaveAnyway?: (value: boolean) => void; @@ -245,6 +340,7 @@ export class AddDataFormManager { onClose, queryClient, getConnectionTab, + getSelectedAuthMethod, setParamsError, setDsnError, setShowSaveAnyway, @@ -263,35 +359,97 @@ export class AddDataFormManager { >; result?: Extract; }) => { - // For non-ClickHouse connectors, expose Save Anyway when a submission starts + const values = event.form.data; + const schema = getConnectorSchema(this.connector.name ?? ""); + const authKey = schema ? findRadioEnumKey(schema) : null; + const selectedAuthMethod = + (authKey && values && values[authKey] != null + ? String(values[authKey]) + : undefined) || + getSelectedAuthMethod?.() || + ""; + const stepState = get(connectorStepStore) as ConnectorStepState; + + // Fast-path: public auth skips validation/test and goes straight to source step. if ( - isConnectorForm && - connector.name !== "clickhouse" && - typeof setShowSaveAnyway === "function" && - event?.result + isMultiStepConnector && + stepState.step === "connector" && + selectedAuthMethod === "public" ) { - setShowSaveAnyway(true); + setConnectorConfig(values); + setStep("source"); + return; } - if (!event.form.valid) return; + // When in the source step of a multi-step flow, the superform still uses + // the connector schema, so it can appear invalid because connector fields + // were intentionally skipped. Allow submission in that case and rely on + // the UI-level required checks for source fields. + if ( + !event.form.valid && + !(isMultiStepConnector && stepState.step === "source") + ) + return; - const values = event.form.data; + if ( + typeof setShowSaveAnyway === "function" && + this.shouldShowSaveAnywayButton({ + isConnectorForm, + event, + stepState, + selectedAuthMethod, + }) + ) { + setShowSaveAnyway(true); + } try { - const stepState = get(connectorStepStore) as ConnectorStepState; if (isMultiStepConnector && stepState.step === "source") { await submitAddSourceForm(queryClient, connector, values); onClose(); } else if (isMultiStepConnector && stepState.step === "connector") { - await submitAddConnectorForm(queryClient, connector, values, false); - setConnectorConfig(values); + // For public auth, skip Test & Connect and go straight to the next step. + if (selectedAuthMethod === "public") { + setConnectorConfig(values); + setStep("source"); + return; + } + const preparedValues = prepareConnectorFormData(connector, values); + await submitAddConnectorForm( + queryClient, + connector, + preparedValues, + false, + ); + + // If mode is "read" (read-only), navigate to explorer step. + // For OLAP connectors without a mode field (Pinot, Druid), also go to explorer. + // Only advance to source step when mode is "readwrite" or undefined (rill-managed). + const mode = values?.mode as string | undefined; + const schema = getConnectorSchema(connector.name ?? ""); + const hasModeField = schema?.properties?.mode !== undefined; + const isReadOnlyOlap = !hasModeField && OLAP_ENGINES.includes(connector.name ?? ""); + + if (mode === "read" || isReadOnlyOlap) { + setConnectorConfig(preparedValues); + setStep("explorer"); + return; + } + + setConnectorConfig(preparedValues); setStep("source"); return; } else if (this.formType === "source") { await submitAddSourceForm(queryClient, connector, values); onClose(); } else { - await submitAddConnectorForm(queryClient, connector, values, false); + const preparedValues = prepareConnectorFormData(connector, values); + await submitAddConnectorForm( + queryClient, + connector, + preparedValues, + false, + ); onClose(); } } catch (e) { @@ -372,9 +530,6 @@ export class AddDataFormManager { isConnectorForm: boolean; paramsFormValues: Record; dsnFormValues: Record; - clickhouseConnectorType?: ClickHouseConnectorType; - clickhouseParamsValues?: Record; - clickhouseDsnValues?: Record; }): string { const connector = this.connector; const { @@ -387,45 +542,90 @@ export class AddDataFormManager { isConnectorForm, paramsFormValues, dsnFormValues, - clickhouseConnectorType, - clickhouseParamsValues, - clickhouseDsnValues, } = ctx; + const connectorPropertiesForPreview = + isMultiStepConnector && stepState?.step === "connector" + ? (connector.configProperties ?? []) + : filteredParamsProperties; + const getConnectorYamlPreview = (values: Record) => { - return compileConnectorYAML(connector, values, { - fieldFilter: (property) => { - if (onlyDsn || connectionTab === "dsn") return true; - return !property.noPrompt; - }, - orderedProperties: + let orderedProperties: ConnectorDriverProperty[] = []; + + // For multi-step connectors with schemas, build properties from schema + const schema = + isMultiStepConnector && + stepState?.step === "connector" && + connector.name + ? getConnectorSchema(connector.name) + : null; + + if (schema) { + // Build ordered properties from schema fields that are visible and on connector step + const schemaProperties: ConnectorDriverProperty[] = []; + const properties = schema.properties ?? {}; + + // Find all grouped enum keys (radio or tabs with grouped fields) - these are UI control fields we don't want in YAML + const groupedEnumKeys = findGroupedEnumKeys(schema); + const authMethodKey = findRadioEnumKey(schema); + + // Ensure all grouped enum keys have values for visibility checks (use actual value or fallback to default) + let valuesForVisibility = { ...values }; + for (const enumKey of groupedEnumKeys) { + if (!valuesForVisibility[enumKey]) { + const enumProp = properties[enumKey]; + if (enumProp?.default) { + valuesForVisibility[enumKey] = enumProp.default; + } else if (enumProp?.enum?.length) { + valuesForVisibility[enumKey] = enumProp.enum[0]; + } + } + } + + for (const [key, prop] of Object.entries(properties)) { + // Skip grouped enum keys (auth_method, connection_method, etc.) - they're just UI control fields + if (groupedEnumKeys.includes(key)) continue; + + // Only include connector step fields that are currently visible + if ( + prop["x-step"] === "connector" && + isVisibleForValues(schema, key, valuesForVisibility) + ) { + const value = values[key]; + const isSecret = prop["x-secret"] || false; + + // Skip if value is undefined/null/empty string + if (value === undefined || value === null || value === "") continue; + + // Should we add other properties, like Array for list of strings, Ie path_prefix, etc. + schemaProperties.push({ + key, + type: + prop.type === "number" + ? ConnectorDriverPropertyType.TYPE_NUMBER + : prop.type === "boolean" + ? ConnectorDriverPropertyType.TYPE_BOOLEAN + : ConnectorDriverPropertyType.TYPE_STRING, + secret: isSecret, + }); + } + } + + orderedProperties = schemaProperties; + } else { + // Non-schema connectors use the old DSN/parameters tab system + orderedProperties = onlyDsn || connectionTab === "dsn" ? filteredDsnProperties - : filteredParamsProperties, - }); - }; + : connectorPropertiesForPreview; + } - const getClickHouseYamlPreview = ( - values: Record, - chType: ClickHouseConnectorType | undefined, - ) => { - // Convert to managed boolean and apply CH Cloud requirements for preview - const managed = chType === "rill-managed"; - const previewValues = { ...values, managed } as Record; - const finalValues = applyClickHouseCloudRequirements( - connector.name, - chType as ClickHouseConnectorType, - previewValues, - ); - return compileConnectorYAML(connector, finalValues, { + return compileConnectorYAML(connector, values, { fieldFilter: (property) => { if (onlyDsn || connectionTab === "dsn") return true; return !property.noPrompt; }, - orderedProperties: - connectionTab === "dsn" - ? filteredDsnProperties - : filteredParamsProperties, + orderedProperties, }); }; @@ -436,9 +636,17 @@ export class AddDataFormManager { const connectorPropertyKeys = new Set( connector.configProperties?.map((p) => p.key).filter(Boolean) || [], ); + + // Also filter out grouped enum keys (auth_method, connection_method, mode, etc.) + const schema = connector.name + ? getConnectorSchema(connector.name) + : null; + const groupedEnumKeys = schema ? findGroupedEnumKeys(schema) : []; + filteredValues = Object.fromEntries( Object.entries(values).filter( - ([key]) => !connectorPropertyKeys.has(key), + ([key]) => + !connectorPropertyKeys.has(key) && !groupedEnumKeys.includes(key), ), ); } @@ -447,6 +655,12 @@ export class AddDataFormManager { connector, filteredValues, ); + + // For multi-step connectors on source step, always show model YAML + if (isMultiStepConnector && stepState?.step === "source") { + return compileSourceYAML(rewrittenConnector, rewrittenFormValues); + } + const isRewrittenToDuckDb = rewrittenConnector.name === "duckdb"; if (isRewrittenToDuckDb) { return compileSourceYAML(rewrittenConnector, rewrittenFormValues); @@ -454,15 +668,6 @@ export class AddDataFormManager { return getConnectorYamlPreview(rewrittenFormValues); }; - // ClickHouse special-case - if (connector.name === "clickhouse") { - const values = - connectionTab === "dsn" - ? clickhouseDsnValues || {} - : clickhouseParamsValues || {}; - return getClickHouseYamlPreview(values, clickhouseConnectorType); - } - // Multi-step connectors if (isMultiStepConnector) { if (stepState?.step === "connector") { @@ -483,20 +688,14 @@ export class AddDataFormManager { } /** - * Save connector anyway (non-ClickHouse), returning a result object for the caller to handle. + * Save connector anyway, bypassing validation. Returns a result object for the caller to handle. */ async saveConnectorAnyway(args: { queryClient: any; values: Record; - clickhouseConnectorType?: ClickHouseConnectorType; }): Promise<{ ok: true } | { ok: false; message: string; details?: string }> { - const { queryClient, values, clickhouseConnectorType } = args; - const processedValues = applyClickHouseCloudRequirements( - this.connector.name, - (clickhouseConnectorType as ClickHouseConnectorType) || - ("self-hosted" as ClickHouseConnectorType), - values, - ); + const { queryClient, values } = args; + const processedValues = prepareConnectorFormData(this.connector, values); try { await submitAddConnectorForm( queryClient, diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index 42c8289d1b1..cfff6ef215b 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -21,7 +21,12 @@ import DuplicateSource from "./DuplicateSource.svelte"; import LocalSourceUpload from "./LocalSourceUpload.svelte"; import RequestConnectorForm from "./RequestConnectorForm.svelte"; - import { OLAP_ENGINES, ALL_CONNECTORS, SOURCES } from "./constants"; + import { + OLAP_ENGINES, + ALL_CONNECTORS, + SOURCES, + MULTI_STEP_CONNECTORS, + } from "./constants"; import { ICONS } from "./icons"; import { resetConnectorStep } from "./connectorStepStore"; @@ -34,22 +39,35 @@ query: { // arrange connectors in the way we would like to display them select: (data) => { - data.connectors = - data.connectors && - data.connectors - .filter( - // Only show connectors in SOURCES or OLAP_ENGINES - (a) => - a.name && - (SOURCES.includes(a.name) || OLAP_ENGINES.includes(a.name)), - ) - .sort( - // CAST SAFETY: we have filtered out any connectors that - // don't have a `name` in the previous filter - (a, b) => - ALL_CONNECTORS.indexOf(a.name as string) - - ALL_CONNECTORS.indexOf(b.name as string), - ); + let connectors = data.connectors || []; + + // Clone clickhouse connector to create clickhousecloud (frontend-only) + const clickhouseConnector = connectors.find( + (c) => c.name === "clickhouse", + ); + if (clickhouseConnector) { + const clickhouseCloudConnector: V1ConnectorDriver = { + ...clickhouseConnector, + name: "clickhousecloud", + displayName: "ClickHouse Cloud", + }; + connectors = [...connectors, clickhouseCloudConnector]; + } + + data.connectors = connectors + .filter( + // Only show connectors in SOURCES or OLAP_ENGINES + (a) => + a.name && + (SOURCES.includes(a.name) || OLAP_ENGINES.includes(a.name)), + ) + .sort( + // CAST SAFETY: we have filtered out any connectors that + // don't have a `name` in the previous filter + (a, b) => + ALL_CONNECTORS.indexOf(a.name as string) - + ALL_CONNECTORS.indexOf(b.name as string), + ); return data; }, }, @@ -121,7 +139,8 @@ // FIXME: excluding salesforce until we implement the table discovery APIs $: isConnectorType = - selectedConnector?.name === "gcs" || + MULTI_STEP_CONNECTORS.includes(selectedConnector?.name ?? "") || + selectedConnector?.implementsObjectStore || selectedConnector?.implementsOlap || selectedConnector?.implementsSqlStore || (selectedConnector?.implementsWarehouse && diff --git a/web-common/src/features/sources/modal/FormValidation.ts b/web-common/src/features/sources/modal/FormValidation.ts index 92c0f87c338..b5853948fd2 100644 --- a/web-common/src/features/sources/modal/FormValidation.ts +++ b/web-common/src/features/sources/modal/FormValidation.ts @@ -1,7 +1,88 @@ +import * as yup from "yup"; import { dsnSchema, getYupSchema } from "./yupSchemas"; +import { getConnectorSchema } from "./connector-schemas"; +import { + getFieldLabel, + getRequiredFieldsByEnumValue, +} from "../../templates/schema-utils"; +import type { AddDataFormType, MultiStepFormSchema } from "./types"; export { dsnSchema }; -export function getValidationSchemaForConnector(name: string) { +export function getValidationSchemaForConnector( + name: string, + formType: AddDataFormType, + opts?: { + isMultiStepConnector?: boolean; + authMethodGetter?: () => string | undefined; + }, +) { + const { isMultiStepConnector, authMethodGetter } = opts || {}; + + // For multi-step source flows, prefer the connector-specific schema when present + // so step 1 (connector) validation doesn't require source-only fields. + if (isMultiStepConnector && formType === "source") { + const connectorKey = `${name}_connector`; + if (connectorKey in getYupSchema) { + return getYupSchema[connectorKey as keyof typeof getYupSchema]; + } + } + + // For multi-step connector step, prefer connector-specific schema when present. + if (isMultiStepConnector && formType === "connector") { + // Generic dynamic schema based on auth options, driven by config. + const dynamicSchema = makeAuthOptionValidationSchema( + name, + authMethodGetter, + ); + if (dynamicSchema) return dynamicSchema; + + const connectorKey = `${name}_connector`; + if (connectorKey in getYupSchema) { + return getYupSchema[connectorKey as keyof typeof getYupSchema]; + } + } + return getYupSchema[name as keyof typeof getYupSchema]; } + +/** + * Build a yup schema that enforces required fields for the selected auth option + * using the multi-step auth config. This keeps validation in sync with the UI + * definitions alongside the schema utilities. + */ +function makeAuthOptionValidationSchema( + connectorName: string, + getAuthMethod?: () => string | undefined, +) { + const schema = getConnectorSchema(connectorName); + if (!schema) return null; + + const fieldValidations: Record = {}; + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { + step: "connector", + }); + + for (const [method, fields] of Object.entries(requiredByMethod || {})) { + for (const fieldId of fields) { + const label = getFieldLabel(schema as MultiStepFormSchema, fieldId); + fieldValidations[fieldId] = ( + fieldValidations[fieldId] || yup.string() + ).test( + `required-${fieldId}-${method}`, + `${label} is required`, + (value) => { + if (!getAuthMethod) return true; + const current = getAuthMethod(); + if (current !== method) return true; + return !!value; + }, + ); + } + } + + // If nothing to validate, skip dynamic schema. + if (!Object.keys(fieldValidations).length) return null; + + return yup.object().shape(fieldValidations); +} diff --git a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte b/web-common/src/features/sources/modal/GCSMultiStepForm.svelte deleted file mode 100644 index e007489218a..00000000000 --- a/web-common/src/features/sources/modal/GCSMultiStepForm.svelte +++ /dev/null @@ -1,129 +0,0 @@ - - - -
-
-
Authentication method
- - - {#if option.value === "credentials"} - - {:else if option.value === "hmac"} -
- - -
- {/if} - - -
- - - {#each filteredParamsProperties as property (property.key)} - {@const propertyKey = property.key ?? ""} - {#if propertyKey !== "path" && propertyKey !== "google_application_credentials" && propertyKey !== "key_id" && propertyKey !== "secret"} -
- {#if property.type === ConnectorDriverPropertyType.TYPE_STRING || property.type === ConnectorDriverPropertyType.TYPE_NUMBER} - onStringInputChange(e)} - alwaysShowError - /> - {:else if property.type === ConnectorDriverPropertyType.TYPE_BOOLEAN} - - {:else if property.type === ConnectorDriverPropertyType.TYPE_INFORMATIONAL} - - {/if} -
- {/if} - {/each} -
diff --git a/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte new file mode 100644 index 00000000000..6466f5132f7 --- /dev/null +++ b/web-common/src/features/sources/modal/JSONSchemaFieldControl.svelte @@ -0,0 +1,71 @@ + + +{#if prop["x-display"] === "file" || prop.format === "file"} + +{:else if prop.type === "boolean"} + +{:else if isSelectEnum && options} + onStringInputChange(e)} + alwaysShowError + /> +{/if} diff --git a/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte new file mode 100644 index 00000000000..989956ba657 --- /dev/null +++ b/web-common/src/features/sources/modal/MultiStepConnectorFlow.svelte @@ -0,0 +1,255 @@ + + +{#if stepState.step === "connector"} + + {#if activeSchema} + + {/if} + +{:else} + + {#if activeSchema} + + {/if} + +{/if} diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts new file mode 100644 index 00000000000..90b228320c2 --- /dev/null +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -0,0 +1,66 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; +import { athenaSchema } from "../../templates/schemas/athena"; +import { azureSchema } from "../../templates/schemas/azure"; +import { bigquerySchema } from "../../templates/schemas/bigquery"; +import { clickhouseSchema } from "../../templates/schemas/clickhouse"; +import { clickhouseCloudSchema } from "../../templates/schemas/clickhouse-cloud"; +import { druidSchema } from "../../templates/schemas/druid"; +import { duckdbSchema } from "../../templates/schemas/duckdb"; +import { gcsSchema } from "../../templates/schemas/gcs"; +import { httpsSchema } from "../../templates/schemas/https"; +import { localFileSchema } from "../../templates/schemas/local_file"; +import { motherduckSchema } from "../../templates/schemas/motherduck"; +import { mysqlSchema } from "../../templates/schemas/mysql"; +import { pinotSchema } from "../../templates/schemas/pinot"; +import { postgresSchema } from "../../templates/schemas/postgres"; +import { redshiftSchema } from "../../templates/schemas/redshift"; +import { s3Schema } from "../../templates/schemas/s3"; +import { salesforceSchema } from "../../templates/schemas/salesforce"; +import { snowflakeSchema } from "../../templates/schemas/snowflake"; +import { sqliteSchema } from "../../templates/schemas/sqlite"; + +export const multiStepFormSchemas: Record = { + s3: s3Schema, + gcs: gcsSchema, + azure: azureSchema, + https: httpsSchema, + postgres: postgresSchema, + mysql: mysqlSchema, + snowflake: snowflakeSchema, + bigquery: bigquerySchema, + redshift: redshiftSchema, + athena: athenaSchema, + clickhousecloud: clickhouseCloudSchema, + clickhouse: clickhouseSchema, + duckdb: duckdbSchema, + motherduck: motherduckSchema, + druid: druidSchema, + pinot: pinotSchema, + // Source-only connectors (no multi-step flow) + salesforce: salesforceSchema, + sqlite: sqliteSchema, + local_file: localFileSchema, +}; + +export function getConnectorSchema( + connectorName: string, +): MultiStepFormSchema | null { + const schema = + multiStepFormSchemas[connectorName as keyof typeof multiStepFormSchemas]; + if (!schema?.properties) return null; + return schema; +} + +export function isStepMatch( + schema: MultiStepFormSchema | null, + key: string, + step?: "connector" | "source" | string, +): boolean { + if (!schema?.properties) return false; + const prop = schema.properties[key]; + if (!prop) return false; + if (!step) return true; + const propStep = prop["x-step"]; + if (!propStep) return true; + return propStep === step; +} diff --git a/web-common/src/features/sources/modal/connectorStepStore.ts b/web-common/src/features/sources/modal/connectorStepStore.ts index 9ce2451f126..9f58cb53645 100644 --- a/web-common/src/features/sources/modal/connectorStepStore.ts +++ b/web-common/src/features/sources/modal/connectorStepStore.ts @@ -1,13 +1,17 @@ import { writable } from "svelte/store"; -export type ConnectorStep = "connector" | "source"; +export type ConnectorStep = "connector" | "source" | "explorer"; -export const connectorStepStore = writable<{ +export type ConnectorStepState = { step: ConnectorStep; connectorConfig: Record | null; -}>({ + selectedAuthMethod: string | null; +}; + +export const connectorStepStore = writable({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); export function setStep(step: ConnectorStep) { @@ -18,9 +22,17 @@ export function setConnectorConfig(config: Record) { connectorStepStore.update((state) => ({ ...state, connectorConfig: config })); } +export function setAuthMethod(method: string | null) { + connectorStepStore.update((state) => ({ + ...state, + selectedAuthMethod: method, + })); +} + export function resetConnectorStep() { connectorStepStore.set({ step: "connector", connectorConfig: null, + selectedAuthMethod: null, }); } diff --git a/web-common/src/features/sources/modal/constants.ts b/web-common/src/features/sources/modal/constants.ts index 4bffc52825d..5453813c369 100644 --- a/web-common/src/features/sources/modal/constants.ts +++ b/web-common/src/features/sources/modal/constants.ts @@ -1,44 +1,8 @@ -export type ClickHouseConnectorType = - | "rill-managed" - | "self-hosted" - | "clickhouse-cloud"; - -export const CONNECTOR_TYPE_OPTIONS: { - value: ClickHouseConnectorType; - label: string; -}[] = [ - { value: "rill-managed", label: "Rill-managed ClickHouse" }, - { value: "self-hosted", label: "Self-hosted ClickHouse" }, - { value: "clickhouse-cloud", label: "ClickHouse Cloud" }, -]; - export const CONNECTION_TAB_OPTIONS: { value: string; label: string }[] = [ { value: "parameters", label: "Enter parameters" }, { value: "dsn", label: "Enter connection string" }, ]; -export type GCSAuthMethod = "credentials" | "hmac"; - -export const GCS_AUTH_OPTIONS: { - value: GCSAuthMethod; - label: string; - description: string; - hint?: string; -}[] = [ - { - value: "credentials", - label: "GCP credentials", - description: - "Upload a JSON key file for a service account with GCS access.", - }, - { - value: "hmac", - label: "HMAC keys", - description: - "Use HMAC access key and secret for S3-compatible authentication.", - }, -]; - // pre-defined order for sources export const SOURCES = [ "athena", @@ -57,6 +21,7 @@ export const SOURCES = [ ]; export const OLAP_ENGINES = [ + "clickhousecloud", "clickhouse", "motherduck", "duckdb", @@ -67,12 +32,34 @@ export const OLAP_ENGINES = [ export const ALL_CONNECTORS = [...SOURCES, ...OLAP_ENGINES]; // Connectors that support multi-step forms (connector -> source) -export const MULTI_STEP_CONNECTORS = ["gcs"]; +export const MULTI_STEP_CONNECTORS = [ + "gcs", + "s3", + "azure", + "https", + "postgres", + "mysql", + "snowflake", + "bigquery", + "redshift", + "athena", + "clickhousecloud", + "clickhouse", + "duckdb", + "motherduck", + "druid", + "pinot", +]; -export const FORM_HEIGHT_TALL = "max-h-[38.5rem] min-h-[38.5rem]"; +export const FORM_HEIGHT_TALL = "max-h-[55.5rem] min-h-[38.5rem]"; +export const FORM_HEIGHT_MEDIUM = "max-h-[47.5rem] min-h-[34.5rem]"; export const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; -export const TALL_FORM_CONNECTORS = new Set([ - "clickhouse", - "snowflake", +export const MEDIUM_FORM_CONNECTORS = new Set([ + "clickhousecloud", "salesforce", + "postgres", + "s3", + "mysql", + "pinot", ]); +export const TALL_FORM_CONNECTORS = new Set(["clickhouse", "snowflake"]); diff --git a/web-common/src/features/sources/modal/icons.ts b/web-common/src/features/sources/modal/icons.ts index 9624685b2a5..2fd0e092ebf 100644 --- a/web-common/src/features/sources/modal/icons.ts +++ b/web-common/src/features/sources/modal/icons.ts @@ -16,7 +16,7 @@ import Postgres from "../../../components/icons/connectors/Postgres.svelte"; import Salesforce from "../../../components/icons/connectors/Salesforce.svelte"; import Snowflake from "../../../components/icons/connectors/Snowflake.svelte"; import SQLite from "../../../components/icons/connectors/SQLite.svelte"; -import ClickHouseCloud from "../../../components/icons/connectors/ClickHouseCloudIcon.svelte"; +import ClickHouseCloud from "../../../components/icons/connectors/ClickHouseCloud.svelte"; export const ICONS = { gcs: GoogleCloudStorage, @@ -34,8 +34,8 @@ export const ICONS = { salesforce: Salesforce, local_file: LocalFile, https: Https, - clickhouse: ClickHouse, clickhousecloud: ClickHouseCloud, + clickhouse: ClickHouse, druid: ApacheDruid, pinot: ApachePinot, }; diff --git a/web-common/src/features/sources/modal/types.ts b/web-common/src/features/sources/modal/types.ts index 5cb0785a5a5..8fd405fe6de 100644 --- a/web-common/src/features/sources/modal/types.ts +++ b/web-common/src/features/sources/modal/types.ts @@ -1,3 +1,51 @@ +import type { MultiStepFormSchema } from "../../templates/schemas/types"; + +export type { + JSONSchemaCondition, + JSONSchemaConditional, + JSONSchemaConstraint, + JSONSchemaField, + JSONSchemaObject, + MultiStepFormSchema, +} from "../../templates/schemas/types"; + export type AddDataFormType = "source" | "connector"; export type ConnectorType = "parameters" | "dsn"; + +export type AuthOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export type AuthField = + | { + type: "credentials"; + id: string; + hint?: string; + optional?: boolean; + accept?: string; + } + | { + type: "input"; + id: string; + label: string; + placeholder?: string; + optional?: boolean; + secret?: boolean; + hint?: string; + }; + +export type MultiStepFormConfig = { + schema: MultiStepFormSchema; + authMethodKey: string; + authOptions: AuthOption[]; + clearFieldsByMethod: Record; + excludedKeys: string[]; + authFieldGroups: Record; + requiredFieldsByMethod: Record; + fieldLabels: Record; + defaultAuthMethod?: string; +}; diff --git a/web-common/src/features/sources/modal/utils.ts b/web-common/src/features/sources/modal/utils.ts index 5a7128802d2..ccaeefaecf3 100644 --- a/web-common/src/features/sources/modal/utils.ts +++ b/web-common/src/features/sources/modal/utils.ts @@ -1,12 +1,21 @@ import { humanReadableErrorMessage } from "../errors/errors"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; -import type { ClickHouseConnectorType } from "./constants"; +import type { MultiStepFormSchema } from "./types"; +import { + findRadioEnumKey, + getRadioEnumOptions, + getRequiredFieldsByEnumValue, +} from "../../templates/schema-utils"; +import { isStepMatch } from "./connector-schemas"; /** * Returns true for undefined, null, empty string, or whitespace-only string. * Useful for validating optional text inputs. */ export function isEmpty(val: any) { + // Booleans are never empty (false is a valid value) + if (typeof val === "boolean") return false; + return ( val === undefined || val === null || @@ -84,24 +93,91 @@ export function hasOnlyDsn( } /** - * Applies ClickHouse Cloud-specific default requirements for connector values. - * - For ClickHouse Cloud: enforces `ssl: true` - * - Otherwise returns values unchanged + * Returns true when the active multi-step auth method has missing or invalid + * required fields. Falls back to configured default/first auth method. */ -export function applyClickHouseCloudRequirements( - connectorName: string | undefined, - connectorType: ClickHouseConnectorType, - values: Record, -): Record { - // Only force SSL for ClickHouse Cloud when the user is using individual params. - // DSN strings encapsulate their own protocol, so we should not inject `ssl` there. - const isDsnBased = "dsn" in values; - const shouldEnforceSSL = - connectorName === "clickhouse" && - connectorType === "clickhouse-cloud" && - !isDsnBased; - if (shouldEnforceSSL) { - return { ...values, ssl: true } as Record; +export function isMultiStepConnectorDisabled( + schema: MultiStepFormSchema | null, + paramsFormValue: Record, + paramsFormErrors: Record, + currentStep: "connector" | "source" = "connector", +) { + if (!schema || !paramsFormValue) return true; + + const authInfo = getRadioEnumOptions(schema); + + // Handle schemas without auth method radio selector (e.g., BigQuery, Athena) + if (!authInfo) { + const requiredFields = (schema.required ?? []).filter((fieldId) => + isStepMatch(schema, fieldId, currentStep), + ); + if (!requiredFields.length) return false; + + return !requiredFields.every((fieldId) => { + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); } - return values; + + const options = authInfo?.options ?? []; + const authKey = authInfo?.key || findRadioEnumKey(schema); + const methodFromForm = + authKey && paramsFormValue?.[authKey] != null + ? String(paramsFormValue[authKey]) + : undefined; + const hasValidFormSelection = options.some( + (opt) => opt.value === methodFromForm, + ); + const method = + (hasValidFormSelection && methodFromForm) || + authInfo?.defaultValue || + options[0]?.value; + + if (!method) return true; + + // Selecting "public" should always enable the button for multi-step auth flows. + if (method === "public") return false; + + // When on source step and auth method isn't set (user skipped connector step), + // validate all source step required fields regardless of auth method + if (currentStep === "source" && !methodFromForm) { + const allSourceRequired = new Set(); + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { + step: currentStep, + }); + + // Collect all source step required fields across all auth methods + for (const fields of Object.values(requiredByMethod)) { + fields.forEach((field) => allSourceRequired.add(field)); + } + + const sourceRequiredFields = Array.from(allSourceRequired); + if (!sourceRequiredFields.length) return false; + + return !sourceRequiredFields.every((fieldId) => { + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); + } + + const requiredByMethod = getRequiredFieldsByEnumValue(schema, { + step: currentStep, + }); + const requiredFields = requiredByMethod[method] ?? []; + + // If no required fields found for this step, button should be enabled + if (!requiredFields.length) return false; + + // Check if all required fields are filled and have no errors + return !requiredFields.every((fieldId) => { + if (!isStepMatch(schema, fieldId, currentStep)) return true; + const value = paramsFormValue[fieldId]; + const errorsForField = paramsFormErrors[fieldId] as any; + const hasErrors = Boolean(errorsForField?.length); + return !isEmpty(value) && !hasErrors; + }); } diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index b12ec332eae..becb927a448 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -5,19 +5,29 @@ import { } from "../../entity-management/name-utils"; export const getYupSchema = { - s3: yup.object().shape({ + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + s3_connector: yup.object().shape({ + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + region: yup.string().optional(), + endpoint: yup.string().optional(), + }), + + s3_source: yup.object().shape({ path: yup .string() .matches(/^s3:\/\//, "Must be an S3 URI (e.g. s3://bucket/path)") - .required("S3 URI is required"), - aws_region: yup.string(), + .required(), name: yup .string() .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) - .required("Source name is required"), + .required(), }), - gcs: yup.object().shape({ + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + gcs_connector: yup.object().shape({ google_application_credentials: yup.string().optional(), key_id: yup.string().optional(), secret: yup.string().optional(), @@ -27,6 +37,34 @@ export const getYupSchema = { .optional(), }), + gcs_source: yup.object().shape({ + path: yup + .string() + .matches(/^gs:\/\//, "Must be a GS URI (e.g. gs://bucket/path)") + .optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + https_connector: yup.object().shape({ + headers: yup.string().optional(), + }), + + https_source: yup.object().shape({ + path: yup + .string() + .matches(/^https?:\/\//, 'Path must start with "http(s)://"') + .required("Path is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Source name is required"), + }), + https: yup.object().shape({ path: yup .string() @@ -41,12 +79,38 @@ export const getYupSchema = { duckdb: yup.object().shape({ path: yup.string().required("path is required"), attach: yup.string().optional(), + sql: yup.string().optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .optional(), + }), + + duckdb_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), }), motherduck: yup.object().shape({ token: yup.string().required("Token is required"), path: yup.string().required("Path is required"), schema_name: yup.string().required("Schema name is required"), + sql: yup.string().optional(), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .optional(), + }), + + motherduck_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), }), sqlite: yup.object().shape({ @@ -65,7 +129,17 @@ export const getYupSchema = { .required("Google application credentials is required"), }), - azure: yup.object().shape({ + // Keep these optional here; per-auth required fields are enforced dynamically + // via multi-step auth configs. This schema acts as a safe fallback (e.g. source + // step selection of `${name}_connector`). + azure_connector: yup.object().shape({ + azure_storage_account: yup.string().optional(), + azure_storage_key: yup.string().optional(), + azure_storage_sas_token: yup.string().optional(), + azure_storage_connection_string: yup.string().optional(), + }), + + azure_source: yup.object().shape({ path: yup .string() .matches( @@ -73,13 +147,184 @@ export const getYupSchema = { "Must be an Azure URI (e.g. azure://container/path)", ) .required("Path is required"), - azure_storage_account: yup.string(), name: yup .string() .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) .required("Source name is required"), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + postgres_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + database: yup.string().optional(), + user: yup.string().optional(), + password: yup.string().optional(), + sslmode: yup.string().optional(), + dsn: yup.string().optional(), + }), + + postgres_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + mysql_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + database: yup.string().optional(), + user: yup.string().optional(), + password: yup.string().optional(), + sslmode: yup.string().optional(), + dsn: yup.string().optional(), + }), + + mysql_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + snowflake_connector: yup.object().shape({ + account: yup.string().optional(), + user: yup.string().optional(), + password: yup.string().optional(), + private_key: yup.string().optional(), + private_key_passphrase: yup.string().optional(), + warehouse: yup.string().optional(), + database: yup.string().optional(), + schema: yup.string().optional(), + role: yup.string().optional(), + dsn: yup.string().optional(), + }), + + snowflake_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + bigquery_connector: yup.object().shape({ + google_application_credentials: yup.string().optional(), + project_id: yup.string().optional(), + }), + + bigquery_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + redshift_connector: yup.object().shape({ + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + region: yup.string().optional(), + database: yup.string().optional(), + workgroup: yup.string().optional(), + cluster_identifier: yup.string().optional(), + }), + + redshift_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + athena_connector: yup.object().shape({ + aws_access_key_id: yup.string().optional(), + aws_secret_access_key: yup.string().optional(), + region: yup.string().optional(), + workgroup: yup.string().optional(), + output_location: yup.string().optional(), + }), + + athena_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + duckdb_connector: yup.object().shape({ + path: yup.string().optional(), + attach: yup.string().optional(), + mode: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + motherduck_connector: yup.object().shape({ + path: yup.string().optional(), + token: yup.string().optional(), + schema_name: yup.string().optional(), + mode: yup.string().optional(), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + druid_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + username: yup.string().optional(), + password: yup.string().optional(), + ssl: yup.boolean().optional(), + dsn: yup.string().optional(), + }), + + druid_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + pinot_connector: yup.object().shape({ + broker_host: yup.string().optional(), + broker_port: yup.number().optional(), + controller_host: yup.string().optional(), + controller_port: yup.number().optional(), + username: yup.string().optional(), + password: yup.string().optional(), + ssl: yup.boolean().optional(), + dsn: yup.string().optional(), + }), + + pinot_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + postgres: yup.object().shape({ dsn: yup.string().optional(), host: yup.string().optional(), @@ -162,6 +407,39 @@ export const getYupSchema = { // .required("Connector name is required"), }), + // Keep base auth fields optional; per-method required fields come from + // multi-step auth configs. This schema is a safe fallback. + clickhousecloud_connector: yup.object().shape({ + host: yup.string().optional(), + port: yup.number().optional(), + username: yup.string().optional(), + password: yup.string().optional(), + database: yup.string().optional(), + cluster: yup.string().optional(), + mode: yup.string().optional(), + }), + + clickhousecloud_source: yup.object().shape({ + sql: yup.string().required("SQL query is required"), + name: yup + .string() + .matches(VALID_NAME_PATTERN, INVALID_NAME_MESSAGE) + .required("Model name is required"), + }), + + clickhousecloud: yup.object().shape({ + host: yup.string(), + port: yup + .string() // Purposefully using a string input, not a numeric input + .matches(/^\d+$/, "Port must be a number"), + username: yup.string(), + password: yup.string(), + database: yup.string(), + cluster: yup.string(), + mode: yup.string(), + name: yup.string(), // Required for typing + }), + druid: yup.object().shape({ host: yup .string() diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 480709142f6..b4c78fd870d 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -6,7 +6,10 @@ import { type V1Source, } from "@rilldata/web-common/runtime-client"; import { makeDotEnvConnectorKey } from "../connectors/code-utils"; +import { getDriverNameForConnector } from "../connectors/connectors-utils"; import { sanitizeEntityName } from "../entity-management/name-utils"; +import { getConnectorSchema } from "./modal/connector-schemas"; +import { findGroupedEnumKeys } from "../templates/schema-utils"; // Helper text that we put at the top of every Model YAML file const SOURCE_MODEL_FILE_TOP = `# Model YAML @@ -51,17 +54,22 @@ export function compileSourceYAML( if (isSecretProperty) { // For source files, we include secret properties return `${key}: "{{ .env.${makeDotEnvConnectorKey( - connector.name as string, + getDriverNameForConnector(connector.name as string), key, )} }}"`; } if (key === "sql") { - // For SQL, we want to use a multi-line string - return `${key}: |\n ${value + // For SQL, we want to use a multi-line string and add a dev section + const sqlLines = value .split("\n") - .map((line) => `${line}`) - .join("\n")}`; + .map((line) => ` ${line}`) + .join("\n"); + const devSqlLines = value + .split("\n") + .map((line) => ` ${line}`) + .join("\n"); + return `${key}: |\n${sqlLines}\n\ndev:\n ${key}: |\n${devSqlLines}\n limit 10000`; } const isStringProperty = stringPropertyKeys.includes(key); @@ -74,7 +82,7 @@ export function compileSourceYAML( .join("\n"); return ( - `${SOURCE_MODEL_FILE_TOP}\n\nconnector: ${connector.name}\n\n` + + `${SOURCE_MODEL_FILE_TOP}\n\nconnector: ${getDriverNameForConnector(connector.name as string)}\n\n` + compiledKeyValues ); } @@ -201,6 +209,130 @@ export function maybeRewriteToDuckDb( return [connectorCopy, formValues]; } +/** + * Prepare connector form values before submission. + * Handles special transformations like ClickHouse auth_method → managed. + */ +export function prepareConnectorFormData( + connector: V1ConnectorDriver, + formValues: Record, +): Record { + const processedValues = { ...formValues }; + + // Get schema to check for grouped fields + const schema = connector.name ? getConnectorSchema(connector.name) : null; + + if (schema) { + // Find all grouped enum keys (auth_method, connection_method, etc.) + const groupedEnumKeys = findGroupedEnumKeys(schema); + + if (groupedEnumKeys.length > 0) { + // Collect all fields that should be included based on active selections + const allowedFields = new Set(); + + // For each grouped enum, find which fields are in the active option's group + for (const enumKey of groupedEnumKeys) { + const enumValue = processedValues[enumKey] as string | undefined; + const prop = schema.properties?.[enumKey]; + const groupedFields = prop?.["x-grouped-fields"]; + + if (enumValue && groupedFields && groupedFields[enumValue]) { + // Add all fields from the active group + for (const fieldKey of groupedFields[enumValue]) { + allowedFields.add(fieldKey); + } + } + } + + // Also include fields that aren't controlled by any grouped enum (standalone fields) + // Collect all fields that ARE controlled by some group + const allGroupedFieldKeys = new Set(); + for (const enumKey of groupedEnumKeys) { + const prop = schema.properties?.[enumKey]; + const groupedFields = prop?.["x-grouped-fields"]; + if (groupedFields) { + for (const fieldArray of Object.values(groupedFields)) { + for (const fieldKey of fieldArray) { + allGroupedFieldKeys.add(fieldKey); + } + } + } + } + + // Filter processedValues to only include allowed fields + const filteredValues: Record = {}; + for (const [key, value] of Object.entries(processedValues)) { + // Include if: + // - It's in the allowed fields for active groups, OR + // - It's not controlled by any group (standalone field), OR + // - It's a grouped enum key itself (we'll remove it later if needed) + if (allowedFields.has(key) || !allGroupedFieldKeys.has(key)) { + filteredValues[key] = value; + } + } + + // ClickHouse: translate auth_method to managed boolean BEFORE removing grouped enum keys + if (connector.name === "clickhouse" && processedValues.auth_method) { + const authMethod = processedValues.auth_method as string; + + if (authMethod === "rill-managed") { + // Rill-managed: set managed=true, mode=readwrite + filteredValues.managed = true; + filteredValues.mode = "readwrite"; + } else if (authMethod === "self-managed") { + // Self-managed: set managed=false + filteredValues.managed = false; + } + } + + // ClickHouse Cloud: set managed=false, ssl will be in filteredValues if using parameters tab + if (connector.name === "clickhousecloud") { + filteredValues.managed = false; + // Only set ssl=true if it's in the filtered values (i.e., using parameters tab) + if ("ssl" in filteredValues) { + filteredValues.ssl = true; + } + } + + // Replace with filtered values + Object.keys(processedValues).forEach( + (key) => delete processedValues[key], + ); + Object.assign(processedValues, filteredValues); + + // Remove the grouped enum keys themselves - they're UI-only fields + for (const enumKey of groupedEnumKeys) { + delete processedValues[enumKey]; + } + } + } else { + // No schema, handle ClickHouse auth_method the old way + if (connector.name === "clickhouse" && processedValues.auth_method) { + const authMethod = processedValues.auth_method as string; + + if (authMethod === "rill-managed") { + // Rill-managed: set managed=true, mode=readwrite + processedValues.managed = true; + processedValues.mode = "readwrite"; + } else if (authMethod === "self-managed") { + // Self-managed: set managed=false + processedValues.managed = false; + } + + // Remove the UI-only auth_method field + delete processedValues.auth_method; + } + + // ClickHouse Cloud: set managed=false and ssl=true (only in non-schema path) + if (connector.name === "clickhousecloud") { + processedValues.managed = false; + processedValues.ssl = true; + } + } + + return processedValues; +} + /** * Process form data for sources, including DuckDB rewrite logic and placeholder handling. * This serves as a single source of truth for both preview and submission. @@ -212,36 +344,47 @@ export function prepareSourceFormData( // Create a copy of form values to avoid mutating the original const processedValues = { ...formValues }; + // Apply DuckDB rewrite logic FIRST (before stripping connector properties) + // This is important for connectors like SQLite that need connector properties + // to build the SQL query before they're removed. + const [rewrittenConnector, rewrittenFormValues] = maybeRewriteToDuckDb( + connector, + processedValues, + ); + // Strip connector configuration keys from the source form values to prevent // leaking connector-level fields (e.g., credentials) into the model file. if (connector.configProperties) { const connectorPropertyKeys = new Set( connector.configProperties.map((p) => p.key).filter(Boolean), ); - for (const key of Object.keys(processedValues)) { + for (const key of Object.keys(rewrittenFormValues)) { if (connectorPropertyKeys.has(key)) { - delete processedValues[key]; + delete rewrittenFormValues[key]; } } } + // Also strip UI-only grouped enum keys (auth_method, connection_method, mode, etc.) + const schema = connector.name ? getConnectorSchema(connector.name) : null; + if (schema) { + const groupedEnumKeys = findGroupedEnumKeys(schema); + for (const key of groupedEnumKeys) { + delete rewrittenFormValues[key]; + } + } + // Handle placeholder values for required source properties - if (connector.sourceProperties) { - for (const prop of connector.sourceProperties) { - if (prop.key && prop.required && !(prop.key in processedValues)) { + if (rewrittenConnector.sourceProperties) { + for (const prop of rewrittenConnector.sourceProperties) { + if (prop.key && prop.required && !(prop.key in rewrittenFormValues)) { if (prop.placeholder) { - processedValues[prop.key] = prop.placeholder; + rewrittenFormValues[prop.key] = prop.placeholder; } } } } - // Apply DuckDB rewrite logic - const [rewrittenConnector, rewrittenFormValues] = maybeRewriteToDuckDb( - connector, - processedValues, - ); - return [rewrittenConnector, rewrittenFormValues]; } diff --git a/web-common/src/features/templates/JSONSchemaFormRenderer.svelte b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte new file mode 100644 index 00000000000..96e218baf7c --- /dev/null +++ b/web-common/src/features/templates/JSONSchemaFormRenderer.svelte @@ -0,0 +1,673 @@ + + +{#if schema} + {#each regularFields as [key, prop]} + {#if isRadioEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + + {#if groupedFields.get(key)} + {@const allGroupedForOption = getGroupedFieldsForOption( + key, + option.value, + )} + {@const regularGrouped = allGroupedForOption.filter( + ([_, prop]) => !prop["x-advanced"], + )} + {@const advancedGrouped = allGroupedForOption.filter( + ([_, prop]) => prop["x-advanced"], + )} + {#each regularGrouped as [childKey, childProp]} + {#if isTabsEnum(childProp)} +
+ {#if childProp.title} +
+ {childProp.title} +
+ {/if} + + {#if groupedFields.get(childKey)} + {#each tabOptions(childProp) as tabOption} + + {#each getGroupedFieldsForOption(childKey, tabOption.value) as [grandchildKey, grandchildProp]} +
+ +
+ {/each} + + {/each} + {/if} + +
+ {:else} +
+ +
+ {/if} + {/each} + {#if advancedGrouped.length > 0} +
+ + Advanced Configuration + +
+ {#each advancedGrouped as [childKey, childProp]} +
+ +
+ {/each} +
+
+ {/if} + {/if} + + +
+ {:else if isTabsEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + {#if groupedFields.get(key)} + {#each tabOptions(prop) as option} + + {@const allGroupedForTabOption = getGroupedFieldsForOption( + key, + option.value, + )} + {@const regularGroupedTab = allGroupedForTabOption.filter( + ([_, prop]) => !prop["x-advanced"], + )} + {@const advancedGroupedTab = allGroupedForTabOption.filter( + ([_, prop]) => prop["x-advanced"], + )} + {#each regularGroupedTab as [childKey, childProp]} + {#if isRadioEnum(childProp)} +
+ {#if childProp.title} +
+ {childProp.title} +
+ {/if} + + + {#if groupedFields.get(childKey)} + {#each getGroupedFieldsForOption(childKey, radioOption.value) as [grandchildKey, grandchildProp]} +
+ +
+ {/each} + {/if} + + +
+ {:else} +
+ +
+ {/if} + {/each} + {#if advancedGroupedTab.length > 0} +
+ + Advanced Configuration + +
+ {#each advancedGroupedTab as [childKey, childProp]} +
+ +
+ {/each} +
+
+ {/if} + + {/each} + {/if} + +
+ {:else} +
+ +
+ {/if} + {/each} + + {#if advancedFields.length > 0} +
+ + Advanced Configuration + +
+ {#each advancedFields as [key, prop]} + {#if isRadioEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + + {#if groupedFields.get(key)} + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} + {#if isTabsEnum(childProp)} +
+ {#if childProp.title} +
+ {childProp.title} +
+ {/if} + + {#if groupedFields.get(childKey)} + {#each tabOptions(childProp) as tabOption} + + {#each getGroupedFieldsForOption(childKey, tabOption.value) as [grandchildKey, grandchildProp]} +
+ +
+ {/each} + + {/each} + {/if} + +
+ {:else} +
+ +
+ {/if} + {/each} + {/if} + + +
+ {:else if isTabsEnum(prop)} +
+ {#if prop.title} +
{prop.title}
+ {/if} + + {#if groupedFields.get(key)} + {#each tabOptions(prop) as option} + + {#each getGroupedFieldsForOption(key, option.value) as [childKey, childProp]} + {#if isRadioEnum(childProp)} +
+ {#if childProp.title} +
+ {childProp.title} +
+ {/if} + + + {#if groupedFields.get(childKey)} + {#each getGroupedFieldsForOption(childKey, radioOption.value) as [grandchildKey, grandchildProp]} +
+ +
+ {/each} + {/if} + + +
+ {:else} +
+ +
+ {/if} + {/each} + + {/each} + {/if} + +
+ {:else} +
+ +
+ {/if} + {/each} +
+
+ {/if} +{/if} diff --git a/web-common/src/features/templates/schema-utils.ts b/web-common/src/features/templates/schema-utils.ts new file mode 100644 index 00000000000..30ea9436346 --- /dev/null +++ b/web-common/src/features/templates/schema-utils.ts @@ -0,0 +1,162 @@ +import type { + JSONSchemaConditional, + MultiStepFormSchema, +} from "./schemas/types"; + +export type RadioEnumOption = { + value: string; + label: string; + description: string; + hint?: string; +}; + +export function isVisibleForValues( + schema: MultiStepFormSchema, + key: string, + values: Record, +): boolean { + const prop = schema.properties?.[key]; + if (!prop) return false; + const conditions = prop["x-visible-if"]; + if (!conditions) return true; + + return Object.entries(conditions).every(([depKey, expected]) => { + const actual = values?.[depKey]; + if (Array.isArray(expected)) { + return expected.map(String).includes(String(actual)); + } + return String(actual) === String(expected); + }); +} + +export function getFieldLabel( + schema: MultiStepFormSchema, + key: string, +): string { + return schema.properties?.[key]?.title || key; +} + +/** + * Find all enum keys (radio or tabs) that have grouped fields. + * These are UI control fields (like auth_method, connection_method) that shouldn't appear in YAML. + */ +export function findGroupedEnumKeys(schema: MultiStepFormSchema): string[] { + if (!schema.properties) return []; + const keys: string[] = []; + for (const [key, value] of Object.entries(schema.properties)) { + if (value.enum && value["x-grouped-fields"]) { + keys.push(key); + } + } + return keys; +} + +export function findRadioEnumKey(schema: MultiStepFormSchema): string | null { + if (!schema.properties) return null; + for (const [key, value] of Object.entries(schema.properties)) { + // Return radio or tabs fields that have grouped fields - those are auth/connection method selectors + // Standalone radio fields (like "mode") should not be considered auth method keys + const display = value["x-display"]; + if ( + value.enum && + (display === "radio" || display === "tabs") && + value["x-grouped-fields"] + ) { + return key; + } + } + return schema.properties.auth_method ? "auth_method" : null; +} + +export function getRadioEnumOptions(schema: MultiStepFormSchema): { + key: string; + options: RadioEnumOption[]; + defaultValue?: string; +} | null { + const enumKey = findRadioEnumKey(schema); + if (!enumKey) return null; + const enumProperty = schema.properties?.[enumKey]; + if (!enumProperty?.enum) return null; + + const labels = enumProperty["x-enum-labels"] ?? []; + const descriptions = enumProperty["x-enum-descriptions"] ?? []; + const options = + enumProperty.enum?.map((value, idx) => ({ + value: String(value), + label: labels[idx] ?? String(value), + description: + descriptions[idx] ?? enumProperty.description ?? "Choose an option", + hint: enumProperty["x-hint"], + })) ?? []; + + const defaultValue = + enumProperty.default !== undefined && enumProperty.default !== null + ? String(enumProperty.default) + : options[0]?.value; + + return { + key: enumKey, + options, + defaultValue: defaultValue || undefined, + }; +} + +export function getRequiredFieldsByEnumValue( + schema: MultiStepFormSchema, + opts?: { step?: "connector" | "source" | string }, +): Record { + const enumInfo = getRadioEnumOptions(schema); + if (!enumInfo) return {}; + + const conditionals = schema.allOf ?? []; + const baseRequired = new Set(schema.required ?? []); + const result: Record = {}; + + const matchesStep = (field: string) => { + if (!opts?.step) return true; + const prop = schema.properties?.[field]; + if (!prop) return false; + const propStep = prop["x-step"]; + if (!propStep) return true; + return propStep === opts.step; + }; + + for (const option of enumInfo.options) { + const required = new Set(); + + baseRequired.forEach((field) => { + if (matchesStep(field)) { + required.add(field); + } + }); + + for (const conditional of conditionals) { + const matches = matchesEnumCondition( + conditional, + enumInfo.key, + option.value, + ); + const target = matches ? conditional.then : conditional.else; + target?.required?.forEach((field) => { + if (matchesStep(field)) { + required.add(field); + } + }); + } + + result[option.value] = Array.from(required); + } + + return result; +} + +function matchesEnumCondition( + conditional: JSONSchemaConditional, + enumKey: string, + value: string, +) { + const conditionProps = conditional.if?.properties; + const constValue = conditionProps?.[enumKey]?.const; + if (constValue === undefined || constValue === null) return false; + return String(constValue) === value; +} diff --git a/web-common/src/features/templates/schemas/athena.ts b/web-common/src/features/templates/schemas/athena.ts new file mode 100644 index 00000000000..245ee2a842c --- /dev/null +++ b/web-common/src/features/templates/schemas/athena.ts @@ -0,0 +1,107 @@ +import type { MultiStepFormSchema } from "./types"; + +export const athenaSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS Access Key ID", + description: "AWS access key ID", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + }, + aws_secret_access_key: { + type: "string", + title: "AWS Secret Access Key", + description: "AWS secret access key", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + }, + output_location: { + type: "string", + title: "S3 Output Location", + description: "S3 URI for query results", + "x-placeholder": "s3://my-bucket/athena-results/", + "x-step": "connector", + }, + region: { + type: "string", + title: "AWS Region", + description: "AWS region where Athena is configured", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-advanced": true, + }, + role_session_name: { + type: "string", + title: "Role Session Name", + description: + "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + "x-placeholder": "my-session-name", + "x-secret": true, + "x-step": "connector", + "x-advanced": true, + }, + external_id: { + type: "string", + title: "External ID", + description: + "Optional external ID to use when assuming an AWS role for cross-account access.", + "x-placeholder": "external-id-123", + "x-secret": true, + "x-step": "connector", + "x-advanced": true, + }, + + workgroup: { + type: "string", + title: "Workgroup", + description: "Athena workgroup name (optional)", + "x-placeholder": "primary", + "x-step": "connector", + "x-advanced": true, + }, + aws_access_token: { + type: "string", + title: "AWS Access Token", + description: "AWS access token for authentication (optional)", + "x-placeholder": "Enter AWS access token", + "x-secret": true, + "x-step": "connector", + "x-advanced": true, + }, + allow_host_access: { + type: "boolean", + title: "Allow Host Access", + description: + "Allow the connector to access the host's network (optional)", + "x-step": "connector", + "x-advanced": true, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Athena", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "output_location", + "sql", + "name", + ], +}; diff --git a/web-common/src/features/templates/schemas/azure.ts b/web-common/src/features/templates/schemas/azure.ts new file mode 100644 index 00000000000..5c0d144b28b --- /dev/null +++ b/web-common/src/features/templates/schemas/azure.ts @@ -0,0 +1,143 @@ +import type { MultiStepFormSchema } from "./types"; + +export const azureSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["connection_string", "account_key", "sas_token", "public"], + default: "connection_string", + description: "Choose how to authenticate to Azure Blob Storage", + "x-display": "radio", + "x-enum-labels": [ + "Connection String", + "Storage Account Key", + "SAS Token", + "Public", + ], + "x-enum-descriptions": [ + "Provide a full Azure Storage connection string.", + "Provide the storage account name and access key.", + "Provide the storage account name and SAS token.", + "Access publicly readable blobs without credentials.", + ], + "x-grouped-fields": { + connection_string: ["azure_storage_connection_string"], + account_key: ["azure_storage_account", "azure_storage_key"], + sas_token: ["azure_storage_account", "azure_storage_sas_token"], + public: [], + }, + "x-step": "connector", + }, + azure_storage_connection_string: { + type: "string", + title: "Connection string", + description: "Paste an Azure Storage connection string", + "x-placeholder": "Enter Azure storage connection string", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + azure_storage_account: { + type: "string", + title: "Storage account", + description: "The name of the Azure storage account", + "x-placeholder": "Enter Azure storage account", + "x-step": "connector", + "x-visible-if": { auth_method: ["account_key", "sas_token"] }, + }, + azure_storage_key: { + type: "string", + title: "Access key", + description: "Primary or secondary access key for the storage account", + "x-placeholder": "Enter Azure storage access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "account_key" }, + }, + azure_storage_sas_token: { + type: "string", + title: "SAS token", + description: + "Shared Access Signature token for the storage account (starting with ?sv=)", + "x-placeholder": "Enter Azure SAS token", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "sas_token" }, + }, + path_prefixes: { + type: "string", + title: "Prefixes", + description: + "List of prefixes to filter the blobs (e.g., ['logs/', 'data/'])", + "x-placeholder": "['logs/', 'data/']", + "x-step": "connector", + "x-advanced": true, + }, + allow_host_access: { + type: "boolean", + title: "Allow host access", + description: + "Allow access to the source from the host, useful for local development", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + path: { + type: "string", + title: "Blob URI", + description: + "URI to the Azure blob container or directory (e.g., https://.blob.core.windows.net/container)", + pattern: "^https?://[A-Za-z0-9.-]+\\.blob\\.core\\.windows\\.net/.+", + errorMessage: { + pattern: + "Enter a blob URL like https://account.blob.core.windows.net/container/path", + }, + "x-placeholder": "https://account.blob.core.windows.net/container", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["azure_storage_connection_string", "path", "name"] }, + }, + { + if: { properties: { auth_method: { const: "account_key" } } }, + then: { + required: [ + "azure_storage_account", + "azure_storage_key", + "path", + "name", + ], + }, + }, + { + if: { properties: { auth_method: { const: "sas_token" } } }, + then: { + required: [ + "azure_storage_account", + "azure_storage_sas_token", + "path", + "name", + ], + }, + }, + { + if: { properties: { auth_method: { const: "public" } } }, + then: { required: ["path", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/bigquery.ts b/web-common/src/features/templates/schemas/bigquery.ts new file mode 100644 index 00000000000..cc0389c1b4a --- /dev/null +++ b/web-common/src/features/templates/schemas/bigquery.ts @@ -0,0 +1,59 @@ +import type { MultiStepFormSchema } from "./types"; + +export const bigquerySchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + google_application_credentials: { + type: "string", + title: "GCP Credentials", + description: + "Upload a JSON key file for a service account with BigQuery access", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + }, + project_id: { + type: "string", + title: "Project ID", + description: + "Google Cloud project ID (optional if specified in credentials)", + "x-placeholder": "my-project-id", + "x-step": "connector", + }, + allow_host_access: { + type: "boolean", + title: "Allow Host Access", + description: + "Allow the connector to access the host machine (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from BigQuery", + "x-placeholder": "SELECT * FROM `project.dataset.table`;", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["google_application_credentials", "sql", "name"], +}; diff --git a/web-common/src/features/templates/schemas/clickhouse-cloud.ts b/web-common/src/features/templates/schemas/clickhouse-cloud.ts new file mode 100644 index 00000000000..cc25b09f72b --- /dev/null +++ b/web-common/src/features/templates/schemas/clickhouse-cloud.ts @@ -0,0 +1,255 @@ +import type { MultiStepFormSchema } from "./types"; + +export const clickhouseCloudSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to ClickHouse Cloud", + "x-display": "tabs", + "x-enum-labels": ["Username & Password", "Connection String"], + "x-grouped-fields": { + parameters: [ + "host", + "port", + "database", + "username", + "password", + "ssl", + "cluster", + "mode", + ], + connection_string: ["dsn", "mode"], + }, + "x-step": "connector", + }, + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the ClickHouse Cloud server", + "x-placeholder": "your-instance.clickhouse.cloud", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + port: { + type: "string", + title: "Port", + description: "Port number of the ClickHouse Cloud server", + enum: ["8443", "9440"], + default: "8443", + "x-display": "select", + "x-enum-labels": ["8443 (HTTPS)", "9440 (Native Secure)"], + "x-hint": "ClickHouse Cloud uses secure ports only", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the ClickHouse Cloud server", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the ClickHouse Cloud server", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + database: { + type: "string", + title: "Database", + description: "Name of the ClickHouse database to connect to", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + cluster: { + type: "string", + title: "Cluster", + description: "Cluster name for distributed tables", + "x-placeholder": "Cluster name", + "x-hint": + "If set, Rill will create models as distributed tables in the cluster", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL to connect to the ClickHouse server", + default: true, + "x-hint": "ClickHouse Cloud always uses SSL", + "x-readonly": true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + database_whitelist: { + type: "string", + title: "Database Whitelist", + description: "List of allowed databases", + "x-placeholder": "db1,db2", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, + }, + optimize_temporary_tables_before_partition_replace: { + type: "string", + title: "Optimize Temporary Tables", + description: "Optimize temporary tables before partition replace", + "x-placeholder": "true", + "x-advanced": true, + "x-step": "connector", + }, + log_queries: { + type: "string", + title: "Log Queries", + description: "Log all queries executed by Rill", + "x-placeholder": "false", + "x-advanced": true, + "x-step": "connector", + }, + query_settings_override: { + type: "object", + title: "Query Settings Override", + description: "Override default query settings", + "x-placeholder": "key1=value1,key2=value2", + "x-advanced": true, + "x-step": "connector", + }, + query_settings: { + type: "object", + title: "Query Settings", + description: "Custom query settings", + "x-placeholder": "key1=value1,key2=value2", + "x-advanced": true, + "x-step": "connector", + }, + embed_port: { + type: "string", + title: "Embed Port", + description: "Port number for embedding the ClickHouse Cloud server", + "x-placeholder": "8443", + "x-step": "connector", + "x-advanced": true, + }, + can_scale_to_zero: { + type: "string", + title: "Can Scale to Zero", + description: "Enable scaling to zero", + "x-placeholder": "false", + "x-advanced": true, + "x-step": "connector", + }, + max_open_conns: { + type: "string", + title: "Max Open Connections", + description: "Maximum number of open connections", + "x-placeholder": "100", + "x-advanced": true, + "x-step": "connector", + }, + max_idle_conns: { + type: "string", + title: "Max Idle Connections", + description: "Maximum number of idle connections", + "x-placeholder": "10", + "x-advanced": true, + "x-step": "connector", + }, + dial_timeout: { + type: "string", + title: "Dial Timeout", + description: "Timeout for establishing a connection", + "x-placeholder": "30s", + "x-advanced": true, + "x-step": "connector", + }, + conn_max_lifetime: { + type: "string", + title: "Connection Max Lifetime", + description: "Maximum lifetime of a connection", + "x-placeholder": "30m", + "x-advanced": true, + "x-step": "connector", + }, + read_timeout: { + type: "string", + title: "Read Timeout", + description: "Timeout for reading from the connection", + "x-placeholder": "30s", + "x-advanced": true, + "x-step": "connector", + }, + dsn: { + type: "string", + title: "Connection String", + description: "ClickHouse connection string (DSN)", + "x-placeholder": + "https://default@your-instance.clickhouse.cloud:8443/default", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + mode: { + type: "string", + title: "Connection Mode", + description: "Database access mode", + enum: ["read", "readwrite"], + default: "read", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from ClickHouse", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["host", "database", "username", "password", "ssl"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + { + if: { properties: { mode: { const: "readwrite" } } }, + then: { required: ["sql", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/clickhouse.ts b/web-common/src/features/templates/schemas/clickhouse.ts new file mode 100644 index 00000000000..7793493a28e --- /dev/null +++ b/web-common/src/features/templates/schemas/clickhouse.ts @@ -0,0 +1,319 @@ +import type { MultiStepFormSchema } from "./types"; + +export const clickhouseSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection type", + enum: ["self-managed", "rill-managed"], + default: "self-managed", + description: "Choose how to connect to ClickHouse", + "x-display": "radio", + "x-enum-labels": ["Self-managed", "Rill-managed"], + "x-enum-descriptions": [ + "Connect to your own self-hosted ClickHouse server.", + "Use a managed ClickHouse instance (starts embedded ClickHouse in development).", + ], + "x-grouped-fields": { + "self-managed": ["connection_method"], + "rill-managed": [], + }, + "x-step": "connector", + }, + connection_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to provide connection details", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-grouped-fields": { + parameters: [ + "host", + "port", + "username", + "password", + "database", + "ssl", + "cluster", + "mode", + ], + connection_string: ["dsn", "mode"], + }, + "x-step": "connector", + "x-visible-if": { auth_method: "self-managed" }, + }, + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the ClickHouse server", + "x-placeholder": "your-server.clickhouse.com", + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, + }, + port: { + type: "number", + title: "Port", + description: "Port number of the ClickHouse server", + default: 9000, + "x-placeholder": "9000", + "x-hint": + "Default: 9000 (native TCP), 8123 (HTTP). Secure: 9440 (TCP+TLS), 8443 (HTTPS)", + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the ClickHouse server", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the ClickHouse server", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, + }, + database: { + type: "string", + title: "Database", + description: "Name of the ClickHouse database to connect to", + default: "default", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, + }, + ssl: { + type: "boolean", + title: "SSL", + description: "Use SSL to connect to the ClickHouse server", + default: true, + "x-hint": "Enable SSL for secure connections", + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, + }, + cluster: { + type: "string", + title: "Cluster", + description: "Cluster name for distributed tables", + "x-placeholder": "Cluster name", + "x-hint": + "If set, Rill will create models as distributed tables in the cluster", + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "parameters", + }, + "x-advanced": true, + }, + dsn: { + type: "string", + title: "Connection String", + description: "ClickHouse connection string (DSN)", + "x-placeholder": "clickhouse://username:password@host:port/database", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { + auth_method: "self-managed", + connection_method: "connection_string", + }, + }, + mode: { + type: "string", + title: "Connection Mode", + description: "Database access mode", + enum: ["read", "readwrite"], + default: "read", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], + "x-step": "connector", + "x-visible-if": { auth_method: "self-managed" }, + }, + managed: { + type: "boolean", + title: "Managed", + description: "Enable managed mode for the ClickHouse server", + default: true, + "x-readonly": true, + "x-hint": "Enable managed mode to manage the server automatically", + "x-step": "connector", + "x-visible-if": { auth_method: "rill-managed" }, + }, + database_whitelist: { + type: "string", + title: "Database Whitelist", + description: "List of allowed databases", + "x-placeholder": "db1,db2", + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + "x-advanced": true, + }, + optimize_temporary_tables_before_partition_replace: { + type: "string", + title: "Optimize Temporary Tables", + description: "Optimize temporary tables before partition replace", + "x-placeholder": "true", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + log_queries: { + type: "string", + title: "Log Queries", + description: "Log all queries executed by Rill", + "x-placeholder": "false", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + query_settings_override: { + type: "object", + title: "Query Settings Override", + description: "Override default query settings", + "x-placeholder": "key1=value1,key2=value2", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + query_settings: { + type: "object", + title: "Query Settings", + description: "Custom query settings", + "x-placeholder": "key1=value1,key2=value2", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + embed_port: { + type: "string", + title: "Embed Port", + description: "Port number for embedding the ClickHouse Cloud server", + "x-placeholder": "8443", + "x-step": "connector", + "x-advanced": true, + "x-visible-if": { connection_method: "self-managed" }, + }, + can_scale_to_zero: { + type: "string", + title: "Can Scale to Zero", + description: "Enable scaling to zero", + "x-placeholder": "false", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + max_open_conns: { + type: "string", + title: "Max Open Connections", + description: "Maximum number of open connections", + "x-placeholder": "100", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + max_idle_conns: { + type: "string", + title: "Max Idle Connections", + description: "Maximum number of idle connections", + "x-placeholder": "10", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + dial_timeout: { + type: "string", + title: "Dial Timeout", + description: "Timeout for establishing a connection", + "x-placeholder": "30s", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + conn_max_lifetime: { + type: "string", + title: "Connection Max Lifetime", + description: "Maximum lifetime of a connection", + "x-placeholder": "30m", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + read_timeout: { + type: "string", + title: "Read Timeout", + description: "Timeout for reading from the connection", + "x-placeholder": "30s", + "x-advanced": true, + "x-step": "connector", + "x-visible-if": { connection_method: "self-managed" }, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from ClickHouse", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "self-managed" } } }, + then: { required: [] }, + }, + { + if: { properties: { auth_method: { const: "rill-managed" } } }, + then: { required: [] }, + }, + { + if: { properties: { mode: { const: "readwrite" } } }, + then: { required: ["sql", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/druid.ts b/web-common/src/features/templates/schemas/druid.ts new file mode 100644 index 00000000000..6214ab0da8f --- /dev/null +++ b/web-common/src/features/templates/schemas/druid.ts @@ -0,0 +1,114 @@ +import type { MultiStepFormSchema } from "./types"; + +export const druidSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to Druid", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-grouped-fields": { + parameters: ["host", "port", "username", "password", "ssl"], + connection_string: ["dsn"], + }, + "x-step": "connector", + }, + host: { + type: "string", + title: "Host", + description: "Hostname or IP address of the Druid server", + "x-placeholder": "localhost", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + port: { + type: "number", + title: "Port", + description: "Port number of the Druid server", + "x-placeholder": "8888", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the Druid server (optional)", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the Druid server (optional)", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + max_open_conns: { + type: "number", + title: "Maximum open connections", + description: "Maximum number of open connections to the Druid server", + default: 10, + "x-step": "connector", + "x-advanced": true, + }, + skip_version_check: { + type: "boolean", + title: "Skip version check", + description: "Skip the version check when connecting to the Druid server", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + skip_query_priority: { + type: "boolean", + title: "Skip query priority", + description: + "Skip the query priority when connecting to the Druid server", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + ssl: { + type: "boolean", + title: "Use SSL", + description: "Use SSL to connect to the Druid server", + default: true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "Druid connection string (DSN)", + "x-placeholder": + "https://example.com/druid/v2/sql/avatica-protobuf?authentication=BASIC&avaticaUser=username&avaticaPassword=password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["host", "ssl"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/duckdb.ts b/web-common/src/features/templates/schemas/duckdb.ts new file mode 100644 index 00000000000..679f6b3256a --- /dev/null +++ b/web-common/src/features/templates/schemas/duckdb.ts @@ -0,0 +1,101 @@ +import type { MultiStepFormSchema } from "./types"; + +export const duckdbSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection type", + enum: ["self-managed", "rill-managed"], + default: "self-managed", + description: "Choose how to connect to DuckDB", + "x-display": "radio", + "x-enum-labels": ["Self-managed", "Rill-managed"], + "x-enum-descriptions": [ + "Connect to your own self-hosted DuckDB server.", + "Use a managed DuckDB instance hosted by Rill.", + ], + "x-grouped-fields": { + "self-managed": ["path", "attach", "mode"], + "rill-managed": ["managed"], + }, + "x-step": "connector", + }, + path: { + type: "string", + title: "Database Path", + description: "Path to external DuckDB database file", + "x-placeholder": "/path/to/main.db", + "x-step": "connector", + }, + attach: { + type: "string", + title: "Attach", + description: + "Attach to an existing DuckDB database with options (alternative to path)", + "x-placeholder": + "'ducklake:metadata.ducklake' AS my_ducklake(DATA_PATH 'datafiles')", + "x-step": "connector", + "x-advanced": true, + }, + mode: { + type: "string", + title: "Connection Mode", + description: "Database access mode", + enum: ["read", "readwrite"], + default: "readwrite", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], + "x-step": "connector", + }, + managed: { + type: "boolean", + title: "Managed", + description: "Enable managed mode for the ClickHouse server", + default: true, + "x-readonly": true, + "x-hint": "Enable managed mode to manage the server automatically", + "x-step": "connector", + "x-visible-if": { auth_method: "rill-managed" }, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from DuckDB", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, + }, + allOf: [ + { + if: { properties: { mode: { const: "readwrite" } } }, + then: { required: ["path", "sql", "name"] }, + }, + { + if: { properties: { mode: { const: "read" } } }, + then: { required: ["path"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/gcs.ts b/web-common/src/features/templates/schemas/gcs.ts new file mode 100644 index 00000000000..c5586dbe34f --- /dev/null +++ b/web-common/src/features/templates/schemas/gcs.ts @@ -0,0 +1,109 @@ +import type { MultiStepFormSchema } from "./types"; + +export const gcsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["credentials", "hmac", "public"], + default: "credentials", + description: "Choose how to authenticate to GCS", + "x-display": "radio", + "x-enum-labels": ["GCP credentials", "HMAC keys", "Public"], + "x-enum-descriptions": [ + "Upload a JSON key file for a service account with GCS access.", + "Use HMAC access key and secret for S3-compatible authentication.", + "Access publicly readable buckets without credentials.", + ], + "x-grouped-fields": { + credentials: ["google_application_credentials"], + hmac: ["key_id", "secret"], + public: [], + }, + "x-step": "connector", + }, + google_application_credentials: { + type: "string", + title: "Service account key", + description: + "Upload a JSON key file for a service account with GCS access.", + format: "file", + "x-display": "file", + "x-accept": ".json", + "x-step": "connector", + "x-visible-if": { auth_method: "credentials" }, + }, + key_id: { + type: "string", + title: "Access Key ID", + description: "HMAC access key ID for S3-compatible authentication", + "x-placeholder": "Enter your HMAC access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + secret: { + type: "string", + title: "Secret Access Key", + description: "HMAC secret access key for S3-compatible authentication", + "x-placeholder": "Enter your HMAC secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "hmac" }, + }, + path_prefixes: { + type: "string", + title: "Path prefixes", + description: + "List of prefixes to filter the files in the GCS bucket. Leave empty to include all files.", + "x-placeholder": "['logs/', 'data/']", + "x-step": "connector", + "x-advanced": true, + }, + allow_host_access: { + type: "boolean", + title: "Allow host access", + description: + "Allow access to the GCS bucket from the host machine. This is useful for debugging and testing.", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + path: { + type: "string", + title: "GCS URI", + description: "Path to your GCS bucket or prefix", + pattern: "^gs://[^/]+(/.*)?$", + errorMessage: { + pattern: "Enter a GCS URI like gs://bucket or gs://bucket/path", + }, + "x-placeholder": "gs://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "credentials" } } }, + then: { required: ["google_application_credentials", "path", "name"] }, + }, + { + if: { properties: { auth_method: { const: "hmac" } } }, + then: { required: ["key_id", "secret", "path", "name"] }, + }, + { + if: { properties: { auth_method: { const: "public" } } }, + then: { required: ["path", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/https.ts b/web-common/src/features/templates/schemas/https.ts new file mode 100644 index 00000000000..ca3299876d0 --- /dev/null +++ b/web-common/src/features/templates/schemas/https.ts @@ -0,0 +1,72 @@ +import type { MultiStepFormSchema } from "./types"; + +export const httpsSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + enum: ["headers", "public"], + default: "headers", + description: "Choose how to authenticate to the REST API", + "x-display": "radio", + "x-enum-labels": ["Custom Headers", "Public"], + "x-enum-descriptions": [ + "Access publicly available APIs without authentication.", + "Provide custom HTTP headers for authentication (e.g., Authorization, API keys).", + ], + "x-grouped-fields": { + headers: ["headers"], + }, + "x-step": "connector", + }, + headers: { + type: "string", + title: "HTTP Headers (JSON)", + description: + 'HTTP headers as JSON object. Example: {"Authorization": "Bearer my-token", "X-API-Key": "value"}', + "x-placeholder": '{"Authorization": "Bearer my-token"}', + "x-step": "connector", + "x-visible-if": { auth_method: "headers" }, + }, + path: { + type: "string", + title: "URL", + description: "HTTP(S) URL to fetch data from", + pattern: "^https?://", + "x-placeholder": "https://api.example.com/data", + "x-step": "source", + }, + format: { + type: "string", + title: "Data Format", + description: "Format of the data returned by the API", + enum: ["json", "csv"], + default: "json", + "x-display": "radio", + "x-enum-labels": ["JSON", "CSV"], + "x-enum-descriptions": ["JSON format", "CSV format"], + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "headers" } } }, + then: { required: ["headers", "path", "name"] }, + }, + { + if: { properties: { auth_method: { const: "public" } } }, + then: { required: ["path", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/local_file.ts b/web-common/src/features/templates/schemas/local_file.ts new file mode 100644 index 00000000000..aba507a9503 --- /dev/null +++ b/web-common/src/features/templates/schemas/local_file.ts @@ -0,0 +1,34 @@ +import type { MultiStepFormSchema } from "./types"; + +export const localFileSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Path", + description: "Path or URL to file", + "x-placeholder": "/path/to/file.csv", + "x-step": "source", + }, + format: { + type: "string", + title: "Format", + description: "File format. Inferred from extension if not set.", + enum: ["csv", "parquet", "json", "ndjson"], + "x-display": "select", + "x-enum-labels": ["CSV", "Parquet", "JSON", "NDJSON"], + "x-placeholder": "csv", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_local_source", + "x-step": "source", + }, + }, + required: ["path", "name"], +}; diff --git a/web-common/src/features/templates/schemas/motherduck.ts b/web-common/src/features/templates/schemas/motherduck.ts new file mode 100644 index 00000000000..6f91a841be9 --- /dev/null +++ b/web-common/src/features/templates/schemas/motherduck.ts @@ -0,0 +1,78 @@ +import type { MultiStepFormSchema } from "./types"; + +export const motherduckSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + path: { + type: "string", + title: "Database Path", + description: "Path to MotherDuck database (must be prefixed with 'md:')", + "x-placeholder": "md:my_db", + "x-step": "connector", + }, + token: { + type: "string", + title: "MotherDuck Token", + description: "Your MotherDuck authentication token", + "x-placeholder": "Enter your MotherDuck token", + "x-secret": true, + "x-step": "connector", + }, + schema_name: { + type: "string", + title: "Schema Name", + description: "Default schema used by the MotherDuck database", + "x-placeholder": "main", + "x-step": "connector", + }, + mode: { + type: "string", + title: "Connection Mode", + description: "Database access mode", + enum: ["read", "readwrite"], + default: "read", + "x-display": "radio", + "x-enum-labels": ["Read-only", "Read-write"], + "x-enum-descriptions": [ + "Only read operations are allowed (recommended for security)", + "Enable model creation and table mutations", + ], + "x-step": "connector", + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from MotherDuck", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + "x-visible-if": { mode: "readwrite" }, + }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + "x-visible-if": { mode: "read" }, + }, + }, + allOf: [ + { + if: { properties: { mode: { const: "readwrite" } } }, + then: { required: ["path", "token", "schema_name", "sql", "name"] }, + }, + { + if: { properties: { mode: { const: "read" } } }, + then: { required: ["path", "token", "schema_name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/mysql.ts b/web-common/src/features/templates/schemas/mysql.ts new file mode 100644 index 00000000000..33cafa3326f --- /dev/null +++ b/web-common/src/features/templates/schemas/mysql.ts @@ -0,0 +1,129 @@ +import type { MultiStepFormSchema } from "./types"; + +export const mysqlSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to MySQL", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-grouped-fields": { + parameters: [ + "", + "host", + "port", + "database", + "user", + "password", + "ssl-mode", + "log_queries", + ], + connection_string: ["dsn", "log_queries"], + }, + "x-step": "connector", + }, + host: { + type: "string", + title: "Host", + description: "Database server hostname or IP address", + "x-placeholder": "localhost", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + port: { + type: "number", + title: "Port", + description: "Database server port", + default: 3306, + "x-placeholder": "3306", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + database: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "my_database", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + user: { + type: "string", + title: "Username", + description: "Database user", + "x-placeholder": "root", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Database password", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + "ssl-mode": { + type: "string", + title: "SSL Mode", + description: "SSL connection mode", + enum: ["", "disabled", "preferred", "required"], + default: "", + "x-display": "select", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, + }, + dsn: { + type: "string", + title: "Connection String", + description: "MySQL connection string (DSN)", + "x-placeholder": "user:password@tcp(host:3306)/dbname?tls=preferred", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from MySQL", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["sql", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { + required: ["host", "database", "user", "password", "sql", "name"], + }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn", "sql", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/pinot.ts b/web-common/src/features/templates/schemas/pinot.ts new file mode 100644 index 00000000000..800321b7761 --- /dev/null +++ b/web-common/src/features/templates/schemas/pinot.ts @@ -0,0 +1,115 @@ +import type { MultiStepFormSchema } from "./types"; + +export const pinotSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to Pinot", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-grouped-fields": { + parameters: [ + "broker_host", + "broker_port", + "controller_host", + "controller_port", + "username", + "password", + "ssl", + ], + connection_string: ["dsn"], + }, + "x-step": "connector", + }, + broker_host: { + type: "string", + title: "Broker Host", + description: "Hostname or IP address of the Pinot broker server", + "x-placeholder": "localhost", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + broker_port: { + type: "number", + title: "Broker Port", + description: "Port number of the Pinot broker server", + "x-placeholder": "8000", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + controller_host: { + type: "string", + title: "Controller Host", + description: "Hostname or IP address of the Pinot controller server", + "x-placeholder": "localhost", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, + }, + controller_port: { + type: "number", + title: "Controller Port", + description: "Port number of the Pinot controller server", + "x-placeholder": "9000", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, + }, + username: { + type: "string", + title: "Username", + description: "Username to connect to the Pinot server (optional)", + "x-placeholder": "default", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Password to connect to the Pinot server (optional)", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + ssl: { + type: "boolean", + title: "Use SSL", + description: "Use SSL to connect to the Pinot server", + default: true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "Pinot connection string (DSN)", + "x-placeholder": + "http(s)://username:password@localhost:8000?controller=localhost:9000", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + explorer_table: { + type: "string", + title: "Select a table", + description: "Select a table to generate metrics from", + "x-step": "explorer", + }, + }, + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["broker_host", "controller_host", "ssl"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/postgres.ts b/web-common/src/features/templates/schemas/postgres.ts new file mode 100644 index 00000000000..6629a505ed2 --- /dev/null +++ b/web-common/src/features/templates/schemas/postgres.ts @@ -0,0 +1,127 @@ +import type { MultiStepFormSchema } from "./types"; + +export const postgresSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to PostgreSQL", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-grouped-fields": { + parameters: [ + "host", + "port", + "dbname", + "user", + "password", + "sslmode", + "log_queries", + ], + connection_string: ["dsn", "log_queries"], + }, + "x-step": "connector", + }, + host: { + type: "string", + title: "Host", + description: "Database server hostname or IP address", + "x-placeholder": "localhost", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + port: { + type: "number", + title: "Port", + description: "Database server port", + default: 5432, + "x-placeholder": "5432", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + dbname: { + type: "string", + title: "Database", + description: "Database name", + "x-placeholder": "my_database", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + user: { + type: "string", + title: "Username", + description: "Database user", + "x-placeholder": "postgres", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Database password", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + }, + sslmode: { + type: "string", + title: "SSL Mode", + description: "SSL connection mode", + enum: ["", "disable", "allow", "prefer", "require"], + default: "", + "x-display": "select", + "x-step": "connector", + "x-visible-if": { auth_method: "parameters" }, + "x-advanced": true, + }, + dsn: { + type: "string", + title: "Connection String", + description: "PostgreSQL connection string (DSN)", + "x-placeholder": + "postgres://user:password@host:5432/dbname?sslmode=require", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "connection_string" }, + }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from PostgreSQL", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["sql", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "parameters" } } }, + then: { required: ["host", "dbname", "user", "password", "sql", "name"] }, + }, + { + if: { properties: { auth_method: { const: "connection_string" } } }, + then: { required: ["dsn", "sql", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/redshift.ts b/web-common/src/features/templates/schemas/redshift.ts new file mode 100644 index 00000000000..94a20829f7c --- /dev/null +++ b/web-common/src/features/templates/schemas/redshift.ts @@ -0,0 +1,97 @@ +import type { MultiStepFormSchema } from "./types"; + +export const redshiftSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + aws_access_key_id: { + type: "string", + title: "AWS Access Key ID", + description: "AWS access key ID for Redshift Data API", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + }, + aws_secret_access_key: { + type: "string", + title: "AWS Secret Access Key", + description: "AWS secret access key for Redshift Data API", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + }, + region: { + type: "string", + title: "AWS Region", + description: "AWS region where the Redshift cluster is located", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-advanced": true, + }, + database: { + type: "string", + title: "Database", + description: "Redshift database name", + "x-placeholder": "dev", + "x-step": "connector", + }, + workgroup: { + type: "string", + title: "Workgroup", + description: "Redshift Serverless workgroup name (for serverless)", + "x-placeholder": "default-workgroup", + "x-step": "connector", + "x-advanced": true, + }, + cluster_identifier: { + type: "string", + title: "Cluster Identifier", + description: + "Redshift provisioned cluster identifier (for provisioned clusters)", + "x-placeholder": "my-redshift-cluster", + "x-hint": + "Provide either workgroup (for serverless) or cluster identifier (for provisioned)", + "x-step": "connector", + "x-advanced": true, + }, + allow_host_access: { + type: "boolean", + title: "Allow Host Access", + description: + "Allow the connector to access the host's network (useful for private clusters)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + log_queries: { + type: "boolean", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + default: false, + "x-step": "connector", + "x-advanced": true, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Redshift", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "database", + "sql", + "name", + ], +}; diff --git a/web-common/src/features/templates/schemas/s3.ts b/web-common/src/features/templates/schemas/s3.ts new file mode 100644 index 00000000000..61d87f14103 --- /dev/null +++ b/web-common/src/features/templates/schemas/s3.ts @@ -0,0 +1,163 @@ +import type { MultiStepFormSchema } from "./types"; + +export const s3Schema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + auth_method: { + type: "string", + title: "Authentication method", + description: "Choose how to authenticate to S3", + enum: ["access_keys", "public"], + default: "access_keys", + "x-display": "radio", + "x-enum-labels": ["Access keys", "Public"], + "x-enum-descriptions": [ + "Use AWS access key ID and secret access key.", + "Access publicly readable buckets without credentials.", + ], + "x-grouped-fields": { + access_keys: [ + "aws_access_key_id", + "aws_secret_access_key", + "region", + "endpoint", + "aws_role_arn", + "aws_role_session_name", + "aws_external_id", + "path_prefixes", + "allow_host_access", + ], + public: [], + }, + "x-step": "connector", + }, + aws_access_key_id: { + type: "string", + title: "Access Key ID", + description: "AWS access key ID for the bucket", + "x-placeholder": "Enter AWS access key ID", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + aws_secret_access_key: { + type: "string", + title: "Secret Access Key", + description: "AWS secret access key for the bucket", + "x-placeholder": "Enter AWS secret access key", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + region: { + type: "string", + title: "Region", + description: + "Rill uses your default AWS region unless you set it explicitly.", + "x-placeholder": "us-east-1", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + }, + endpoint: { + type: "string", + title: "Endpoint", + description: + "Override the S3 endpoint (for S3-compatible services like R2/MinIO).", + "x-placeholder": "https://s3.example.com", + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, + }, + aws_role_arn: { + type: "string", + title: "AWS Role ARN", + description: "AWS Role ARN to assume", + "x-placeholder": "arn:aws:iam::123456789012:role/MyRole", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, + }, + aws_role_session_name: { + type: "string", + title: "Role Session Name", + description: + "Optional session name to use when assuming an AWS role. Defaults to 'rill-session'.", + "x-placeholder": "my-session-name", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, + }, + aws_external_id: { + type: "string", + title: "External ID", + description: + "Optional external ID to use when assuming an AWS role for cross-account access.", + "x-placeholder": "external-id-123", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { auth_method: "access_keys" }, + "x-advanced": true, + }, + path_prefixes: { + type: "string", + title: "Path prefixes", + description: "Prefixes to include in the model", + "x-placeholder": "['logs/', 'data/']", + "x-step": "connector", + "x-advanced": true, + "x-visible-if": { auth_method: "access_keys" }, + }, + allow_host_access: { + type: "boolean", + title: "Allow host access", + description: + "Allow the Rill instance to access the S3 bucket from the host machine.", + default: false, + "x-step": "connector", + "x-advanced": true, + "x-visible-if": { auth_method: "access_keys" }, + }, + path: { + type: "string", + title: "S3 URI", + description: "Path to your S3 bucket or prefix", + pattern: "^s3://[^/]+(/.*)?$", + errorMessage: { + pattern: "Enter an S3 URI like s3://bucket or s3://bucket/path", + }, + "x-placeholder": "s3://bucket/path", + "x-step": "source", + }, + name: { + type: "string", + title: "Model name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["path", "name"], + allOf: [ + { + if: { properties: { auth_method: { const: "access_keys" } } }, + then: { + required: [ + "aws_access_key_id", + "aws_secret_access_key", + "path", + "name", + ], + }, + }, + { + if: { properties: { auth_method: { const: "public" } } }, + then: { + required: ["path", "name"], + }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/salesforce.ts b/web-common/src/features/templates/schemas/salesforce.ts new file mode 100644 index 00000000000..429279a584c --- /dev/null +++ b/web-common/src/features/templates/schemas/salesforce.ts @@ -0,0 +1,74 @@ +import type { MultiStepFormSchema } from "./types"; + +export const salesforceSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + username: { + type: "string", + title: "Username", + description: "Salesforce username", + "x-placeholder": "Enter your Salesforce username", + "x-step": "source", + }, + password: { + type: "string", + title: "Password", + description: "Salesforce password", + "x-placeholder": "Enter your Salesforce password", + "x-secret": true, + "x-step": "source", + }, + key: { + type: "string", + title: "Security Token", + description: "Salesforce security token", + "x-placeholder": "Enter your security token", + "x-secret": true, + "x-step": "source", + }, + endpoint: { + type: "string", + title: "Endpoint", + description: "Salesforce endpoint URL (optional, defaults to production)", + "x-placeholder": "https://login.salesforce.com", + "x-step": "source", + "x-advanced": true, + }, + client_id: { + type: "string", + title: "Client ID", + description: "Connected App client ID (optional)", + "x-placeholder": "Enter client ID", + "x-step": "source", + "x-advanced": true, + }, + soql: { + type: "string", + title: "SOQL Query", + description: "SOQL Query to extract data from Salesforce", + "x-placeholder": "SELECT Id, CreatedDate, Name FROM Opportunity", + "x-hint": + "Write a SOQL query to retrieve data from your Salesforce object.", + "x-step": "source", + }, + sobject: { + type: "string", + title: "SObject", + description: "SObject to query in Salesforce", + "x-placeholder": "Opportunity", + "x-hint": + "Enter the name of the Salesforce object you want to query (e.g., Opportunity, Lead, Account).", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_salesforce_source", + "x-step": "source", + }, + }, + required: ["soql", "sobject", "name"], +}; diff --git a/web-common/src/features/templates/schemas/snowflake.ts b/web-common/src/features/templates/schemas/snowflake.ts new file mode 100644 index 00000000000..f064c97a58e --- /dev/null +++ b/web-common/src/features/templates/schemas/snowflake.ts @@ -0,0 +1,205 @@ +import type { MultiStepFormSchema } from "./types"; + +export const snowflakeSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + connection_method: { + type: "string", + title: "Connection method", + enum: ["parameters", "connection_string"], + default: "parameters", + description: "Choose how to connect to Snowflake", + "x-display": "tabs", + "x-enum-labels": ["Enter parameters", "Enter connection string"], + "x-grouped-fields": { + parameters: ["auth_method"], + connection_string: ["dsn"], + }, + "x-step": "connector", + }, + auth_method: { + type: "string", + title: "Authentication method", + enum: ["password", "keypair"], + default: "password", + description: "Choose how to authenticate to Snowflake", + "x-display": "radio", + "x-enum-labels": ["Username & Password", "Key Pair"], + "x-enum-descriptions": [ + "Authenticate with username and password.", + "Authenticate with RSA key pair (more secure).", + ], + "x-grouped-fields": { + password: [ + "account", + "user", + "password", + "warehouse", + "database", + "schema", + "role", + ], + keypair: [ + "account", + "user", + "private_key", + "private_key_passphrase", + "warehouse", + "database", + "schema", + "role", + ], + }, + "x-step": "connector", + "x-visible-if": { connection_method: "parameters" }, + }, + account: { + type: "string", + title: "Account", + description: "Snowflake account identifier (e.g., abc12345.us-east-1)", + "x-placeholder": "abc12345.us-east-1", + "x-step": "connector", + "x-visible-if": { connection_method: "parameters" }, + }, + user: { + type: "string", + title: "Username", + description: "Snowflake username", + "x-placeholder": "Enter username", + "x-step": "connector", + "x-visible-if": { connection_method: "parameters" }, + }, + password: { + type: "string", + title: "Password", + description: "Snowflake password", + "x-placeholder": "Enter password", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { + connection_method: "parameters", + auth_method: "password", + }, + }, + private_key: { + type: "string", + title: "Private Key", + description: "RSA private key in PEM format", + "x-placeholder": + "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { + connection_method: "parameters", + auth_method: "keypair", + }, + }, + warehouse: { + type: "string", + title: "Warehouse", + description: "Snowflake warehouse name", + "x-placeholder": "COMPUTE_WH", + "x-step": "connector", + "x-visible-if": { connection_method: "parameters" }, + }, + database: { + type: "string", + title: "Database", + description: "Snowflake database name", + "x-placeholder": "MY_DATABASE", + "x-step": "connector", + "x-visible-if": { connection_method: "parameters" }, + }, + schema: { + type: "string", + title: "Schema", + description: "Snowflake schema name", + "x-placeholder": "PUBLIC", + "x-step": "connector", + "x-visible-if": { connection_method: "parameters" }, + }, + role: { + type: "string", + title: "Role", + description: "Optional Snowflake role to assume", + "x-placeholder": "ANALYST", + "x-step": "connector", + "x-visible-if": { connection_method: "parameters" }, + }, + authenticator: { + type: "string", + title: "Authenticator", + description: "Optional authenticator method (e.g., 'externalbrowser')", + "x-placeholder": "externalbrowser", + "x-step": "connector", + "x-advanced": true, + "x-visible-if": { connection_method: "parameters" }, + }, + dsn: { + type: "string", + title: "Connection String", + description: "Snowflake connection string (DSN)", + "x-placeholder": "user:password@account/database/schema?warehouse=wh", + "x-secret": true, + "x-step": "connector", + "x-visible-if": { connection_method: "connection_string" }, + }, + parallel_fetch_limit: { + type: "number", + title: "Parallel Fetch Limit", + description: "Maximum number of parallel fetches for query results", + "x-placeholder": "10", + "x-step": "connector", + "x-advanced": true, + }, + log_queries: { + type: "string", + title: "Log Queries", + description: "Enable logging of all SQL queries (useful for debugging)", + "x-placeholder": "false", + "x-step": "connector", + "x-advanced": true, + }, + sql: { + type: "string", + title: "SQL Query", + description: "SQL query to extract data from Snowflake", + "x-placeholder": "SELECT * FROM my_table;", + "x-step": "source", + }, + name: { + type: "string", + title: "Model Name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_model", + "x-step": "source", + }, + }, + required: ["sql", "name"], + allOf: [ + { + if: { + properties: { + connection_method: { const: "parameters" }, + auth_method: { const: "password" }, + }, + }, + then: { required: ["account", "user", "password", "sql", "name"] }, + }, + { + if: { + properties: { + connection_method: { const: "parameters" }, + auth_method: { const: "keypair" }, + }, + }, + then: { required: ["account", "user", "private_key", "sql", "name"] }, + }, + { + if: { properties: { connection_method: { const: "connection_string" } } }, + then: { required: ["dsn", "sql", "name"] }, + }, + ], +}; diff --git a/web-common/src/features/templates/schemas/sqlite.ts b/web-common/src/features/templates/schemas/sqlite.ts new file mode 100644 index 00000000000..6d4b63b349a --- /dev/null +++ b/web-common/src/features/templates/schemas/sqlite.ts @@ -0,0 +1,31 @@ +import type { MultiStepFormSchema } from "./types"; + +export const sqliteSchema: MultiStepFormSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + db: { + type: "string", + title: "Database Path", + description: "Path to SQLite database file", + "x-placeholder": "/path/to/database.db", + "x-step": "source", + }, + table: { + type: "string", + title: "Table", + description: "SQLite table name", + "x-placeholder": "my_table", + "x-step": "source", + }, + name: { + type: "string", + title: "Source name", + description: "Name for the source model", + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + "x-placeholder": "my_sqlite_source", + "x-step": "source", + }, + }, + required: ["db", "table", "name"], +}; diff --git a/web-common/src/features/templates/schemas/types.ts b/web-common/src/features/templates/schemas/types.ts new file mode 100644 index 00000000000..6366cdad2fa --- /dev/null +++ b/web-common/src/features/templates/schemas/types.ts @@ -0,0 +1,68 @@ +export type JSONSchemaVisibleIfValue = + | string + | number + | boolean + | Array; + +export type JSONSchemaField = { + type?: "string" | "number" | "boolean" | "object"; + title?: string; + description?: string; + enum?: Array; + const?: string | number | boolean; + default?: string | number | boolean; + pattern?: string; + format?: string; + /** + * Custom error messages for validation failures. + * Use `pattern` key to provide a human-readable message when pattern validation fails. + */ + errorMessage?: { + pattern?: string; + format?: string; + }; + properties?: Record; + required?: string[]; + "x-display"?: "radio" | "select" | "textarea" | "file" | "tabs"; + "x-step"?: "connector" | "source" | "explorer"; + "x-secret"?: boolean; + "x-readonly"?: boolean; + "x-visible-if"?: Record; + "x-enum-labels"?: string[]; + "x-enum-descriptions"?: string[]; + "x-placeholder"?: string; + "x-hint"?: string; + "x-accept"?: string; + "x-advanced"?: boolean; + /** + * Explicit grouping for radio/select options: maps an option value to the + * child field keys that should render beneath that option. + */ + "x-grouped-fields"?: Record; +}; + +export type JSONSchemaCondition = { + properties?: Record; +}; + +export type JSONSchemaConstraint = { + required?: string[]; +}; + +export type JSONSchemaConditional = { + if?: JSONSchemaCondition; + then?: JSONSchemaConstraint; + else?: JSONSchemaConstraint; +}; + +export type JSONSchemaObject = { + $schema?: string; + type: "object"; + title?: string; + description?: string; + properties?: Record; + required?: string[]; + allOf?: JSONSchemaConditional[]; +}; + +export type MultiStepFormSchema = JSONSchemaObject; diff --git a/web-local/tests/connectors/test-connection.spec.ts b/web-local/tests/connectors/test-connection.spec.ts index e571c62d3bf..1ed49d8276c 100644 --- a/web-local/tests/connectors/test-connection.spec.ts +++ b/web-local/tests/connectors/test-connection.spec.ts @@ -4,6 +4,73 @@ import { test } from "../setup/base"; test.describe("Test Connection", () => { test.use({ project: "Blank" }); + test("Azure connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#azure").click(); + await page.waitForSelector('form[id*="azure"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Select Storage Account Key (default may be different) -> requires account + key. + await page.getByRole("radio", { name: "Storage Account Key" }).click(); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Storage account" }).fill("acct"); + await expect(button).toBeDisabled(); + + await page.getByRole("textbox", { name: "Access key" }).fill("key"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch to Connection String -> requires connection string, so disabled until filled. + await page.getByRole("radio", { name: "Connection String" }).click(); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Connection string" }) + .fill("DefaultEndpointsProtocol=https;"); + await expect(button).toBeEnabled(); + }); + + test("S3 connector - auth method specific required fields", async ({ + page, + }) => { + await page.getByRole("button", { name: "Add Asset" }).click(); + await page.getByRole("menuitem", { name: "Add Data" }).click(); + await page.locator("#s3").click(); + await page.waitForSelector('form[id*="s3"]'); + + const button = page.getByRole("dialog").getByRole("button", { + name: /(Test and Connect|Continue)/, + }); + + // Default method is Access keys -> requires access key id + secret. + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Access Key ID" }) + .fill("AKIA_TEST"); + await expect(button).toBeDisabled(); + await page + .getByRole("textbox", { name: "Secret Access Key" }) + .fill("SECRET"); + await expect(button).toBeEnabled(); + + // Switch to Public (no required fields) -> button should stay enabled. + await page.getByRole("radio", { name: "Public" }).click(); + await expect(button).toBeEnabled(); + + // Switch back to Access keys -> fields cleared, so disabled until refilled. + await page.getByRole("radio", { name: "Access keys" }).click(); + await expect(button).toBeDisabled(); + }); + test("GCS connector - HMAC", async ({ page }) => { // Skip test if environment variables are not set if (