Bump fivetran sdk to v2 (#34452)

Updated relevant proto items.
Updated logging to use the warning/task system.
Was more thorough with source connector. Destination connector is not yet shipped - so ok if that has shortcomings - we can resolve later.

GitOrigin-RevId: 0a35089cc54dd42c430a11b551eab45d35605b5d

Author: nipunn1313

crates/fivetran_common/build.rs

@@ -11,7 +11,7 @@ use fxhash::FxHasher32;
 
 // Make sure to select a rev off the `production` branch of the sdk
 // https://github.com/fivetran/fivetran_sdk/tree/production
-const REV: &str = "1fabb7626b6ec81a4f56d49a16a654210cb1d0be";
+const REV: &str = "466a61bddfc0e541bfec3cb0cc6a3cf3704d64be";
 
 const FILES: &[&str] = &[
     "common.proto",
@@ -21,7 +21,7 @@ const FILES: &[&str] = &[
 
 // File hash to protect against accidental changes to the vendored files:
 // Update this when updating `REV` above.
-const FILE_HASH: u64 = 3447879181;
+const FILE_HASH: u64 = 1411440539;
 
 cfg_if::cfg_if! {
     if #[cfg(target_os = "macos")] {

crates/fivetran_common/fivetran_sdk/1fabb7626b6ec81a4f56d49a16a654210cb1d0be/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Fivetran Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/fivetran_common/fivetran_sdk/466a61bddfc0e541bfec3cb0cc6a3cf3704d64be/LICENSE

@@ -1,8 +1,8 @@
 syntax = "proto3";
 option optimize_for = SPEED;
 option java_multiple_files = true;
-option go_package = "fivetran.com/fivetran_sdk";
-package fivetran_sdk;
+option go_package = "fivetran.com/fivetran_sdk_v2";
+package fivetran_sdk.v2;
 
 import "google/protobuf/timestamp.proto";
 
@@ -18,12 +18,29 @@ message ConfigurationFormResponse {
 message FormField {
   string name = 1;
   string label = 2;
-  bool required = 3;
+  optional bool required = 3;
   optional string description = 4;
   oneof type {
     TextField text_field = 5;
     DropdownField dropdown_field = 6;
     ToggleField toggle_field = 7;
+    ConditionalFields conditional_fields = 10;
+  }
+  optional string default_value = 8;
+  optional string placeholder = 9;
+}
+
+message ConditionalFields {
+  VisibilityCondition condition = 1;
+  repeated FormField fields = 2;
+}
+
+message VisibilityCondition {
+  string condition_field = 1;
+  oneof visible_when {
+    bool bool_value = 2;
+    string string_value = 3;
+    bool empty_value = 4;
   }
 }
 
@@ -53,7 +70,6 @@ message TestResponse {
   oneof response {
     bool success = 1;
     string failure = 2;
-    // potential future warning
   }
 }
 
@@ -86,14 +102,22 @@ enum DataType {
   XML = 12;
   STRING = 13;
   JSON = 14;
+  NAIVE_TIME = 15;
+}
+
+message DataTypeParams {
+  oneof params {
+    DecimalParams decimal = 1;
+    int32 string_byte_length = 2;
+  }
 }
 
 message DecimalParams {
   uint32 precision = 1;
   uint32 scale = 2;
 }
 
-enum OpType {
+enum RecordType {
   UPSERT = 0;
   UPDATE = 1;
   DELETE = 2;
@@ -117,6 +141,7 @@ message ValueType {
     string string = 13;
     string json = 14;
     string xml = 15;
+    google.protobuf.Timestamp naive_time = 16;
   }
 }
 
@@ -129,5 +154,13 @@ message Column {
   string name = 1;
   DataType type = 2;
   bool primary_key = 3;
-  optional DecimalParams decimal = 4;
+  optional DataTypeParams params = 4;
+}
+
+message Warning {
+  string message = 1;
+}
+
+message Task {
+  string message = 1;
 }

crates/fivetran_common/fivetran_sdk/1fabb7626b6ec81a4f56d49a16a654210cb1d0be/common.proto renamed to crates/fivetran_common/fivetran_sdk/466a61bddfc0e541bfec3cb0cc6a3cf3704d64be/common.proto

@@ -1,13 +1,13 @@
 syntax = "proto3";
 option optimize_for = SPEED;
 option java_multiple_files = true;
-option go_package = "fivetran.com/fivetran_sdk";
-package fivetran_sdk;
+option go_package = "fivetran.com/fivetran_sdk_v2";
+package fivetran_sdk.v2;
 
 import "common.proto";
 
-// Fivetran (grpc client) <> Connector (grpc server)
-service Connector {
+// Fivetran (grpc client) <> SourceConnector (grpc server)
+service SourceConnector {
     rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {}
     rpc Test (TestRequest) returns (TestResponse) {}
     rpc Schema (SchemaRequest) returns (SchemaResponse) {}
@@ -65,28 +65,12 @@ message TableSelection {
 }
 
 message UpdateResponse {
-    oneof response {
-        LogEntry log_entry = 1;
-        Operation operation = 2;
-    }
-}
-
-enum LogLevel {
-    INFO = 0;
-    WARNING = 1;
-    SEVERE = 2;
-}
-
-message LogEntry {
-    LogLevel level = 1;
-    string message = 2;
-}
-
-message Operation {
-    oneof op {
+    oneof operation {
         Record record = 1;
         SchemaChange schema_change = 2;
         Checkpoint checkpoint = 3;
+        Warning warning = 4;
+        Task task = 5;
     }
 }
 
@@ -100,7 +84,7 @@ message SchemaChange {
 message Record {
     optional string schema_name = 1;
     string table_name = 2;
-    OpType type = 3;
+    RecordType type = 3;
     map<string, ValueType> data = 4;
 }
 

crates/fivetran_common/fivetran_sdk/1fabb7626b6ec81a4f56d49a16a654210cb1d0be/connector_sdk.proto renamed to crates/fivetran_common/fivetran_sdk/466a61bddfc0e541bfec3cb0cc6a3cf3704d64be/connector_sdk.proto

@@ -1,21 +1,29 @@
 syntax = "proto3";
 option optimize_for = SPEED;
 option java_multiple_files = true;
-option go_package = "fivetran.com/fivetran_sdk";
-package fivetran_sdk;
+option go_package = "fivetran.com/fivetran_sdk_v2";
+package fivetran_sdk.v2;
 
 import "google/protobuf/timestamp.proto";
 import "common.proto";
 
-// Fivetran (grpc client) <> Destination (grpc server)
-service Destination {
+// Fivetran (grpc client) <> DestinationConnector (grpc server)
+service DestinationConnector {
     rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {}
+    rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {}
     rpc Test (TestRequest) returns (TestResponse) {}
     rpc DescribeTable (DescribeTableRequest) returns (DescribeTableResponse) {}
     rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {}
     rpc AlterTable(AlterTableRequest) returns (AlterTableResponse) {}
     rpc Truncate(TruncateRequest) returns (TruncateResponse) {}
     rpc WriteBatch (WriteBatchRequest) returns (WriteBatchResponse) {}
+    rpc WriteHistoryBatch (WriteHistoryBatchRequest) returns (WriteBatchResponse) {}
+}
+
+message CapabilitiesRequest {}
+
+message CapabilitiesResponse {
+    BatchFileFormat batch_file_format = 1;
 }
 
 message DescribeTableRequest {
@@ -27,8 +35,9 @@ message DescribeTableRequest {
 message DescribeTableResponse {
     oneof response {
         bool not_found = 1;
-        string failure = 2;
-        Table table = 3;
+        Table table = 2;
+        Warning warning = 3;
+        Task task = 4;
     }
 }
 
@@ -41,7 +50,8 @@ message CreateTableRequest {
 message CreateTableResponse {
     oneof response {
         bool success = 1;
-        string failure = 2;
+        Warning warning = 2;
+        Task task = 3;
     }
 }
 
@@ -54,7 +64,8 @@ message AlterTableRequest {
 message AlterTableResponse {
     oneof response {
         bool success = 1;
-        string failure = 2;
+        Warning warning = 2;
+        Task task = 3;
     }
 }
 
@@ -68,13 +79,14 @@ message TruncateRequest {
 }
 
 message SoftTruncate {
-    string deleted_column = 3;
+    string deleted_column = 1;
 }
 
 message TruncateResponse {
     oneof response {
         bool success = 1;
-        string failure = 2;
+        Warning warning = 2;
+        Task task = 3;
     }
 }
 
@@ -86,12 +98,22 @@ message WriteBatchRequest {
     repeated string replace_files = 5;
     repeated string update_files = 6;
     repeated string delete_files = 7;
-    oneof file_params {
-        CsvFileParams csv = 8;
-    }
+    FileParams file_params = 8;
+}
+
+message WriteHistoryBatchRequest {
+    map<string, string> configuration = 1;
+    string schema_name = 2;
+    Table table = 3;
+    map<string, bytes> keys = 4;
+    repeated string earliest_start_files = 5;
+    repeated string replace_files = 6;
+    repeated string update_files = 7;
+    repeated string delete_files = 8;
+    FileParams file_params = 9;
 }
 
-message CsvFileParams {
+message FileParams {
     Compression compression = 1;
     Encryption encryption = 2;
     string null_string = 3;
@@ -103,6 +125,11 @@ enum Encryption {
     AES = 1;
 }
 
+enum BatchFileFormat {
+    CSV = 0;
+    PARQUET = 1;
+}
+
 enum Compression {
     OFF = 0;
     ZSTD = 1;
@@ -112,6 +139,7 @@ enum Compression {
 message WriteBatchResponse {
     oneof response {
         bool success = 1;
-        string failure = 2;
+        Warning warning = 2;
+        Task task = 3;
     }
 }

crates/fivetran_common/fivetran_sdk/1fabb7626b6ec81a4f56d49a16a654210cb1d0be/destination_sdk.proto renamed to crates/fivetran_common/fivetran_sdk/466a61bddfc0e541bfec3cb0cc6a3cf3704d64be/destination_sdk.proto

@@ -2,5 +2,8 @@ This folder is a mirror of https://github.com/fivetran/fivetran_sdk at a pinned
 revision. Vendoring this source lets us avoid depending on the network during
 our builds.
 
-When upgrading, be sure to select a rev off the `production` branch of the sdk
-https://github.com/fivetran/fivetran_sdk/tree/production
+When upgrading, be sure to select a rev off the `production` or `v2` tag of the
+sdk :
+
+- https://github.com/fivetran/fivetran_sdk/tree/production
+- https://github.com/fivetran/fivetran_sdk/tree/v2

crates/fivetran_common/fivetran_sdk/README.md

@@ -33,24 +33,28 @@ impl Config {
             FormField {
                 name: CONFIG_KEY_DEPLOYMENT_URL.to_string(),
                 label: "Deployment URL".to_string(),
-                required: true,
+                required: Some(true),
                 description: Some(
-                    "The domain where the deployment is hosted (\"https://….convex.cloud\"). You \
-                     can find it in the deployment settings page of the Convex dashboard."
+                    "The domain where the deployment is hosted. You can find it in the deployment \
+                     settings page of the Convex dashboard."
                         .to_string(),
                 ),
                 r#type: Some(Type::TextField(TextField::PlainText as i32)),
+                default_value: None,
+                placeholder: Some("https://aware-llama-900.convex.cloud".to_string()),
             },
             FormField {
                 name: CONFIG_KEY_DEPLOYMENT_KEY.to_string(),
                 label: "Deploy Key".to_string(),
-                required: true,
+                required: Some(true),
                 description: Some(
                     "The key giving access to your deployment. You can find it in the deployment \
                      settings page of the Convex dashboard."
                         .to_string(),
                 ),
                 r#type: Some(Type::TextField(TextField::Password as i32)),
+                default_value: None,
+                placeholder: None,
             },
         ]
     }

crates/fivetran_common/src/config.rs

@@ -1,2 +1,2 @@
 #![allow(clippy::enum_variant_names)]
-tonic::include_proto!("fivetran_sdk");
+tonic::include_proto!("fivetran_sdk.v2");