fix(query): RawValueSource::parse_fallback fully parse input (#17340)

* test

* update

* fix test

* fix

* add test

Author: forsaken628

Cargo.lock

@@ -29,6 +29,7 @@ base64 = { workspace = true }
 bstr = { workspace = true }
 chrono-tz = { workspace = true }
 geozero = { workspace = true }
+goldenfile = { workspace = true }
 hex = { workspace = true }
 jiff = { workspace = true }
 jsonb = { workspace = true }

src/query/formats/Cargo.toml

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::io::Write;
+
 use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
 use databend_common_expression::types::DataType;
@@ -23,6 +25,7 @@ use databend_common_formats::FastFieldDecoderValues;
 use databend_common_formats::FastValuesDecodeFallback;
 use databend_common_formats::FastValuesDecoder;
 use databend_common_io::prelude::FormatSettings;
+use goldenfile::Mint;
 
 struct DummyFastValuesDecodeFallback {}
 
@@ -35,23 +38,22 @@ impl FastValuesDecodeFallback for DummyFastValuesDecodeFallback {
 
 #[tokio::test]
 async fn test_fast_values_decoder_multi() -> Result<()> {
+    let mut mint = Mint::new("tests/it/testdata");
+    let file = &mut mint.new_goldenfile("fast_values.txt").unwrap();
+
     struct Test {
         data: &'static str,
         column_types: Vec<DataType>,
-        output: Result<&'static str>,
     }
 
-    let tests = vec![
+    let cases = vec![
         Test {
             data: "(0, 1, 2), (3,4,5)",
             column_types: vec![
                 DataType::Number(NumberDataType::Int16),
                 DataType::Number(NumberDataType::Int16),
                 DataType::Number(NumberDataType::Int16),
             ],
-            output: Ok(
-                "+----------+----------+----------+\n| Column 0 | Column 1 | Column 2 |\n+----------+----------+----------+\n| 0        | 1        | 2        |\n| 3        | 4        | 5        |\n+----------+----------+----------+",
-            ),
         },
         Test {
             data: "(0, 1, 2), (3,4,5), ",
@@ -60,9 +62,6 @@ async fn test_fast_values_decoder_multi() -> Result<()> {
                 DataType::Number(NumberDataType::Int16),
                 DataType::Number(NumberDataType::Int16),
             ],
-            output: Err(ErrorCode::BadDataValueType(
-                "Must start with parentheses".to_string(),
-            )),
         },
         Test {
             data: "('', '', '')",
@@ -71,7 +70,6 @@ async fn test_fast_values_decoder_multi() -> Result<()> {
                 DataType::Number(NumberDataType::Int16),
                 DataType::Number(NumberDataType::Int16),
             ],
-            output: Err(ErrorCode::Unimplemented("fallback".to_string())),
         },
         Test {
             data: "( 1, '', '2022-10-01')",
@@ -80,9 +78,6 @@ async fn test_fast_values_decoder_multi() -> Result<()> {
                 DataType::String,
                 DataType::Date,
             ],
-            output: Ok(
-                "+----------+----------+--------------+\n| Column 0 | Column 1 | Column 2     |\n+----------+----------+--------------+\n| 1        | ''       | '2022-10-01' |\n+----------+----------+--------------+",
-            ),
         },
         Test {
             data: "(1, 2, 3), (1, 1, 1), (1, 1, 1);",
@@ -91,9 +86,6 @@ async fn test_fast_values_decoder_multi() -> Result<()> {
                 DataType::Number(NumberDataType::Int16),
                 DataType::Number(NumberDataType::Int16),
             ],
-            output: Ok(
-                "+----------+----------+----------+\n| Column 0 | Column 1 | Column 2 |\n+----------+----------+----------+\n| 1        | 2        | 3        |\n| 1        | 1        | 1        |\n| 1        | 1        | 1        |\n+----------+----------+----------+",
-            ),
         },
         Test {
             data: "(1, 2, 3), (1, 1, 1), (1, 1, 1);  ",
@@ -102,9 +94,6 @@ async fn test_fast_values_decoder_multi() -> Result<()> {
                 DataType::Number(NumberDataType::Int16),
                 DataType::Number(NumberDataType::Int16),
             ],
-            output: Ok(
-                "+----------+----------+----------+\n| Column 0 | Column 1 | Column 2 |\n+----------+----------+----------+\n| 1        | 2        | 3        |\n| 1        | 1        | 1        |\n| 1        | 1        | 1        |\n+----------+----------+----------+",
-            ),
         },
         Test {
             data: "(1.2, -2.9, 3.55), (3.12e2, 3.45e+3, -1.9e-3);",
@@ -113,35 +102,38 @@ async fn test_fast_values_decoder_multi() -> Result<()> {
                 DataType::Number(NumberDataType::Int16),
                 DataType::Number(NumberDataType::Int16),
             ],
-            output: Ok(
-                "+----------+----------+----------+\n| Column 0 | Column 1 | Column 2 |\n+----------+----------+----------+\n| 1        | -3       | 4        |\n| 312      | 3450     | 0        |\n+----------+----------+----------+",
-            ),
         },
     ];
 
-    for tt in tests {
+    for case in cases {
+        writeln!(file, "---------- Input Data ----------")?;
+        writeln!(file, "{:?}", case.data)?;
+
+        writeln!(file, "---------- Input Column Types ----------")?;
+        writeln!(file, "{:?}", case.column_types)?;
+
         let field_decoder =
             FastFieldDecoderValues::create_for_insert(FormatSettings::default(), true);
-        let mut values_decoder = FastValuesDecoder::new(tt.data, &field_decoder);
+        let mut values_decoder = FastValuesDecoder::new(case.data, &field_decoder);
         let fallback = DummyFastValuesDecodeFallback {};
-        let mut columns = tt
+        let mut columns = case
             .column_types
             .into_iter()
             .map(|dt| ColumnBuilder::with_capacity(&dt, values_decoder.estimated_rows()))
             .collect::<Vec<_>>();
         let result = values_decoder.parse(&mut columns, &fallback).await;
-        match tt.output {
-            Err(err) => {
-                assert!(result.is_err());
-                assert_eq!(err.to_string(), result.unwrap_err().to_string())
-            }
-            Ok(want) => {
-                let columns = columns.into_iter().map(|cb| cb.build()).collect::<Vec<_>>();
-                let got = DataBlock::new_from_columns(columns);
-                assert!(result.is_ok(), "{:?}", result);
-                assert_eq!(got.to_string(), want.to_string())
-            }
+
+        writeln!(file, "---------- Output ---------")?;
+
+        if let Err(err) = result {
+            writeln!(file, "{}", err)?;
+        } else {
+            let columns = columns.into_iter().map(|cb| cb.build()).collect::<Vec<_>>();
+            let got = DataBlock::new_from_columns(columns);
+            writeln!(file, "{}", got)?;
         }
+        writeln!(file, "\n")?;
     }
+
     Ok(())
 }

src/query/formats/tests/it/field_decoder/fast_values.rs

@@ -0,0 +1,82 @@
+---------- Input Data ----------
+"(0, 1, 2), (3,4,5)"
+---------- Input Column Types ----------
+[Number(Int16), Number(Int16), Number(Int16)]
+---------- Output ---------
++----------+----------+----------+
+| Column 0 | Column 1 | Column 2 |
++----------+----------+----------+
+| 0        | 1        | 2        |
+| 3        | 4        | 5        |
++----------+----------+----------+
+
+
+---------- Input Data ----------
+"(0, 1, 2), (3,4,5), "
+---------- Input Column Types ----------
+[Number(Int16), Number(Int16), Number(Int16)]
+---------- Output ---------
+BadDataValueType. Code: 1010, Text = Must start with parentheses.
+
+
+---------- Input Data ----------
+"('', '', '')"
+---------- Input Column Types ----------
+[Number(Int16), Number(Int16), Number(Int16)]
+---------- Output ---------
+Unimplemented. Code: 1002, Text = fallback.
+
+
+---------- Input Data ----------
+"( 1, '', '2022-10-01')"
+---------- Input Column Types ----------
+[Number(Int16), String, Date]
+---------- Output ---------
++----------+----------+--------------+
+| Column 0 | Column 1 | Column 2     |
++----------+----------+--------------+
+| 1        | ''       | '2022-10-01' |
++----------+----------+--------------+
+
+
+---------- Input Data ----------
+"(1, 2, 3), (1, 1, 1), (1, 1, 1);"
+---------- Input Column Types ----------
+[Number(Int16), Number(Int16), Number(Int16)]
+---------- Output ---------
++----------+----------+----------+
+| Column 0 | Column 1 | Column 2 |
++----------+----------+----------+
+| 1        | 2        | 3        |
+| 1        | 1        | 1        |
+| 1        | 1        | 1        |
++----------+----------+----------+
+
+
+---------- Input Data ----------
+"(1, 2, 3), (1, 1, 1), (1, 1, 1);  "
+---------- Input Column Types ----------
+[Number(Int16), Number(Int16), Number(Int16)]
+---------- Output ---------
++----------+----------+----------+
+| Column 0 | Column 1 | Column 2 |
++----------+----------+----------+
+| 1        | 2        | 3        |
+| 1        | 1        | 1        |
+| 1        | 1        | 1        |
++----------+----------+----------+
+
+
+---------- Input Data ----------
+"(1.2, -2.9, 3.55), (3.12e2, 3.45e+3, -1.9e-3);"
+---------- Input Column Types ----------
+[Number(Int16), Number(Int16), Number(Int16)]
+---------- Output ---------
++----------+----------+----------+
+| Column 0 | Column 1 | Column 2 |
++----------+----------+----------+
+| 1        | -3       | 4        |
+| 312      | 3450     | 0        |
++----------+----------+----------+
+
+

src/query/formats/tests/it/testdata/fast_values.txt

@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use databend_common_ast::parser::parse_comma_separated_exprs;
+use databend_common_ast::parser::parse_values_with_placeholder;
 use databend_common_ast::parser::tokenize_sql;
 use databend_common_base::base::tokio::sync::Semaphore;
 use databend_common_catalog::table::Table;
@@ -488,12 +488,12 @@ impl AsyncSource for RawValueSource {
         }
 
         let format = self.ctx.get_format_settings()?;
-        let numeric_cast_option = self
+        let rounding_mode = self
             .ctx
             .get_settings()
             .get_numeric_cast_option()
-            .unwrap_or("rounding".to_string());
-        let rounding_mode = numeric_cast_option.as_str() == "rounding";
+            .map(|s| s == "rounding")
+            .unwrap_or(true);
         let field_decoder = FastFieldDecoderValues::create_for_insert(format, rounding_mode);
 
         let mut values_decoder = FastValuesDecoder::new(&self.data, &field_decoder);
@@ -528,7 +528,13 @@ impl FastValuesDecodeFallback for RawValueSource {
             let mut bind_context = self.bind_context.clone();
             let metadata = self.metadata.clone();
 
-            let exprs = parse_comma_separated_exprs(&tokens[1..tokens.len()], sql_dialect)?;
+            let exprs = parse_values_with_placeholder(&tokens, sql_dialect)?
+                .into_iter()
+                .map(|expr| match expr {
+                    Some(expr) => Ok(expr),
+                    None => Err(ErrorCode::SyntaxException("unexpected placeholder")),
+                })
+                .collect::<Result<Vec<_>>>()?;
 
             bind_context
                 .exprs_to_scalar(

src/query/service/src/pipelines/builders/builder_replace_into.rs

@@ -46,6 +46,9 @@ INSERT INTO t_str(a) values( 'a"b\"c\'d')
 statement ok
 INSERT INTO t_str(a) values( 'a"b\"c\\\'d')
 
+statement error (?s)1005
+INSERT INTO t_str values('aaa','bbb'''sssd...);
+
 onlyif mysql
 query T
 select * from t_str order by a
@@ -113,6 +116,13 @@ select max(column1), max(column2), max(column3), count() from my_table
 ----
 value9_1 value9_2 value9_3 32
 
+statement error (?s)1005(.*)unexpected `AS`
+INSERT INTO my_table (column1, column2, column3) VALUES (
+  Rand() AS string, -- c24
+  Rand() AS string, -- c25
+  Rand() AS string  -- c26
+);
+
 statement ok
 drop table my_table
 

tests/sqllogictests/suites/base/03_common/03_0016_insert_into_values.test

@@ -76,7 +76,7 @@ INSERT INTO t0(c2varchar, c1varchar) VALUES ('diu7Pk', '')
 statement error 1065
 INSERT INTO t0(c2varchar) VALUES (max('1'))
 
-statement error 1065
+statement error 1005
 INSERT INTO t0(c2varchar) VALUES (max('1') partition by('1'))
 
 statement error 1065

tests/sqllogictests/suites/base/03_common/03_0027_insert_default.test

@@ -522,7 +522,7 @@ select count(*) from t_decimal_4_2;
 ----
 0
 
-statement error 1303
+statement error 1005
 insert into t_decimal_4_2 values (.);
 
 statement ok
@@ -533,7 +533,7 @@ select count(*) from t_decimal_4_2;
 ----
 0
 
-statement error 1303
+statement error 1005
 insert into t_decimal_4_2 values (+);
 
 statement ok
@@ -544,7 +544,7 @@ select count(*) from t_decimal_4_2;
 ----
 0
 
-statement error 1303
+statement error 1005
 insert into t_decimal_4_2 values (-);
 
 statement ok
@@ -708,13 +708,13 @@ create table t_decimal_76_2(a Decimal(76,2) not null);
 statement error 1065
 insert into t_decimal_76_2 values (1.1.1);
 
-statement error 1303
+statement error 1005
 insert into t_decimal_76_2 values (.);
 
-statement error 1303
+statement error 1005
 insert into t_decimal_76_2 values (+);
 
-statement error 1303
+statement error 1005
 insert into t_decimal_76_2 values (-);
 
 statement error 1303