fix

Author: zhyass

src/query/expression/src/utils/block_thresholds.rs

@@ -78,6 +78,29 @@ impl BlockThresholds {
             return self.max_rows_per_block;
         }
 
-        total_rows.div_ceil(block_num_by_size)
+        let mut rows_per_block = total_rows.div_ceil(block_num_by_size);
+        let max_bytes_per_block = match rows_per_block {
+            v if v < self.max_rows_per_block / 10 => {
+                // If block rows < 100_000, max_bytes_per_block set to 200M
+                2 * self.max_bytes_per_block
+            }
+            v if v < self.max_rows_per_block / 2 => {
+                // If block rows < 500_000, max_bytes_per_block set to 150M
+                3 * self.max_bytes_per_block / 2
+            }
+            v if v < self.min_rows_per_block => {
+                // If block rows < 800_000, max_bytes_per_block set to 125M
+                5 * self.max_bytes_per_block / 4
+            }
+            _ => self.max_bytes_per_block,
+        };
+
+        if max_bytes_per_block > self.max_bytes_per_block {
+            rows_per_block = std::cmp::max(
+                total_rows / (std::cmp::max(total_bytes / max_bytes_per_block, 1)),
+                1,
+            );
+        }
+        rows_per_block
     }
 }

src/query/service/src/interpreters/interpreter_table_recluster.rs

@@ -249,11 +249,8 @@ impl ReclusterTableInterpreter {
                 let block_thresholds = tbl.get_block_thresholds();
                 let total_bytes = recluster_info.removed_statistics.uncompressed_byte_size as usize;
                 let total_rows = recluster_info.removed_statistics.row_count as usize;
-                let block_size = settings.get_max_block_size()?;
-                let rows_per_block = block_thresholds
-                    .calc_rows_per_block(total_bytes, total_rows)
-                    .max(block_size as usize);
-                let total_partitions = std::cmp::max(total_rows / rows_per_block, 1000);
+                let rows_per_block = block_thresholds.calc_rows_per_block(total_bytes, total_rows);
+                let total_partitions = total_rows / rows_per_block;
 
                 let ast_exprs = tbl.resolve_cluster_keys(self.ctx.clone()).unwrap();
                 let cluster_keys_len = ast_exprs.len();