adjust max number of lines for CSV file imports

Author: whummer

.github/workflows/build_test.yml

@@ -26,6 +26,7 @@ concurrency:
 
 env:
   USE_PASSWORD: "true"
+  SF_CSV_IMPORT_MAX_ROWS: "30000"
 
 jobs:
   lint:
@@ -168,7 +169,7 @@ jobs:
           # image_name=whummer/snowflake-tmp
           docker pull $image_name &
           pip install localstack
-          IMAGE_NAME=$image_name DEBUG=1 DOCKER_FLAGS='-e SF_LOG=trace' localstack start -d
+          IMAGE_NAME=$image_name DEBUG=1 DOCKER_FLAGS='-e SF_LOG=trace -e SF_CSV_IMPORT_MAX_ROWS=30000' localstack start -d
           localstack wait
 
 #      - name: Setup parameters file

test/integ/test_large_put.py

@@ -18,6 +18,10 @@ def test_put_copy_large_files(tmpdir, conn_cnx, db_parameters):
     # generates N files
     number_of_files = 2
     number_of_lines = 200000
+
+    # note: adjusted for compatibility with LocalStack emulator tests
+    number_of_lines_expected = int(os.getenv("SF_CSV_IMPORT_MAX_ROWS"))
+
     tmp_dir = generate_k_lines_of_n_files(
         number_of_lines, number_of_files, tmp_dir=str(tmpdir.mkdir("data"))
     )
@@ -91,7 +95,9 @@ def mocked_file_agent(*args, **kwargs):
                 cnt = 0
                 for rec in c:
                     cnt += rec[0]
-                assert cnt == number_of_files * number_of_lines, "Number of rows"
+                assert (
+                    cnt == number_of_files * number_of_lines_expected
+                ), "Number of rows"
             finally:
                 c.close()
     finally: