Unify bounds checks across all sheet-like format (#233)

* More bounds checks

* Add comments

* Fix comment

* Fix comment

* Fixes for rendering missing data

* Fix for fmt

* Clarify more

* Fix for fmt

* Embed version in runner binary

* Fix override variables

* Fix for gofmt

* Try separate installs

* More specific versions

* Bump everything to 1.18.1

* All the retries

* Drop effects

* Fixes for tsc

* Fixes for dark mode

* Drop useless r curl

* Revert select and radio defaults

* Rebuild windows

Author: eatonphil

desktop/panel/file.test.js

@@ -199,8 +199,8 @@ for (const subprocessName of RUNNERS) {
       file: path.join('testdata', 'csv', 'missing_columns.csv'),
       expected: [
         { a: '3', b: '4' },
-        { a: '1', b: '2' },
-        { a: '5', b: '6' },
+        { a: '1', b: '2', C: '3' },
+        { a: '5', b: '6', C: null },
       ],
       name: 'missing columns in csv',
     },

desktop/scripts/desktop.build

@@ -2,7 +2,8 @@ setenv UI_CONFIG_OVERRIDES "window.DS_CONFIG_MODE = 'desktop';"
 yarn build-ui
 
 # Flags from various package management guidelines: https://wiki.archlinux.org/title/Go_package_guidelines
-cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w" -o ../build/go_desktop_runner{required_ext} cmd/main.go
+setenv_default VERSION "development"
+cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w -X main.VERSION={VERSION}" -o ../build/go_desktop_runner{required_ext} cmd/main.go
 
 yarn esbuild desktop/preload.ts --external:electron --sourcemap --bundle --outfile=build/preload.js
 yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop_runner.js

desktop/scripts/release.build

@@ -1,6 +1,7 @@
 rm -rf build
 yarn
 setenv UI_ESBUILD_ARGS "--minify"
+setenv VERSION {arg0}
 yarn build-desktop
 prepend "window.DS_CONFIG_MODE='desktop';" build/ui.js
 prepend "window.DS_CONFIG_VERSION='{arg0}';" build/ui.js

runner/cmd/main.go

@@ -6,8 +6,11 @@ import (
 	"github.com/multiprocessio/datastation/runner"
 )
 
-const VERSION = "development"
-const APP_NAME = "DataStation Runner (Go)"
+// Overridden during build
+var (
+	VERSION  = "development"
+	APP_NAME = "DataStation Runner (Go)"
+)
 
 func main() {
 	runner.Verbose = true

runner/file.go

@@ -25,6 +25,33 @@ import (
 
 var preferredParallelism = runtime.NumCPU() * 2
 
+func indexToExcelColumn(i int) string {
+	i -= 1
+
+	if i/26 > 0 {
+		return indexToExcelColumn(i/26) + string(rune(i%26+65))
+	}
+
+	return string(rune(i%26 + 65))
+}
+
+func recordToMap[T any](row map[string]any, fields *[]string, record []T) {
+	i := -1 // This is only set to 0 if len(record) > 0
+	var el T
+	for i, el = range record {
+		if i >= len(*fields) {
+			*fields = append(*fields, indexToExcelColumn(i+1))
+		}
+
+		(row)[(*fields)[i]] = el
+	}
+
+	// If the record has less fields than we've seen already, set all unseen fields to nil
+	for _, field := range (*fields)[i+1:] {
+		(row)[field] = nil
+	}
+}
+
 func transformCSV(in io.Reader, out io.Writer, delimiter rune) error {
 	r := csv.NewReader(in)
 	r.Comma = delimiter
@@ -55,13 +82,7 @@ func transformCSV(in io.Reader, out io.Writer, delimiter rune) error {
 				continue
 			}
 
-			for i, field := range fields {
-				if i < len(record) {
-					row[field] = record[i]
-				} else {
-					row[field] = nil
-				}
-			}
+			recordToMap(row, &fields, record)
 
 			err = w.EncodeRow(row)
 			if err != nil {
@@ -159,9 +180,8 @@ func transformORC(in *orc.Reader, out io.Writer) error {
 		for c.Stripes() {
 			for c.Next() {
 				r := c.Row()
-				for i, col := range cols {
-					row[col] = r[i]
-				}
+
+				recordToMap(row, &cols, r)
 
 				err := w.EncodeRow(row)
 				if err != nil {
@@ -197,13 +217,7 @@ func writeSheet(rows [][]string, w *jsonutil.StreamEncoder) error {
 			continue
 		}
 
-		for i, name := range header {
-			var cell any = nil
-			if i < len(r) {
-				cell = r[i]
-			}
-			row[name] = cell
-		}
+		recordToMap(row, &header, r)
 
 		err := w.EncodeRow(row)
 		if err != nil {

runner/file_test.go

@@ -17,6 +17,68 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func Test_indexToExcelColumn(t *testing.T) {
+	tests := []struct {
+		input  int
+		output string
+	}{
+		{26, "Z"},
+		{51, "AY"},
+		{52, "AZ"},
+		{80, "CB"},
+		{676, "YZ"},
+		{702, "ZZ"},
+		{705, "AAC"},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, indexToExcelColumn(test.input), test.output)
+	}
+}
+
+func Test_recordToMap(t *testing.T) {
+	tests := []struct {
+		fields []string
+		record []string
+		expect map[string]any
+	}{
+		{
+			[]string{"a", "b"},
+			[]string{"1"},
+			map[string]any{"a": "1", "b": nil},
+		},
+		{
+			[]string{"a", "b"},
+			[]string{"1", "2", "3"},
+			map[string]any{"a": "1", "b": "2", "C": "3"},
+		},
+		{
+			[]string{"a", "b"},
+			[]string{},
+			map[string]any{"a": nil, "b": nil},
+		},
+		{
+			[]string{},
+			[]string{
+				"1", "2", "3", "4", "5", "6", "7", "8", "9", "10",
+				"11", "12", "13", "14", "15", "16", "17", "18", "19", "20",
+				"21", "22", "23", "24", "25", "26", "27", "28", "29", "30",
+			},
+			map[string]any{
+				"A": "1", "B": "2", "C": "3", "D": "4", "E": "5", "F": "6", "G": "7", "H": "8", "I": "9", "J": "10",
+				"K": "11", "L": "12", "M": "13", "N": "14", "O": "15", "P": "16", "Q": "17", "R": "18", "S": "19", "T": "20",
+				"U": "21", "V": "22", "W": "23", "X": "24", "Y": "25", "Z": "26", "AA": "27", "AB": "28", "AC": "29", "AD": "30",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		m := map[string]any{}
+		recordToMap(m, &test.fields, test.record)
+		assert.Equal(t, test.expect, m)
+	}
+}
+
 func Test_transformJSONLines(t *testing.T) {
 	longString := strings.Repeat("Omnis ut ut voluptatem provident eaque necessitatibus quia. Eos veniam qui. ", 1024) // 76kb
 	tests := []struct {

scripts/build.py

@@ -123,7 +123,7 @@ def eval_line(line):
         quote_line(line, show=True)
         return
     if line[0] == 'setenv':
-        os.environ[line[1]] = line[2]
+        os.environ[line[1]] = line[2].format(**BUILTIN_VARIABLES, **os.environ)
         quote_line(line, show=True)
         return
     if line[0] == 'setenv_default':

scripts/ci/prepare_go.sh

@@ -3,8 +3,8 @@
 set -ex
 
 # Set up Go
-sudo curl -LO https://go.dev/dl/go1.18.linux-amd64.tar.gz
+sudo curl -LO https://go.dev/dl/go1.18.1.linux-amd64.tar.gz
 sudo rm -rf /usr/local/go
-sudo tar -C /usr/local -xzf go1.18.linux-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.18.1.linux-amd64.tar.gz
 sudo ln -s /usr/local/go/bin/go /usr/local/bin/go
 sudo ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt

scripts/ci/prepare_linux_integration_test_setup_only.sh

@@ -146,16 +146,16 @@ function retry {
 docker ps
 
 # Load influx1 data
-retry 3 'curl -XPOST "http://localhost:8087/query?u=test&p=testtest" --data-urlencode "q=CREATE DATABASE test"'
-retry 3 "curl -XPOST 'http://localhost:8087/write?db=test&u=test&p=testtest' --data-binary @testdata/influx/noaa-ndbc-data-sample.lp"
+retry 10 'curl -XPOST "http://localhost:8087/query?u=test&p=testtest" --data-urlencode "q=CREATE DATABASE test"'
+retry 10 "curl -XPOST 'http://localhost:8087/write?db=test&u=test&p=testtest' --data-binary @testdata/influx/noaa-ndbc-data-sample.lp"
 
 # Load influx2 data
-retry 3 "curl -XPOST 'http://localhost:8086/api/v2/write?org=test&bucket=test&precision=ns' --header 'Authorization: Token test' --data-binary @testdata/influx/noaa-ndbc-data-sample.lp"
+retry 10 "curl -XPOST 'http://localhost:8086/api/v2/write?org=test&bucket=test&precision=ns' --header 'Authorization: Token test' --data-binary @testdata/influx/noaa-ndbc-data-sample.lp"
 
 # Load Elasticsearch data
-retry 3 "curl -X PUT http://localhost:9200/test"
+retry 10 "curl -X PUT http://localhost:9200/test"
 for t in $(ls testdata/documents/*.json); do
-    retry 3 "curl -X POST -H 'Content-Type: application/json' -d @$t http://localhost:9200/test/_doc"
+    retry 10 "curl -X POST -H 'Content-Type: application/json' -d @$t http://localhost:9200/test/_doc"
 done
 
 # Configure scylla

scripts/ci/prepare_macos.sh

@@ -7,9 +7,9 @@ brew install cmake jq r julia node@16 npm go-jsonnet
 brew link --overwrite node@16
 
 # Install go
-sudo curl -LO https://go.dev/dl/go1.18.darwin-amd64.tar.gz
+sudo curl -LO https://go.dev/dl/go1.18.1.darwin-amd64.tar.gz
 sudo rm -rf /usr/local/go
-sudo tar -C /usr/local -xzf go1.18.darwin-amd64.tar.gz
+sudo tar -C /usr/local -xzf go1.18.1.darwin-amd64.tar.gz
 sudo mv /usr/local/go/bin/go /usr/local/bin/go
 sudo mv /usr/local/go/bin/gofmt /usr/local/bin/gofmt