Bump JSONL line length limit to 1MB (from 64k) (#209)

Without explicitly setting a maximum size of the buffer that may be
allocated during scanning, Go's `bufio` defaults to 64k.

See
- https://pkg.go.dev/bufio#pkg-constants, and
- https://pkg.go.dev/bufio#Scanner.Buffer

Author: fritzgrabo

runner/file.go

@@ -437,10 +437,19 @@ func transformJSONConcatFile(in string, out io.Writer) error {
 	return transformJSONConcat(r, out)
 }
 
+func newLargeLineScanner(in io.Reader) *bufio.Scanner {
+	scanner := bufio.NewScanner(in)
+
+	// Bump line length limit to 1MB (from 64k)
+	buf := make([]byte, 4096)
+	scanner.Buffer(buf, 1024*1024)
+	return scanner
+}
+
 func transformJSONLines(in io.Reader, out io.Writer) error {
 	first := true
 	return withJSONOutWriter(out, "[", "]", func() error {
-		scanner := bufio.NewScanner(in)
+		scanner := newLargeLineScanner(in)
 		for scanner.Scan() {
 			if !first {
 				_, err := out.Write([]byte(",\n"))
@@ -477,7 +486,7 @@ var BUILTIN_REGEX = map[MimeType]*regexp.Regexp{
 }
 
 func transformRegexp(in io.Reader, out io.Writer, re *regexp.Regexp) error {
-	scanner := bufio.NewScanner(in)
+	scanner := newLargeLineScanner(in)
 	return withJSONArrayOutWriterFile(out, func(w *jsonutil.StreamEncoder) error {
 		row := map[string]any{}
 		for scanner.Scan() {

runner/file_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -14,36 +15,59 @@ import (
 )
 
 func Test_transformJSONLines(t *testing.T) {
-	tmp, err := ioutil.TempFile("", "")
-	defer os.Remove(tmp.Name())
-	assert.Nil(t, err)
+	longString := strings.Repeat("Omnis ut ut voluptatem provident eaque necessitatibus quia. Eos veniam qui. ", 1024) // 76kb
+	tests := []struct {
+		input  string
+		output []map[string]any
+	}{
+		{
+			`{"a": 1, "b": 2}
+{"a": 2, "b": 3}`,
+			[]map[string]any{
+				{
+					"a": float64(1),
+					"b": float64(2),
+				},
+				{
+					"a": float64(2),
+					"b": float64(3),
+				},
+			},
+		},
+		{
+			`{"a": 1, "b": "` + longString + `"}`,
+			[]map[string]any{
+				{
+					"a": float64(1),
+					"b": longString,
+				},
+			},
+		},
+	}
 
-	tmp.WriteString(`{"a": 1, "b": 2}
-{"a": 2, "b": 3}`)
+	for _, test := range tests {
+		tmp, err := ioutil.TempFile("", "")
+		assert.Nil(t, err)
 
-	tmp2, err := ioutil.TempFile("", "")
-	defer os.Remove(tmp2.Name())
-	assert.Nil(t, err)
+		tmp.WriteString(test.input)
 
-	err = transformJSONLinesFile(tmp.Name(), tmp2)
-	assert.Nil(t, err)
+		tmp2, err := ioutil.TempFile("", "")
+		assert.Nil(t, err)
 
-	var m []map[string]any
-	tmp2Bs, err := ioutil.ReadFile(tmp2.Name())
-	assert.Nil(t, err)
-	err = json.Unmarshal(tmp2Bs, &m)
-	assert.Nil(t, err)
+		err = transformJSONLinesFile(tmp.Name(), tmp2)
+		assert.Nil(t, err)
 
-	assert.Equal(t, []map[string]any{
-		{
-			"a": float64(1),
-			"b": float64(2),
-		},
-		{
-			"a": float64(2),
-			"b": float64(3),
-		},
-	}, m)
+		var m []map[string]any
+		tmp2Bs, err := ioutil.ReadFile(tmp2.Name())
+		assert.Nil(t, err)
+		err = json.Unmarshal(tmp2Bs, &m)
+		assert.Nil(t, err)
+
+		assert.Equal(t, test.output, m)
+
+		os.Remove(tmp.Name())
+		os.Remove(tmp2.Name())
+	}
 }
 
 func Test_parquet(t *testing.T) {