Add opaque Payload property to AuditMessage (#153)

* Add opaque Payload property to AuditMessage

* Add opaque Payload property to AuditMessage
* Replace deprecated ioutil
* Get rid of go.uber.org/multierr dependency
* Update go to 1.21

* Rollback the golden file check, apparently it creates the different file on Mac vs Linux, so the test on Mac fails while works on Linux

* Address code review feedback

Author: aleksmaus

aucoalesce/coalesce_test.go

@@ -21,7 +21,6 @@ import (
 	"bufio"
 	"encoding/json"
 	"flag"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -110,7 +109,7 @@ func testCoalesceEvent(t *testing.T, file string) {
 }
 
 func readEventsFromYAML(t testing.TB, name string) []testEvent {
-	file, err := ioutil.ReadFile(name)
+	file, err := os.ReadFile(name)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -184,7 +183,7 @@ func writeGoldenFile(name string, events []testEventOutput) error {
 func readGoldenFile(name string) ([]map[string]interface{}, error) {
 	name = strings.TrimSuffix(name, ".yaml")
 
-	data, err := ioutil.ReadFile(name + ".json.golden")
+	data, err := os.ReadFile(name + ".json.golden")
 	if err != nil {
 		return nil, err
 	}

aucoalesce/normalize_test.go

@@ -18,7 +18,7 @@
 package aucoalesce
 
 import (
-	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -31,7 +31,7 @@ func TestNormInit(t *testing.T) {
 }
 
 func TestLoadNormalizationConfig(t *testing.T) {
-	b, err := ioutil.ReadFile("normalizations.yaml")
+	b, err := os.ReadFile("normalizations.yaml")
 	if err != nil {
 		t.Fatal(err)
 	}

audit.go

@@ -30,8 +30,6 @@ import (
 	"time"
 	"unsafe"
 
-	"go.uber.org/multierr"
-
 	"github.com/elastic/go-libaudit/v2/auparse"
 )
 
@@ -441,7 +439,7 @@ func (c *AuditClient) Close() error {
 			err = c.set(status, NoWait)
 		}
 
-		err = multierr.Append(err, c.Netlink.Close())
+		err = errors.Join(err, c.Netlink.Close())
 	})
 
 	return err

auparse/auparse.go

@@ -56,6 +56,8 @@ type AuditMessage struct {
 	Sequence   uint32           // Sequence parsed from payload.
 	RawData    string           // Raw message as a string.
 
+	Payload interface{} // Opaque payload. This can be anything that is needed to be preserved along with the message and returned back after aggregation.
+
 	offset int               // offset is the index into RawData where the header ends and message begins.
 	data   map[string]string // The key value pairs parsed from the message.
 	tags   []string          // The keys associated with the event (e.g. the values set in rules with -F key=exec).

auparse/auparse_test.go

@@ -22,7 +22,6 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -436,7 +435,7 @@ func writeGoldenFile(sourceName string, events []*AuditMessage) error {
 }
 
 func readGoldenFile(name string) ([]*StoredAuditMessage, error) {
-	data, err := ioutil.ReadFile(name)
+	data, err := os.ReadFile(name)
 	if err != nil {
 		return nil, err
 	}
@@ -480,7 +479,7 @@ func BenchmarkParseLogLine(b *testing.B) {
 	require.NoError(b, err)
 	var msgs []string
 	for _, f := range files {
-		data, err := ioutil.ReadFile(f)
+		data, err := os.ReadFile(f)
 		require.NoError(b, err)
 		for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
 			if _, err = ParseLogLine(line); err == nil {

auparse/mk_audit_exit_codes.go

@@ -25,7 +25,6 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -144,7 +143,7 @@ func readErrorNumbers() ([]ErrorNumber, error) {
 }
 
 func run() error {
-	tmp, err := ioutil.TempDir("", "mk_audit_exit_codes")
+	tmp, err := os.MkdirTemp("", "mk_audit_exit_codes")
 	if err != nil {
 		return err
 	}
@@ -202,7 +201,7 @@ func run() error {
 		}
 	}
 
-	if err = ioutil.WriteFile(flagOut, buf.Bytes(), 0o644); err != nil {
+	if err = os.WriteFile(flagOut, buf.Bytes(), 0o644); err != nil {
 		return err
 	}
 

auparse/mk_audit_msg_types.go

@@ -26,7 +26,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
@@ -246,7 +245,7 @@ func readRecordTypes() (map[string]int, error) {
 }
 
 func run() error {
-	tmp, err := ioutil.TempDir("", "mk_audit_msg_types")
+	tmp, err := os.MkdirTemp("", "mk_audit_msg_types")
 	if err != nil {
 		return err
 	}

go.mod

@@ -1,12 +1,17 @@
 module github.com/elastic/go-libaudit/v2
 
-go 1.16
+go 1.21
 
 require (
 	github.com/elastic/go-licenser v0.4.1
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/stretchr/testify v1.7.0
-	go.uber.org/multierr v1.7.0
 	golang.org/x/sys v0.11.0
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

go.sum

@@ -8,14 +8,9 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
-go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=

reassembler_test.go

@@ -128,6 +128,9 @@ func testReassembler(t testing.TB, file string, expected *results) {
 			continue
 		}
 
+		// Attach some predictable Payload
+		msg.Payload = createTestPayload(msg)
+
 		reassmbler.PushMessage(msg)
 	}
 
@@ -144,11 +147,21 @@ func testReassembler(t testing.TB, file string, expected *results) {
 
 		for _, msg := range stream.events[i] {
 			assert.EqualValues(t, expectedEvent.seq, msg.Sequence, "sequence number")
+
+			// Verify that custom payload is preserved
+			assert.Equal(t, createTestPayload(msg), msg.Payload)
 		}
 		assert.Equal(t, expectedEvent.count, len(stream.events[i]), "message count")
 	}
 }
 
+func createTestPayload(msg *auparse.AuditMessage) map[string]interface{} {
+	return map[string]interface{}{
+		"seq": msg.Sequence,
+		"typ": msg.RecordType,
+	}
+}
+
 func TestSequenceNumSliceSort(t *testing.T) {
 	expected := sequenceNumSlice{maxSeq - 5, maxSeq - 4, maxSeq - 3, maxSeq - 2, maxSeq, 0, 1, 2, 3, 4}
 	seqs := sequenceNumSlice{maxSeq - 5, maxSeq - 4, 0, 1, 2, maxSeq - 3, maxSeq - 2, maxSeq, 3, 4}