added dynamic rule engine

Author: dustin-decker

Gopkg.lock

@@ -1,7 +1,9 @@
 ---
-- description: fetching external resources
-  event_type: PROCESS_EVENT_TYPE_EXEC
-  actions:
+- 
+  name: insider_threat
+  description: "IT'S COMING FROM INSIDE THE HOUSE!"
+  query: ClientAddr='127.0.0.1'
+  actions: 
     - alert
+  indicator_type: custom
   score: 60
-  query: 'exec_filename ~= (wget || curl)'

config/dynamic_rules.yaml

@@ -79,7 +79,7 @@ func Start() {
 		}
 
 		// handle connection in goroutine so we can accept new TCP connections
-		go server.handleConn(conn, eventChan, incomingConn.RemoteAddr().String())
+		go server.handleConn(conn, eventChan, incomingConn.RemoteAddr().(*net.TCPAddr).IP.String())
 	}
 }
 

server/daemon/server.go

@@ -5,39 +5,74 @@ import (
 	"log"
 	"os"
 
+	"github.com/caibirdme/yql"
 	"github.com/dustin-decker/threatseer/server/event"
+	"github.com/fatih/structs"
 	yaml "gopkg.in/yaml.v2"
 )
 
-type dynamicRule struct {
-	eventType   string   `yaml:"event_type"`
-	description string   `yaml:"description"`
-	actions     []string `yaml:"actions"`
-	query       string   `yaml:"query"`
-	score       int      `yaml:"score"`
+type DynamicRules []struct {
+	Name          string   `yaml:"name"`
+	Description   string   `yaml:"description"`
+	EventType     string   `yaml:"event_type"`
+	Query         string   `yaml:"query"`
+	Actions       []string `yaml:"actions"`
+	IndicatorType string   `yaml:"indicator_type"`
+	Score         int      `yaml:"score"`
 }
 
-type DynamicRulesEngine struct {
+// RulesEngine stores engine state
+type RulesEngine struct {
 	Out          chan event.Event
-	dynamicRules []dynamicRule
+	DynamicRules DynamicRules
 }
 
 // Run initiates the engine on the pipeline
-func (engine *DynamicRulesEngine) Run(in chan event.Event) {
+func (engine *RulesEngine) Run(in chan event.Event) {
 	for {
+		// incoming event from the pipeline
 		e := <-in
-		// log.Print(e)
+
+		// convert struct to map[string]interface{}
+		evnt := structs.Map(e)
+
+		for _, rule := range engine.DynamicRules {
+			if len(rule.Query) > 0 {
+				result, err := yql.Match(rule.Query, evnt)
+				if err != nil {
+					if err.Error() == "interface conversion: interface is nil, not antlr.ParserRuleContext" {
+						log.Print("incorrect syntax for dynamic engine rule, got: ", rule.Query)
+					} else {
+						log.Print("dynamic engine got error while testing rule: ", err)
+					}
+				}
+				if result {
+					e.Indicators = append(
+						e.Indicators,
+						event.Indicator{
+							Engine:        "dynamic",
+							IndicatorType: rule.IndicatorType,
+							Description:   rule.Description,
+							Score:         rule.Score,
+							RuleName:      rule.Name,
+						},
+					)
+				}
+			}
+		}
+
+		// make event available to the next pipeline engine
 		engine.Out <- e
 	}
 }
 
 // NewDynamicRulesEngine returns engine with configs loaded
-func NewDynamicRulesEngine() DynamicRulesEngine {
-	var e DynamicRulesEngine
+func NewDynamicRulesEngine() RulesEngine {
+	var e RulesEngine
 
 	// load risky_process.yaml information
 	filename := "config/dynamic_rules.yaml"
-	var dr []dynamicRule
+	var dr DynamicRules
 	if _, err := os.Stat(filename); os.IsNotExist(err) {
 		log.Printf("%s does not exist, not loading any data for that check", filename)
 	} else {
@@ -50,7 +85,7 @@ func NewDynamicRulesEngine() DynamicRulesEngine {
 			log.Fatalf("could not parse %s, got %s", filename, err.Error())
 		}
 	}
-	e.dynamicRules = dr
+	e.DynamicRules = dr
 	e.Out = make(chan event.Event, 0)
 
 	return e

server/engines/dynamic/dynamic.go

@@ -31,7 +31,5 @@ func BenchmarkYQL(b *testing.B) {
 		json.Unmarshal([]byte(jsonString), &temp)
 
 		yql.Match(rawYQL, temp)
-		// result, _ := yql.Match(rawYQL, temp)
-		// fmt.Println(result)
 	}
 }

server/engines/dynamic/query_stream_test.go

@@ -17,14 +17,14 @@ type riskyProcess struct {
 
 func (rp *riskyProcess) Indicator() event.Indicator {
 	return event.Indicator{
-		Engine:      "static",
-		Type:        "risky_process",
-		Description: fmt.Sprintf("%s is a risky process often used for %s", rp.Name, rp.Reason),
-		Score:       rp.Score,
+		Engine:        "static",
+		IndicatorType: "risky_process",
+		Description:   fmt.Sprintf("%s is a risky process often used for %s", rp.Name, rp.Reason),
+		Score:         rp.Score,
 	}
 }
 
-func (engine *StaticRulesEngine) checkRiskyProcess(processInfo *api.ProcessEvent) *riskyProcess {
+func (engine *RulesEngine) checkRiskyProcess(processInfo *api.ProcessEvent) *riskyProcess {
 	if processInfo != nil {
 		cl := processInfo.GetExecFilename()
 		for _, rp := range engine.riskyProcesses {

server/engines/static/risky_processes.go

@@ -10,18 +10,20 @@ import (
 	"github.com/dustin-decker/threatseer/server/event"
 )
 
-type StaticRulesEngine struct {
+// RulesEngine stores engine state
+type RulesEngine struct {
 	Out            chan event.Event
 	riskyProcesses []riskyProcess
 }
 
 // Run initiates the engine on the pipeline
-func (engine *StaticRulesEngine) Run(in chan event.Event) {
+func (engine *RulesEngine) Run(in chan event.Event) {
 	for {
+		// incoming event from the pipeline
 		e := <-in
 
+		// process checks
 		processInfo := e.Event.GetProcess()
-
 		if processInfo != nil {
 			// check for risky processes
 			rp := engine.checkRiskyProcess(processInfo)
@@ -30,14 +32,14 @@ func (engine *StaticRulesEngine) Run(in chan event.Event) {
 			}
 		}
 
-		// log.Print(e.Event)
+		// make event available to the next pipeline engine
 		engine.Out <- e
 	}
 }
 
 // NewStaticRulesEngine returns engine with configs loaded
-func NewStaticRulesEngine() StaticRulesEngine {
-	var e StaticRulesEngine
+func NewStaticRulesEngine() RulesEngine {
+	var e RulesEngine
 
 	// load risky_process.yaml information
 	filename := "config/risky_processes.yaml"

server/engines/static/static.go

@@ -1,8 +1,6 @@
 package event
 
 import (
-	"net"
-
 	api "github.com/capsule8/capsule8/api/v0"
 )
 
@@ -12,14 +10,15 @@ import (
 type Event struct {
 	Event      *api.TelemetryEvent
 	Indicators []Indicator
-	ClientAddr net.Addr
+	ClientAddr string
 }
 
 // Indicator is an individual result from an engine
 type Indicator struct {
-	Engine      string
-	Type        string
-	Description string
-	ExtraInfo   string
-	Score       int
+	Engine        string
+	RuleName      string
+	IndicatorType string
+	Description   string
+	ExtraInfo     string
+	Score         int
 }