Refactored the code a bit and added an inline_matches field to allow …

…Top N vizualisation of all matches

internal/events/base.go

@@ -1,6 +1,7 @@
 package events
 
 import (
+	"github.com/bonjourmalware/melody/internal/loggable"
 	"time"
 )
 
@@ -15,6 +16,7 @@ type BaseEvent struct {
 	Timestamp  time.Time
 	Additional map[string]string
 	Event
+	loggable.Loggable
 }
 
 // Tags is an abstraction of map[string]interface{} allowing for the use of a set-like structure and a more graceful
@@ -36,6 +38,16 @@ func (ev BaseEvent) GetDestPort() uint16 {
 	return ev.DestPort
 }
 
+// GetSession fetches the Session of an event
+func (ev BaseEvent) GetSession() string {
+	return ev.Session
+}
+
+// GetTags fetches the Tags of an event
+func (ev BaseEvent) GetTags() map[string][]string {
+	return ev.Tags
+}
+
 // AddAdditional fetches the Additional values of an event
 func (ev *BaseEvent) AddAdditional(add map[string]string) {
 	for key, values := range add {
@@ -69,23 +81,15 @@ func (ev *BaseEvent) AddTags(tags map[string]string) {
 	}
 }
 
-// ToArray converts an optimized Tags to an array
-//func (t *Tags) ToArray() []string {
-//	var ret []string
-//	for tag := range *t {
-//		ret = append(ret, tag)
-//	}
+////ToInlineArray converts a Tags map to an array of its values with the keys and values merged with a '.'
+//func (t *Tags) ToInlineArray() []string {
+//	var inlineTags []string
 //
-//	return ret
-//}
-
-// ToArray converts an optimized Tags to an array
-//func (t *Tags) ToJSON() map[string][]string {
-//	var ret []string
-//	for tag := range *t {
-//		ret = append(ret, tag)
+//	for key, values := range *t {
+//		for _, val := range values {
+//			inlineTags = append(inlineTags, fmt.Sprintf("%s.%s", key, val))
+//		}
 //	}
 //
-//	return ret
+//	return inlineTags
 //}
-//

internal/events/event.go

@@ -1,14 +1,14 @@
 package events
 
-import "github.com/google/gopacket/layers"
+import (
+	"github.com/bonjourmalware/melody/internal/loggable"
+	"github.com/google/gopacket/layers"
+)
 
 // Event is the interface implementing common methods to generated events
 type Event interface {
 	//Match(rule rules.Rule) bool
 	ToLog() EventLog
-	GetKind() string
-	GetSourceIP() string
-	GetDestPort() uint16
 	GetIPHeader() *layers.IPv4
 	GetICMPv6Header() *layers.ICMPv6
 	GetICMPv4Header() *layers.ICMPv4
@@ -18,6 +18,7 @@ type Event interface {
 
 	AddTags(tags map[string]string)
 	AddAdditional(add map[string]string)
+	loggable.Loggable
 }
 
 // EventLog is the interface implementing common methods to generated events' log data

internal/events/http.go

@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/bonjourmalware/melody/internal/events/logdata"
+	"github.com/bonjourmalware/melody/internal/logdata"
 
 	"github.com/rs/xid"
 
@@ -56,16 +56,18 @@ func (ev HTTPEvent) ToLog() EventLog {
 	ev.LogData = logdata.HTTPEventLog{}
 	ev.LogData.Timestamp = time.Now().Format(time.RFC3339Nano)
 	//ev.LogData.NsTimestamp = strconv.FormatInt(time.Now().UnixNano(), 10)
-	ev.LogData.Type = ev.Kind
-	ev.LogData.SourceIP = ev.SourceIP
-	ev.LogData.DestPort = ev.DestPort
-	ev.LogData.Session = ev.Session
-
-	if len(ev.Tags) == 0 {
-		ev.LogData.Tags = make(map[string][]string)
-	} else {
-		ev.LogData.Tags = ev.Tags
-	}
+	//ev.LogData.Type = ev.Kind
+	//ev.LogData.SourceIP = ev.SourceIP
+	//ev.LogData.DestPort = ev.DestPort
+	//ev.LogData.Session = ev.Session
+	//
+	//if len(ev.Tags) == 0 {
+	//	ev.LogData.Tags = make(map[string][]string)
+	//} else {
+	//	ev.LogData.Tags = ev.Tags
+	//}
+
+	ev.LogData.Init(ev.BaseEvent)
 
 	ev.LogData.Session = ev.Session
 	ev.LogData.HTTP.Verb = ev.Verb

internal/events/icmpv4.go

@@ -4,7 +4,7 @@ import (
 	"time"
 
 	"github.com/bonjourmalware/melody/internal/events/helpers"
-	"github.com/bonjourmalware/melody/internal/events/logdata"
+	"github.com/bonjourmalware/melody/internal/logdata"
 
 	"github.com/bonjourmalware/melody/internal/config"
 
@@ -46,16 +46,18 @@ func (ev ICMPv4Event) ToLog() EventLog {
 	ev.LogData = logdata.ICMPv4EventLog{}
 	ev.LogData.Timestamp = ev.Timestamp.Format(time.RFC3339Nano)
 
-	ev.LogData.Type = ev.Kind
-	ev.LogData.SourceIP = ev.SourceIP
-	ev.LogData.DestPort = ev.DestPort
-	ev.LogData.Session = ev.Session
-
-	if len(ev.Tags) == 0 {
-		ev.LogData.Tags = make(map[string][]string)
-	} else {
-		ev.LogData.Tags = ev.Tags
-	}
+	//ev.LogData.Type = ev.Kind
+	//ev.LogData.SourceIP = ev.SourceIP
+	//ev.LogData.DestPort = ev.DestPort
+	//ev.LogData.Session = ev.Session
+	//
+	//if len(ev.Tags) == 0 {
+	//	ev.LogData.Tags = make(map[string][]string)
+	//} else {
+	//	ev.LogData.Tags = ev.Tags
+	//}
+
+	ev.LogData.Init(ev.BaseEvent)
 
 	ev.LogData.ICMPv4 = logdata.ICMPv4LogData{
 		TypeCode:     ev.ICMPv4Layer.Header.TypeCode,

internal/events/icmpv6.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/bonjourmalware/melody/internal/config"
 	"github.com/bonjourmalware/melody/internal/events/helpers"
-	"github.com/bonjourmalware/melody/internal/events/logdata"
+	"github.com/bonjourmalware/melody/internal/logdata"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 )
@@ -45,16 +45,18 @@ func (ev ICMPv6Event) ToLog() EventLog {
 	//ev.LogData.NsTimestamp = strconv.FormatInt(time.Now().UnixNano(), 10)
 	ev.LogData.Timestamp = ev.Timestamp.Format(time.RFC3339Nano)
 
-	ev.LogData.Type = ev.Kind
-	ev.LogData.SourceIP = ev.SourceIP
-	ev.LogData.DestPort = ev.DestPort
-	ev.LogData.Session = ev.Session
+	//ev.LogData.Type = ev.Kind
+	//ev.LogData.SourceIP = ev.SourceIP
+	//ev.LogData.DestPort = ev.DestPort
+	//ev.LogData.Session = ev.Session
+	//
+	//if len(ev.Tags) == 0 {
+	//	ev.LogData.Tags = make(map[string][]string)
+	//} else {
+	//	ev.LogData.Tags = ev.Tags
+	//}
 
-	if len(ev.Tags) == 0 {
-		ev.LogData.Tags = make(map[string][]string)
-	} else {
-		ev.LogData.Tags = ev.Tags
-	}
+	ev.LogData.Init(ev.BaseEvent)
 
 	ev.LogData.ICMPv6 = logdata.ICMPv6LogData{
 		TypeCode:     ev.ICMPv6Layer.Header.TypeCode,

internal/events/tcp.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/bonjourmalware/melody/internal/events/helpers"
 
-	"github.com/bonjourmalware/melody/internal/events/logdata"
+	"github.com/bonjourmalware/melody/internal/logdata"
 
 	"github.com/bonjourmalware/melody/internal/sessions"
 
@@ -64,16 +64,18 @@ func (ev TCPEvent) ToLog() EventLog {
 
 	ev.LogData = logdata.TCPEventLog{}
 	ev.LogData.Timestamp = ev.Timestamp.Format(time.RFC3339Nano)
-	ev.LogData.Type = ev.Kind
-	ev.LogData.SourceIP = ev.SourceIP
-	ev.LogData.DestPort = ev.DestPort
-	ev.LogData.Session = ev.Session
-
-	if len(ev.Tags) == 0 {
-		ev.LogData.Tags = make(map[string][]string)
-	} else {
-		ev.LogData.Tags = ev.Tags
-	}
+	//ev.LogData.Type = ev.Kind
+	//ev.LogData.SourceIP = ev.SourceIP
+	//ev.LogData.DestPort = ev.DestPort
+	//ev.LogData.Session = ev.Session
+	//
+	//if len(ev.Tags) == 0 {
+	//	ev.LogData.Tags = make(map[string][]string)
+	//} else {
+	//	ev.LogData.Tags = ev.Tags
+	//}
+	//
+	ev.LogData.Init(ev.BaseEvent)
 
 	switch ev.IPVersion {
 	case 4:

internal/events/udp.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/bonjourmalware/melody/internal/events/helpers"
-	"github.com/bonjourmalware/melody/internal/events/logdata"
+	"github.com/bonjourmalware/melody/internal/logdata"
 
 	"github.com/bonjourmalware/melody/internal/config"
 
@@ -59,19 +59,19 @@ func (ev UDPEvent) ToLog() EventLog {
 	var ipFlagsStr []string
 
 	ev.LogData = logdata.UDPEventLog{}
-	//ev.LogData.Timestamp = time.Now().Format(time.RFC3339)
-	//ev.LogData.NsTimestamp = strconv.FormatInt(time.Now().UnixNano(), 10)
 	ev.LogData.Timestamp = ev.Timestamp.Format(time.RFC3339Nano)
-	ev.LogData.Type = ev.Kind
-	ev.LogData.SourceIP = ev.SourceIP
-	ev.LogData.DestPort = ev.DestPort
-	ev.LogData.Session = ev.Session
-
-	if len(ev.Tags) == 0 {
-		ev.LogData.Tags = make(map[string][]string)
-	} else {
-		ev.LogData.Tags = ev.Tags
-	}
+	//ev.LogData.Type = ev.Kind
+	//ev.LogData.SourceIP = ev.SourceIP
+	//ev.LogData.DestPort = ev.DestPort
+	//ev.LogData.Session = ev.Session
+	//
+	//if len(ev.Tags) == 0 {
+	//	ev.LogData.Tags = make(map[string][]string)
+	//} else {
+	//	ev.LogData.Tags = ev.Tags
+	//}
+
+	ev.LogData.Init(ev.BaseEvent)
 
 	switch ev.IPVersion {
 	case 4:

internal/logdata/base.go

@@ -0,0 +1,51 @@
+package logdata
+
+import (
+	"fmt"
+
+	"github.com/bonjourmalware/melody/internal/loggable"
+)
+
+// IPLogData is the interface used by packet structs supporting an IP layer
+type IPLogData interface{}
+
+// BaseLogData is used as the base packet log and contains common data, such as the timestamp
+type BaseLogData struct {
+	Timestamp  string              `json:"timestamp"`
+	Session    string              `json:"session"`
+	Type       string              `json:"type"`
+	SourceIP   string              `json:"src_ip"`
+	DestPort   uint16              `json:"dst_port"`
+	Tags       map[string][]string `json:"matches"`
+	InlineTags []string            `json:"inline_matches"`
+	Additional map[string]string   `json:"embedded"`
+}
+
+// Init takes the common BaseEvent attributes to setup the BaseLogData struct
+func (l *BaseLogData) Init(ev loggable.Loggable) {
+	l.Type = ev.GetKind()
+	l.SourceIP = ev.GetSourceIP()
+	l.DestPort = ev.GetDestPort()
+	l.Session = ev.GetSession()
+	l.InlineTags = []string{}
+
+	if len(ev.GetTags()) == 0 {
+		l.Tags = make(map[string][]string)
+	} else {
+		l.Tags = ev.GetTags()
+		l.InlineTags = makeInlineArray(ev.GetTags())
+	}
+}
+
+//makeInlineArray converts a Tags map to an array of its values with the keys and values merged with a '.'
+func makeInlineArray(tags map[string][]string) []string {
+	var inlineTags []string
+
+	for key, values := range tags {
+		for _, val := range values {
+			inlineTags = append(inlineTags, fmt.Sprintf("%s.%s", key, val))
+		}
+	}
+
+	return inlineTags
+}

internal/loggable/loggable.go

@@ -0,0 +1,10 @@
+package loggable
+
+// Loggable is an interface to allow mutual use of events.BaseEvent for logdata.BaseLogData
+type Loggable interface {
+	GetSession() string
+	GetTags() map[string][]string
+	GetKind() string
+	GetSourceIP() string
+	GetDestPort() uint16
+}