feat: new directive option: all_rules_always_active (#181)

* add dockerbuild-dev.sh script

* add AllRulesAlwaysActive flag to directive struct

* add backlog processing for AllRulesALwaysActive

* update test files

* remove event count restriction in UI

* fix ng lint err

internal/pkg/dsiem/siem/backlog.go

@@ -106,9 +106,28 @@ func (b *backLog) newEventProcessor() {
 		lenSDiffInt := len(b.Directive.StickyDiffs[idx].SDiffInt)
 		b.RUnlock()
 		if !rule.DoesEventMatch(evt, currRule, currSDiff, evt.ConnID) {
-			b.debug("backlog doeseventmatch false", evt.ConnID)
-			b.chFound <- false
-			continue
+			// if flag is set, check if event match previous stage
+			if b.Directive.AllRulesAlwaysActive && idx != 0 {
+				prevFound := false
+				for i := 0; i < idx; i++ {
+					b.RLock()
+					prevRule := b.Directive.Rules[i]
+					prevSDiff := &b.Directive.StickyDiffs[i]
+					b.RUnlock()
+					if rule.DoesEventMatch(evt, prevRule, prevSDiff, evt.ConnID) {
+						b.debug("backlog "+b.ID+" previous rule "+strconv.Itoa(i)+" consumes matching event", evt.ConnID)
+						// just add the event to the stage, no need to process other steps in processMatchedEvent
+						b.appendandWriteEvent(evt, i, nil)
+						prevFound = true
+						break
+					}
+				}
+				b.chFound <- prevFound
+			} else {
+				b.debug("backlog doeseventmatch false", evt.ConnID)
+				b.chFound <- false
+			}
+			continue // main for loop
 		}
 		b.chFound <- true // answer quickly
 
@@ -369,16 +388,16 @@ func (b *backLog) appendandWriteEvent(e event.NormalizedEvent, idx int, tx *apm.
 	// dont wait for I/O
 	// b.RLock()
 	// go func(b *backLog) {
-	err := b.updateElasticsearch(e)
+	err := b.updateElasticsearch(e, idx)
 	if err != nil {
 		b.warn("failed to update Elasticsearch! "+err.Error(), e.ConnID)
-		if apm.Enabled() {
+		if apm.Enabled() && tx != nil {
 			tx.SetError(err)
 			tx.Result("Failed to append and write event")
 			tx.End()
 		}
 	} else {
-		if apm.Enabled() {
+		if apm.Enabled() && tx != nil {
 			tx.Result("Event appended to backlog")
 			tx.End()
 		}
@@ -488,11 +507,12 @@ func (b *backLog) setStatusTime() {
 	b.Unlock()
 }
 
-func (b *backLog) updateElasticsearch(e event.NormalizedEvent) error {
+func (b *backLog) updateElasticsearch(e event.NormalizedEvent, idx int) error {
 	b.debug("backlog updating Elasticsearch", e.ConnID)
 	b.Lock()
 	b.StatusTime = time.Now().Unix()
-	v := siemAlarmEvents{b.ID, b.CurrentStage, e.EventID}
+	stage := idx + 1
+	v := siemAlarmEvents{b.ID, stage, e.EventID}
 	b.Unlock()
 	bLogFileMutex.Lock()
 	f, err := os.OpenFile(bLogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)

internal/pkg/dsiem/siem/backlog_test.go

@@ -80,7 +80,7 @@ func TestBackLog(t *testing.T) {
 
 	fDir := path.Join(testDir, "internal", "pkg", "dsiem", "siem", "fixtures")
 
-	// use directive that expires fast and has only 2 stages
+	// use directive that expires fast and has only 3 stages
 	dirs, _, err := LoadDirectivesFromFile(path.Join(fDir, "directive4"), directiveFileGlob, false)
 	if err != nil {
 		t.Fatal(err)
@@ -96,7 +96,7 @@ func TestBackLog(t *testing.T) {
 	e.ConnID = 1
 	dctives := dirs.Dirs[0]
 	e.PluginID = dctives.Rules[0].PluginID
-	e.PluginSID = 2100384
+	e.PluginSID = dctives.Rules[0].PluginSID[0]
 
 	e.Timestamp = time.Now().UTC().Format(time.RFC3339)
 
@@ -136,27 +136,38 @@ func TestBackLog(t *testing.T) {
 		}
 	}()
 
+	// will raise stage to 2nd
 	fmt.Println("first event (by start)")
-	go b.start(e,0)
+	fmt.Println("using backlog: ", b.Directive.Name)
+	fmt.Println("all_rules_always_active flag: ", b.Directive.AllRulesAlwaysActive)
 
-	// will also raise stage
+	go b.start(e, 0)
+
+	// will also raise stage to 3rd
 	fmt.Print("under pressure ..")
-	e.ConnID = 1
+	e.ConnID = 2
 	e.RcvdTime = time.Now().Add(-700 * time.Second).Unix()
+	e.PluginSID = b.Directive.Rules[1].PluginSID[0]
 	verifyEventOutput(t, e, b.chData, "backlog is under pressure")
 
+	fmt.Print("previous rule consuming event ..")
+	e.ConnID = 3
+	e.PluginSID = b.Directive.Rules[0].PluginSID[0]
+	verifyEventOutput(t, e, b.chData, "consumes matching event")
+
 	e.RcvdTime = time.Now().Add(-time.Second).Unix()
-	e.ConnID = 2
+	e.ConnID = 4
+	e.PluginSID = b.Directive.Rules[1].PluginSID[0]
 	fmt.Print("reached max stage ..")
 	verifyEventOutput(t, e, b.chData, "reached max stage and occurrence")
 
 	fmt.Print("out of order event ..")
-	e.ConnID = 4
+	e.ConnID = 5
 	e.Timestamp = time.Now().Add(time.Second * -300).UTC().Format(time.RFC3339)
 	verifyEventOutput(t, e, b.chData, "event timestamp out of order")
 
 	fmt.Print("invalid timestamp ..")
-	e.ConnID = 5
+	e.ConnID = 6
 	e.Timestamp = "#"
 	verifyEventOutput(t, e, b.chData, "cannot parse event timestamp")
 
@@ -167,7 +178,6 @@ func TestBackLog(t *testing.T) {
 	}
 	fmt.Println("OK")
 
-
 	fmt.Print("Check deletion ..")
 	verifyFuncOutput(t, func() {
 		b.delete()

internal/pkg/dsiem/siem/backlogmgr.go

@@ -290,11 +290,24 @@ func createNewBackLog(d Directive, e event.NormalizedEvent) (bp *backLog, err er
 }
 
 func initBackLogRules(d *Directive, e event.NormalizedEvent) {
+
 	for i := range d.Rules {
-		// the first rule cannot use reference to other
 		if i == 0 {
+			// if flag is active, replace ANY and HOME_NET on the first rule with specific addresses from event
+			if d.AllRulesAlwaysActive {
+				ref := d.Rules[i].From
+				if ref == "ANY" || ref == "HOME_NET" || ref == "!HOME_NET" {
+					d.Rules[i].From = e.SrcIP
+				}
+				ref = d.Rules[i].To
+				if ref == "ANY" || ref == "HOME_NET" || ref == "!HOME_NET" {
+					d.Rules[i].To = e.DstIP
+				}
+			}
+			// the first rule cannot use reference to other
 			continue
 		}
+
 		// for the rest, refer to the referenced stage if its not ANY or HOME_NET or !HOME_NET
 		// if the reference is ANY || HOME_NET || !HOME_NET then refer to event if its in the format of
 		// :ref

internal/pkg/dsiem/siem/directive.go

@@ -43,14 +43,15 @@ const (
 
 // Directive represents a SIEM use case that has several correlation rules
 type Directive struct {
-	ID          int                   `json:"id"`
-	Name        string                `json:"name"`
-	Priority    int                   `json:"priority"`
-	Disabled    bool                  `json:"disabled"`
-	Kingdom     string                `json:"kingdom"`
-	Category    string                `json:"category"`
-	Rules       []rule.DirectiveRule  `json:"rules"`
-	StickyDiffs []rule.StickyDiffData `json:"-"`
+	ID                   int                   `json:"id"`
+	Name                 string                `json:"name"`
+	Priority             int                   `json:"priority"`
+	Disabled             bool                  `json:"disabled"`
+	AllRulesAlwaysActive bool                  `json:"all_rules_always_active"`
+	Kingdom              string                `json:"kingdom"`
+	Category             string                `json:"category"`
+	Rules                []rule.DirectiveRule  `json:"rules"`
+	StickyDiffs          []rule.StickyDiffData `json:"-"`
 }
 
 // Directives group directive together
@@ -307,6 +308,7 @@ func copyDirective(dst *Directive, src Directive, e event.NormalizedEvent) {
 	dst.Priority = src.Priority
 	dst.Kingdom = src.Kingdom
 	dst.Category = src.Category
+	dst.AllRulesAlwaysActive = src.AllRulesAlwaysActive
 
 	// replace SRC_IP and DST_IP with the asset name or IP address
 	title := src.Name

internal/pkg/dsiem/siem/fixtures/directive4/directives_dsiem-backend-0_testing1.json

@@ -1,19 +1,65 @@
 {
   "directives": [
     {
-      "name": "Correct Rule, Attack from SRC_IP to DST_IP",
+      "name": "Correct Rule with Always Active Option, Attack from SRC_IP to DST_IP",
       "kingdom": "Reconnaissance & Probing",
       "category": "Misc Activity",
+      "all_rules_always_active": true,
       "id": 5,
       "priority": 3,
-      "rules" : [
-        { "name": "ICMP Ping", "type": "PluginRule", "stage": 1, "plugin_id": 1001, "plugin_sid": [ 2100384 ], "occurrence": 1, 
-          "from": "HOME_NET", "to": "ANY", "port_from": "ANY", "port_to": "ANY", "protocol": "ICMP", "reliability": 1, "timeout": 0 },
-        { "name": "ICMP Ping", "type": "PluginRule", "stage": 2, "plugin_id": 1001, "plugin_sid": [ 2100384 ], "occurrence": 1,
-          "from": ":1", "to": ":1", "port_from": ":1", "port_to": "ANY", "protocol": "ICMP", "reliability": 6, "timeout": 3 },
-        { "name": "ICMP Ping", "type": "PluginRule", "stage": 3, "plugin_id": 1001, "plugin_sid": [ 2100384 ], "occurrence": 1,
-          "from": ":1", "to": ":1", "port_from": ":1", "port_to": "ANY", "protocol": "ICMP", "reliability": 6, "timeout": 3 }
+      "rules": [
+        {
+          "name": "ICMP Ping",
+          "type": "PluginRule",
+          "stage": 1,
+          "plugin_id": 1001,
+          "plugin_sid": [
+            2100384
+          ],
+          "occurrence": 1,
+          "from": "HOME_NET",
+          "to": "ANY",
+          "port_from": "ANY",
+          "port_to": "ANY",
+          "protocol": "ICMP",
+          "reliability": 1,
+          "timeout": 0
+        },
+        {
+          "name": "ICMP Ping",
+          "type": "PluginRule",
+          "stage": 2,
+          "plugin_id": 1001,
+          "plugin_sid": [
+            2100385
+          ],
+          "occurrence": 1,
+          "from": ":1",
+          "to": ":1",
+          "port_from": ":1",
+          "port_to": "ANY",
+          "protocol": "ICMP",
+          "reliability": 6,
+          "timeout": 3
+        },
+        {
+          "name": "ICMP Ping",
+          "type": "PluginRule",
+          "stage": 3,
+          "plugin_id": 1001,
+          "plugin_sid": [
+            2100385
+          ],
+          "occurrence": 1,
+          "from": ":1",
+          "to": ":1",
+          "port_from": ":1",
+          "port_to": "ANY",
+          "protocol": "ICMP",
+          "reliability": 6,
+          "timeout": 3
+        }
       ]
     }
   ]
-}
+}

web/ui/src/app/views/base/detailalarm.component.ts

@@ -111,12 +111,12 @@ export class DetailalarmComponent implements OnInit, OnDestroy {
       tempAlarms = resp.hits.hits;
       await Promise.all(tempAlarms.map(async (e) => {
         await Promise.all(e['_source']['rules'].map(async (r) => {
-          if (r['status'] === 'finished') {
-            r['events_count'] = r['occurrence'];
-          } else {
+          // if (r['status'] === 'finished') {
+          //  r['events_count'] = r['occurrence'];
+          // } else {
             const response = await this.es.countEvents(this.es.esIndexAlarmEvent, alarmID, r['stage']);
             r['events_count'] = response.count;
-          }
+          // }
         }));
       }));
     } catch (err) {