Merge remote-tracking branch 'origin/2.4/dev' into 2.4/soupchanges

salt/common/tools/sbin/so-log-check

@@ -206,7 +206,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse"                 # Suricata encountering a malformed rule
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed"       # Detections: Exclude false positive due to automated testing
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors"                   # Detections: Not an actual error
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Provided Grok expressions do not match field value\\: \\[unprovisioned\\]"  # SOC log: before fields.status was changed to fields.licenseStatus
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager"  # SOC log: before fields.status was changed to fields.licenseStatus
 fi
 
 RESULT=0

salt/common/tools/sbin_jinja/so-raid-status

@@ -9,58 +9,80 @@
 
 . /usr/sbin/so-common
 
+software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
+hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")
+
 {%- if salt['grains.get']('sosmodel', '') %}
 {%- set model = salt['grains.get']('sosmodel') %}
 model={{ model }}
 # Don't need cloud images to use this
 if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then
     exit 0
 fi
+
+for i in "${software_raid[@]}"; do
+  if [[ "$model" == $i ]]; then
+    is_softwareraid=true
+    is_hwraid=false
+    break
+  fi
+done
+
+for i in "${hardware_raid[@]}"; do
+  if [[ "$model" == $i ]]; then
+    is_softwareraid=false
+    is_hwraid=true
+    break
+  fi
+done
+
 {%- else %}
 echo "This is not an appliance"
 exit 0
 {%- endif %}
-if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then
-	is_bossraid=true
-fi
-if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then
-	is_swraid=true
-fi
-if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then
-	is_hwraid=true
-fi
 
 check_nsm_raid() {
   PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
   MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
-
-  if [[ $APPLIANCE == '1' ]]; then
+  if [[ "$model" == "SOS500" || "$model" == "SOS500-DE02" ]]; then
+    #This doesn't have raid
+    HWRAID=0
+  else
     if [[ -n $PERCCLI ]]; then
       HWRAID=0
     elif [[ -n $MEGACTL ]]; then
       HWRAID=0
     else
       HWRAID=1
-    fi
-
+    fi   
   fi
 
 }
 
 check_boss_raid() {
   MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
   MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter")
+  BOSSNVMECLI=$(/usr/local/bin/mnv_cli info -o vd -i 0 | grep Functional)
 
-  # Check to see if this is a SM based system
-  if [[ -z $MVTEST ]]; then
-    if [[ -n $MVCLI ]]; then
+  # Is this NVMe Boss Raid?
+  if [[ "$model" =~ "-DE02" ]]; then
+    if [[ -n $BOSSNVMECLI ]]; then
       BOSSRAID=0
     else
       BOSSRAID=1
     fi
   else
-    # This doesn't have boss raid so lets make it 0
-    BOSSRAID=0
+    # Check to see if this is a SM based system
+    if [[ -z $MVTEST ]]; then
+      if [[ -n $MVCLI ]]; then
+        BOSSRAID=0
+      else
+        BOSSRAID=1
+      fi
+    else
+      # This doesn't have boss raid so lets make it 0
+      BOSSRAID=0
+    fi
   fi
 }
 
@@ -79,14 +101,13 @@ SWRAID=0
 BOSSRAID=0
 HWRAID=0
 
-if [[ $is_hwraid ]]; then
+if [[ "$is_hwraid" == "true" ]]; then
 	check_nsm_raid
+  check_boss_raid
 fi
-if [[ $is_bossraid ]]; then
-	check_boss_raid
-fi
-if [[ $is_swraid ]]; then
+if [[ "$is_softwareraid" == "true" ]]; then
 	check_software_raid
+  check_boss_raid
 fi
 
 sum=$(($SWRAID + $BOSSRAID + $HWRAID))

salt/elastalert/soc_elastalert.yaml

@@ -3,8 +3,8 @@ elastalert:
     description: You can enable or disable Elastalert.
     helpLink: elastalert.html
   alerter_parameters:
-    title: Alerter Parameters
-    description: Optional configuration parameters for additional alerters that can be enabled for all Sigma rules. Filter for 'Alerter' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key.
+    title: Custom Configuration Parameters
+    description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key.
     global: True
     multiline: True
     syntax: yaml

salt/elasticfleet/defaults.yaml

@@ -97,6 +97,7 @@ elasticfleet:
     - symantec_endpoint
     - system
     - tcp
+    - tenable_io
     - tenable_sc
     - ti_abusech
     - ti_anomali

salt/elasticsearch/defaults.yaml

@@ -9282,6 +9282,190 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-tenable_io_x_asset:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-tenable_io.asset-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-tenable_io.asset-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-tenable_io.asset@package"
+          - "logs-tenable_io.asset@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+        ignore_missing_component_templates:
+        - logs-tenable_io.asset@custom
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-tenable_io_x_plugin:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-tenable_io.plugin-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-tenable_io.plugin-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-tenable_io.plugin@package"
+          - "logs-tenable_io.plugin@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+        ignore_missing_component_templates:
+        - logs-tenable_io.plugin@custom
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-tenable_io_x_scan:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-tenable_io.scan-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-tenable_io.scan-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-tenable_io.scan@package"
+          - "logs-tenable_io.scan@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+        ignore_missing_component_templates:
+        - logs-tenable_io.scan@custom
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-tenable_io_x_vulnerability:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-tenable_io.vulnerability-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-tenable_io.vulnerability-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-tenable_io.vulnerability@package"
+          - "logs-tenable_io.vulnerability@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+        ignore_missing_component_templates:
+        - logs-tenable_io.vulnerability@custom
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-tenable_sc_x_asset:
       index_sorting: false
       index_template:

salt/elasticsearch/files/ingest-dynamic/common

@@ -62,6 +62,7 @@
     { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
     { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
     { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
+    { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
     { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
 {%- endraw %}
 {%- if HIGHLANDER %}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -466,6 +466,13 @@ elasticsearch:
     so-logs-sonicwall_firewall_x_log: *indexSettings
     so-logs-snort_x_log: *indexSettings
     so-logs-symantec_endpoint_x_log: *indexSettings
+    so-logs-tenable_io_x_asset: *indexSettings
+    so-logs-tenable_io_x_plugin: *indexSettings
+    so-logs-tenable_io_x_scan: *indexSettings
+    so-logs-tenable_io_x_vulnerability: *indexSettings
+    so-logs-tenable_sc_x_asset: *indexSettings
+    so-logs-tenable_sc_x_plugin: *indexSettings
+    so-logs-tenable_sc_x_vulnerability: *indexSettings
     so-logs-ti_abusech_x_malware: *indexSettings
     so-logs-ti_abusech_x_malwarebazaar: *indexSettings
     so-logs-ti_abusech_x_threatfox: *indexSettings

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -10,10 +10,16 @@
 
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
+{%-     if index == 'so-logs-detections.alerts' %}
 echo
-echo "Setting up {{ index }}-logs policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-echo
+  echo "Setting up so-logs-detections.alerts-so policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+{%-   else %}
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+{%-     endif %}
 {%-   endif %}
 {%- endfor %}
 echo

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -5,7 +5,6 @@
 # Elastic License 2.0.
 {%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 {%  from 'vars/globals.map.jinja' import GLOBALS %}
-{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
 
 STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
 STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
@@ -68,9 +67,9 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
     echo -n "Waiting for ElasticSearch..."
     retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
     {% if GLOBALS.role != 'so-heavynode' %}
-    SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-    INSTALLED=$(elastic_fleet_package_is_installed endpoint }} )
-    if [ "$INSTALLED" != "installed" ]; then
+    TEMPLATE="logs-endpoint.alerts@package"
+    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
+    if [ "$INSTALLED" != "$TEMPLATE" ]; then
       echo
       echo "Packages not yet installed."
       echo