Merge pull request #12195 from Security-Onion-Solutions/2.4/dev

2.4.40

.github/workflows/contrib.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: "Contributor Check"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: cla-assistant/github-action@v2.1.3-beta
+        uses: cla-assistant/github-action@v2.3.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.30-20231228 ISO image released on 2024/01/02
+### 2.4.40-20240116 ISO image released on 2024/01/17
 
 
 ### Download and Verify
 
-2.4.30-20231228 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.30-20231228.iso
+2.4.40-20240116 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.40-20240116.iso
 
-MD5: DBD47645CD6FA8358C51D8753046FB54  
-SHA1: 2494091065434ACB028F71444A5D16E8F8A11EDF  
-SHA256: 3345AE1DC58AC7F29D82E60D9A36CDF8DE19B7DFF999D8C4F89C7BD36AEE7F1D  
+MD5: AC55D027B663F3CE0878FEBDAD9DD78B  
+SHA1: C2B51723B17F3DC843CC493EB80E93B123E3A3E1  
+SHA256: C5F135FCF45A836BBFF58C231F95E1EA0CD894898322187AD5FBFCD24BC2F123  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.30-20231228.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.40-20240116.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.30-20231228.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.40-20240116.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.30-20231228.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.40-20240116.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.30-20231228.iso.sig securityonion-2.4.30-20231228.iso
+gpg --verify securityonion-2.4.40-20240116.iso.sig securityonion-2.4.40-20240116.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Thu 28 Dec 2023 10:08:31 AM EST using RSA key ID FE507013
+gpg: Signature made Tue 16 Jan 2024 07:34:40 PM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-20231204

VERSION

@@ -1 +1 @@
-2.4.30
+2.4.40

pillar/top.sls

@@ -61,8 +61,6 @@ base:
     - elastalert.adv_elastalert
     - backup.soc_backup
     - backup.adv_backup
-    - curator.soc_curator
-    - curator.adv_curator
     - soctopus.soc_soctopus
     - soctopus.adv_soctopus
     - minions.{{ grains.id }}
@@ -113,8 +111,6 @@ base:
     - kibana.adv_kibana
     - strelka.soc_strelka
     - strelka.adv_strelka
-    - curator.soc_curator
-    - curator.adv_curator
     - kratos.soc_kratos
     - kratos.adv_kratos
     - redis.soc_redis
@@ -172,8 +168,6 @@ base:
     - kibana.adv_kibana
     - strelka.soc_strelka
     - strelka.adv_strelka
-    - curator.soc_curator
-    - curator.adv_curator
     - backup.soc_backup
     - backup.adv_backup
     - zeek.soc_zeek
@@ -194,8 +188,6 @@ base:
     - logstash.adv_logstash
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
-    - curator.soc_curator
-    - curator.adv_curator
     - redis.soc_redis
     - redis.adv_redis
     - zeek.soc_zeek
@@ -268,8 +260,6 @@ base:
     - soctopus.adv_soctopus
     - kibana.soc_kibana
     - kibana.adv_kibana
-    - curator.soc_curator
-    - curator.adv_curator
     - backup.soc_backup
     - backup.adv_backup
     - kratos.soc_kratos

salt/allowed_states.map.jinja

@@ -219,10 +219,6 @@
     {% do allowed_states.append('kibana.secrets') %}
   {% endif %}
 
-  {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-    {% do allowed_states.append('curator') %}
-  {% endif %}
-
   {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
     {% do allowed_states.append('elastalert') %}
   {% endif %}

salt/common/init.sls

@@ -179,6 +179,14 @@ so-status_check_cron:
     - month: '*'
     - dayweek: '*'
 
+# This cronjob/script runs a check if the node needs restarted, but should be used for future status checks as well
+common_status_check_cron:
+  cron.present:
+    - name: '/usr/sbin/so-common-status-check > /dev/null 2>&1'
+    - identifier: common_status_check
+    - user: root
+    - minute: '*/10'
+
 remove_post_setup_cron:
   cron.absent:
     - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate'

salt/common/tools/sbin/so-common-status-check

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import sys
+import subprocess
+import os
+
+sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
+import salt.config
+import salt.loader
+
+__opts__ = salt.config.minion_config('/etc/salt/minion')
+__grains__ = salt.loader.grains(__opts__)
+
+def check_needs_restarted():
+  osfam = __grains__['os_family']
+  val = '0'
+  outfile = "/opt/so/log/sostatus/needs_restarted"
+
+  if osfam == 'Debian':
+    if os.path.exists('/var/run/reboot-required'):
+      val = '1'
+  elif osfam == 'RedHat':
+    cmd = 'needs-restarting -r > /dev/null 2>&1'
+    try:
+      needs_restarting = subprocess.check_call(cmd, shell=True)
+    except subprocess.CalledProcessError:
+      val = '1'
+  else:
+    fail("Unsupported OS")
+
+  with open(outfile, 'w') as f:
+    f.write(val)
+
+def fail(msg):
+  print(msg, file=sys.stderr)
+  sys.exit(1)
+
+
+def main():
+  proc = subprocess.run(['id', '-u'], stdout=subprocess.PIPE, encoding="utf-8")
+  if proc.stdout.strip() != "0":
+    fail("This program must be run as root")
+
+  check_needs_restarted()
+
+if __name__ == "__main__":
+  main()

salt/common/tools/sbin/so-image-common

@@ -42,7 +42,6 @@ container_list() {
     )
   elif [ $MANAGERCHECK != 'so-helix' ]; then
     TRUSTED_CONTAINERS=(
-      "so-curator"
       "so-elastalert"
       "so-elastic-agent"
       "so-elastic-agent-builder"

salt/common/tools/sbin/so-log-check

@@ -109,6 +109,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout exceeded"             # server not yet ready (telegraf waiting on elasticsearch)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes"            # server not yet ready (telegraf waiting on influx)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at"            # server not yet ready (telegraf waiting on health data)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection timed out"         # server not yet ready (telegraf plugin unable to connect)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|command timed out"            # server not yet ready (telegraf plugin waiting for script to finish)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key"        # server not yet ready (salt minion waiting on key acceptance)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes"              # server not yet ready (logstash waiting on elastic)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll"               # server not yet ready (sensoroni waiting on soc)
@@ -119,6 +121,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving container" # Telegraf unable to reach Docker engine, rare
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error while communicating"    # Elasticsearch MS -> HN "sensor" temporarily unavailable
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error"          # Docker registry container when new node comes onlines
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -167,6 +170,9 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature"      # Malformed Suricata rule, from upstream provider
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to determine destination index stats"  # Elastic transform temporary error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # InfluxDB flux query, import nodes
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exhausting result iterator"   # InfluxDB flux query mismatched table results (temporary data issue)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to finish run"         # InfluxDB rare error, self-recoverable
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
@@ -223,6 +229,9 @@ exclude_log "spool"        # disregard zeek analyze logs as this is data specifi
 exclude_log "import"       # disregard imported test data the contains error strings
 exclude_log "update.log"   # ignore playbook updates due to several known issues
 exclude_log "playbook.log" # ignore due to several playbook known issues
+exclude_log "cron-cluster-delete.log" # ignore since Curator has been removed
+exclude_log "cron-close.log" # ignore since Curator has been removed
+exclude_log "curator.log" # ignore since Curator has been removed
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

salt/common/tools/sbin/so-nsm-clear

@@ -41,8 +41,13 @@ done
 if [ $SKIP -ne 1 ]; then
         # Inform user we are about to delete all data
         echo
-        echo "This script will delete all NIDS data (PCAP, Suricata, Zeek)" 
-        echo "If you would like to proceed, please type "AGREE" and hit ENTER."
+        echo "This script will delete all NSM data from /nsm."
+        echo
+        echo "This includes Suricata data, Zeek data, and full packet capture (PCAP)."
+        echo
+        echo "This will NOT delete any Suricata or Zeek logs that have already been ingested into Elasticsearch."
+        echo
+        echo "If you would like to proceed, then type AGREE and press ENTER."
         echo
         # Read user input
         read INPUT
@@ -54,8 +59,8 @@ delete_pcap() {
   [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
 }
 delete_suricata() {
-  SURI_LOG="/opt/so/log/suricata/eve.json"
-  [ -f $SURI_LOG ] && so-suricata-stop && rm -f $SURI_LOG && so-suricata-start
+  SURI_LOG="/nsm/suricata/"
+  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
 }
 delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"

salt/common/tools/sbin_jinja/so-raid-status

@@ -49,11 +49,18 @@ check_nsm_raid() {
 
 check_boss_raid() {
   MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
+  MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter")
 
-  if [[ -n $MVCLI ]]; then
-    BOSSRAID=0
+  # Check to see if this is a SM based system
+  if [[ -z $MVTEST ]]; then
+    if [[ -n $MVCLI ]]; then
+      BOSSRAID=0
+    else
+      BOSSRAID=1
+    fi
   else
-    BOSSRAID=1
+    # This doesn't have boss raid so lets make it 0
+    BOSSRAID=0
   fi
 }
 
@@ -90,4 +97,4 @@ else
   RAIDSTATUS=1
 fi
 
-echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
+echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log