Merge Main into Dev

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.4.30-20231121 ISO image released on 2023/11/21
+### 2.4.30-20231204 ISO image released on 2023/12/06
 
 
 
 ### Download and Verify
 
-2.4.30-20231121 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.30-20231121.iso
+2.4.30-20231204 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.30-20231204.iso
 
-MD5: 09DB0A6B3A75435C855E777272FC03F8  
-SHA1: A68868E67A3F86B77E01F54067950757EFD3BA72  
-SHA256: B3880C0302D9CDED7C974585B14355544FC9C3279F952EC79FC2BA9AEC7CB749  
+MD5: 596A164241D0C62AEBBE23D7883F505E  
+SHA1: 139FE16DC3B13B1F1A748EE57BC2C5FEBADAEB07  
+SHA256: D5730F9952F5AC6DF06D4E02A9EF5C43B16AC85D8072C6D60AEFF03281122C71  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.30-20231121.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.30-20231204.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.30-20231121.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.30-20231204.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.30-20231121.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.30-20231204.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.30-20231121.iso.sig securityonion-2.4.30-20231121.iso
+gpg --verify securityonion-2.4.30-20231204.iso.sig securityonion-2.4.30-20231204.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 21 Nov 2023 01:21:38 PM EST using RSA key ID FE507013
+gpg: Signature made Tue 05 Dec 2023 11:46:42 AM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -20,8 +20,8 @@
             ],
             "data_stream.dataset": "import",
             "custom": "",
-	    "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.34.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.24.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.34.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.34.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.24.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
-	    "tags": [
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.43.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.38.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.43.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.43.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.38.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "tags": [
               "import"
             ]
           }

salt/manager/tools/sbin/soup

@@ -450,7 +450,10 @@ post_to_2.4.20() {
 post_to_2.4.30() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
+  # there is an occasional error with this state: pki_public_ca_crt: TypeError: list indices must be integers or slices, not str
+  set +e
   salt-call state.apply ca queue=True
+  set -e
   stop_salt_minion
   mv /etc/pki/managerssl.crt /etc/pki/managerssl.crt.old
   mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old
@@ -594,7 +597,11 @@ unmount_update() {
 
 update_airgap_rules() {
   # Copy the rules over to update them for airgap.
-  rsync -av $UPDATE_DIR/agrules/* /nsm/repo/rules/
+  rsync -av $UPDATE_DIR/agrules/suricata/* /nsm/rules/suricata/
+  rsync -av $UPDATE_DIR/agrules/yara/* /nsm/rules/yara/
+  if [ -d /nsm/repo/rules/sigma ]; then
+    rsync -av $UPDATE_DIR/agrules/sigma/* /nsm/repo/rules/sigma/
+  fi
 }
 
 update_airgap_repo() {
@@ -753,6 +760,9 @@ apply_hotfix() {
     elastic_fleet_integration_remove endpoints-initial elastic-defend-endpoints
     /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
   elif [[ "$INSTALLEDVERSION" == "2.4.30" ]] ; then
+    if [[ $is_airgap -eq 0 ]]; then
+      update_airgap_rules
+    fi
     if [[ -f /etc/pki/managerssl.key.old ]]; then
       echo "Skipping Certificate Generation"
     else
@@ -768,6 +778,7 @@ apply_hotfix() {
       mv /etc/pki/managerssl.crt /etc/pki/managerssl.crt.old
       mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old
       systemctl_func "start" "salt-minion"
+      (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
     fi  
   else
    echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
@@ -875,7 +886,6 @@ main() {
     echo "Hotfix applied"
     update_version
     enable_highstate
-    (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
     highstate
   else
     echo ""