Merge pull request #13391 from Security-Onion-Solutions/2.4/dev

2.4.90

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.80-20240624 ISO image released on 2024/06/25
+### 2.4.90-20240729 ISO image released on 2024/07/29
 
 
 ### Download and Verify
 
-2.4.80-20240624 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+2.4.90-20240729 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso
 
-MD5: 139F9762E926F9CB3C4A9528A3752C31  
-SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9  
-SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD  
+MD5: 9A7714F5922EE555F08675D25E6237D5  
+SHA1: D3B331452627DB716906BA9F3922574DFA3852DC  
+SHA256: 5B0CE32543944DBC50C4E906857384211E1BE83EF409619778F18FC62017E0E0  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso
+gpg --verify securityonion-2.4.90-20240729.iso.sig securityonion-2.4.90-20240729.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013
+gpg: Signature made Thu 25 Jul 2024 06:51:11 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.4.80
+2.4.90

pillar/elasticsearch/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='elasticsearch:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+elasticsearch:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/logstash/nodes.sls

@@ -1,16 +1,15 @@
 {% set node_types = {} %}
-{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
+    tgt='logstash:enabled:true',
     fun='network.ip_addrs',
-    tgt_type='compound') | dictsort()
+    tgt_type='pillar') | dictsort()
 %}
 
 # only add a node to the pillar if it returned an ip from the mine
 {%   if ip | length > 0%}
-{%     set hostname = cached_grains[minionid]['host'] %}
-{%     set node_type = minionid.split('_')[1] %}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
 {%     if node_type not in node_types.keys() %}
 {%       do node_types.update({node_type: {hostname: ip[0]}}) %}
 {%     else %}

pillar/redis/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='redis:enabled:true',
+    fun='network.ip_addrs',
+    tgt_type='pillar') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+redis:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/top.sls

@@ -47,10 +47,12 @@ base:
     - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -147,10 +149,12 @@ base:
     - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -215,18 +219,22 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
     - soc.license
     - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_receiver':
     - logstash.nodes

salt/allowed_states.map.jinja

@@ -136,7 +136,9 @@
           'firewall',
           'schedule',
           'docker_clean',
-          'stig'
+          'stig',
+          'kafka.ca',
+          'kafka.ssl'
           ],
       'so-standalone': [
           'salt.master',
@@ -195,7 +197,6 @@
           'schedule',
           'docker_clean',
           'kafka',
-          'elasticsearch.ca',
           'stig'
           ],
       'so-desktop': [

salt/common/tools/sbin/so-log-check

@@ -241,6 +241,7 @@ exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on
 exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
 exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
 exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
+exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

salt/elasticsearch/config.map.jinja

@@ -1,23 +1,37 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
 
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
-{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #}
-{% set ES_LOGSTASH_NODES = [] %}
-{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
+{% set ELASTICSEARCH_SEED_HOSTS = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 {% for node_type, node_details in node_data.items() | sort %}
-{%   if node_type not in ['heavynode', 'fleet'] %}
+{%   if node_type != 'heavynode' %}
 {%     for hostname in node_data[node_type].keys() %}
-{%       do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%       do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %}
 {%     endfor %}
 {%   endif %}
 {% endfor %}
 
+{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #}
+{% set ELASTICSEARCH_NODES = [] %}
+{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{% for node_type, node_details in node_data.items() %}
+{%   for hostname in node_data[node_type].keys() %}
+{%     do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %}
+{%   endfor %}
+{% endfor %}
+
 {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
-    {% if ES_LOGSTASH_NODES | length > 1 %}
+    {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
       {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
-      {% for NODE in ES_LOGSTASH_NODES %}
+      {% for NODE in ELASTICSEARCH_SEED_HOSTS %}
         {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
       {% endfor %}
     {% endif %}

salt/elasticsearch/defaults.yaml

@@ -296,7 +296,7 @@ elasticsearch:
                   limit: 1500
               number_of_replicas: 0
               number_of_shards: 1
-              refresh_interval: 30s
+              refresh_interval: 1s
               sort:
                 field: '@timestamp'
                 order: desc

salt/elasticsearch/enabled.sls

@@ -7,8 +7,8 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
-{%   from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -27,7 +27,7 @@ so-elasticsearch:
       - sobridge:
         - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
     - extra_hosts:
-    {% for node in LOGSTASH_NODES %}
+    {% for node in ELASTICSEARCH_NODES %}
     {%   for hostname, ip in node.items() %}
       - {{hostname}}:{{ip}}
     {%   endfor %}
@@ -38,7 +38,7 @@ so-elasticsearch:
       {% endfor %}
     {% endif %}
     - environment:
-      {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
+      {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true

salt/elasticsearch/template.map.jinja

@@ -1,3 +1,8 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
 
@@ -17,10 +22,26 @@
 {% set ES_INDEX_SETTINGS = {} %}
 {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
-{#   if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #}
-{#   this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%   if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%     do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+
+{#   prevent this action from being performed on custom defined indices. #}
+{#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+
+{#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
+{#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     endif %}
+
+{#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%         endif %}
+{%       endfor %}
+{%     endif %}
+
 {%   endif %}
 
 {%   if settings.index_template is defined %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 {%- from 'vars/globals.map.jinja' import GLOBALS %}
-{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 
 . /usr/sbin/so-common
 

salt/idstools/tools/sbin_jinja/so-rule-update

@@ -11,17 +11,17 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 {%- set proxy = salt['pillar.get']('manager:proxy') %}
 {%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %}
 
-# Download the rules from the internet
 {%- if proxy %}
+# Download the rules from the internet
     export http_proxy={{ proxy }}
     export https_proxy={{ proxy }}
     export no_proxy="{{ noproxy }}"
 {%- endif %}
 
     mkdir -p /nsm/rules/suricata
     chown -R socore:socore /nsm/rules/suricata
+{%- if not GLOBALS.airgap %}
 # Download the rules from the internet
-{%- if GLOBALS.airgap != 'True' %}
 {%-   if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %}
     docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force
 {%-   elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %}