Skip to content

Commit

Permalink
Merge pull request #12513 from Security-Onion-Solutions/newsuripcap
Browse files Browse the repository at this point in the history 
Change Factoring for so-minion pcap disk space
  • Loading branch information
@TOoSmOotH
TOoSmOotH committed Mar 7, 2024
2 parents 8f36a8a + 70f3ce0 commit b5f1733
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 75 deletions.
24 changes: 14 additions & 10 deletions salt/manager/tools/sbin/so-minion
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ function getinstallinfo() {

function pcapspace() {
if [[ "$OPERATION" == "setup" ]]; then
# Use 25% for PCAP
PCAP_PERCENTAGE=1
DFREEPERCENT=75
local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n)
else

Expand All @@ -96,10 +99,9 @@ function pcapspace() {
fi

local s=$(( $SPACESIZE / 1000000 ))
local s1=$(( $s / 4 ))
local s2=$(( $s1 / $CORECOUNT ))
local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))

MAXPCAPFILES=$s2
MAX_PCAP_SPACE=$s1

}

Expand Down Expand Up @@ -268,20 +270,18 @@ function add_sensor_to_minion() {
echo " lb_procs: '$CORECOUNT'" >> $PILLARFILE
echo "suricata:" >> $PILLARFILE
echo " enabled: True " >> $PILLARFILE
if [[ $is_pcaplimit ]]; then
echo " pcap:" >> $PILLARFILE
echo " maxsize: $MAX_PCAP_SPACE" >> $PILLARFILE
fi
echo " config:" >> $PILLARFILE
echo " af-packet:" >> $PILLARFILE
echo " threads: '$CORECOUNT'" >> $PILLARFILE
if [[ $is_pcaplimit ]]; then
echo " output:" >> $PILLARFILE
echo " pcap-log:" >> $PILLARFILE
echo " max-files: '$MAXPCAPFILES'" >> $PILLARFILE
fi
echo "pcap:" >> $PILLARFILE
echo " enabled: True" >> $PILLARFILE
if [[ $is_pcaplimit ]]; then
echo " config:" >> $PILLARFILE
echo " diskfreepercentage: 75" >> $PILLARFILE
pcapspace
echo " diskfreepercentage: $DFREEPERCENT" >> $PILLARFILE
fi
echo " " >> $PILLARFILE
}
Expand Down Expand Up @@ -563,6 +563,8 @@ function createIDH() {

function createHEAVYNODE() {
is_pcaplimit=true
PCAP_PERCENTAGE=1
DFREEPERCENT=75
pcapspace
add_elasticsearch_to_minion
add_elastic_agent_to_minion
Expand All @@ -575,6 +577,8 @@ function createHEAVYNODE() {

function createSENSOR() {
is_pcaplimit=true
DFREEPERCENT=10
PCAP_PERCENTAGE=3
pcapspace
add_sensor_to_minion
add_strelka_to_minion
Expand Down
21 changes: 11 additions & 10 deletions salt/suricata/defaults.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
suricata:
enabled: False
pcap:
filesize: 1000mb
maxsize: 25
compression: "none"
lz4-checksum: "no"
lz4-level: 8
filename: "%n/so-pcap.%t"
mode: "multi"
use-stream-depth: "no"
conditional: "all"
dir: "/nsm/suripcap"
config:
threading:
set-cpu-affinity: "no"
Expand Down Expand Up @@ -128,16 +139,6 @@ suricata:
enabled: "no"
pcap-log:
enabled: "no"
compression: "none"
lz4-checksum: "no"
lz4-level: 8
filename: "%n/so-pcap.%t"
limit: "1000mb"
mode: "multi"
max-files: 10
use-stream-depth: "no"
conditional: "all"
dir: "/nsm/suripcap"
alert-debug:
enabled: "no"
alert-prelude:
Expand Down
23 changes: 18 additions & 5 deletions salt/suricata/map.jinja
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,24 @@
{% set surimeta_evelog_index = [] %}
{% set surimeta_filestore_index = [] %}

{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
{% if GLOBALS.pcap_engine == "SURICATA" %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
{# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %}
{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #}
{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %}
{% endif %}

{# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #}
{# we are limited to only one iterface #}
{% load_yaml as afpacket %}
Expand Down Expand Up @@ -60,11 +78,6 @@
{% do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
{% endif %}

{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
{% if GLOBALS.pcap_engine == "SURICATA" %}
{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
{% endif %}

{# outputs is a list but we convert to dict in defaults to work with ui #}
{# below they are converted back to lists #}
{% load_yaml as outputs %}
Expand Down
98 changes: 50 additions & 48 deletions salt/suricata/soc_suricata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,55 @@ suricata:
multiline: True
title: Classifications
helpLink: suricata.html
pcap:
filesize:
description: Max file size for individual PCAP files written by Suricata. Increasing this number could improve write performance at the expense of pcap retrieval times.
advanced: True
helplink: suricata.html
maxsize:
description: Size in GB for total usage size of PCAP on disk.
helplink: suricata.html
compression:
description: Enable compression of Suricata PCAP. Currently unsupported
advanced: True
readonly: True
helpLink: suricata.html
lz4-checksum:
description: Enable PCAP lz4 checksum. Currently unsupported
advanced: True
readonly: True
helpLink: suricata.html
lz4-level:
description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported
advanced: True
readonly: True
helpLink: suricata.html
filename:
description: Filename output for Suricata PCAP.
advanced: True
readonly: True
helpLink: suricata.html
mode:
description: Suricata PCAP mode. Currently only multi is supported.
advanced: True
readonly: True
helpLink: suricata.html
use-stream-depth:
description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth.
advanced: True
regex: ^(yes|no)$
regexFailureMessage: You must enter either yes or no.
helpLink: suricata.html
conditional:
description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules.
regex: ^(all|alert|tag)$
regexFailureMessage: You must enter either all, alert or tag.
helpLink: suricata.html
dir:
description: Parent directory to store PCAP.
advanced: True
readonly: True
helpLink: suricata.html
config:
af-packet:
interface:
Expand Down Expand Up @@ -167,54 +216,7 @@ suricata:
description: This value is ignored by SO. pcapengine in globals takes precidence.
readonly: True
helpLink: suricata.html
advanced: True
compression:
description: Enable compression of Suricata PCAP. Currently unsupported
advanced: True
readonly: True
helpLink: suricata.html
lz4-checksum:
description: Enable PCAP lz4 checksum. Currently unsupported
advanced: True
readonly: True
helpLink: suricata.html
lz4-level:
description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported
advanced: True
readonly: True
helpLink: suricata.html
filename:
description: Filename output for Suricata PCAP.
advanced: True
readonly: True
helpLink: suricata.html
limit:
description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit.
helpLink: suricata.html
mode:
description: Suricata PCAP mode. Currently only multi is supported.
advanced: True
readonly: True
helpLink: suricata.html
max-files:
description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit.
helpLink: suricata.html
use-stream-depth:
description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth.
advanced: True
regex: ^(yes|no)$
regexFailureMessage: You must enter either yes or no.
helpLink: suricata.html
conditional:
description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules.
regex: ^(all|alert|tag)$
regexFailureMessage: You must enter either all, alert or tag.
helpLink: suricata.html
dir:
description: Parent directory to store PCAP.
advanced: True
readonly: True
helpLink: suricata.html
advanced: True
asn1-max-frames:
description: Maximum nuber of asn1 frames to decode.
helpLink: suricata.html
Expand Down
4 changes: 2 additions & 2 deletions setup/so-functions
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1811,8 +1811,8 @@ repo_sync_local() {
info "Adding Repo Download Configuration"
mkdir -p /nsm/repo
mkdir -p /opt/so/conf/reposync/cache
echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9.3" > /opt/so/conf/reposync/mirror.txt
echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9.3" >> /opt/so/conf/reposync/mirror.txt
echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt
echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9" >> /opt/so/conf/reposync/mirror.txt
echo "[main]" > /opt/so/conf/reposync/repodownload.conf
echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf
Expand Down

0 comments on commit b5f1733

Please sign in to comment.