Change Factoring for so-minion pcap disk space

salt/manager/tools/sbin/so-minion

@@ -81,6 +81,9 @@ function getinstallinfo() {
 
 function pcapspace() {
 	if [[ "$OPERATION" == "setup" ]]; then
+		# Use 25% for PCAP
+		PCAP_PERCENTAGE=1
+		DFREEPERCENT=75
 		local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n)
 	else
 
@@ -96,10 +99,9 @@ function pcapspace() {
 	fi
 
 	local s=$(( $SPACESIZE / 1000000 ))
-	local s1=$(( $s / 4 ))
-	local s2=$(( $s1 / $CORECOUNT ))
+	local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
 
-	MAXPCAPFILES=$s2
+	MAX_PCAP_SPACE=$s1
 
 }
 
@@ -268,20 +270,18 @@ function add_sensor_to_minion() {
 	echo "      lb_procs: '$CORECOUNT'" >> $PILLARFILE
 	echo "suricata:" >> $PILLARFILE
 	echo "  enabled: True " >> $PILLARFILE
+	if [[ $is_pcaplimit ]]; then
+	echo "  pcap:" >> $PILLARFILE
+	echo "    maxsize: $MAX_PCAP_SPACE" >> $PILLARFILE
+	fi
 	echo "  config:" >> $PILLARFILE
 	echo "    af-packet:" >> $PILLARFILE
 	echo "      threads: '$CORECOUNT'" >> $PILLARFILE
-	if [[ $is_pcaplimit ]]; then
-		echo "    output:" >> $PILLARFILE
-		echo "      pcap-log:" >> $PILLARFILE
-		echo "        max-files: '$MAXPCAPFILES'" >> $PILLARFILE
-	fi
 	echo "pcap:" >> $PILLARFILE
 	echo "  enabled: True" >> $PILLARFILE
 	if [[ $is_pcaplimit ]]; then
 		echo "  config:" >> $PILLARFILE
-		echo "    diskfreepercentage: 75" >> $PILLARFILE
-		pcapspace
+		echo "    diskfreepercentage: $DFREEPERCENT" >> $PILLARFILE
 	fi
 	echo "  " >> $PILLARFILE
 }
@@ -563,6 +563,8 @@ function createIDH() {
 
 function createHEAVYNODE() {
 	is_pcaplimit=true
+	PCAP_PERCENTAGE=1
+	DFREEPERCENT=75
 	pcapspace
 	add_elasticsearch_to_minion
 	add_elastic_agent_to_minion
@@ -575,6 +577,8 @@ function createHEAVYNODE() {
 
 function createSENSOR() {
 	is_pcaplimit=true
+	DFREEPERCENT=10
+	PCAP_PERCENTAGE=3
 	pcapspace
 	add_sensor_to_minion
 	add_strelka_to_minion

salt/suricata/defaults.yaml

@@ -1,5 +1,16 @@
 suricata:
   enabled: False
+  pcap:
+    filesize: 1000mb
+    maxsize: 25
+    compression: "none"
+    lz4-checksum: "no"
+    lz4-level: 8
+    filename: "%n/so-pcap.%t"
+    mode: "multi"
+    use-stream-depth: "no"
+    conditional: "all"
+    dir: "/nsm/suripcap"
   config:
     threading:
       set-cpu-affinity: "no"
@@ -128,16 +139,6 @@ suricata:
         enabled: "no"
       pcap-log:
         enabled: "no"
-        compression: "none"
-        lz4-checksum: "no"
-        lz4-level: 8
-        filename: "%n/so-pcap.%t"
-        limit: "1000mb"
-        mode: "multi"
-        max-files: 10
-        use-stream-depth: "no"
-        conditional: "all"
-        dir: "/nsm/suripcap"
       alert-debug:
         enabled: "no"
       alert-prelude:

salt/suricata/map.jinja

@@ -8,6 +8,24 @@
 {% set surimeta_evelog_index = [] %}
 {% set surimeta_filestore_index = [] %}
 
+{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
+{% if GLOBALS.pcap_engine == "SURICATA" %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
+{#   move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %}
+{#   multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #}
+{%   set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %}
+{% endif %}
+
 {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #}
 {# we are limited to only one iterface #}
 {% load_yaml as afpacket %}
@@ -60,11 +78,6 @@
 {%   do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
 {% endif %}
 
-{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
-{% if GLOBALS.pcap_engine == "SURICATA" %}
-{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
-{% endif %}
-
 {# outputs is a list but we convert to dict in defaults to work with ui #}
 {# below they are converted back to lists #}
 {% load_yaml as outputs %}

salt/suricata/soc_suricata.yaml

@@ -19,6 +19,55 @@ suricata:
       multiline: True
       title: Classifications
       helpLink: suricata.html
+  pcap:
+    filesize: 
+      description: Max file size for individual PCAP files written by Suricata. Increasing this number could improve write performance at the expense of pcap retrieval times.
+      advanced: True
+      helplink: suricata.html 
+    maxsize:
+      description: Size in GB for total usage size of PCAP on disk. 
+      helplink: suricata.html
+    compression: 
+      description: Enable compression of Suricata PCAP. Currently unsupported
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    lz4-checksum: 
+      description: Enable PCAP lz4 checksum. Currently unsupported
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    lz4-level: 
+      description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    filename:
+      description: Filename output for Suricata PCAP.
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    mode: 
+      description: Suricata PCAP mode. Currently only multi is supported.
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    use-stream-depth: 
+      description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. 
+      advanced: True
+      regex: ^(yes|no)$
+      regexFailureMessage: You must enter either yes or no.
+      helpLink: suricata.html
+    conditional: 
+      description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules.
+      regex: ^(all|alert|tag)$
+      regexFailureMessage: You must enter either all, alert or tag.
+      helpLink: suricata.html
+    dir:
+      description: Parent directory to store PCAP.
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
   config:
     af-packet:
       interface:
@@ -167,54 +216,7 @@ suricata:
           description: This value is ignored by SO. pcapengine in globals takes precidence.
           readonly: True
           helpLink: suricata.html
-          advanced: True
-        compression: 
-          description: Enable compression of Suricata PCAP. Currently unsupported
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        lz4-checksum: 
-          description: Enable PCAP lz4 checksum. Currently unsupported
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        lz4-level: 
-          description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        filename:
-          description: Filename output for Suricata PCAP.
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        limit:
-          description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit.
-          helpLink: suricata.html
-        mode: 
-          description: Suricata PCAP mode. Currently only multi is supported.
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        max-files: 
-          description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit.
-          helpLink: suricata.html
-        use-stream-depth: 
-          description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. 
-          advanced: True
-          regex: ^(yes|no)$
-          regexFailureMessage: You must enter either yes or no.
-          helpLink: suricata.html
-        conditional: 
-          description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules.
-          regex: ^(all|alert|tag)$
-          regexFailureMessage: You must enter either all, alert or tag.
-          helpLink: suricata.html
-        dir:
-          description: Parent directory to store PCAP.
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
+          advanced: True        
     asn1-max-frames:
       description: Maximum nuber of asn1 frames to decode.
       helpLink: suricata.html

setup/so-functions

@@ -1811,8 +1811,8 @@ repo_sync_local() {
 		info "Adding Repo Download Configuration"
 		mkdir -p /nsm/repo
 		mkdir -p /opt/so/conf/reposync/cache
-		echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9.3" > /opt/so/conf/reposync/mirror.txt
-		echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9.3" >> /opt/so/conf/reposync/mirror.txt
+		echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt
+		echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9" >> /opt/so/conf/reposync/mirror.txt
 		echo "[main]" > /opt/so/conf/reposync/repodownload.conf
 		echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 		echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf