Skip to content
Snippets Groups Projects
Commit 6ae1cdac authored by Rajmund Hruška's avatar Rajmund Hruška
Browse files

Feature: Merge inspector configs into one. (Redmine issue: #6949)

parent ac5e2122
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
conf/mentat-controller.py.conf
+ 1
− 19
View file @ 6ae1cdac
......@@ -52,25 +52,7 @@
},
#
# [CHAIN A]: Message inspection module - event validations
#
{
"name": "mentat-inspector-b.py",
"exec": "mentat-inspector.py",
# Enable multiple instances working the same queue directory
#"paralel": true,
# In case of paralel mode, you MUST set the required number of instances
#"count": 3,
"args": [
# Enable debug information before daemonization
#"--debug"
# Force logging level ['debug', 'info', 'warning', 'error', 'critical']
#"--log-level=debug"
]
},
#
# [CHAIN A|ENTRY]: Message inspection module - event classifications
# [CHAIN A|ENTRY]: Message inspection module - event classifications and validations
#
{
"exec": "mentat-inspector.py",
......
conf/mentat-inspector-b.py.conf deleted 100644 → 0
+ 0
− 196
View file @ ac5e2122
#-------------------------------------------------------------------------------
#
# CONFIGURATION FILE FOR MENTAT-INSPECTOR-B.PY MODULE
#
# Event validation module.
#
# This configuration attempts to perform event validation and basic attribute
# sanity checks.
#
#-------------------------------------------------------------------------------
{
#---------------------------------------------------------------------------
# Custom daemon configurations
#---------------------------------------------------------------------------
# List of the inspection rules. The rules will be processed in
# following order.
"inspection_rules": [
{
"name": "Check: EventTime > DetectTime",
"rule": "exists EventTime and exists DetectTime and EventTime > DetectTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "EventTime_gt_DetectTime", "unique": true}}
]
},
{
"name": "Check: CeaseTime > DetectTime",
"rule": "exists CeaseTime and exists DetectTime and CeaseTime > DetectTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "CeaseTime_gt_DetectTime", "unique": true}}
]
},
{
"name": "Check: WinStartTime > DetectTime",
"rule": "exists WinStartTime and exists DetectTime and WinStartTime > DetectTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "WinStartTime_gt_DetectTime", "unique": true}}
]
},
{
"name": "Check: DetectTime > CreateTime",
"rule": "exists DetectTime and exists CreateTime and DetectTime > CreateTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "DetectTime_gt_CreateTime", "unique": true}}
]
},
{
"name": "Check: EventTime > CeaseTime",
"rule": "exists EventTime and exists CeaseTime and EventTime > CeaseTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "EventTime_gt_CeaseTime", "unique": true}}
]
},
{
"name": "Check: WinStartTime > WinEndTime",
"rule": "exists WinStartTime and exists WinEndTime and WinStartTime > WinEndTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "WinStartTime_gt_WinEndTime", "unique": true}}
]
},
{
"name": "Check: WinStartTime > EventTime",
"rule": "exists WinStartTime and exists EventTime and WinStartTime > EventTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "WinStartTime_gt_EventTime", "unique": true}}
]
},
{
"name": "Check: Source port and TCP/UDP",
"rule": "exists Source.Port and not (Source.Proto in ['tcp', 'udp'])",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Used_Source_Port_and_missing_Proto", "unique": true}}
]
},
{
"name": "Check: Target port and TCP/UDP",
"rule": "exists Target.Port and not (Target.Proto in ['tcp', 'udp'])",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Used_Target_Port_and_missing_Proto", "unique": true}}
]
},
{
"name": "Check: DetectTime too old",
"rule": "DetectTime < (utcnow() - 3D00:00:00)",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "DetectTime_too_old", "unique": true}}
]
},
{
"name": "Check: DetectTime in the future",
"rule": "DetectTime > (utcnow() + 01:00:00)",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "DetectTime_in_the_future", "unique": true}}
]
},
{
"name": "Check: Category Test only",
"rule": "Category is ['Test']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Category_Test_only", "unique": true}}
]
},
{
"name": "Check: Category Unknown",
"rule": "not Category in ['Abusive', 'Abusive.Spam', 'Abusive.Harassment', 'Abusive.Child', 'Abusive.Sexual', 'Abusive.Violence', 'Malware', 'Malware.Virus', 'Malware.Worm', 'Malware.Trojan', 'Malware.Spyware', 'Malware.Dialer', 'Malware.Rootkit', 'Recon', 'Recon.Scanning', 'Recon.Sniffing', 'Recon.SocialEngineering', 'Recon.Searching', 'Attempt', 'Attempt.Exploit', 'Attempt.Login', 'Attempt.NewSignature', 'Intrusion', 'Intrusion.AdminCompromise', 'Intrusion.UserCompromise', 'Intrusion.AppCompromise', 'Intrusion.Botnet', 'Availability', 'Availability.DoS', 'Availability.DDoS', 'Availability.Sabotage', 'Availability.Outage', 'Information', 'Information.UnauthorizedAccess', 'Information.UnauthorizedModification', 'Fraud', 'Fraud.UnauthorizedUsage', 'Fraud.Copyright', 'Fraud.Masquerade', 'Fraud.Phishing', 'Fraud.Scam', 'Vulnerable', 'Vulnerable.Open', 'Vulnerable.Config', 'Anomaly', 'Anomaly.Traffic', 'Anomaly.Connection', 'Anomaly.Protocol', 'Anomaly.System', 'Anomaly.Application', 'Anomaly.Behaviour', 'Other', 'Test']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Category_unknown", "unique": true}}
]
},
{
"name": "Check: Source Type Unknown",
"rule": "exists Source.Type and not Source.Type in ['Proxy', 'OriginMalware', 'OriginSandbox', 'OriginSpam', 'OriginBlacklist', 'Phishing', 'Malware', 'MITM', 'Spam', 'Backscatter', 'Open', 'Poisoned', 'FastFlux', 'Botnet', 'CC', 'Tor', 'Incomplete', 'Anonymised']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Source_Type_unknown", "unique": true}}
]
},
{
"name": "Check: Target Type Unknown",
"rule": "exists Target.Type and not Target.Type in ['Proxy', 'OriginMalware', 'OriginSandbox', 'OriginSpam', 'Phishing', 'Malware', 'MITM', 'Spam', 'Backscatter', 'Open', 'Poisoned', 'FastFlux', 'Botnet', 'CC', 'Tor', 'Incomplete', 'Anonymised']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Target_Type_unknown", "unique": true}}
]
},
{
"name": "Check: Node Type Unknown",
"rule": "exists Node.Type and not Node.Tag in ['Connection', 'Datagram', 'Content', 'Data', 'File', 'Flow', 'Log', 'Protocol', 'Host', 'Network', 'Correlation', 'External', 'Reporting', 'Blackhole', 'Signature', 'Statistical', 'Heuristic', 'Integrity', 'Policy', 'Honeypot', 'Tarpit', 'Recon', 'Monitor']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Node_Type_unknown", "unique": true}}
]
},
{
"name": "Check: ID suspiciously short",
"rule": "strlen(ID) < 8",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "ID_too_short", "unique": true}}
]
},
{
"name": "Check: Description suspiciously short or missing",
"rule": "strlen(Description) < 8",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Description_short_or_missing", "unique": true}}
]
}
],
# List of fallback actions, that will be performed with given message
# in case it does NOT match any of the inspection rules.
"fallback_actions": [],
#---------------------------------------------------------------------------
# Common piper daemon configurations
#---------------------------------------------------------------------------
#"queue_in_dir": "mentat-inspector-b.py",
"queue_out_dir": "mentat-enricher.py",
#"queue_in_wait": 3,
#"queue_out_wait": 10,
#"queue_out_limit": 5000,
#---------------------------------------------------------------------------
# Common daemon configurations
#---------------------------------------------------------------------------
#"no_daemon": false,
#"chroot-dir": null,
#"work_dir": "/",
#"pid_file": "/var/mentat/run/mentat-inspector-b.py.pid",
#"state_file": "/var/mentat/run/mentat-inspector-b.py.state",
#"umask": "0o002",
#"stats_interval": 300,
#"paralel": false,
#---------------------------------------------------------------------------
# Common application configurations
#---------------------------------------------------------------------------
#"quiet": false,
#"verbosity": 0,
#"log_file": "/var/mentat/log/mentat-inspector-b.py.log",
#"log_level": "info",
#"runlog_dir": "/var/mentat/run/mentat-inspector-b.py",
#"runlog_dump": false,
#"runlog_log": false,
#"pstate_file": "/var/mentat/run/mentat-inspector-b.py.pstate",
#"pstate_dump": false,
#"pstate_log": false,
#"action": null,
#"user": "mentat",
#"group": "mentat",
# This is a dummy last configuration so that the user does not have to fix
# the commas in the whole configuration file after each change.
"_dummy_": "_dummy_"
}
conf/mentat-inspector.py.conf
+ 128
− 6
View file @ 6ae1cdac
......@@ -201,23 +201,145 @@
{"action": "tag", "args": {"path": "_Mentat.EventClass", "value": "recon-scanning", "overwrite": false} },
{"action": "tag", "args": {"path": "_Mentat.EventSeverity", "value": "low", "overwrite": false} }
]
},
{
"name": "Check: EventTime > DetectTime",
"rule": "exists EventTime and exists DetectTime and EventTime > DetectTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "EventTime_gt_DetectTime", "unique": true}}
]
},
{
"name": "Check: CeaseTime > DetectTime",
"rule": "exists CeaseTime and exists DetectTime and CeaseTime > DetectTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "CeaseTime_gt_DetectTime", "unique": true}}
]
},
{
"name": "Check: WinStartTime > DetectTime",
"rule": "exists WinStartTime and exists DetectTime and WinStartTime > DetectTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "WinStartTime_gt_DetectTime", "unique": true}}
]
},
{
"name": "Check: DetectTime > CreateTime",
"rule": "exists DetectTime and exists CreateTime and DetectTime > CreateTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "DetectTime_gt_CreateTime", "unique": true}}
]
},
{
"name": "Check: EventTime > CeaseTime",
"rule": "exists EventTime and exists CeaseTime and EventTime > CeaseTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "EventTime_gt_CeaseTime", "unique": true}}
]
},
{
"name": "Check: WinStartTime > WinEndTime",
"rule": "exists WinStartTime and exists WinEndTime and WinStartTime > WinEndTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "WinStartTime_gt_WinEndTime", "unique": true}}
]
},
{
"name": "Check: WinStartTime > EventTime",
"rule": "exists WinStartTime and exists EventTime and WinStartTime > EventTime",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "WinStartTime_gt_EventTime", "unique": true}}
]
},
{
"name": "Check: Source port and TCP/UDP",
"rule": "exists Source.Port and not (Source.Proto in ['tcp', 'udp'])",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Used_Source_Port_and_missing_Proto", "unique": true}}
]
},
{
"name": "Check: Target port and TCP/UDP",
"rule": "exists Target.Port and not (Target.Proto in ['tcp', 'udp'])",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Used_Target_Port_and_missing_Proto", "unique": true}}
]
},
{
"name": "Check: DetectTime too old",
"rule": "DetectTime < (utcnow() - 3D00:00:00)",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "DetectTime_too_old", "unique": true}}
]
},
{
"name": "Check: DetectTime in the future",
"rule": "DetectTime > (utcnow() + 01:00:00)",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "DetectTime_in_the_future", "unique": true}}
]
},
{
"name": "Check: Category Test only",
"rule": "Category is ['Test']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Category_Test_only", "unique": true}}
]
},
{
"name": "Check: Category Unknown",
"rule": "not Category in ['Abusive', 'Abusive.Spam', 'Abusive.Harassment', 'Abusive.Child', 'Abusive.Sexual', 'Abusive.Violence', 'Malware', 'Malware.Virus', 'Malware.Worm', 'Malware.Trojan', 'Malware.Spyware', 'Malware.Dialer', 'Malware.Rootkit', 'Recon', 'Recon.Scanning', 'Recon.Sniffing', 'Recon.SocialEngineering', 'Recon.Searching', 'Attempt', 'Attempt.Exploit', 'Attempt.Login', 'Attempt.NewSignature', 'Intrusion', 'Intrusion.AdminCompromise', 'Intrusion.UserCompromise', 'Intrusion.AppCompromise', 'Intrusion.Botnet', 'Availability', 'Availability.DoS', 'Availability.DDoS', 'Availability.Sabotage', 'Availability.Outage', 'Information', 'Information.UnauthorizedAccess', 'Information.UnauthorizedModification', 'Fraud', 'Fraud.UnauthorizedUsage', 'Fraud.Copyright', 'Fraud.Masquerade', 'Fraud.Phishing', 'Fraud.Scam', 'Vulnerable', 'Vulnerable.Open', 'Vulnerable.Config', 'Anomaly', 'Anomaly.Traffic', 'Anomaly.Connection', 'Anomaly.Protocol', 'Anomaly.System', 'Anomaly.Application', 'Anomaly.Behaviour', 'Other', 'Test']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Category_unknown", "unique": true}}
]
},
{
"name": "Check: Source Type Unknown",
"rule": "exists Source.Type and not Source.Type in ['Proxy', 'OriginMalware', 'OriginSandbox', 'OriginSpam', 'OriginBlacklist', 'Phishing', 'Malware', 'MITM', 'Spam', 'Backscatter', 'Open', 'Poisoned', 'FastFlux', 'Botnet', 'CC', 'Tor', 'Incomplete', 'Anonymised']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Source_Type_unknown", "unique": true}}
]
},
{
"name": "Check: Target Type Unknown",
"rule": "exists Target.Type and not Target.Type in ['Proxy', 'OriginMalware', 'OriginSandbox', 'OriginSpam', 'Phishing', 'Malware', 'MITM', 'Spam', 'Backscatter', 'Open', 'Poisoned', 'FastFlux', 'Botnet', 'CC', 'Tor', 'Incomplete', 'Anonymised']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Target_Type_unknown", "unique": true}}
]
},
{
"name": "Check: Node Type Unknown",
"rule": "exists Node.Type and not Node.Tag in ['Connection', 'Datagram', 'Content', 'Data', 'File', 'Flow', 'Log', 'Protocol', 'Host', 'Network', 'Correlation', 'External', 'Reporting', 'Blackhole', 'Signature', 'Statistical', 'Heuristic', 'Integrity', 'Policy', 'Honeypot', 'Tarpit', 'Recon', 'Monitor']",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Node_Type_unknown", "unique": true}}
]
},
{
"name": "Check: ID suspiciously short",
"rule": "strlen(ID) < 8",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "ID_too_short", "unique": true}}
]
},
{
"name": "Check: Description suspiciously short or missing",
"rule": "strlen(Description) < 8",
"actions": [
{"action": "tag", "args": {"path": "_Mentat.InspectionErrors[*]", "value": "Description_short_or_missing", "unique": true}}
]
}
],
# List of fallback actions, that will be performed with given message
# in case it does NOT match any of the inspection rules.
"fallback_actions": [
{"action": "log", "args": {"label":"Fallback rule match, event did not match any of the classification rules"}}
],
"fallback_actions": [],
#---------------------------------------------------------------------------
# Common piper daemon configurations
#---------------------------------------------------------------------------
#"queue_in_dir": "mentat-inspector.py",
"queue_out_dir": "mentat-inspector-b.py",
"queue_out_dir": "mentat-enricher.py",
#"queue_in_wait": 3,
#"queue_out_wait": 10,
#"queue_out_limit": 5000,
......
doc/sphinx/_doclib/administration.rst
+ 2
− 7
View file @ 6ae1cdac
......@@ -201,8 +201,8 @@ To enable these scripts please configure them to be launched periodically via
going suddenly offline.
``/etc/mentat/scripts/mentat-check-inspectionerrors.sh``
Query the IDEA event database and detect list of all inspection errors along
with example events. One of the :ref:`section-bin-mentat-inspector` modules
is by default configured to perform event sanity inspection and logs errors
with example events. The :ref:`section-bin-mentat-inspector` module
is by default configured to perform event sanity inspection and logging errors
it finds directly into the event. This script can provide summary of all
current inspection errors, so you can go and fix malfunctioning detectors.
``/etc/mentat/scripts/mentat-check-noeventclass.sh``
......@@ -262,7 +262,6 @@ of Mentat modules:
2018-09-26 13:31:17,981 INFO: Status of configured Mentat real-time modules:
2018-09-26 13:31:17,981 INFO: Real-time module 'mentat-storage.py': 'Process is running or service is OK (1)'
2018-09-26 13:31:17,981 INFO: Real-time module 'mentat-enricher.py': 'Process is running or service is OK (1)'
2018-09-26 13:31:17,982 INFO: Real-time module 'mentat-inspector-b.py': 'Process is running or service is OK (1)'
2018-09-26 13:31:17,982 INFO: Real-time module 'mentat-inspector.py': 'Process is running or service is OK (1)'
2018-09-26 13:31:17,982 INFO: Overall real-time module status: 'All modules are running OK'
2018-09-26 13:31:17,982 INFO: Status of configured Mentat cronjob modules:
......@@ -304,10 +303,6 @@ the following as your NRPE configuration:
command[check_mentat_inspector_a_pending_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-inspector.py/pending -w 100 -c 1000
command[check_mentat_inspector_a_incoming_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-inspector.py/incoming -w 5000 -c 10000
command[check_mentat_inspector_b_errors_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-inspector-b.py/errors -w 100 -c 1000
command[check_mentat_inspector_b_pending_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-inspector-b.py/pending -w 100 -c 1000
command[check_mentat_inspector_b_incoming_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-inspector-b.py/incoming -w 5000 -c 10000
command[check_mentat_enricher_errors_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-enricher.py/errors -w 100 -c 1000
command[check_mentat_enricher_pending_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-enricher.py/pending -w 100 -c 1000
command[check_mentat_enricher_incoming_dir]=/usr/lib/nagios/plugins/check_file_count -d /var/mentat/spool/mentat-enricher.py/incoming -w 5000 -c 10000
......
doc/sphinx/_doclib/migration.rst
+ 0
− 2
View file @ 6ae1cdac
......@@ -182,11 +182,9 @@ sure everything is in order.
# processing modules by inspecting the log files. Look for obvious
# errors and warnings.
tail -f /var/mentat/log/mentat-inspector.py.log
tail -f /var/mentat/log/mentat-inspector-b.py.log
tail -f /var/mentat/log/mentat-enricher.py.log
tail -f /var/mentat/log/mentat-storage.py.log
grep ERROR /var/mentat/log/mentat-inspector.py.log
grep ERROR /var/mentat/log/mentat-inspector-b.py.log
grep ERROR /var/mentat/log/mentat-enricher.py.log
grep ERROR /var/mentat/log/mentat-storage.py.log
......
lib/mentat/module/controller.py
+ 1
− 1
View file @ 6ae1cdac
......@@ -60,7 +60,7 @@ Usage examples
# Work with particular modules.
mentat-controller.py --command start --target mentat-storage.py
mentat-controller.py --command stop --target mentat-enricher.py mentat-inspector.py
mentat-controller.py --command signal-usr1 --target mentat-inspector-b.py
mentat-controller.py --command signal-usr1 --target mentat-inspector.py
Available script commands
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment