Initial version of periodical anomaly detection scripts.

I have added prototype periodical anomaly detection scripts created by Pavel Kácha into Mentat package. So far these are just simple shell scripts intended to be executed periodically via cron that will simply use the PSQL utility to query database and send the result via email. Possible improvements might be to envelope these into Mentat script to make use of common configurations. (Redmine issue: #4222)

etc/default/mentat

@@ -9,3 +9,4 @@
 MENTAT_IS_ENABLED=yes
 MENTAT_USER=mentat
 MENTAT_GROUP=mentat
+MENTAT_CHECKS_MAIL_TO=(root)

scripts/mentat-check-alive.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Example utility script for Mentat database and event data sanity check:
+#	Detectors that are dead over 2 days (but seen last week)
+#
+# Author: Pavel Kácha <ph@cesnet.cz>
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+. /etc/default/mentat
+cd /
+
+#sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF
+sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF | mail -s 'Mentat: Detectors dead over 2 days (but seen last week)' ${MENTAT_CHECKS_MAIL_TO[@]}
+SET timezone TO 'utc';
+SELECT
+	node_name as "Node Name",
+	max(cesnet_storagetime) as "Storage Time"
+FROM
+	events
+WHERE
+	cesnet_storagetime > LOCALTIMESTAMP - INTERVAL '7 day'
+GROUP BY
+	node_name
+HAVING
+	MAX(cesnet_storagetime) < LOCALTIMESTAMP - INTERVAL '2 day';
+EOF

scripts/mentat-check-inspectionerrors.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Example utility script for Mentat database and event data sanity check:
+#	IDEA message inspection errors
+#
+# Author: Pavel Kácha <ph@cesnet.cz>
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+. /etc/default/mentat
+cd /
+
+#sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF
+sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF | mail -s 'Mentat: IDEA inspection errors' ${MENTAT_CHECKS_MAIL_TO[@]}
+SET timezone TO 'utc';
+SELECT
+	node_name AS "Node Name",
+	cesnet_inspectionerrors AS "Inspection Errors",
+	'https://mentat-hub.cesnet.cz/mentat/events/show/' || max(id) AS "Example event"
+FROM
+	events
+WHERE
+	cesnet_inspectionerrors!='{}'
+	AND cesnet_storagetime > localtimestamp - INTERVAL '1 day'
+GROUP BY
+	node_name, cesnet_inspectionerrors
+ORDER BY
+	node_name;
+EOF

scripts/mentat-check-no-eventclass.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Example utility script for Mentat database and event data sanity check:
+#	Events that did not fit any of our classes
+#
+# Author: Pavel Kácha <ph@cesnet.cz>
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+. /etc/default/mentat
+cd /
+
+#sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF
+sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF | mail -s 'Mentat: Events do not fit any of our classes' ${MENTAT_CHECKS_MAIL_TO[@]}
+SET timezone TO 'utc';
+SELECT
+	node_name AS "Node Name",
+	'https://mentat-hub.cesnet.cz/mentat/events/show/' || max(id) AS "Example event",
+	COUNT(*) as Count
+FROM
+	events
+WHERE
+	(cesnet_eventclass IS NULL OR cesnet_eventclass='')
+	AND cesnet_storagetime > localtimestamp - INTERVAL '1 day'
+GROUP BY
+	node_name
+ORDER BY
+	node_name;
+EOF

scripts/mentat-check-test.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Example utility script for Mentat database and event data sanity check:
+#	Clients still sending messages with Test category
+#
+# Author: Pavel Kácha <ph@cesnet.cz>
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+. /etc/default/mentat
+cd /
+
+#sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF
+sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF | mail -s 'Mentat: Detectors still sending Test' ${MENTAT_CHECKS_MAIL_TO[@]}
+SET timezone TO 'utc';
+SELECT
+	node_name AS "Node Name",
+	max(category) AS "Category",
+	'https://mentat-hub.cesnet.cz/mentat/events/show/' || max(id) AS "Example event"
+FROM
+	events
+WHERE
+	'Test' = ANY(category)
+GROUP BY
+	node_name
+ORDER BY
+	node_name;
+EOF

scripts/mentat-check-volatile-description.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Example utility script for Mentat database and event data sanity check:
+#	Clients sending non static Descriptions (dynamic text like IPs should go to Note)
+#
+# Author: Pavel Kácha <ph@cesnet.cz>
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+. /etc/default/mentat
+cd /
+
+#sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF
+sudo --user=postgres psql --dbname=mentat_events --expanded <<EOF | mail -s 'Mentat: Detectors sending non static Descriptions (dynamic text like IPs should go to Note)' ${MENTAT_CHECKS_MAIL_TO[@]}
+SET timezone TO 'utc';
+SELECT
+	node_name AS "Node Name",
+	category AS "Category",
+	COUNT(*) AS "Count",
+	MAX(description) as "Example Description",
+	'https://mentat-hub.cesnet.cz/mentat/events/show/' || MAX(id) AS "Example event"
+FROM (
+	SELECT
+		node_name, category, description, MAX(id) as id
+	FROM
+		events
+	WHERE
+		cesnet_storagetime > localtimestamp - INTERVAL '1 day'
+	GROUP BY
+		node_name, category, description
+) AS subquery
+GROUP BY
+	node_name, category
+HAVING
+	COUNT(*) > 5
+ORDER BY
+	node_name, category
+EOF