Merge branch 'devel' into release

conf/requirements-latest-dev.pip

@@ -4,5 +4,6 @@ nose
 coverage
 pyflakes
 pylint
+sendria
 sphinx
 sphinx-rtd-theme

conf/requirements.pip

@@ -6,7 +6,7 @@ babel==2.8.0
 wtforms==2.2.1
 sqlalchemy==1.3.15
 alembic==1.4.2
-jinja2==2.10.3
+jinja2==3.0.3
 blinker==1.4
 bsddb3==6.2.7
 flask==1.1.1
@@ -27,7 +27,7 @@ requests==2.23.0
 rrdtool==0.1.15
 pyyaml==5.3.1
 pydgets==0.9
-pyzenkit==0.59
+pyzenkit==0.61
 pynspect==0.17
 ipranges==0.1.10
 typedcols==0.1.13

conf/scripts/devsnapshot.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+# Check exit status of any command and fail on error
+set -e
+
+# Source global configurations
+source /etc/default/mentat
+source /etc/mentat/scripts/lib.sh
+
+print_title '<BEGIN> CREATING DEVELOPMENT SNAPSHOT'
+
+BACKUP_DATE_FROM=`date --date="8 days ago" --iso-8601=seconds`
+BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/main
+mkdir -p $BACKUP_DIR
+chown mentat:mentat $BACKUP_DIR
+cd $BACKUP_DIR
+print_subtitle 'Dumping main tables from metadata database'
+sudo -u mentat pg_dump --clean --create --if-exists --format=d --jobs=3 --dbname=mentat_main --username=mentat --no-password --exclude-table=reports_events --exclude-table=statistics_events --exclude-table=changelogs_items --file=$BACKUP_DIR
+BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/chunks
+mkdir -p $BACKUP_DIR
+for table_name in reports_events statistics_events changelogs_items
+do
+   print_subtitle "Dumping chunk of table ${table_name} from metadata database"
+   sudo -u mentat psql mentat_main -c "COPY (SELECT * FROM ${table_name} WHERE createtime >= '${BACKUP_DATE_FROM}'::timestamptz) TO stdout" > ${BACKUP_DIR}/${table_name}.dat
+   sudo -u mentat psql mentat_main -c "COPY (SELECT string_agg(data.column_name, ',') FROM (SELECT column_name FROM information_schema.columns WHERE table_name = '${table_name}' ORDER BY ordinal_position) AS data) TO stdout" > ${BACKUP_DIR}/${table_name}.cols
+done
+
+BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/events
+BACKUP_DATE_FROM_EVENTS=`date --date="1 days ago" --iso-8601=seconds`
+mkdir -p $BACKUP_DIR
+chown mentat:mentat $BACKUP_DIR
+cd $BACKUP_DIR
+
+table_name='events'
+print_subtitle "Dumping chunk of table ${table_name} from event database"
+sudo -u mentat psql mentat_events -c "COPY (SELECT * FROM ${table_name} WHERE storagetime >= '${BACKUP_DATE_FROM_EVENTS}'::timestamptz) TO stdout" > ${BACKUP_DIR}/${table_name}.dat
+sudo -u mentat psql mentat_events -c "COPY (SELECT string_agg(data.column_name, ',') FROM (SELECT column_name FROM information_schema.columns WHERE table_name = '${table_name}' ORDER BY ordinal_position) AS data) TO stdout" > ${BACKUP_DIR}/${table_name}.cols
+
+table_name='events_json'
+print_subtitle "Dumping chunk of table ${table_name} from event database"
+sudo -u mentat psql mentat_events -c "COPY (SELECT events_json.* FROM events_json JOIN events ON events_json.id = events.id WHERE events.storagetime >= '${BACKUP_DATE_FROM_EVENTS}'::timestamptz) TO stdout" > ${BACKUP_DIR}/${table_name}.dat
+sudo -u mentat psql mentat_events -c "COPY (SELECT string_agg(data.column_name, ',') FROM (SELECT column_name FROM information_schema.columns WHERE table_name = '${table_name}' ORDER BY ordinal_position) AS data) TO stdout" > ${BACKUP_DIR}/${table_name}.cols
+
+print_subtitle "Size before packaging:"
+du -csh /var/mentat/devsnapshots/devsnapshot
+print_subtitle "Packaging created database snapshots"
+cd /var/mentat
+SNAPSHOT_FILE_NAME="/var/mentat/devsnapshots/devsnapshot-${BACKUP_DATE_FROM}.tar.gz"
+LATEST_SNAPSHOT_FILE_NAME="/var/mentat/devsnapshots/devsnapshot-latest.tar.gz"
+tar -czPf "${SNAPSHOT_FILE_NAME}" /var/mentat/devsnapshots/devsnapshot/
+rm -f "${LATEST_SNAPSHOT_FILE_NAME}"
+ln -s "${SNAPSHOT_FILE_NAME}" "${LATEST_SNAPSHOT_FILE_NAME}"
+rm -rf /var/mentat/devsnapshots/devsnapshot/
+
+print_subtitle "Created DB snapshot file:"
+ls -alh ${SNAPSHOT_FILE_NAME}
+
+find /var/mentat/devsnapshots/ -type f -mtime +7 -name '*.tar.gz' -delete
+
+print_title '<DONE> CREATING DEVELOPMENT SNAPSHOT'

scripts/fetch-negistry.sh → conf/scripts/fetch-negistry.sh

@@ -61,7 +61,7 @@ NC='\033[0m'
 FLAG_DEVEL=NO
 FLAG_QUIET=NO
 FLAG_STUB=NO
-LINK="https://negistry.cesnet.cz/db/api/v1/negistry-ripe-cesnet-ipranges.json"
+LINK="https://negistry.cesnet.cz/db/api/v1/negistry-multifeed-abuses-full-ipranges.json"
 LINK_DEV="https://irgames.cesnet.cz/negistry/db/api/v1/negistry-ripe-cesnet-ipranges.json"
 TARGET="/var/mentat/whois-negistry.json"
 

conf/scripts/init-runenv.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+# Check exit status of any command and fail on error
+set -e
+
+# Source global configurations
+. /etc/default/mentat
+
+# Make sure required directory structure exists and has proper permissions
+mentat_dir_list=(
+    /var/mentat
+    /var/mentat/backups
+    /var/mentat/cache
+    /var/mentat/charts
+    /var/mentat/log
+    /var/mentat/reports
+    /var/mentat/reports/reporter
+    /var/mentat/reports/informant
+    /var/mentat/reports/statistician
+    /var/mentat/rrds
+    /var/mentat/run
+    /var/mentat/spool
+    /var/mentat/spool/mentat-inspector.py
+    /var/mentat/spool/mentat-inspector.py/incoming
+    /var/mentat/spool/mentat-inspector.py/pending
+    /var/mentat/spool/mentat-inspector.py/tmp
+    /var/mentat/spool/mentat-inspector.py/errors
+    /var/mentat/spool/mentat-enricher.py
+    /var/mentat/spool/mentat-enricher.py/incoming
+    /var/mentat/spool/mentat-enricher.py/pending
+    /var/mentat/spool/mentat-enricher.py/tmp
+    /var/mentat/spool/mentat-enricher.py/errors
+    /var/mentat/spool/mentat-storage.py
+    /var/mentat/spool/mentat-storage.py/incoming
+    /var/mentat/spool/mentat-storage.py/pending
+    /var/mentat/spool/mentat-storage.py/tmp
+    /var/mentat/spool/mentat-storage.py/errors
+)
+for d in "${mentat_dir_list[@]}"
+do
+    mkdir -p "$d"
+    chown "$MENTAT_USER" "$d"
+    chgrp "$MENTAT_GROUP" "$d"
+    chmod 775 "$d"
+done

conf/scripts/init-venv.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+# Check exit status of any command and fail on error
+set -e
+
+# Source global configurations
+. /etc/default/mentat
+
+echo -e "\n*** Initializing Python virtual environment for local development ***\n"
+echo "Requested version: $PYTHON_VERSION"
+echo "Path to binary:    `which $PYTHON_VERSION`"
+echo "Path to venv:      $MENTAT_VENV"
+echo ""
+if [ -d $MENTAT_VENV ]; then
+    echo "Virtual environment already exists in '$MENTAT_VENV'.";
+else
+    $PYTHON_VERSION -m venv $MENTAT_VENV;
+    echo "Virtual environment successfully created in '$MENTAT_VENV'.";
+fi
+
+echo -e "\n*** Upgrading 'pip' and 'wheel' within the virtual environment to latest versions ***\n"
+$MENTAT_VENV/bin/pip install --upgrade pip
+$MENTAT_VENV/bin/pip install --upgrade wheel
+echo ""
+echo "Venv path: `. $MENTAT_VENV/bin/activate && python -c 'import sys; print(sys.prefix)'`"
+echo "Python stuff versions:"
+echo ""
+$MENTAT_VENV/bin/python --version
+$MENTAT_VENV/bin/pip --version
+echo ""
+
+echo -e "\nYour development environment is ready in `. $MENTAT_VENV/bin/activate && python -c 'import sys; print(sys.prefix)'`.\n"
+echo -e "Please activate it manually with following command:\n"
+echo -e "\t$MENTAT_VENV/bin/activate\n"
+echo -e "Consider adding following alias to your ~/.bashrc file for easier environment activation:\n"
+echo -e "\talias entervenv='. venv/bin/activate'"
+echo -e "\n!!! Please keep in mind, that all makefile targets except this one ('develop') leave it up to you to activate the correct virtual environment !!!\n"

conf/scripts/lib-mentat.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+COLCODE_BLACK=$(tput setaf 0)
+COLCODE_RED=$(tput setaf 1)
+COLCODE_GREEN=$(tput setaf 2)
+COLCODE_ORANGE=$(tput setaf 3)
+COLCODE_BLUE=$(tput setaf 4)
+COLCODE_PURPLE=$(tput setaf 5)
+COLCODE_CYAN=$(tput setaf 6)
+COLCODE_WHITE=$(tput setaf 7)
+COLCODE_NC=$(tput sgr0)
+
+COLCODE_B_BLACK=$(tput setab 0)
+COLCODE_B_RED=$(tput setab 1)
+COLCODE_B_GREEN=$(tput setab 2)
+COLCODE_B_ORANGE=$(tput setab 3)
+COLCODE_B_BLUE=$(tput setab 4)
+COLCODE_B_PURPLE=$(tput setab 5)
+COLCODE_B_CYAN=$(tput setab 6)
+COLCODE_B_WHITE=$(tput setab 7)
+
+COLCODE_BOLD=$(tput bold)
+COLCODE_FAINT=$(tput dim)

conf/scripts/lib.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Bash function library for Mentat system.
+#
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+function ensure_link {
+	if [ ! -L $2 ] ; then
+		if [ -f $2 ] ; then
+			rm -f $2
+		fi
+		if [ -d $2 ] ; then
+			rmdir $2
+		fi
+		ln -s $1 $2
+	fi
+}
+
+function print_title {
+	echo "================================================================================"
+	echo " $1"
+	echo "================================================================================"
+	date
+}
+
+function print_subtitle {
+	echo "==========> $1"
+}

scripts/lwchroot-init.sh → conf/scripts/lwchroot-init.sh

 #!/bin/bash
 #-------------------------------------------------------------------------------
-# Utility script for creating local lightweight chroot substructure.
+# Utility script for creating local lightweight chroot directory structure.
 #
 # Copyright (C) since 2011 CESNET, z.s.p.o
 # Use of this source is governed by the MIT license, see LICENSE file.
@@ -31,6 +31,6 @@ if [ ! -f .env ]; then
 	echo "FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf)" >> .env
 fi
 
-if [ ! -L ./migrations-events/.env ]; then
-	ln -s $(realpath ./.env) $(realpath ./migrations-events)/.env;
-fi
+if [ ! -L ./conf/migrations-events/.env ]; then
+	ln -s $(realpath ./.env) $(realpath ./conf/migrations-events)/.env;
+fi
\ No newline at end of file