From bd653d6913ade190d74c93b73e4119d85e3d584b Mon Sep 17 00:00:00 2001
From: Jan Mach <jan.mach@cesnet.cz>
Date: Thu, 6 Sep 2018 15:01:48 +0200
Subject: [PATCH] Created presentation - Mentat: Introduction for developers.
This presentation can be used to introduce new developers into the Mentat system development. (Redmine issue: #3361)
---
.gitignore | 8 +
.../mentat-developer-intro/Makefile | 29 +
.../mentat-developer-intro.tex | 848 ++++++++++++++++++
3 files changed, 885 insertions(+)
create mode 100644 doc/presentations/mentat-developer-intro/Makefile
create mode 100644 doc/presentations/mentat-developer-intro/mentat-developer-intro.tex
diff --git a/.gitignore b/.gitignore
index 8e47c2a6..982fd730 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,13 @@
!.gitignore
.directory
*~
+*.aux
*.log
+*.nav
+*.out
+*.snm
+*.toc
+*.vrb
*.pem
*.cert
*.key
@@ -41,6 +47,8 @@ vendor/
# Ignore Sphinx-doc documentation build directory.
/doc/sphinx/_build/
+/doc/presentations/*/*.pdf
+
# Ignore submodule documentation.
/doc/sphinx/_doclib/apidoc/pyzenkit*.rst
/doc/sphinx/_doclib/apidoc/pynspect*.rst
diff --git a/doc/presentations/mentat-developer-intro/Makefile b/doc/presentations/mentat-developer-intro/Makefile
new file mode 100644
index 00000000..121e0d90
--- /dev/null
+++ b/doc/presentations/mentat-developer-intro/Makefile
@@ -0,0 +1,29 @@
+# Makefile.pdflatex
+# Author: Johannes Ranke <jranke@uni-bremen.de>
+# Last Change: 2006 Apr 18
+# based on the Makefiles of Tadeusz Pietraszek
+# posted on his blog on March 24, 2006
+# SVN: $Id$
+
+TEXFILES=$(wildcard *.tex)
+TARGETS=$(patsubst %.tex,%.pdf,$(TEXFILES))
+
+RERUN = "No file.*"
+
+all: all-recursive $(TARGETS)
+
+clean: clean-recursive
+ rm -f *.aux *.log *.bbl *.blg *.brf *.cb *.ind *.idx *.ilg \
+ *.inx *.ps *.dvi *.toc *.out *.lot *~ *.lof *.ttt *.fff *.nav *.snm *.vrb
+
+all-recursive:
+ for dir in $(wildcard *); do if [ -d $$dir ] && [ -f $$dir/Makefile ]; then cd $$dir; $(MAKE) all; cd ..; fi; done
+
+clean-recursive:
+ for dir in $(wildcard *); do if [ -d $$dir ] && [ -f $$dir/Makefile ]; then cd $$dir; $(MAKE) clean; cd ..; fi; done
+
+%.pdf: %.tex ../../img/logo/* ../../img/mentat/*
+ vlna -v KkSsVvZzOoUuAaI -r -l $<
+ xelatex $<
+ egrep $(RERUN) $*.log && sleep 1 && (xelatex $<) ; true
+ egrep $(RERUN) $*.log && sleep 2 && (xelatex $<) ; true
diff --git a/doc/presentations/mentat-developer-intro/mentat-developer-intro.tex b/doc/presentations/mentat-developer-intro/mentat-developer-intro.tex
new file mode 100644
index 00000000..6ee58729
--- /dev/null
+++ b/doc/presentations/mentat-developer-intro/mentat-developer-intro.tex
@@ -0,0 +1,848 @@
+%
+% Original latex template: https://github.com/oskar456/beamer-skeleton
+%
+
+% Wide screen (aspect ratio 16:9)
+%\documentclass[hyperref={pdfpagelabels=false},xcolor=dvipsnames,12pt,aspectratio=169]{beamer}
+
+% Normal screen (aspect ratio 4:3)
+\documentclass[hyperref={pdfpagelabels=false},xcolor=dvipsnames,12pt]{beamer}
+\let\Tiny=\tiny
+
+%\usepackage[czech]{babel}
+\usepackage{xltxtra}
+\usepackage{hyperref}
+\usepackage{listings}
+\hypersetup{colorlinks=false,hidelinks}
+\setromanfont{DejaVu Serif}
+\setsansfont{DejaVu Sans}
+\setmonofont{DejaVu Sans Mono}
+\lstset{language=Python}
+
+\setbeamertemplate{navigation symbols}{}
+
+\mode<presentation>
+{
+ \usetheme{Madrid}
+ \usecolortheme[RGB={0,104,162}]{structure}
+}
+\AtBeginSection[]{
+ \begin{frame}
+ \centering
+ \begin{beamercolorbox}[sep=12pt,center]{part title}
+ \usebeamerfont{section title}\insertsection\par
+ \end{beamercolorbox}
+ \end{frame}
+}
+
+\title[Mentat System]{Mentat System}
+\subtitle{Introduction for developers}
+\author{Jan Mach}
+\institute[CESNET, a.l.e.]{\includegraphics[width=4cm]{../../img/logo/cesnet_RGB_raw}}
+\logo{\includegraphics[width=1.5cm]{../../img/logo/cesnet_RGB_lrprotect}}
+\date{v1.0, 2018-09-06}
+
+% Automatic TOC generation at the beginning of each section (must be in preamble)
+\AtBeginSection[]
+{
+ \begin{frame}
+ \frametitle{Agenda}
+ \tableofcontents[currentsection]
+ \end{frame}
+}
+\AtBeginSubsection[]
+{
+ \begin{frame}
+ \frametitle{Agenda}
+ \tableofcontents[currentsection,currentsubsection]
+ \end{frame}
+}
+
+\begin{document}
+
+
+%-------------------------------------------------------------------------------
+
+
+\section*{Opening}
+
+{
+\setbeamertemplate{logo}{}
+\begin{frame}
+ \titlepage
+ \begin{figure}
+ \includegraphics[width=2ex]{../../img/logo/cc}
+ \hspace*{0.5ex}
+ \includegraphics[width=2ex]{../../img/logo/by}
+
+ \tiny Attribution 3.0 Unported (CC BY 3.0)
+ \end{figure}
+\end{frame}
+}
+
+\begin{frame}
+ \frametitle{Agenda}
+ \tableofcontents
+\end{frame}
+
+
+%-------------------------------------------------------------------------------
+
+
+\section{Introduction}
+
+\begin{frame}
+ \frametitle{Motivation}
+ \begin{alertblock}{Key ideas}
+ \begin{itemize}
+ \item Aid for CESNET-CERTS security team
+ \item Resource consolidation
+ \end{itemize}
+ \end{alertblock}
+ \begin{block}{Main features}
+ \begin{itemize}
+ \item Gathering/receiving information from various sources
+ \item Long term searchable persistent information storage
+ \item Real-time and back information processing with various methods
+ \item Fully automatic processing, enable performing of automatic actions on specific conditions
+ \end{itemize}
+ \end{block}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Latest news}
+ \begin{itemize}
+ \item \alert{Released version 2.0.x}
+ \item Migrated completely to \href{https://www.python.org/}{\alert{Python3}}
+ \item Migrated database to \href{https://www.postgresql.org/}{\alert{PostgreSQL}}
+ \item Automated build system \href{https://alchemist.cesnet.cz/}{\alert{Alchemist}}
+ \item Autogenerated \href{https://alchemist.cesnet.cz/mentat/doc/production/html/manual.html}{\alert{documentation}}
+ \item Public Git code \href{https://alchemist.cesnet.cz/mentat/repo.git}{\alert{repository}} and \href{https://homeproj.cesnet.cz/projects/mentat}{\alert{issue tracker}}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Alchemist build system}
+ \href{https://alchemist.cesnet.cz/}{\texttt{https://alchemist.cesnet.cz/}}
+ \vspace{1cm}
+ \begin{itemize}
+ \item \alert{Automated build system for Mentat and related libraries}
+ \item Contents:
+ \begin{itemize}
+ \item General information
+ \item Build environment settings
+ \item Autogenerated documentation
+ \item Git repositories
+ \item Debian packages
+ \item Python wheels
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+
+%-------------------------------------------------------------------------------
+
+
+\section{Overview}
+
+\begin{frame}
+ \frametitle{System overview}
+ \begin{figure}
+ \includegraphics[width=12cm]{../../img/mentat/mentat-overview}
+ \end{figure}
+ \begin{itemize}
+ \item Implementation language: \href{https://www.python.org/}{\alert{Python3}}
+ \item Data model: \href{https://idea.cesnet.cz/en/index}{\alert{IDEA}}
+ \item Data storage: \href{https://www.postgresql.org/}{\alert{PostgreSQL}}
+ \item Network communication protocol: \href{https://warden.cesnet.cz/en/index}{\alert{Warden}}
+ \end{itemize}
+\end{frame}
+
+
+%-------------------------------------------------------------------------------
+
+
+\section{Design}
+
+\subsection{Technologies}
+
+\begin{frame}
+ \frametitle{Warden}
+ \href{https://warden.cesnet.cz/en/index}{\texttt{https://warden.cesnet.cz/en/index}}
+ \vspace{1cm}
+ \begin{itemize}
+ \item A~system for efficient sharing information about detected events (threats)
+ \item Simple client-server architecture
+ \item Sending and receiving clients
+ \item Based on HTTPS protocol with bidirectional certificate authentication
+ \item Communication possible with any HTTPS capable library
+ \item Python client library and simple filer daemon in distribution
+ \item Community approach in data sharing
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Data model: IDEA}
+ \href{https://idea.cesnet.cz/en/index}{\texttt{https://idea.cesnet.cz/en/index}}
+ \vspace{1cm}
+ \begin{itemize}
+ \item Intrusion Detection Extensible Alert
+ \item JSON based format (NoSQL friendly)
+ \item Shallow structure, strong typed (SQL friendly)
+ \item Easily extendable and customizable
+ \item Possibility to mark anonymised, inaccurate, incomplete or forged data
+ \item Support for aggregated, correlated events
+ \item Support for various data attachments
+ \item Dictionaries for description of various event attributes (Category, Source/Target type, etc.)
+ \end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{IDEA: Example message}
+ \begin{itemize}
+ \item Example Botnet C\&C report event
+ \end{itemize}
+ \fontsize{10pt}{7.2}\selectfont
+\begin{verbatim}
+{
+ "Format": "IDEA0",
+ "ID": "cca3325c-a989-4f8c-998f-5b0e971f6ef0",
+ "DetectTime": "2014-03-05T15:52:22Z",
+ "Category": ["Intrusion.Botnet"],
+ "Description": "Botnet Command and Control",
+ "Source": [
+ {
+ "Type": ["Botnet", "CC"],
+ "IP4": ["93.184.216.119"],
+ "Proto": ["tcp", "ircu"],
+ "Port": [6667]
+ }
+ ]
+}
+\end{verbatim}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Libraries}
+ \begin{itemize}
+ \item Utils: \href{https://pypi.org/project/geoip2/}{\texttt{geoip2}}, \href{https://pypi.org/project/ply/}{\texttt{ply}}, \href{https://pypi.org/project/geoip2/}{\texttt{rrdtool}}, \href{https://pypi.org/project/psycopg2/}{\texttt{psycopg2}}
+ \item Web: \href{https://pypi.org/project/Flask/}{\texttt{Flask}}, \href{https://pypi.org/project/Jinja2/}{\texttt{Jinja2}}, \href{https://pypi.org/project/Babel/}{\texttt{Babel}}, \href{https://pypi.org/project/WTForms/}{\texttt{WTForms}}, \href{https://pypi.org/project/SQLAlchemy/}{\texttt{SQLAlchemy}}
+ \item \href{https://pypi.org/project/idea-format/}{\alert{idea-format}}: Library for working with IDEA messages
+ \item \href{https://pypi.org/project/pynspect/}{\alert{pynspect}}: Data filtering library
+ \item \href{https://pypi.org/project/pyzenkit/}{\alert{pyzenkit}}: Application development framework
+ \end{itemize}
+\end{frame}
+
+\subsection{Architecture}
+
+\begin{frame}
+ \frametitle{System architecture}
+ \begin{figure}
+ \includegraphics[width=9cm]{../../img/mentat/mentat-architecture}
+ \end{figure}
+\end{frame}
+
+\begin{frame}
+ \frametitle{System modules}
+ \begin{itemize}
+ \item Real-time event processing modules
+ \begin{itemize}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/bin_mentat-inspector.html}{\texttt{mentat-inspector}}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/bin_mentat-enricher.html}{\texttt{mentat-enricher}}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/bin_mentat-storage.html}{\texttt{mentat-storage}}
+ \end{itemize}
+ \item Event post processing modules (via database)
+ \begin{itemize}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/bin_mentat-reporter.html}{\texttt{mentat-reporter}}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/bin_mentat-statistician.html}{\texttt{mentat-statistician}}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/bin_mentat-informant.html}{\texttt{mentat-informant}}
+ \item (management scripts)
+ \end{itemize}
+ \item Control modules and user interfaces
+ \begin{itemize}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/bin_mentat-controller.html}{\texttt{mentat-controller}}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/hawat.html}{\texttt{Hawat}}
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Module design}
+ \begin{itemize}
+ \item Design inspired by \href{http://www.postfix.org/}{\alert{Postfix MTA}}
+ \begin{itemize}
+ \item Hierarchical structure of many small one task daemons
+ \item Filesystem directory message queues (aka. filer protocol)
+ \end{itemize}
+ \item Process-level paralelization support, more instances can work with the same queue
+ \item \href{https://alchemist.cesnet.cz/pyzenkit/doc/production/html/manual.html}{\alert{PyZenKit}} as common framework for module development
+ \begin{itemize}
+ \item Application life cycle
+ \item Configuration loading, validation and merging (JSON)
+ \item Deamonisation, logging setup
+ \item Database abstract layer
+ \item IDEA message abstract layer
+ \item Filtering library, statistical data processing library
+ \item WHOIS library, DNS resolving library, reporting library
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Message exchange queue (1)}
+ \begin{itemize}
+ \item aka. filer protocol
+ \item simple filesystem directory with substructure:
+ \begin{itemize}
+ \item incoming: input queue, only complete messages
+ \item pending: daemon work directory, messages in progress
+ \item tmp: work directory
+ \item errors: messages causing problems during processing
+ \end{itemize}
+ \item \alert{key requirement: atomic move}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Message exchange queue (2)}
+ \begin{figure}
+ \includegraphics[width=7cm]{../../img/mentat/mentat-queue-protocol}
+ \end{figure}
+ \begin{itemize}
+ \item Inserting message into queue:
+ \begin{itemize}
+ \item create new file in \alert{tmp} subdirectory
+ \item filename is arbitrary, but must be unique within all subdirectories
+ \item when done writing, move/rename the file to \alert{incoming}
+ \item move must be atomic, so all subdirectories must be on same partition
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\subsection{PyZenKit framework}
+
+\begin{frame}
+ \frametitle{Design goals}
+ \begin{itemize}
+ \item provide feature rich application out of the box
+ \item enable customizability and extendability
+ \begin{itemize}
+ \item built-in features are configurable by text files, and/or command line arguments
+ \item callback hooks for subclasses
+ \item prepared for inheritance and method overloading
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.jsonconf}
+ \begin{itemize}
+ \item reading and writing of JSON configuration files
+ \item merging multiple JSON configuration files
+ \item support for configuration directories
+ \item support for single line comments in JSON files
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.daemonizer}
+ \begin{itemize}
+ \item setup directories and limits
+ \item setup user and group permissions
+ \item double fork and split session
+ \item setup signal handlers
+ \item close all open file descriptors (except for possible log files)
+ \item redirect stdin, stdout, stderr to /dev/null
+ \item detect current PID and store it to appropriate PID file
+ \item at exit remove PID file
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.baseapp (1)}
+ \begin{itemize}
+ \item \alert{base implementation for generic console application}
+ \item Features:
+ \begin{itemize}
+ \item application life-cycle management
+ \item application configuration service
+ \item command line argument parsing service
+ \item logging service
+ \item persistent state service
+ \item application runlog service
+ \item plugin system (experimental)
+ \item application actions
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.baseapp (2)}
+ \begin{itemize}
+ \item Application usage modes:
+ \begin{itemize}
+ \item run
+ \item plugin
+ \end{itemize}
+ \item Application life cycle:
+ \begin{itemize}
+ \item init
+ \item setup
+ \item process
+ \item evaluate
+ \item teardown
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.baseapp (3)}
+ \begin{itemize}
+ \item Built-in actions:
+ \begin{itemize}
+ \item config-view
+ \item runlog-dump
+ \item runlog-view
+ \item runlogs-dump
+ \item runlogs-list
+ \item runlogs-evaluate
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{pyzenkit.baseapp (4)}
+ \begin{itemize}
+ \item \alert{example implementation can be found in module source code}
+ \item documentation: \href{https://alchemist.cesnet.cz/pyzenkit/doc/production/html/doc/_doclib/api_pyzenkit.baseapp.html}{\texttt{https://alchemist.cesnet.cz/}}
+ \end{itemize}
+ \fontsize{10pt}{7.2}\selectfont
+\begin{verbatim}
+# On Debian Jessie try following (as root):
+cd /usr/local/lib/python3.4/dist-packages
+python3 pyzenkit/baseapp.py --help
+python3 pyzenkit/baseapp.py
+python3 pyzenkit/baseapp.py --action runlogs-evaluate
+\end{verbatim}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.zenscript (1)}
+ \begin{itemize}
+ \item \alert{base implementation for generic console script application}
+ \item based on \alert{pyzenkit.baseapp}
+ \item Additional features:
+ \begin{itemize}
+ \item support for executing multiple different \alert{commands}
+ \item execution modes: default, regular, shell
+ \item support for executions in regular time intervals
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{pyzenkit.zenscript (2)}
+ \begin{itemize}
+ \item \alert{example implementation can be found in module source code}
+ \item documentation: \href{https://alchemist.cesnet.cz/pyzenkit/doc/production/html/doc/_doclib/api_pyzenkit.zenscript.html}{\texttt{https://alchemist.cesnet.cz/}}
+ \end{itemize}
+ \fontsize{10pt}{7.2}\selectfont
+\begin{verbatim}
+# On Debian Jessie try following (as root):
+cd /usr/local/lib/python3.4/dist-packages
+python3 pyzenkit/zenscript.py --help
+python3 pyzenkit/zenscript.py
+python3 pyzenkit/zenscript.py --command alternative
+python3 pyzenkit/zenscript.py --action runlogs-evaluate
+\end{verbatim}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.zendaemon (1)}
+ \begin{itemize}
+ \item \alert{base implementation for generic daemon application}
+ \item based on \alert{pyzenkit.baseapp}
+ \item Additional features:
+ \begin{itemize}
+ \item fully automated daemonization process
+ \item event driven design
+ \item support for handling arbitrary signals
+ \item support for modularity with daemon components
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.zendaemon (2)}
+ \begin{itemize}
+ \item Event driven design:
+ \begin{itemize}
+ \item infinite event loop and event scheduler
+ \item events are being emited in different parts of application
+ \item event callbacks must be registered to handle events
+ \item multiple event callback may handle single event (pipeline)
+ \end{itemize}
+ \item Event scheduling:
+ \begin{itemize}
+ \item \texttt{schedule}
+ \item \texttt{schedule\_next}
+ \item \texttt{schedule\_after}
+ \item \texttt{schedule\_at}
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{pyzenkit.zendaemon (3)}
+ \begin{itemize}
+ \item Signal handling:
+ \begin{itemize}
+ \item SIGINT
+ \item SIGUSR1
+ \item SIGUSR2
+ \end{itemize}
+ \item Sending signals:
+ \end{itemize}
+ \fontsize{10pt}{7.2}\selectfont
+\begin{verbatim}
+# On Debian Jessie try following (as root):
+cd /usr/local/lib/python3.4/dist-packages
+python3 pyzenkit/zendaemon.py --no-daemon
+python3 pyzenkit/zendaemon.py --action signal-usr1
+python3 pyzenkit/zendaemon.py --action=signal-usr2
+\end{verbatim}
+\end{frame}
+
+\begin{frame}
+ \frametitle{pyzenkit.zendaemon (4)}
+ \begin{figure}
+ \includegraphics[width=7cm]{../../img/mentat/mentat-daemon-architecture}
+ \end{figure}
+ \begin{itemize}
+ \item Daemon components:
+ \begin{itemize}
+ \item actual workers in the design
+ \item the daemon object is in fact only a~container for components
+ \item components must be registered into the daemon object
+ \item great for code reusability
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{pyzenkit.zendaemon (5)}
+ \begin{itemize}
+ \item \alert{example implementations can be found in module source code}
+ \item documentation: \href{https://alchemist.cesnet.cz/pyzenkit/doc/production/html/doc/_doclib/api_pyzenkit.zendaemon.html}{\texttt{https://alchemist.cesnet.cz/}}
+ \end{itemize}
+ \fontsize{10pt}{7.2}\selectfont
+\begin{verbatim}
+# On Debian Jessie try following (as root):
+cd /usr/local/lib/python3.4/dist-packages
+python3 pyzenkit/zendaemon.py --help
+python3 pyzenkit/zendaemon.py --no-daemon
+python3 pyzenkit/zendaemon.py --action runlogs-evaluate
+\end{verbatim}
+\end{frame}
+
+\subsection{Mentat framework}
+
+\begin{frame}
+ \frametitle{mentat.daemon.piper (1)}
+ \begin{itemize}
+ \item \alert{base implementation pipe-like message processing daemon}
+ \item based on \alert{pyzenkit.zendaemon}
+ \item Additional features:
+ \begin{itemize}
+ \item preconfigured message queue features:
+ \begin{itemize}
+ \item automated inclusion and bootstrapping of \alert{mentat.daemon.component.filer} daemon component
+ \item additional configurations and command line arguments related to filer protocol.
+ \end{itemize}
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{mentat.daemon.piper (2)}
+ \begin{itemize}
+ \item \alert{example implementation can be found in module source code}
+ \item documentation: \href{https://alchemist.cesnet.cz/mentat/doc/production/html/_doclib/api_mentat.daemon.piper.html}{\texttt{https://alchemist.cesnet.cz/}}
+ \end{itemize}
+ \fontsize{10pt}{7.2}\selectfont
+\begin{verbatim}
+# On Debian Jessie try following (as root):
+cd /usr/lib/python3/dist-packages
+python3 mentat/daemon/piper.py --help
+python3 mentat/daemon/piper.py --no-daemon
+python3 mentat/daemon/piper.py --action runlogs-evaluate
+\end{verbatim}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Remarks}
+ \begin{itemize}
+ \item \alert{project is still evolving}
+ \item \alert{there are many examples directly in the module source code}
+ \item \alert{use existing modules as templates for creating new ones}
+ \item local Makefile may be usefull:
+ \begin{itemize}
+ \item \texttt{make pyflakes}
+ \item \texttt{make pylint}
+ \item \texttt{make test}
+ \item \texttt{make docs}
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Mentat repository structure}
+ \begin{itemize}
+ \item \alert{/bin}: executables (simple)
+ \item \alert{/conf}: configuration files and cron scripts
+ \item \alert{/lib}: Python libraries
+ \item \alert{/submodules}: local copies of some custom libraries
+ \end{itemize}
+\end{frame}
+
+%-------------------------------------------------------------------------------
+
+
+\section{Creating daemon module}
+
+\subsection{Overview}
+
+\begin{frame}
+ \frametitle{Overview}
+ \begin{itemize}
+ \item Option 1: real-time processing module can be anything that can work according to the \alert{filer protocol}
+ \item Option 2: use \alert{pyzenkit} and \alert{mentat} frameworks
+ \end{itemize}
+\end{frame}
+
+\subsection{DemoPiperDaemon.py}
+
+\begin{frame}[fragile]
+ \frametitle{DemoPiperDaemon (2)}
+ \fontsize{6pt}{7.2}\selectfont
+\begin{lstlisting}
+import pyzenkit
+import mentat.const
+import mentat.daemon.piper
+
+class DemoPrintComponent(pyzenkit.zendaemon.ZenDaemonComponent):
+
+ def get_events(self):
+ return [
+ {
+ 'event': 'message_process',
+ 'callback': self.cbk_event_message_process,
+ 'prepend': False
+ }
+ ]
+
+ def cbk_event_message_process(self, daemon, args):
+ daemon.logger.info(
+ "Processing message: '{}': '{}'".format(
+ args['id'], str(args['data']).strip()
+ )
+ )
+ daemon.queue.schedule('message_commit', args)
+ self.inc_statistic('cnt_printed')
+ return (daemon.FLAG_CONTINUE, None)
+\end{lstlisting}
+\end{frame}
+
+
+\begin{frame}[fragile]
+ \frametitle{DemoPiperDaemon (2)}
+ \fontsize{6pt}{7.2}\selectfont
+\begin{lstlisting}
+class DemoPiperDaemon(mentat.daemon.piper.PiperDaemon):
+
+ def __init__(self):
+ super().__init__(
+ name = 'mentat-demopiper.py',
+ description = 'DemoPiperDaemon - Demonstration daemon',
+ path_bin = '/usr/local/bin',
+ path_cfg = '/tmp',
+ path_log = '/var/mentat/log',
+ path_run = '/var/mentat/run',
+ path_tmp = '/tmp',
+
+ default_config_dir = None,
+ default_queue_in_dir = '/var/mentat/spool/mentat-demopiper.py',
+ default_queue_out_dir = None,
+
+ schedule = [
+ ('message_enqueue', {'data': '{"testA1": 1, "testA2": 2}'}),
+ ('message_enqueue', {'data': '{"testB1": 1, "testB2": 2}'}),
+ (mentat.const.DFLT_EVENT_START,)
+ ],
+ schedule_after = [
+ (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS)
+ ],
+
+ components = [
+ DemoPrintComponent()
+ ]
+ )
+\end{lstlisting}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{DemoPiperDaemon (3)}
+ \fontsize{6pt}{7.2}\selectfont
+\begin{lstlisting}
+if __name__ == "__main__":
+ DemoPiperDaemon().run()
+\end{lstlisting}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{DemoPiperDaemon (4)}
+ \begin{itemize}
+ \item save previous code to file: \texttt{/etc/mentat/examples/mentat-demopiper.py}
+ \item create configuration file: \texttt{/tmp/mentat-demopiper.py}
+ \item add module pipeline in: \texttt{/etc/mentat/conf/mentat-storage.py.conf}
+ \item add module to \texttt{/etc/mentat/conf/mentat-controller.py.conf}
+ \end{itemize}
+ \fontsize{6pt}{7.2}\selectfont
+\begin{lstlisting}
+# Create symlink to example:
+ln -s /etc/mentat/examples/mentat-demopiper.py /usr/local/bin/mentat-demopiper.py
+# Stop all currently running components
+mentat-controller.py --command stop
+# Start all currently components
+mentat-controller.py --command start
+# Generate test messages
+mentat-ideagen.py --count 10
+# View log file
+tail -f /var/mentat/log/mentat-demopiper.py.log
+\end{lstlisting}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{DemoPiperDaemon (5)}
+ \begin{itemize}
+ \item adding more command line arguments:
+ \end{itemize}
+ \fontsize{6pt}{7.2}\selectfont
+\begin{lstlisting}
+
+ # Add to DemoPiperDaemon class
+ def _init_argparser(self, **kwargs):
+ """
+ :param kwargs: Various additional parameters passed down from object constructor.
+ :return: Valid argument parser object.
+ :rtype: argparse.ArgumentParser
+ """
+ argparser = super()._init_argparser(**kwargs)
+
+ arggroup_daemon = argparser.add_argument_group('custom daemon arguments')
+ arggroup_daemon.add_argument(
+ '--reload-interval',
+ type = int,
+ default = None,
+ help = 'time interval for reloading internal plugins in seconds')
+
+ return argparser
+\end{lstlisting}
+\end{frame}
+
+\begin{frame}[fragile]
+ \frametitle{DemoPiperDaemon (6)}
+ \begin{itemize}
+ \item default values for configurations
+ \end{itemize}
+ \fontsize{6pt}{7.2}\selectfont
+\begin{lstlisting}
+
+ # Add to DemoPiperDaemon class
+ def _init_config(self, cfgs, **kwargs):
+ """
+ :param list cfgs: Additional set of configurations.
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Default configuration structure.
+ :rtype: dict
+ """
+ cfgs = (
+ ('something', None),
+ ('reload_interval', 300)
+ ) + cfgs
+ return super()._init_config(cfgs, **kwargs)
+\end{lstlisting}
+\end{frame}
+
+
+%-------------------------------------------------------------------------------
+
+
+\section{Hawat: Web interface}
+
+\begin{frame}
+ \frametitle{Hawat}
+ \begin{itemize}
+ \item Topic for another day
+ \item Implemented using \href{http://flask.pocoo.org/docs/1.0/}{\alert{Flask}}, \href{http://jinja.pocoo.org/docs/2.10/}{\alert{Jinja2}}, \href{http://babel.pocoo.org/en/latest/}{\alert{Babel}}, \href{http://docs.sqlalchemy.org/en/latest/}{\alert{SQLAlchemy}} and Mentat frameworks
+ \item Modularization using Flask blueprints
+ \item Customized Flask classes for deeper integration
+ \begin{itemize}
+ \item View classes for common tasks (item management, searching, ...)
+ \item Application menu, item context menus, ...
+ \end{itemize}
+ \item \alert{Read Flask`s documentation!}
+ \end{itemize}
+\end{frame}
+
+
+%-------------------------------------------------------------------------------
+
+
+\section{Resources}
+
+\begin{frame}
+ \frametitle{Essential resources}
+ \begin{itemize}
+ \item \href{https://homeproj.cesnet.cz/projects/mentat}{Homeproj: Project issue tracker}
+ \item \href{https://homeproj.cesnet.cz/git/mentat-ng.git/}{Primary code repository}
+ \item \href{https://alchemist.cesnet.cz/mentat/doc/production/html/manual.html}{Official documentation}
+ \item \href{https://alchemist.cesnet.cz}{Alchemist: automated build system}
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Additional resources}
+ \begin{itemize}
+ \item \href{https://mentat.cesnet.cz/en/index}{Project Mentat: official website}
+ \item \href{https://warden.cesnet.cz/en/index}{Project Warden: official website}
+ \item \href{https://idea.cesnet.cz/en/index}{IDEA: official website}
+ \item \href{https://www.postgresql.org/}{PostgreSQL: official website}
+ \item \href{http://www.sphinx-doc.org/en/stable/}{Sphinx: official website}
+ \end{itemize}
+\end{frame}
+
+
+%-------------------------------------------------------------------------------
+
+
+\section*{Conclusion}
+{
+\setbeamertemplate{logo}{}
+\begin{frame}
+ \begin{center}
+ \large Thank you for your attention
+
+ \vspace{1cm}
+ Jan Mach
+
+ \href{mailto:Jan.Mach@cesnet.cz}{\texttt{Jan.Mach@cesnet.cz}} \\
+ \begin{figure}
+ \includegraphics[width=8cm]{../../img/logo/cesnet_RGB_lrprotect}
+ \end{figure}
+ \end{center}
+\end{frame}
+}
+
+\end{document}
+
--
GitLab