Compare revisions

# Python related

*~

*.py[cod]

*.egg-info

__pycache__

# Safety net

*.log

*.pem

*.cert

*.key

*.gpg

*.tmp

# Archives

*.tar

*.gz

*.bz2

*.xz

*.tgz

*.tbz2

*.txz

*.rpm

*.deb

---

server_admin: "{{ root@inventory_hostname }}"

warden_filer_bin_path: /opt/warden-filer

warden_filer_lib_path: /var/lib/warden_filer

warden_filer_run_path: /run/warden_filer

warden_client_cert_path: /etc/ssl/certs/warden.cert.pem

warden_client_key_path: /etc/ssl/private/warden.key.pem

warden_client_id_store: /var/lib/warden_filer/warden_filer.id

warden_filer_pid_file: /run/warden_filer/receiver.pid

warden_filer_uid: 1

warden_filer_gid: 1

 No newline at end of file

---

- name: Checkout Warden repository

  git:

    repo: https://gitlab.cesnet.cz/713/warden/warden.git

    version: warden-client-3.0-beta3

    dest: /tmp/warden_client_repository

- name: Create bin dir for warden_filer

  file:

    path: "{{ warden_filer_bin_path }}"

    state: directory

    owner: root

    group: root

    mode: "755"

- name: Create lib and run dir for warden_filer

  file:

    path: "{{ item }}"

    state: directory

    owner: "{{ warden_filer_uid }}"

    group: "{{ warden_filer_gid }}"

    mode: "755"

  with_items: 

    - "{{ warden_filer_lib_path }}"

    - "{{ warden_filer_run_path }}"

- name: Install Filer binaries

  copy:

    remote_src: true

    src: "/tmp/warden_client_repository/{{ item.src }}"

    dest: "{{ warden_filer_bin_path }}/{{ item.dest }}"

    mode: "755"

  with_items:

    - src: warden_client/warden_client.py

      dest: warden_client.py

    - src: warden_filer/warden_filer.py

      dest: warden_filer.py

    - src: warden_filer/check_file_count

      dest: check_file_count

- name: Link Filer binary to /usr/local/bin

  file:

    src: "{{ warden_filer_bin_path }}/warden_filer.py"

    dest: "/usr/local/bin/warden_filer.py"

    state: link

    owner: root

    group: root

    mode: "755"

- name: Install Warden Filer config

  template:

    src: "{{ item }}"

    dest: "/{{ item }}"

  with_items:

    - etc/warden_filer.cfg

    - etc/default/warden_filer_receiver

- name: Install Warden Filer init script

  copy:

    remote_src: true

    src: /tmp/warden_client_repository/warden_filer/warden_filer_receiver

    dest: /etc/init.d/warden_filer_receiver

    mode: "755"

PYTHONPATH={{ warden_filer_bin_path }}

{

    // Warden config can be also referenced as:

    // "warden": "/path/to/warden_client.cfg"

    "warden": {

        "url": "{{ warden_server_url | mandatory }}",

        "keyfile": "{{ warden_client_key_path }}",

        "certfile": "{{ warden_client_cert_path }}",

        "timeout": 30,

        "send_events_limit": 1000,

        "get_events_limit": 1000,

        "syslog": {"level": "debug", "facility": "local7"},

        "idstore": "{{ warden_client_id_store }}",

        "name": "{{ warden_client_name | mandatory }}"

    },

    "receiver": {

        "dir": "{{ warden_filer_output_dir | mandatory }}",

        "pid_file": "{{ warden_filer_pid_file }}",

        "uid": {{ warden_filer_uid }},

        "gid": {{ warden_filer_gid }},

        "file_limit": 10000,

        "limit_wait_time": 20

    }

}

---

server_admin: "{{ root@inventory_hostname }}"

warden_server_hostname: "{{ inventory_hostname }}"

warden_server_virtual_host: "{{ ansible_default_ipv4 }}:443 {{ ansible_default_ipv6 }}:443"

warden_server_dir_path: /opt/warden-server

warden_ra_dir_path: /opt/warden-ra

warden_db_name: warden3

warden_db_user: warden

warden_ra_ejbca_url: https://ejbca.cesnet-ca.cz:8443/ejbca/ejbcaws/ejbcaws?wsdl

warden_ra_cert: /etc/ssl/certs/warden_ra.cert.pem

warden_ra_key: /etc/ssl/private/warden_ra.key.pem

warden_ra_ca_name: "Warden CA"

warden_ra_ejbca_certificate_profile: "Warden"

warden_ra_ejbca_end_entity_profile: "Warden EE"

warden_ra_subject_dn_template: "DC=test,DC=snakeoil,DC=warden,CN=%s"

warden_ra_ejbca_username_suffix: "@warden"

---

- name: Checkout Warden repository

  git:

    repo: https://gitlab.cesnet.cz/713/warden/warden.git

    version: warden-server-3.0-beta3

    dest: /tmp/warden_server_repository

- name: Populate Warden server directory

  copy:

    src: "/tmp/warden_server_repository/warden_server"

    dest: "{{ warden_server_dir_path }}"

- name: Populate Warden RA directory

  copy:

    src: "/tmp/warden_server_repository/warden_ra"

    dest: "{{ warden_ra_dir_path }}"

- name: Install Warden server config

  template:

    src: opt/warden-server/warden_server.cfg

    dest: "{{ warden_server_dir_path }}/warden_server.cfg"

- name: Install Warden RA config

  template:

    src: opt/warden-ra/warden_ra.cfg

    dest: "{{ warden_ra_dir_path }}/warden_ra.cfg"

- name: Ensure PyMySQL module

  apt:

    pkg: python-mysqldb

    state: present

- name: Check whether Warden database already exists

  command: |

    mysql

      --batch --skip-column-names

      --user="{{ warden_db_user }}" --password="{{ warden_db_password | mandatory }}"

      "{{ warden_db_name }}"

      --execute "SELECT 1;"

  register: warden_db_exists

  changed_when: False

- name: Create Warden database

  mysql_db:

    name: "{{ warden_db_name }}"

    state: present

- name: Create Warden database user

  mysql_user:

    name: "{{ warden_db_user }}"

    password: "{{ warden_db_password | mandatory }}"

    priv: "{{ warden_db_name }}.*:ALL"

    state: present

- name: Prepare initial Warden tables and fixtures

  mysql_db:

    name: "{{ warden_db_name }}"

    login_user: "{{ warden_db_user }}"

    login_password: "{{ warden_db_password | mandatory }}"

    state: import

    target: "{{ warden_server_dir_path }}/warden_3.0.sql"

  # Import is not idempotent, so run it only when db does not exist

  when: "warden_db_exists is defined and warden_db_exists.rc != 0 and warden_db_exists.stderr.find('ERROR 1049')"

- name: Install https config

  template:

    src: "etc/apache2/sites-available/warden.conf"

    dest: "/etc/apache2/sites-available/warden.conf"

    validate: "{{ ansible_apache_include_check.dest }} sites-enabled/ %s"

  notify: Reload Apache

- name: Activate http/s config

  command: a2ensite warden

  args:

    creates: /etc/apache2/sites-enabled/warden.conf

  notify: Reload Apache

- name: Deactivate default site

  command: a2dissite default-ssl

  args:

    removes: /etc/apache2/sites-enabled/default-ssl.conf

  notify: Reload Apache

<VirtualHost {{ warden_server_virtual_host }} >

    ServerAdmin {{ server_admin }}

    DocumentRoot /var/www

    <Directory />

            Options FollowSymLinks

            AllowOverride None

    </Directory>

    ServerName {{ warden_server_hostname }}

    ErrorLog /var/log/apache2/ssl_error_warden3.log

    CustomLog /var/log/apache2/ssl_access_warden3.log common

    SSLEngine on

    SSLVerifyClient optional

    SSLVerifyDepth 4

    SSLOptions +StdEnvVars +ExportCertData

    SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA

    SSLProtocol all -SSLv2 -SSLv3

    SSLHonorCipherOrder On

    SSLCertificateFile      /etc/ssl/certs/cert.pem

    SSLCertificateKeyFile   /etc/ssl/certs/key.pem

    SSLCACertificateFile    /etc/ssl/certs/root_cert_chain.pem

    Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains"

    WSGIScriptAlias /warden3 {{ warden_server_dir_path }}/warden_server.wsgi

    <Directory /opt/warden-server/warden_server.wsgi>

        Require all granted

    </Directory>

    WSGIScriptAlias /warden-ra {{ warden_ra_dir_path }}/warden_ra.wsgi

    <Directory /opt/warden-ra/warden_ra.wsgi>

        Require all granted

    </Directory>

</VirtualHost>

{

    "Log": {

        "type": "SysLogger",

        "facility": "local6",

        "level": "debug"

    },

    "Registry": {

        "type": "EjbcaRegistry",

        "url": "{{ warden_ra_ejbca_url }}

        "cert": "{{ warden_ra_cert }}",

        "key": "{{ warden_ra_key }}",

        "ca_name": "{{ warden_ra_ca_name }}",

        "certificate_profile_name": "{{ warden_ra_ejbca_certificate_profile }}",

        "end_entity_profile_name": "{{ warden_ra_ejbca_end_entity_profile }}",

        "subject_dn_template": "{{ warden_ra_subject_dn_template }}",

        "username_suffix": "{{ warden_ra_ejbca_username_suffix }}"

    }

}

{

    "Log": {

        "type": "SysLogger",

	"facility": "local7",

        "level": "debug"

    },

    "Auth": {

        "type": "X509MixMatchAuthenticator"

    },

    "Handler": {

        "send_events_limit": 1000,

        "get_events_limit": 1000,

        "description": "Warden 3 Server"

    },

    "DB": {

        "user": "warden",

        "password": "{{ warden_db_password }}",

        "dbname": "{{ warden_db_user }}"

    }

}

#!/usr/bin/env python

# -*- coding: utf-8 -*-

#

#  banner.py

#

#  Copyright 2015 CESNET z. s. p. o.

#  Author Jakub Cegan cegan@ics.muni.cz

#

#

def main(args):

    SVGNS = "http://www.w3.org/2000/svg"

    # We set up path and names

    banner_path = "/var/www/banner/"

    banner_name_cz = "banner-cz.svg"

    banner_name_en = "banner-en.svg"

    template_name  = "banner-template.svg"

    banners = [{'name': banner_name_en, 'database' : "Database Size:", 'events' : "Number of Events:", 'senders' : "Number of Senders:", 'receivers' : "Number of Receivers:", 'created' : "Banner Created:"}, {'name': banner_name_cz, 'database' : "Velikost databáze:", 'events' : "Suma všech událostí:", 'senders' : "Odesílající klienti:", 'receivers' : "Přijímající klienti:", 'created' : "Banner vytvořen:"}]

    # We have DB credentials

    host, database, user, password = sys.argv[1:]

    db = MySQLdb.connect(host = host, user = user, passwd = password, db = database)

    cursor = db.cursor()

    cursor.execute('SELECT count(*) AS reader_count FROM clients WHERE clients.read <> 0 AND clients.valid <> 0 AND clients.test = 0;')

    row = cursor.fetchone()

    receivers = str(row[0])

    #receivers = str(random.randint(0,100))

    cursor.execute('SELECT count(*) AS writer_count FROM clients WHERE clients.write <> 0 AND clients.valid <> 0 AND clients.test = 0;')

    row = cursor.fetchone()

    senders = str(row[0])

    #senders = str(random.randint(0,100))

    cursor.execute('SELECT sum(round(((data_length + index_length) / 1024 / 1024 / 1024), 2)) AS db_size FROM information_schema.tables WHERE table_schema = "warden3" AND table_name="events"')

    row = cursor.fetchone()

    database_size = str(row[0]) + ' GB'

    #database_size = str(random.randint(0,50)) + ' GB'

    cursor.execute('SELECT max(id) - min(id) AS event_count FROM events;')

    row = cursor.fetchone()

    events =  str(row[0])

    #events = str(random.randint(0,10000000))

    #cursor.execute('SELECT max(id) AS last_id FROM events;')

    #row = cursor.fetchone()

    #last_event =  str(row[0])

    time = datetime.datetime.today().strftime("%Y-%m-%dT%H:%M:%S%Z")

    for banner in banners:

      xml_data = etree.parse(template_name)

      # We search for element 'text' with id='tile_text' in SVG namespace

      # Fill texts

      find_text = etree.ETXPath("//{%s}text[@id='database-text']" % (SVGNS))

      find_text(xml_data)[0].text = banner['database']

      find_text = etree.ETXPath("//{%s}text[@id='events-text']" % (SVGNS))

      find_text(xml_data)[0].text = banner['events']

      find_text = etree.ETXPath("//{%s}text[@id='senders-text']" % (SVGNS))

      find_text(xml_data)[0].text = banner['senders']

      find_text = etree.ETXPath("//{%s}text[@id='receivers-text']" % (SVGNS))

      find_text(xml_data)[0].text = banner['receivers']

      find_text = etree.ETXPath("//{%s}text[@id='latest-text']" % (SVGNS))

      find_text(xml_data)[0].text = banner['created']

      # Insert values from database

      find_text = etree.ETXPath("//{%s}text[@id='database']" % (SVGNS))

      find_text(xml_data)[0].text = database_size

      find_text = etree.ETXPath("//{%s}text[@id='events']" % (SVGNS))

      find_text(xml_data)[0].text = events

      find_text = etree.ETXPath("//{%s}text[@id='senders']" % (SVGNS))

      find_text(xml_data)[0].text = senders

      find_text = etree.ETXPath("//{%s}text[@id='receivers']" % (SVGNS))

      find_text(xml_data)[0].text = receivers

      find_text = etree.ETXPath("//{%s}text[@id='latest']" % (SVGNS))

      find_text(xml_data)[0].text = time

      # Write edited svg into file

      new_svg = etree.tostring(xml_data)

      xml_data.write(banner_path + banner['name'])

    # We will not use pygal graphs for now

    #chart = pygal.StackedLine(fill=True, style=CleanStyle, x_label_rotation=40, tooltip_border_radius=10) # Setting style here is not necessary

    #chart.title = 'Events in last 24 hours'

    #chart.x_labels = map(lambda d: d.strftime('%H:%M:%S'), reversed([base - datetime.timedelta(hours=x) for x in range(0, 24)]))

    #chart.add('Event type  A', [random.randint(0,5000) for r in xrange(24)])

    #chart.add('Event type B', [random.randint(0,5000) for r in xrange(24)])

    #chart.add('Event type C', [random.randint(0,5000) for r in xrange(24)])

    #chart.add('Other types',  [random.randint(0,5000) for r in xrange(24)])

    #chart.render_to_file('chart_hours.svg') # Save the svg to a file

    #chart = pygal.StackedLine(fill=True, style=CleanStyle, x_label_rotation=40, tooltip_border_radius=10) # Setting style here is not necessary

    #chart.title = 'Events in last month'

    #chart.x_labels = map(lambda d: d.strftime('%d. %m. %Y'), reversed([base - datetime.timedelta(days=x) for x in range(0, 31)]))

    #chart.add('Event type A', [random.randint(0,5000) for r in xrange(31)])

    #chart.add('Event type B', [random.randint(0,5000) for r in xrange(31)])

    #chart.add('Event type C', [random.randint(0,5000) for r in xrange(31)])

    #chart.add('Other types',  [random.randint(0,5000) for r in xrange(31)])

    #chart.render_to_file('chart_month.svg') # Save the svg to a file

    return 0

if __name__ == '__main__':

    import sys

    import random

    import datetime

    import MySQLdb

    from lxml import etree

    #import pygal

    #from pygal.style import CleanStyle

    sys.exit(main(sys.argv))

# haas2warden

Warden connector for data of [CZ.NIC HaaS project](https://haas.nic.cz/).

It downloads daily [HaaS data dumps](https://haas.nic.cz/stats/export/),

converts them to IDEA messages and sends them to CESNET's Warden server.

It should be run from `cron` every night when data from previous day are

available (at 3:30).

The script just writes IDEA messages as files into a "filer" directory.

A _warden_filer_ daemon must be configured to pick up the messages

and send them to Warden server.

There is a systemd file which can be used to run the warden_filer.

# Run every day at 03:30

30 03 * * * haas2warden python3 /data/haas2warden/haas2warden.py -p /data/haas2warden/warden_filer/ -n org.example.ext.cznic_haas -t >> /data/haas2warden/haas2warden.log 2>&1 

#!/usr/bin/env python3

from gzip import decompress

from json import loads

from datetime import datetime, timedelta

import argparse

import logging

import uuid

import json

import os

import requests

data_date = datetime.date(datetime.utcnow()) - timedelta(days=1)

LOGFORMAT = "%(asctime)-15s,%(name)s [%(levelname)s] %(message)s"

LOGDATEFORMAT = "%Y-%m-%dT%H:%M:%S"

logging.basicConfig(level=logging.INFO, format=LOGFORMAT, datefmt=LOGDATEFORMAT)

logger = logging.getLogger('haas2warden')

def createIDEAFile(idea_id, idea_msg):

    """

    Creates file for IDEA message in .../tmp folder, then move it to .../incoming folder

    """

    tmp_dir_path = os.path.join(args.path, "tmp")

    idea_file_path = os.path.join(tmp_dir_path, idea_id+".idea")

    os.makedirs(tmp_dir_path, exist_ok=True)

    idea_file = open(idea_file_path, "w")

    idea_file.write(idea_msg)

    idea_file.close()

    incoming_dir_path = os.path.join(args.path, "incoming")

    incoming_file_path = os.path.join(incoming_dir_path,idea_id+".idea")

    os.makedirs(incoming_dir_path, exist_ok=True) 

    os.rename(idea_file_path,incoming_file_path)

def createIDEA(time, time_closed, ip, login_successful, commands):

    """

    Creates IDEA message 

    """