Compare revisions

    "certfile": "cert.pem",

    "certfile": "cert.pem",

    "keyfile": "key.pem",

    "keyfile": "key.pem",

    "filelog": {"level": "debug"},

    "filelog": {"level": "debug"},

    "name": "org.example.warden_client",

    "name": "org.example.warden_client"

    "secret": "ToP_SeCrEt"

}

}

  A. Introduction

  A. Introduction

  B. Dependencies

  B. Dependencies

  C. Installation

  C. Installation

  D. Configuration

  D. Upgrading

  E. Command line

  E. Configuration

  F. Command line

------------------------------------------------------------------------------

------------------------------------------------------------------------------

A. Introduction

A. Introduction

 2. Python modules

 2. Python modules

    python-mysqldb 5.3.3+

    python-mysqldb 5.3.3+ | python-psycopg2 2.8.6+

    python-m2crypto 0.20+

    python-m2crypto 0.20+

    jsonschema 2.4+

    jsonschema 2.4+

 3. Database

 3. Database

    MySQL | MariaDB >= 5.5

    MySQL | MariaDB >= 5.5 | PostgreSQL >= 13

------------------------------------------------------------------------------

------------------------------------------------------------------------------

C. Installation

C. Installation

   > GRANT ALL ON warden3.* TO `warden`@`localhost`;

   > GRANT ALL ON warden3.* TO `warden`@`localhost`;

   > FLUSH PRIVILEGES;

   > FLUSH PRIVILEGES;

   # psql

   > CREATE DATABASE warden3 ENCODING 'UTF-8';

   > CREATE ROLE "warden" LOGIN PASSWORD 'example';

   > GRANT ALL ON DATABASE "warden3" TO "warden";

 * Create necessary table structure

 * Create necessary table structure

   mysql -p -u warden warden3 < warden_3.0.sql

   mysql -p -u warden warden3 < warden_3.0_mysql.sql

   or

   psql -U warden -h localhost warden3 < warden_3.0_postgres.sql

 * Get up to date Idea schema

 * Get up to date Idea schema

   wget -O warden_server/idea.schema https://idea.cesnet.cz/_media/en/idea0.schema

   wget -O warden_server/idea.schema https://idea.cesnet.cz/_media/en/idea0.schema

 * Load category and tag maps into database (This step is optional for MySQL dbms)

   ./warden_server.py loadmaps

 * Enable mod_wsgi, mod_ssl, include Warden configuration

 * Enable mod_wsgi, mod_ssl, include Warden configuration

   This depends heavily on your distribution and Apache configuration.

   This depends heavily on your distribution and Apache configuration.

 * Reload Apache

 * Reload Apache

------------------------------------------------------------------------------

------------------------------------------------------------------------------

D. Configuration

D. Upgrading

 * When upgrading from Warden 3.0, run the following migrations:

 mysql -p -u warden warden3 < migrations/mysql_migration_0001_upgrade_add_manage_flag.sql

 or

 psql -U warden -h localhost warden3 < migrations/postgresql_migration_0001_upgrade_add_manage_flag.sql

------------------------------------------------------------------------------

E. Configuration

   Configuration is JSON object in file (warden_server.cfg by default),

   Configuration is JSON object in file (warden_server.cfg by default),

however, lines starting with "#" or "//" are allowed and will be ignored as

however, lines starting with "#" or "//" are allowed and will be ignored as

   Sections and their "type" objects can be:

   Sections and their "type" objects can be:

		Log: FileLogger, SysLogger

		Log: FileLogger, SysLogger

		DB: MySQL

		DB: MySQL, PostgreSQL

		Auth: X509Authenticator, X509NameAuthenticator,

		Auth: X509Authenticator, X509NameAuthenticator,

              X509MixMatchAuthenticator,PlainAuthenticator

              X509MixMatchAuthenticator,PlainAuthenticator

		Validator: JSONSchemaValidator, NoValidator

		Validator: JSONSchemaValidator, NoValidator

      retry_count: number of retries, defaults to 3

      retry_count: number of retries, defaults to 3

      event_size_limit: max size of serialized event, defaults to 5 MB

      event_size_limit: max size of serialized event, defaults to 5 MB

      catmap_filename: IDEA category mapping to database ids, defaults to

      catmap_filename: IDEA category mapping to database ids, defaults to

                       "catmap_mysql.json" at installation directory

                       "catmap_db.json" at installation directory

      tagmap_filename": IDEA node type mapping to database ids, defaults to

                        "tagmap_db.json" at installation directory

   PostgreSQL: database storage backend

      host: database server host, default "localhost"

      user: database user, default "warden"

      password: database password

      dbname: database name, default "warden3"

      port: database server port, default 5432

      retry_pause: retry in case of database errors, in seconds, defaults to 5

      retry_count: number of retries, defaults to 3

      event_size_limit: max size of serialized event, defaults to 5 MB

      catmap_filename: IDEA category mapping to database ids, defaults to

                       "catmap_db.json" at installation directory

      tagmap_filename": IDEA node type mapping to database ids, defaults to

      tagmap_filename": IDEA node type mapping to database ids, defaults to

                        "tagmap_mysql.json" at installation directory

                        "tagmap_db.json" at installation directory

   WardenHandler: Main Warden RPC worker

   WardenHandler: Main Warden RPC worker

      send_events_limit: max events sent in one bunch, defaults to 10000

      send_events_limit: max events sent in one bunch, defaults to 10000

      description: human readable description, sent in server info

      description: human readable description, sent in server info

------------------------------------------------------------------------------

------------------------------------------------------------------------------

E. Command line

F. Command line

   When run from command line, server offers set of commands and options for

   When run from command line, server offers set of commands and options for

runtime and database management. You can also use --help option for each

runtime and database management. You can also use --help option for each

                          [-s SECRET] [--note NOTE]

                          [-s SECRET] [--note NOTE]

                          [--valid | --novalid] [--read | --noread]

                          [--valid | --novalid] [--read | --noread]

                          [--nowrite | --write] [--nodebug | --debug]

                          [--nowrite | --write] [--nodebug | --debug]

                          [--test | --notest]

                          [--nomanage | --manage] [--test | --notest]

   Add new client registration entry.

   Add new client registration entry.

     --write

     --write

     --nodebug             client is allowed receive debug output (default - no)

     --nodebug             client is allowed receive debug output (default - no)

     --debug

     --debug

     --nomanage            client is allowed to manage other clients (default - no)

     --manage

     --test                client is yet in testing phase (default - yes)

     --test                client is yet in testing phase (default - yes)

     --notest

     --notest

                        [-r REQUESTOR] [-s SECRET] [--note NOTE]

                        [-r REQUESTOR] [-s SECRET] [--note NOTE]

                        [--valid | --novalid] [--read | --noread]

                        [--valid | --novalid] [--read | --noread]

                        [--nowrite | --write] [--nodebug | --debug]

                        [--nowrite | --write] [--nodebug | --debug]

                        [--test | --notest]

                        [--nomanage | --manage] [--test | --notest]

   Modify details of client registration entry.

   Modify details of client registration entry.

warden_server.py loadmaps [--help]

warden_server.py loadmaps [--help]

   Load 'categories' and 'tags' table from 'catmap_mysql.json' and

   Load 'categories' and 'tags' table from 'catmap_db.json' and

   'tagmap_mysql.json'. Note that this is NOT needed for server at all, load

   'tagmap_db.json'. Note also that previous content of both tables

   them into db at will, should you need to run your own specific SQL queries

   will be lost.

   on data directly. Note also that previous content of both tables will be

   lost.

------------------------------------------------------------------------------

------------------------------------------------------------------------------

Copyright (C) 2011-2015 Cesnet z.s.p.o

Copyright (C) 2011-2015 Cesnet z.s.p.o

* The test suite, just like the Warden Server, is compatible with both Python2

* The test suite, just like the Warden Server, is compatible with both Python2

  (tested on 2.7) and Python3 (tested on 3.6).

  (tested on 2.7) and Python3 (tested on 3.6).

* Just like Warden Server, the test suite requires a local MySQL installation.

* Just like Warden Server, the test suite requires a local MySQL or PostgreSQL

  installation.

* It is safe to run the test suite on a production system. For testing,

* It is safe to run the test suite on a production system. For testing,

  a database distinct from the default production one is used. Also, the user

  a database distinct from the default production one is used. Also, the user

  account used for accessing the testing database is set for local login only.

  account used for accessing the testing database is set for local login only.

Before running the tests (for the first time), a DB user with required rights

Before running the tests (for the first time), a DB user with required rights

must be created. An easy way to do it is:

must be created. An easy way to do it is:

	./test_warden_server.py --init

	./test_warden_server.py --init

This will prompt for MySQL root password.

This will prompt for the database administrator account ('root' for MySQL and

'postgres' for PostgreSQL) password. Please note that by default, the user

'postgres' can only be authenticated using the peer authentication method,

which requires that the script is run by the operating system user 'postgres'.

When this is the case, the password is not required.

Standard usage for testing:

Standard usage for testing:

	./test_warden_server.py

	./test_warden_server.py

Advanced usage:

Advanced usage:

	./test_warden_server.py --help

	./test_warden_server.py --help

	usage: test_warden_server.py [-h] [-i] [-n]

	usage: test_warden_server.py [-h] [-i] [-n] [-d {MySQL,PostgreSQL}]

	Warden3 Server Test Suite

	Warden3 Server Test Suite

	optional arguments:

	optional arguments:

	  -h, --help            show this help message and exit

	  -h, --help            show this help message and exit

	  -i, --init     Set up an user with rights to CREATE/DROP the

	  -d {MySQL,PostgreSQL}, --dbms {MySQL,PostgreSQL}

			 test database

	                        Database management system to use for

	  -n, --nopurge  Skip the database purge after running the tests

	                        testing

	  -i, --init            Set up an user with rights to

	                        CREATE/DROP the test database

	  -n, --nopurge         Skip the database purge after running

	                        the tests

Option -d (--dbms) sets the databse management system to use for testing.

If this option is not provided, MySQL is used as default.

Option -n (--nopurge) is meant for debugging purposes and test development, it

Option -n (--nopurge) is meant for debugging purposes and test development, it

keeps the test database around for inspection after running the tests.

keeps the test database around for inspection after running the tests.

ALTER TABLE clients DROP COLUMN manage;

ALTER TABLE clients ADD COLUMN manage tinyint UNSIGNED NOT NULL DEFAULT 0;

ALTER TABLE clients DROP COLUMN manage;

ALTER TABLE clients ADD COLUMN manage BOOL NOT NULL DEFAULT FALSE;

  `debug` tinyint UNSIGNED NOT NULL DEFAULT '0',

  `debug` tinyint UNSIGNED NOT NULL DEFAULT '0',

  `write` tinyint UNSIGNED NOT NULL DEFAULT '0',

  `write` tinyint UNSIGNED NOT NULL DEFAULT '0',

  `test` tinyint UNSIGNED NOT NULL DEFAULT '0',

  `test` tinyint UNSIGNED NOT NULL DEFAULT '0',

  `manage` tinyint UNSIGNED NOT NULL DEFAULT '0',

  PRIMARY KEY (`id`),

  PRIMARY KEY (`id`),

  KEY `clients_1` (`valid`, `secret`, `hostname`),

  KEY `clients_1` (`valid`, `secret`, `hostname`),

  KEY `clients_2` (`valid`, `name`)

  KEY `clients_2` (`valid`, `name`)

SET TimeZone='+00:00';

-- ---------------------------------------------------------

--

-- Database: "warden3"

--

-- --------------------------------------------------------

--

-- Table structure for table "categories"

--

CREATE TABLE IF NOT EXISTS "categories" (

  "id" int NOT NULL UNIQUE,

  "category" text NOT NULL,

  "subcategory" text DEFAULT NULL,

  "cat_subcat" text NOT NULL

);

CREATE INDEX IF NOT EXISTS "cat_sub" ON "categories" ("cat_subcat");

-- --------------------------------------------------------

--

-- Table structure for table "clients"

--

CREATE TABLE IF NOT EXISTS "clients" (

  "id" SERIAL PRIMARY KEY,

  "registered" timestamp NOT NULL DEFAULT '1970-01-01 00:00:00',

  "requestor" text NOT NULL,

  "hostname" text NOT NULL,

  "note" text NULL,

  "valid" boolean NOT NULL DEFAULT true,

  "name" text NOT NULL,

  "secret" text NULL,

  "read" boolean NOT NULL DEFAULT true,

  "debug" boolean NOT NULL DEFAULT false,

  "write" boolean NOT NULL DEFAULT false,

  "test" boolean NOT NULL DEFAULT false,

  "manage" boolean NOT NULL DEFAULT false

);

CREATE INDEX IF NOT EXISTS "clients_1" ON "clients" ("valid", "secret", "hostname");

CREATE INDEX IF NOT EXISTS "clients_2" ON "clients" ("valid", "name");

-- --------------------------------------------------------

--

-- Table structure for table "events"

--

CREATE TABLE IF NOT EXISTS "events" (

  "id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (MINVALUE 2),

  "received" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,

  "client_id" int NOT NULL REFERENCES "clients" ("id"),

  "data" bytea NOT NULL,

  "valid" boolean NOT NULL DEFAULT true

);

CREATE INDEX IF NOT EXISTS "id" ON "events" ("id", "client_id");

CREATE INDEX IF NOT EXISTS "received" ON "events" ("received");

-- --------------------------------------------------------

--

-- Table structure for table "event_category_mapping"

--

CREATE TABLE IF NOT EXISTS "event_category_mapping" (

  "event_id" bigint NOT NULL REFERENCES "events" ("id") ON DELETE CASCADE,

  "category_id" int NOT NULL,

  PRIMARY KEY ("event_id", "category_id"),

  CONSTRAINT "event_category_mapping_category_id_fk" FOREIGN KEY ("category_id") REFERENCES "categories" ("id")

);

-- --------------------------------------------------------

--

-- Table structure for table "last_events"

--

CREATE TABLE IF NOT EXISTS "last_events" (

  "id" SERIAL PRIMARY KEY,

  "client_id" int NOT NULL REFERENCES "clients" ("id"),

  "event_id" bigint REFERENCES "events" ("id"),

  "timestamp" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP

);

CREATE INDEX IF NOT EXISTS "client_id" ON "last_events" ("client_id", "event_id");

-- --------------------------------------------------------

--

-- Table structure for table "tags"

--

CREATE TABLE IF NOT EXISTS "tags" (

  "id" int NOT NULL UNIQUE,

  "tag" text NOT NULL

);

CREATE INDEX IF NOT EXISTS "id_tag_name" ON "tags" ("id", "tag");

CREATE INDEX IF NOT EXISTS "tag_name" ON "tags" ("tag");

-- --------------------------------------------------------

--

-- Table structure for table "event_tag_mapping"

--

CREATE TABLE IF NOT EXISTS "event_tag_mapping" (

  "event_id" bigint NOT NULL REFERENCES "events" ("id") ON DELETE CASCADE,

  "tag_id" int NOT NULL,

  PRIMARY KEY ("event_id", "tag_id"),

  CONSTRAINT "event_tag_mapping_tag_id_fk" FOREIGN KEY ("tag_id") REFERENCES "tags" ("id")

);